From 2a02a9125d2a044911a3250747dd20e95564ed7f Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 11:09:34 -0400 Subject: [PATCH 001/102] plugin/fs: initial experiment --- docs/experimental-features.md | 29 +++++++++++++++++ plugin/plugins/filesystem/filesystem.go | 42 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 plugin/plugins/filesystem/filesystem.go diff --git a/docs/experimental-features.md b/docs/experimental-features.md index f7121850e73..98a3a02a10d 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -30,6 +30,7 @@ the above issue. - [AutoRelay](#autorelay) - [TLS 1.3 Handshake](#tls-13-as-default-handshake-protocol) - [Strategic Providing](#strategic-providing) +- [IPFS filesystem API plugin](#filesystem-plugin) --- @@ -705,3 +706,31 @@ ipfs config --json Experimental.StrategicProviding true - [ ] provide roots - [ ] provide all - [ ] provide strategic + +## Filesystem API Plugin + +### State + +Experimental, not included by default. + +This daemon plugin wraps the IPFS node and exposes file system services over a multiaddr listener. +Currently we use the 9P2000.L protocol, and offer read-only support for `/ipfs` requests. With support for other (writable) systems coming next. +You may connect to this service using the `v9fs` client used in the Linux kernel. +`mount -t 9p -o trans=unix $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` +To configure the listening address and more, see the [package documentation](https://godoc.org/github.com/ipfs/go-ipfs/plugin/plugins/filesystem) + + +At the moment, [a modified 9P library](https://github.com/djdv/p9/tree/diff) is being used to connect golang clients to the service. +In the future, we plan to add a client library that wraps this and provides a standard client interface. + +### How to enable + +See the Plugin documentation [here]https://github.com/ipfs/go-ipfs/blob/master/docs/plugins.md#installing-plugins +You will likely want to add the plugin to the `go-ipfs` preload list +`filesystem github.com/ipfs/go-ipfs/plugin/plugins/filesystem *` + +### Road to being a real feature + +- [ ] needs testing + - Technically correct (`go test`, sharness, etc.) + - Practically/API correct (does this service provide clients with what they need) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go new file mode 100644 index 00000000000..41d130b04d8 --- /dev/null +++ b/plugin/plugins/filesystem/filesystem.go @@ -0,0 +1,42 @@ +package filesystem + +import ( + "fmt" + + plugin "github.com/ipfs/go-ipfs/plugin" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +// Plugins is an exported list of plugins that will be loaded by go-ipfs. +var Plugins = []plugin.Plugin{ + &FileSystemPlugin{}, +} + +// impl check +var _ plugin.PluginDaemon = (*FileSystemPlugin)(nil) + +type FileSystemPlugin struct{} + +func (*FileSystemPlugin) Name() string { + return "filesystem" +} + +func (*FileSystemPlugin) Version() string { + return "0.0.1" +} + +func (*FileSystemPlugin) Init() error { + return nil +} + +func (*FileSystemPlugin) Start(_ coreiface.CoreAPI) error { + fmt.Println("Initialising file system...") + //TODO: print mountpoints, sockets, etc. + fmt.Println("FS okay") + return nil +} + +func (*FileSystemPlugin) Close() error { + fmt.Println("Closing file system handles...") + return nil +} From ef757862ecf7c15a52dbb4cb2d800a04c336496f Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 26 Jul 2019 15:47:37 -0400 Subject: [PATCH 002/102] plugin/fs: Basic Readdir --- go.mod | 3 ++ go.sum | 4 ++ plugin/plugins/filesystem/filesystem.go | 63 +++++++++++++++++++++---- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index faaaf8eabd2..a120b5fced9 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,8 @@ require ( github.com/golangci/golangci-lint v1.17.1 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/golang-lru v0.5.3 + github.com/hugelgupf/p9 v0.0.0-20190806022330-7ba0920fba11 + github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 // indirect github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.0 @@ -112,6 +114,7 @@ require ( golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac // indirect golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect + google.golang.org/appengine v1.4.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools/gotestsum v0.3.4 ) diff --git a/go.sum b/go.sum index 35dac5e5a46..35c64b30e82 100644 --- a/go.sum +++ b/go.sum @@ -193,6 +193,10 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hugelgupf/p9 v0.0.0-20190806022330-7ba0920fba11 h1:WEZFyl8SZ3T4zQtVHKbAgx0mwdWTy+max9u9NjqcHjA= +github.com/hugelgupf/p9 v0.0.0-20190806022330-7ba0920fba11/go.mod h1:lQha5pHJCBOvLDb0yaewOSud/TGvhlp9jtfQuIbgzJU= +github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8= +github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 41d130b04d8..9e9a04ca7af 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -1,9 +1,14 @@ package filesystem import ( + "context" "fmt" + "net" + "github.com/hugelgupf/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" + fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" + logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" ) @@ -15,7 +20,12 @@ var Plugins = []plugin.Plugin{ // impl check var _ plugin.PluginDaemon = (*FileSystemPlugin)(nil) -type FileSystemPlugin struct{} +type FileSystemPlugin struct { + ctx context.Context + cancel context.CancelFunc + root string + addr string +} func (*FileSystemPlugin) Name() string { return "filesystem" @@ -25,18 +35,55 @@ func (*FileSystemPlugin) Version() string { return "0.0.1" } -func (*FileSystemPlugin) Init() error { +func (fs *FileSystemPlugin) Init() error { + fs.ctx, fs.cancel = context.WithCancel(context.Background()) return nil } -func (*FileSystemPlugin) Start(_ coreiface.CoreAPI) error { - fmt.Println("Initialising file system...") - //TODO: print mountpoints, sockets, etc. - fmt.Println("FS okay") +func init() { +} + +func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { + logger := logging.Logger("plugin/filesystem") + logger.Info("Initialising 9p resource server...") + + // construct 9p resource server / config + proto, addr, err := getAddr() + if err != nil { + logger.Errorf("9P server error: %s", err) + return err + } + + p9pFSS, err := fsnodes.NewRoot(fs.ctx, core, logger) + if err != nil { + logger.Errorf("9P server error: %s", err) + return err + } + + // Bind and listen on the socket. + serverSocket, err := net.Listen(proto, addr) + if err != nil { + logger.Errorf("9P server error: %s", err) + return err + } + + // Run the server. + s := p9.NewServer(p9pFSS) + go func() { + if err := s.Serve(serverSocket); err != nil { + logger.Errorf("9P server error: %s", err) + return + } + }() + + //TODO: prettier print; mountpoints, socket/addr, confirm start actually succeeded, etc. + logger.Infof("9P service started on %s", addr) return nil } -func (*FileSystemPlugin) Close() error { - fmt.Println("Closing file system handles...") +func (fs *FileSystemPlugin) Close() error { + //TODO: fmt.Println("Closing file system handles...") + fmt.Println("9P server requested to close") + fs.cancel() return nil } From 43a38b67d13b220147e4b346485e9484ce5e813d Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 14 Aug 2019 10:19:50 -0400 Subject: [PATCH 003/102] plugin/fs: Basic ReadAt --- go.mod | 2 + go.sum | 9 +- plugin/plugins/filesystem/cmd/main.go | 114 ++++++++++++ plugin/plugins/filesystem/nodes/ipfs.go | 175 ++++++++++++++++++ plugin/plugins/filesystem/nodes/pinfs.go | 118 +++++++++++++ plugin/plugins/filesystem/nodes/root.go | 153 ++++++++++++++++ plugin/plugins/filesystem/nodes/utils.go | 214 +++++++++++++++++++++++ 7 files changed, 783 insertions(+), 2 deletions(-) create mode 100644 plugin/plugins/filesystem/cmd/main.go create mode 100644 plugin/plugins/filesystem/nodes/ipfs.go create mode 100644 plugin/plugins/filesystem/nodes/pinfs.go create mode 100644 plugin/plugins/filesystem/nodes/root.go create mode 100644 plugin/plugins/filesystem/nodes/utils.go diff --git a/go.mod b/go.mod index a120b5fced9..c6e4023ea93 100644 --- a/go.mod +++ b/go.mod @@ -121,4 +121,6 @@ require ( replace github.com/golangci/golangci-lint => github.com/mhutchinson/golangci-lint v1.17.2-0.20190819125825-d18f2136e32b +replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20190814142810-aa6caf006bd3 + go 1.12 diff --git a/go.sum b/go.sum index 35c64b30e82..81e2cb201f6 100644 --- a/go.sum +++ b/go.sum @@ -20,7 +20,9 @@ github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3o github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -70,6 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/djdv/p9 v0.0.0-20190814142810-aa6caf006bd3 h1:O4Sy8ycHbPfhddtyipjq8ufJJPdUjouGFAGfkr24m/c= +github.com/djdv/p9 v0.0.0-20190814142810-aa6caf006bd3/go.mod h1:akLqomEjJslbyK2h+TEXmVabYavEatmOdB6iektRfEM= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= @@ -193,8 +197,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hugelgupf/p9 v0.0.0-20190806022330-7ba0920fba11 h1:WEZFyl8SZ3T4zQtVHKbAgx0mwdWTy+max9u9NjqcHjA= -github.com/hugelgupf/p9 v0.0.0-20190806022330-7ba0920fba11/go.mod h1:lQha5pHJCBOvLDb0yaewOSud/TGvhlp9jtfQuIbgzJU= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= @@ -865,6 +867,8 @@ golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190730183949-1393eb018365 h1:SaXEMXhWzMJThc05vu6uh61Q245r4KaWMrsTedk0FDc= +golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -895,6 +899,7 @@ google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/plugin/plugins/filesystem/cmd/main.go b/plugin/plugins/filesystem/cmd/main.go new file mode 100644 index 00000000000..2482825d163 --- /dev/null +++ b/plugin/plugins/filesystem/cmd/main.go @@ -0,0 +1,114 @@ +package main + +import ( + "fmt" + "log" + "net" + "strings" + + "github.com/hugelgupf/p9/p9" + "github.com/ipfs/go-ipfs/plugin/plugins/filesystem" +) + +func main() { + + log.SetFlags(log.Flags() | log.Lshortfile) + + conn, err := net.Dial(filesystem.DefaultProtocol, filesystem.DefaultAddress) + if err != nil { + log.Fatal(err) + } + + client, err := p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) + if err != nil { + log.Fatal(err) + } + defer client.Close() + log.Printf("Connected to server supporting version:\n%v\n\n", client.Version()) + + rootRef, err := client.Attach("") + if err != nil { + log.Fatal(err) + } + log.Printf("Attached to root:\n%#v\n\n", rootRef) + + readDirDBG("/", rootRef) + readDirDBG("/ipfs", rootRef) + readDirDBG("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", rootRef) + readDBG("/ipfs/QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB", rootRef) + + client.Close() +} + +func readDirDBG(path string, fsRef p9.File) { + components := strings.Split(strings.TrimPrefix(path, "/"), "/") + if len(components) == 1 && components[0] == "" { + components = nil + } + + fmt.Printf("components: %#v\n", components) + + qids, targetRef, err := fsRef.Walk(components) + if err != nil { + log.Fatal(err) + } + log.Printf("walked to %q :\nQIDs:%v, FID:%v\n\n", path, qids, targetRef) + + refQid, ioUnit, err := targetRef.Open(0) + if err != nil { + log.Fatal(err) + } + log.Printf("%q Opened:\nQID:%v, iounit:%v\n\n", path, refQid, ioUnit) + + ents, err := targetRef.Readdir(0, ^uint32(0)) + if err != nil { + log.Fatal(err) + } + + log.Printf("%q Readdir:\n[%d]%v\n\n", path, len(ents), ents) + if err = targetRef.Close(); err != nil { + log.Fatal(err) + } + + log.Printf("%q closed:\n%#v\n\n", path, targetRef) +} + +func readDBG(path string, fsRef p9.File) { + components := strings.Split(strings.TrimPrefix(path, "/"), "/") + if len(components) == 1 && components[0] == "" { + components = nil + } + + fmt.Printf("components: %#v\n", components) + + qids, targetRef, err := fsRef.Walk(components) + if err != nil { + log.Fatal(err) + } + log.Printf("walked to %q :\nQIDs:%v, FID:%v\n\n", path, qids, targetRef) + + _, _, attr, err := targetRef.GetAttr(p9.AttrMask{Size: true}) + if err != nil { + log.Fatal(err) + } + log.Printf("Getattr for %q :\n%v\n\n", path, attr) + + refQid, ioUnit, err := targetRef.Open(0) + if err != nil { + log.Fatal(err) + } + log.Printf("%q Opened:\nQID:%v, iounit:%v\n\n", path, refQid, ioUnit) + + buf := make([]byte, attr.Size) + readBytes, err := targetRef.ReadAt(buf, 0) + if err != nil { + log.Fatal(err) + } + + log.Printf("%q Read:\n[%d bytes]\n%s\n\n", path, readBytes, buf) + if err = targetRef.Close(); err != nil { + log.Fatal(err) + } + + log.Printf("%q closed:\n%#v\n\n", path, targetRef) +} diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go new file mode 100644 index 00000000000..86b7bf61eb2 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -0,0 +1,175 @@ +package fsnodes + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/hugelgupf/p9/p9" + files "github.com/ipfs/go-ipfs-files" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" + corepath "github.com/ipfs/interface-go-ipfs-core/path" +) + +type IPFS struct { + IPFSBase +} + +//TODO: [review] check fields; better wrappers around inheritance init, etc. +func initIPFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { + id := &IPFS{ + IPFSBase: IPFSBase{ + core: core, + Base: Base{ + Logger: logger, + Ctx: ctx, + Qid: p9.QID{ + Type: p9.TypeDir, + Version: 1, + Path: uint64(pIPFSRoot)}}}} + return id +} + +func (id *IPFS) Attach() (p9.File, error) { + id.Logger.Debugf("ID Attach") + //TODO: check core connection here + return id, nil +} + +func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + id.Logger.Debugf("ID GetAttr") + qid := p9.QID{ + Type: p9.TypeDir, + Version: 1} + + var attr p9.Attr + var attrMask p9.AttrMask + + //TODO: make this impossible; initalize a valid CID for roots + if id.Path != nil { + qid.Path = cidToQPath(id.Path.Cid()) + if err := coreGetAttr(id.Ctx, &attr, &attrMask, id.core, id.Path); err != nil { + return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err + } + } else { + qid.Path = uint64(pIPFSRoot) + attr.Mode = p9.ModeDirectory | p9.Read | p9.Exec + attr.RDev, attrMask.RDev = dIPFS, true + } + + return qid, attrMask, attr, nil +} + +func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { + id.Logger.Debugf("ID Walk names %v", names) + id.Logger.Debugf("ID Walk myself: %v", id.Qid) + + if doClone(names) { + id.Logger.Debugf("ID Walk cloned") + return []p9.QID{id.Qid}, id, nil + } + + var ipfsPath corepath.Path + var resolvedPath corepath.Resolved + var err error + qids := make([]p9.QID, 0, len(names)) + + for _, name := range names { + //TODO: convert this into however Go deals with do;while + if ipfsPath == nil { + ipfsPath = corepath.New("/ipfs/" + name) + } else { + ipfsPath = corepath.Join(ipfsPath, name) + } + //TODO: timerctx; don't want to hang forever + resolvedPath, err = id.core.ResolvePath(id.Ctx, ipfsPath) + if err != nil { + return nil, nil, err + } + + //XXX: generate QID more directly + dirEnt := &p9.Dirent{} + if err = coreStat(id.Ctx, dirEnt, id.core, ipfsPath); err != nil { + return nil, nil, err + } + + qids = append(qids, dirEnt.QID) + } + + id.Path = resolvedPath + + id.Logger.Debugf("ID Walk reg ret %v, %v", qids, id) + return qids, id, nil +} + +func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + id.Logger.Debugf("ID Open") + return id.Qid, 0, nil +} + +func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + id.Logger.Debugf("ID Readdir") + + entChan, err := coreLs(id.Ctx, id.Path, id.core) + if err != nil { + return nil, err + } + + var ents []p9.Dirent + + var off uint64 = 0 + for ent := range entChan { + id.Logger.Debugf("got ent: %v\n", ent) + if ent.Err != nil { + return nil, err + } + off++ + + nineEnt := coreEntTo9Ent(ent) + nineEnt.Offset = off + ents = append(ents, nineEnt) + + if uint32(len(ents)) == count { + break + } + } + + id.Logger.Debugf("ID Readdir returning ents:%v", ents) + return ents, nil +} + +func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { + id.Logger.Debugf("ID ReadAt") + const replaceMe = -1 //TODO: proper error codes + + apiNode, err := id.core.Unixfs().Get(id.Ctx, id.Path) + if err != nil { + return replaceMe, err + } + + fIo, ok := apiNode.(files.File) + if !ok { + return replaceMe, fmt.Errorf("%q is not a file", id.Path.String()) + } + + if fileBound, err := fIo.Size(); err == nil { + if int64(offset) >= fileBound { + return replaceMe, errors.New("read offset extends past end of file") + } + } + + if offset != 0 { + _, err = fIo.Seek(int64(offset), io.SeekStart) + if err != nil { + return replaceMe, fmt.Errorf("Read - seek error: %s", err) + } + } + + readBytes, err := fIo.Read(p) + if err != nil && err != io.EOF { + return replaceMe, err + } + return readBytes, nil +} diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go new file mode 100644 index 00000000000..3ef1ac168d1 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -0,0 +1,118 @@ +package fsnodes + +import ( + "context" + gopath "path" + + "github.com/hugelgupf/p9/p9" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" + coreoptions "github.com/ipfs/interface-go-ipfs-core/options" +) + +type PinFS struct { + IPFSBase +} + +//TODO: [review] check fields +func initPinFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { + pd := &PinFS{ + IPFSBase: IPFSBase{ + core: core, + Base: Base{ + Logger: logger, + Ctx: ctx, + Qid: p9.QID{ + Type: p9.TypeDir, + Version: 1, + Path: uint64(pPinRoot)}}}} + return pd +} + +func (pd *PinFS) Attach() (p9.File, error) { + pd.Logger.Debugf("PD Attach") + //TODO: check core connection here + return pd, nil +} + +func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + pd.Logger.Debugf("PD GetAttr") + qid := p9.QID{ + Type: p9.TypeDir, + Version: 1, + Path: uint64(pPinRoot), + } + + //TODO: [metadata] quick hack; revise + attr := p9.Attr{ + Mode: p9.ModeDirectory, + RDev: dMemory, + } + + attrMask := p9.AttrMask{ + Mode: true, + RDev: true, + } + + return qid, attrMask, attr, nil +} + +func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { + pd.Logger.Debugf("PD Walk names %v", names) + pd.Logger.Debugf("PD Walk myself: %v", pd.Qid) + + if doClone(names) { + pd.Logger.Debugf("PD Walk cloned") + return []p9.QID{pd.Qid}, pd, nil + } + +//NOTE: if doClone is false, it implies len(names) > 0 + var tailFile p9.File = pd + var subQids []p9.QID + qids := make([]p9.QID, 0, len(names)) + + ipfsDir, err := initIPFS(pd.Ctx, pd.core, pd.Logger).Attach() + if err != nil { + return nil,nil,err + } + if subQids, tailFile, err = ipfsDir.Walk(names); err != nil { + return nil, nil, err + } + qids = append(qids, subQids...) + pd.Logger.Debugf("PD Walk reg ret %v, %v", qids, tailFile) + return qids, tailFile, nil +} + +func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + pd.Logger.Debugf("PD Open") + return pd.Qid, 0, nil +} + +func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + pd.Logger.Debugf("PD Readdir") + + pins, err := pd.core.Pin().Ls(pd.Ctx, coreoptions.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + + if count < uint32(len(pins)) { + pins = pins[:count] + } + ents := make([]p9.Dirent, 0, len(pins)) + + for i, pin := range pins { + dirEnt := &p9.Dirent{ + Name: gopath.Base(pin.Path().String()), + Offset: uint64(i), + } + + if err = coreStat(pd.Ctx, dirEnt, pd.core, pin.Path()); err != nil { + return nil, err + } + ents = append(ents, *dirEnt) + } + + pd.Logger.Debugf("PD Readdir returning ents:%v", ents) + return ents, err +} diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go new file mode 100644 index 00000000000..ed30d2f3cd6 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/root.go @@ -0,0 +1,153 @@ +package fsnodes + +import ( + "context" + "fmt" + + "github.com/hugelgupf/p9/p9" + cid "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" + corepath "github.com/ipfs/interface-go-ipfs-core/path" +) + +var _ p9.File = (*RootIndex)(nil) + +//TODO: this shouldn't be necessary, consider how best to path the virtual roots and maintain compat with IPFS core +func newRootPath() corepath.Resolved { + return rootNode("/") +} + +type rootNode string + +func (rn rootNode) String() string { return string(rn) } +func (rootNode) Namespace() string { return "virtual" } +func (rootNode) Mutable() bool { return true } +func (rootNode) IsValid() error { return nil } +func (rootNode) Cid() cid.Cid { return cid.Cid{} } +func (rootNode) Root() cid.Cid { return cid.Cid{} } +func (rootNode) Remainder() string { return "" } + +// + +type RootIndex struct { + IPFSBase +} + +func NewRoot(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) (*RootIndex, error) { + ri := &RootIndex{ + IPFSBase: IPFSBase{ + Base: Base{ + Ctx: ctx, + Logger: logger, + Qid: p9.QID{ + Type: p9.TypeDir, + Version: 1, + Path: uint64(pVirtualRoot)}}, + Path: newRootPath(), + core: core, + }, + } + + return ri, nil +} + +func (ri *RootIndex) Attach() (p9.File, error) { + ri.Logger.Debugf("RI Attach") + return ri, nil +} + +func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + ri.Logger.Debugf("RI GetAttr") + qid := p9.QID{ + Type: p9.TypeDir, + Version: 1, + Path: uint64(pVirtualRoot), + } + + //ri.Logger.Errorf("RI mask: %v", req) + + //TODO: [metadata] quick hack; revise + attr := p9.Attr{ + Mode: p9.ModeDirectory, + RDev: dMemory, + } + + attrMask := p9.AttrMask{ + Mode: true, + RDev: true, + } + + return qid, attrMask, attr, nil +} +func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { + ri.Logger.Debugf("RI Walk names %v", names) + ri.Logger.Debugf("RI Walk myself: %v", ri.Qid) + + if doClone(names) { + ri.Logger.Debugf("RI Walk cloned") + return []p9.QID{ri.Qid}, ri, nil + } + + //NOTE: if doClone is false, it implies len(names) > 0 + + var tailFile p9.File + var subQids []p9.QID + qids := make([]p9.QID, 0, len(names)) + + switch names[0] { + case "ipfs": + pinDir, err := initPinFS(ri.Ctx, ri.core, ri.Logger).Attach() + if err != nil { + return nil, nil, err + } + if subQids, tailFile, err = pinDir.Walk(names[1:]); err != nil { + return nil, nil, err + } + qids = append(qids, subQids...) + default: + return nil, nil, fmt.Errorf("%q is not provided by us", names[0]) //TODO: Err vars + } + ri.Logger.Debugf("RI Walk reg ret %v, %v", qids, tailFile) + return qids, tailFile, nil +} + +// TODO: check specs for directory iounit size, +// if it's undefined we should repurpose it to return the count to the client +func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + ri.Logger.Debugf("RI Open") + return ri.Qid, 0, nil +} + +func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + ri.Logger.Debugf("RI Readdir") + + subsystems := [...]indexPath{pIPFSRoot} + + ents := make([]p9.Dirent, 0, len(subsystems)) + for i, subsystem := range subsystems { + version := 1 //TODO: generate dynamically + + // TODO: allocate root ents array elsewhere + // modify dynamic fields only here + + ents = append(ents, p9.Dirent{ + Name: subsystem.String(), + Offset: uint64(i), + Type: p9.TypeDir, //TODO: resolve dynamically + QID: p9.QID{ + Type: p9.TypeDir, + Version: uint32(version), //TODO: maintain version + Path: uint64(subsystem), + }, + }) + } + + //TODO: [all instances; double check] the underlying API may already have do this after we return the slice to it + if uint32(len(ents)) > count { + ents = ents[:count] + } + + ri.Logger.Debugf("RI Readdir returning [%d]ents:%v", len(ents), ents) + return ents, nil +} diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go new file mode 100644 index 00000000000..9c8c0ca74f3 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -0,0 +1,214 @@ +package fsnodes + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "time" + + "github.com/hugelgupf/p9/p9" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-unixfs" + coreiface "github.com/ipfs/interface-go-ipfs-core" + coreoptions "github.com/ipfs/interface-go-ipfs-core/options" + corepath "github.com/ipfs/interface-go-ipfs-core/path" +) + +func doClone(names []string) bool { + l := len(names) + if l < 1 { + return true + } + //TODO: double check the spec to make sure dot handling is correct + if pc := names[0]; l == 1 && (pc == ".." || pc == "." || pc == "") { + return true + } + return false +} + +//TODO: rename this and/or extend +// it only does some of the stat and not what people probably expect +func coreStat(ctx context.Context, dirEnt *p9.Dirent, core coreiface.CoreAPI, path corepath.Path) (err error) { + var ipldNode ipld.Node + if ipldNode, err = core.ResolveNode(ctx, path); err != nil { + return + } + err = ipldStat(dirEnt, ipldNode) + return +} + +//TODO: consider how we want to use AttrMask +// instead of filling it we can use it to only populate requested fields (as is intended) +func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask *p9.AttrMask, core coreiface.CoreAPI, path corepath.Path) (err error) { + ipldNode, err := core.ResolveNode(ctx, path) + if err != nil { + return err + } + ufsNode, err := unixfs.ExtractFSNode(ipldNode) + if err != nil { + return err + } + attr.Mode = p9.Read | p9.Exec + switch t := ufsNode.Type(); t { + case unixfs.TFile: + attr.Mode |= p9.ModeRegular + case unixfs.TDirectory, unixfs.THAMTShard: + attr.Mode |= p9.ModeDirectory + case unixfs.TSymlink: + attr.Mode |= p9.ModeSymlink + default: + return fmt.Errorf("unexpected node type %d", t) + } + attrMask.Mode = true + + if bs := ufsNode.BlockSizes(); len(bs) != 0 { + attr.BlockSize = bs[0] //NOTE: this value is to be used as a hint only; subsequent child block size may differ + } + + attr.Size, attrMask.Size = ufsNode.FileSize(), true + + switch path.Namespace() { + case "ipfs": + attr.RDev, attrMask.RDev = dIPFS, true + //case "ipns": + //attr.RDev, attrMask.RDev = dIPNS, true + //etc. + } + + //TODO: rdev; switch off namespace => dIpfs, dIpns, etc. + //Blocks + return nil +} + +func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { + ufsNode, err := unixfs.ExtractFSNode(node) + + if err != nil { + return err + } + + nodeType := coreTypeToQtype(coreiface.FileType(ufsNode.Type())) //eww + + dirEnt.Type = nodeType + dirEnt.QID.Type = nodeType + dirEnt.QID.Version = 1 + dirEnt.QID.Path = cidToQPath(node.Cid()) + + return nil +} + +func cidToQPath(cid cid.Cid) (path uint64) { + buf := bytes.NewReader(cid.Bytes()) + err := binary.Read(buf, binary.LittleEndian, &path) + if err != nil { + panic(err) + } + return +} + +func coreLs(ctx context.Context, corePath corepath.Path, core coreiface.CoreAPI) (<-chan coreiface.DirEntry, error) { + + //FIXME: asyncContext hangs on reset + //asyncContext := deriveTimerContext(ctx, 10*time.Second) + asyncContext := ctx + + coreChan, err := core.Unixfs().Ls(asyncContext, corePath, coreoptions.Unixfs.ResolveChildren(false)) + if err != nil { + //asyncContext.Cancel() + return nil, err + } + + oStat, err := core.Object().Stat(asyncContext, corePath) + if err != nil { + return nil, err + } + + relayChan := make(chan coreiface.DirEntry) + go func() { + //defer asyncContext.Cancel() + defer close(relayChan) + + for i := 0; i != oStat.NumLinks; i++ { + select { + case <-asyncContext.Done(): + return + case msg, ok := <-coreChan: + if !ok { + return + } + if msg.Err != nil { + relayChan <- msg + return + } + relayChan <- msg + //asyncContext.Reset() //reset timeout for each entry we receive successfully + } + } + }() + + return relayChan, err +} + +func coreTypeToQtype(ct coreiface.FileType) p9.QIDType { + switch ct { + // case coreiface.TDirectory, unixfs.THAMTShard: Should we account for this? + case coreiface.TDirectory: + return p9.TypeDir + case coreiface.TSymlink: + return p9.TypeSymlink + default: + return p9.TypeRegular + } +} + +func coreEntTo9Ent(coreEnt coreiface.DirEntry) p9.Dirent { + entType := coreTypeToQtype(coreEnt.Type) + + return p9.Dirent{ + Name: coreEnt.Name, + Type: entType, + QID: p9.QID{ + Type: entType, + Version: 1, + Path: cidToQPath(coreEnt.Cid)}} +} + +type timerContextActual struct { + context.Context + cancel context.CancelFunc + timer time.Timer + grace time.Duration +} + +func (tctx timerContextActual) Reset() { + if !tctx.timer.Stop() { + <-tctx.timer.C + } + tctx.timer.Reset(tctx.grace) +} + +func (tctx timerContextActual) Cancel() { + tctx.cancel() + if !tctx.timer.Stop() { + <-tctx.timer.C + } +} + +type timerContext interface { + context.Context + Reset() + Cancel() +} + +func deriveTimerContext(ctx context.Context, grace time.Duration) timerContext { + asyncContext, cancel := context.WithCancel(ctx) + timer := time.AfterFunc(grace, cancel) + tctx := timerContextActual{Context: asyncContext, + cancel: cancel, + grace: grace, + timer: *timer} + + return tctx +} From 7a8053233a40dfc9e97e2829d85c1f775006a556 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 15 Aug 2019 09:22:53 -0400 Subject: [PATCH 004/102] plugin/fs: Fix a bunch of path related issues --- go.mod | 2 +- go.sum | 6 +- plugin/plugins/filesystem/nodes/base.go | 113 +++++++++++++++++++++ plugin/plugins/filesystem/nodes/ipfs.go | 95 +++++++++--------- plugin/plugins/filesystem/nodes/pinfs.go | 55 +++++------ plugin/plugins/filesystem/nodes/root.go | 120 +++++++++++------------ plugin/plugins/filesystem/nodes/utils.go | 72 +++++++++++--- plugin/plugins/filesystem/utils.go | 30 ++++++ 8 files changed, 338 insertions(+), 155 deletions(-) create mode 100644 plugin/plugins/filesystem/nodes/base.go create mode 100644 plugin/plugins/filesystem/utils.go diff --git a/go.mod b/go.mod index c6e4023ea93..5e6224a9f2c 100644 --- a/go.mod +++ b/go.mod @@ -121,6 +121,6 @@ require ( replace github.com/golangci/golangci-lint => github.com/mhutchinson/golangci-lint v1.17.2-0.20190819125825-d18f2136e32b -replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20190814142810-aa6caf006bd3 +replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20190822034506-af913a6d8082 go 1.12 diff --git a/go.sum b/go.sum index 81e2cb201f6..4284d34d200 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/p9 v0.0.0-20190814142810-aa6caf006bd3 h1:O4Sy8ycHbPfhddtyipjq8ufJJPdUjouGFAGfkr24m/c= -github.com/djdv/p9 v0.0.0-20190814142810-aa6caf006bd3/go.mod h1:akLqomEjJslbyK2h+TEXmVabYavEatmOdB6iektRfEM= +github.com/djdv/p9 v0.0.0-20190822034506-af913a6d8082 h1:Ml5E7tP7gL/A9CdCc7iRZf5cNIWFVDc9WeMYMP3F4ck= +github.com/djdv/p9 v0.0.0-20190822034506-af913a6d8082/go.mod h1:akLqomEjJslbyK2h+TEXmVabYavEatmOdB6iektRfEM= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= @@ -890,8 +890,6 @@ golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go new file mode 100644 index 00000000000..191c8a09aa7 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/base.go @@ -0,0 +1,113 @@ +package fsnodes + +import ( + "context" + "io" + "sync" + + "github.com/hugelgupf/p9/p9" + "github.com/hugelgupf/p9/unimplfs" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" + corepath "github.com/ipfs/interface-go-ipfs-core/path" +) + +type FSNode interface { + corepath.Path + //RWLocker + Stat() (p9.QID, error) +} + +const ( //type + tVirtual = iota + tIPFS +) +const ( //device + dMemory = iota + dIPFS +) + +const ( //FS namespaces + nRoot = "root" +) +const ( //9P paths + pVirtualRoot uint64 = iota + //pIPFSRoot + //pIpnsRoot + pPinRoot + //pKeyRoot +) + +var _ p9.File = (*Base)(nil) + +//var _ FSNode = (*Base)(nil) + +//TODO: docs +// Base is a foundational node, intended to be embedded/extended +type Base struct { + unimplfs.NoopFile + p9.DefaultWalkGetAttr + Qid p9.QID + meta p9.Attr + + Ctx context.Context + Logger logging.EventLogger +} + +type IPFSBase struct { + Base + + //Path corepath.Path + Path corepath.Resolved + core coreiface.CoreAPI +} + +type ResourceRef struct { + sync.Mutex + //meta p9p.Dir + closer io.Closer +} + +/* TODO: [current] master attach(with the name) { + global staticRoot = &root{} + reference := staticRoot + + } + + walk { + switch names[0] { + if "ipfs" { + return fs.roots[ipfs].walk(names[1:]) + } + } + } +} +*/ + +/* +func (bn *Base) Stat(ctx context.Context) (p9.QID, error) { + var ( + qid p9.QID + fi os.FileInfo + err error + ) + + // Stat the file. + if l.file != nil { + fi, err = l.file.Stat() + } else { + fi, err = os.Lstat(l.path) + } + if err != nil { + log.Printf("error stating %#v: %v", l, err) + return qid, nil, err + } + + // Construct the QID type. + qid.Type = p9.ModeFromOS(fi.Mode()).QIDType() + + // Save the path from the Ino. + qid.Path = fi.Sys().(*syscall.Stat_t).Ino + return qid, fi, nil +} +*/ diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 86b7bf61eb2..74f1f75c29e 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -2,7 +2,6 @@ package fsnodes import ( "context" - "errors" "fmt" "io" @@ -21,14 +20,14 @@ type IPFS struct { func initIPFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { id := &IPFS{ IPFSBase: IPFSBase{ + Path: newRootPath("/ipfs"), core: core, Base: Base{ Logger: logger, Ctx: ctx, - Qid: p9.QID{ - Type: p9.TypeDir, - Version: 1, - Path: uint64(pIPFSRoot)}}}} + Qid: p9.QID{Type: p9.TypeDir}}}} + + id.Qid.Path = cidToQPath(id.Path.Cid()) return id } @@ -40,68 +39,62 @@ func (id *IPFS) Attach() (p9.File, error) { func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { id.Logger.Debugf("ID GetAttr") - qid := p9.QID{ - Type: p9.TypeDir, - Version: 1} + id.Logger.Debugf("ID GetAttr path: %v", id.Path) + + if id.Path.Namespace() == nRoot { + _, mask := defaultRootAttr() + return id.Qid, mask, id.meta, nil + } var attr p9.Attr var attrMask p9.AttrMask + qid := p9.QID{Path: cidToQPath(id.Path.Cid())} - //TODO: make this impossible; initalize a valid CID for roots - if id.Path != nil { - qid.Path = cidToQPath(id.Path.Cid()) - if err := coreGetAttr(id.Ctx, &attr, &attrMask, id.core, id.Path); err != nil { - return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err - } - } else { - qid.Path = uint64(pIPFSRoot) - attr.Mode = p9.ModeDirectory | p9.Read | p9.Exec - attr.RDev, attrMask.RDev = dIPFS, true + if err := coreGetAttr(id.Ctx, &attr, &attrMask, id.core, id.Path); err != nil { + return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err } - return qid, attrMask, attr, nil } func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { id.Logger.Debugf("ID Walk names %v", names) - id.Logger.Debugf("ID Walk myself: %v", id.Qid) + id.Logger.Debugf("ID Walk myself: %s:%v", id.Path, id.Qid) if doClone(names) { id.Logger.Debugf("ID Walk cloned") return []p9.QID{id.Qid}, id, nil } - var ipfsPath corepath.Path - var resolvedPath corepath.Resolved - var err error qids := make([]p9.QID, 0, len(names)) + //TODO: [spec check] make sure we're not messing things up by doing this instead of mutating + walkedNode := &IPFS{} // operate on a copy + *walkedNode = *id + + var err error for _, name := range names { - //TODO: convert this into however Go deals with do;while - if ipfsPath == nil { - ipfsPath = corepath.New("/ipfs/" + name) - } else { - ipfsPath = corepath.Join(ipfsPath, name) - } //TODO: timerctx; don't want to hang forever - resolvedPath, err = id.core.ResolvePath(id.Ctx, ipfsPath) + if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, corepath.Join(walkedNode.Path, name)); err != nil { + return qids, nil, err + } + + ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) if err != nil { - return nil, nil, err + return qids, nil, err } - //XXX: generate QID more directly + //TODO: this is too opague; we want core path => qid, dirent isn't necessary dirEnt := &p9.Dirent{} - if err = coreStat(id.Ctx, dirEnt, id.core, ipfsPath); err != nil { - return nil, nil, err + if err = ipldStat(dirEnt, ipldNode); err != nil { + return qids, nil, err } - qids = append(qids, dirEnt.QID) + walkedNode.Qid = dirEnt.QID + qids = append(qids, walkedNode.Qid) } - id.Path = resolvedPath - - id.Logger.Debugf("ID Walk reg ret %v, %v", qids, id) - return qids, id, nil + id.Logger.Debugf("ID Walk ret %v, %v", qids, walkedNode) + return qids, walkedNode, err } func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { @@ -112,6 +105,8 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { id.Logger.Debugf("ID Readdir") + //FIXME: we read the entire directory for each readdir call; this is very wasteful with small requests (Unix `ls`) + //TODO [quick hack]: append only entry list stored on id; likely going to be problematic for large directories (test: Wikipedia) entChan, err := coreLs(id.Ctx, id.Path, id.core) if err != nil { return nil, err @@ -119,9 +114,8 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { var ents []p9.Dirent - var off uint64 = 0 + var off uint64 = 1 for ent := range entChan { - id.Logger.Debugf("got ent: %v\n", ent) if ent.Err != nil { return nil, err } @@ -136,8 +130,20 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { } } - id.Logger.Debugf("ID Readdir returning ents:%v", ents) - return ents, nil + //FIXME: I don't think order is gauranteed from Ls + eLen := uint64(len(ents)) + if offset >= eLen { + return nil, nil + } + + offsetIndex := ents[offset:] + if len(offsetIndex) > int(count) { + id.Logger.Debugf("ID Readdir returning [%d]%v\n", count, offsetIndex[:count]) + return offsetIndex[:count], nil + } + + id.Logger.Debugf("ID Readdir returning [%d]%v\n", len(offsetIndex), offsetIndex) + return offsetIndex, nil } func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { @@ -156,7 +162,8 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { if fileBound, err := fIo.Size(); err == nil { if int64(offset) >= fileBound { - return replaceMe, errors.New("read offset extends past end of file") + //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. + return 0, nil } } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 3ef1ac168d1..7ea979ad51e 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -18,14 +18,14 @@ type PinFS struct { func initPinFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { pd := &PinFS{ IPFSBase: IPFSBase{ + Path: newRootPath("/ipfs"), core: core, Base: Base{ Logger: logger, Ctx: ctx, - Qid: p9.QID{ - Type: p9.TypeDir, - Version: 1, - Path: uint64(pPinRoot)}}}} + Qid: p9.QID{Type: p9.TypeDir}}}} + + pd.Qid.Path = cidToQPath(pd.Path.Cid()) return pd } @@ -43,16 +43,7 @@ func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) Path: uint64(pPinRoot), } - //TODO: [metadata] quick hack; revise - attr := p9.Attr{ - Mode: p9.ModeDirectory, - RDev: dMemory, - } - - attrMask := p9.AttrMask{ - Mode: true, - RDev: true, - } + attr, attrMask := defaultRootAttr() return qid, attrMask, attr, nil } @@ -66,21 +57,12 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { return []p9.QID{pd.Qid}, pd, nil } -//NOTE: if doClone is false, it implies len(names) > 0 - var tailFile p9.File = pd - var subQids []p9.QID - qids := make([]p9.QID, 0, len(names)) + ipfsDir, err := initIPFS(pd.Ctx, pd.core, pd.Logger).Attach() + if err != nil { + return nil, nil, err + } - ipfsDir, err := initIPFS(pd.Ctx, pd.core, pd.Logger).Attach() - if err != nil { - return nil,nil,err - } - if subQids, tailFile, err = ipfsDir.Walk(names); err != nil { - return nil, nil, err - } - qids = append(qids, subQids...) - pd.Logger.Debugf("PD Walk reg ret %v, %v", qids, tailFile) - return qids, tailFile, nil + return ipfsDir.Walk(names) } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { @@ -96,15 +78,22 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, err } - if count < uint32(len(pins)) { - pins = pins[:count] + pLen := uint64(len(pins)) + if offset >= pLen { + return nil, nil + } + + //TODO: we should initialize and store entries during Open() to assure order is maintained through read calls + offsetIndex := pins[offset:] + if len(offsetIndex) > int(count) { + offsetIndex = offsetIndex[:count] } - ents := make([]p9.Dirent, 0, len(pins)) - for i, pin := range pins { + ents := make([]p9.Dirent, 0, len(offsetIndex)) + for i, pin := range offsetIndex { dirEnt := &p9.Dirent{ Name: gopath.Base(pin.Path().String()), - Offset: uint64(i), + Offset: uint64(i + 1), } if err = coreStat(pd.Ctx, dirEnt, pd.core, pin.Path()); err != nil { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index ed30d2f3cd6..b660355d0f1 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -9,44 +9,76 @@ import ( logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/multiformats/go-multihash" ) var _ p9.File = (*RootIndex)(nil) -//TODO: this shouldn't be necessary, consider how best to path the virtual roots and maintain compat with IPFS core -func newRootPath() corepath.Resolved { - return rootNode("/") +func newRootPath(path string) corepath.Resolved { + return rootNode(path) } type rootNode string func (rn rootNode) String() string { return string(rn) } -func (rootNode) Namespace() string { return "virtual" } +func (rootNode) Namespace() string { return nRoot } func (rootNode) Mutable() bool { return true } func (rootNode) IsValid() error { return nil } -func (rootNode) Cid() cid.Cid { return cid.Cid{} } -func (rootNode) Root() cid.Cid { return cid.Cid{} } +func (rn rootNode) Cid() cid.Cid { + prefix := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN} + c, err := prefix.Sum([]byte(rn)) + if err != nil { + panic(err) //invalid root + } + return c +} +func (rootNode) Root() cid.Cid { //TODO: this should probably reference a package variable set during init `rootCid` + prefix := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN} + c, err := prefix.Sum([]byte("/")) + if err != nil { + panic(err) //invalid root + } + return c +} func (rootNode) Remainder() string { return "" } // type RootIndex struct { IPFSBase + subsystems []p9.Dirent } func NewRoot(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) (*RootIndex, error) { + //XXX: syntax abuse below, try not to look here ri := &RootIndex{ IPFSBase: IPFSBase{ + Path: newRootPath("/"), + core: core, Base: Base{ Ctx: ctx, Logger: logger, - Qid: p9.QID{ - Type: p9.TypeDir, - Version: 1, - Path: uint64(pVirtualRoot)}}, - Path: newRootPath(), - core: core, - }, + Qid: p9.QID{Type: p9.TypeDir}}}, + subsystems: make([]p9.Dirent, 0, 1)} //TODO: [const]: dirent count + ri.Qid.Path = cidToQPath(ri.Path.Cid()) + + rootDirTemplate := p9.Dirent{ + Type: p9.TypeDir, + QID: p9.QID{Type: p9.TypeDir}} + + for i, pathUnion := range [...]struct { + string + p9.Dirent + }{ + {"ipfs", rootDirTemplate}, + //{"ipns", rootDirTemplate}, + } { + pathUnion.Dirent.Offset = uint64(i + 1) + pathUnion.Dirent.Name = pathUnion.string + + pathNode := newRootPath("/"+pathUnion.string) + pathUnion.Dirent.QID.Path = cidToQPath(pathNode.Cid()) + ri.subsystems = append(ri.subsystems, pathUnion.Dirent) } return ri, nil @@ -65,18 +97,9 @@ func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, err Path: uint64(pVirtualRoot), } - //ri.Logger.Errorf("RI mask: %v", req) - - //TODO: [metadata] quick hack; revise - attr := p9.Attr{ - Mode: p9.ModeDirectory, - RDev: dMemory, - } + ri.Logger.Debugf("RI mask: %v", req) - attrMask := p9.AttrMask{ - Mode: true, - RDev: true, - } + attr, attrMask := defaultRootAttr() return qid, attrMask, attr, nil } @@ -90,26 +113,16 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { } //NOTE: if doClone is false, it implies len(names) > 0 - - var tailFile p9.File - var subQids []p9.QID - qids := make([]p9.QID, 0, len(names)) - switch names[0] { case "ipfs": pinDir, err := initPinFS(ri.Ctx, ri.core, ri.Logger).Attach() if err != nil { return nil, nil, err } - if subQids, tailFile, err = pinDir.Walk(names[1:]); err != nil { - return nil, nil, err - } - qids = append(qids, subQids...) + return pinDir.Walk(names[1:]) default: return nil, nil, fmt.Errorf("%q is not provided by us", names[0]) //TODO: Err vars } - ri.Logger.Debugf("RI Walk reg ret %v, %v", qids, tailFile) - return qids, tailFile, nil } // TODO: check specs for directory iounit size, @@ -120,34 +133,19 @@ func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - ri.Logger.Debugf("RI Readdir") - - subsystems := [...]indexPath{pIPFSRoot} - - ents := make([]p9.Dirent, 0, len(subsystems)) - for i, subsystem := range subsystems { - version := 1 //TODO: generate dynamically - - // TODO: allocate root ents array elsewhere - // modify dynamic fields only here - - ents = append(ents, p9.Dirent{ - Name: subsystem.String(), - Offset: uint64(i), - Type: p9.TypeDir, //TODO: resolve dynamically - QID: p9.QID{ - Type: p9.TypeDir, - Version: uint32(version), //TODO: maintain version - Path: uint64(subsystem), - }, - }) + ri.Logger.Debugf("RI Readdir {%d}", count) + sLen := uint64(len(ri.subsystems)) + + if offset >= sLen { + return nil, nil //TODO: [spec] should we error here? } - //TODO: [all instances; double check] the underlying API may already have do this after we return the slice to it - if uint32(len(ents)) > count { - ents = ents[:count] + offsetIndex := ri.subsystems[offset:] + if len(offsetIndex) > int(count) { + ri.Logger.Debugf("RI Readdir returning [%d]%v\n", count, offsetIndex[:count]) + return offsetIndex[:count], nil } - ri.Logger.Debugf("RI Readdir returning [%d]ents:%v", len(ents), ents) - return ents, nil + ri.Logger.Debugf("RI Readdir returning [%d]%v\n", len(offsetIndex), offsetIndex) + return offsetIndex, nil } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 9c8c0ca74f3..7027fdfc2be 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -1,16 +1,16 @@ package fsnodes import ( - "bytes" "context" - "encoding/binary" "fmt" + "hash/fnv" "time" "github.com/hugelgupf/p9/p9" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-unixfs" + unixpb "github.com/ipfs/go-unixfs/pb" coreiface "github.com/ipfs/interface-go-ipfs-core" coreoptions "github.com/ipfs/interface-go-ipfs-core/options" corepath "github.com/ipfs/interface-go-ipfs-core/path" @@ -89,7 +89,7 @@ func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { return err } - nodeType := coreTypeToQtype(coreiface.FileType(ufsNode.Type())) //eww + nodeType := unixfsTypeToQType(ufsNode.Type()) dirEnt.Type = nodeType dirEnt.QID.Type = nodeType @@ -99,13 +99,12 @@ func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { return nil } -func cidToQPath(cid cid.Cid) (path uint64) { - buf := bytes.NewReader(cid.Bytes()) - err := binary.Read(buf, binary.LittleEndian, &path) - if err != nil { +func cidToQPath(cid cid.Cid) uint64 { + hasher := fnv.New64a() + if _, err := hasher.Write(cid.Bytes()); err != nil { panic(err) } - return + return hasher.Sum64() } func coreLs(ctx context.Context, corePath corepath.Path, core coreiface.CoreAPI) (<-chan coreiface.DirEntry, error) { @@ -151,20 +150,33 @@ func coreLs(ctx context.Context, corePath corepath.Path, core coreiface.CoreAPI) return relayChan, err } -func coreTypeToQtype(ct coreiface.FileType) p9.QIDType { +func coreTypeToQType(ct coreiface.FileType) p9.QIDType { switch ct { - // case coreiface.TDirectory, unixfs.THAMTShard: Should we account for this? + // case coreiface.TDirectory, unixfs.THAMTShard // Should we account for this? case coreiface.TDirectory: return p9.TypeDir case coreiface.TSymlink: return p9.TypeSymlink - default: + default: //TODO: probably a bad assumption to make + return p9.TypeRegular + } +} + +//TODO: see if we can remove the need for this; rely only on the core if we can +func unixfsTypeToQType(ut unixpb.Data_DataType) p9.QIDType { + switch ut { + // case unixpb.Data_DataDirectory, unixpb.Data_DataHAMTShard // Should we account for this? + case unixpb.Data_Directory: + return p9.TypeDir + case unixpb.Data_Symlink: + return p9.TypeSymlink + default: //TODO: probably a bad assumption to make return p9.TypeRegular } } func coreEntTo9Ent(coreEnt coreiface.DirEntry) p9.Dirent { - entType := coreTypeToQtype(coreEnt.Type) + entType := coreTypeToQType(coreEnt.Type) return p9.Dirent{ Name: coreEnt.Name, @@ -212,3 +224,39 @@ func deriveTimerContext(ctx context.Context, grace time.Duration) timerContext { return tctx } + +func defaultRootAttr() (p9.Attr, p9.AttrMask) { + now := time.Now() + + return p9.Attr{ + //Mode: p9.ModeDirectory, + Mode: p9.ModeDirectory | p9.Read | p9.Exec, + //NLink: 1, + RDev: dMemory, + //UID: p9.NoUID, + //GID: p9.NoGID, + ATimeSeconds: uint64(now.Nanosecond()), + ATimeNanoSeconds: uint64(now.Second()), + MTimeSeconds: uint64(now.Nanosecond()), + MTimeNanoSeconds: uint64(now.Second()), + CTimeSeconds: uint64(now.Nanosecond()), + CTimeNanoSeconds: uint64(now.Second()), + BTimeSeconds: uint64(now.Nanosecond()), + BTimeNanoSeconds: uint64(now.Second()), + }, p9.AttrMask{ + Mode: true, + NLink: true, + //UID: true, + //GID: true, + RDev: true, + ATime: true, + MTime: true, + CTime: true, + INo: true, + Size: true, + Blocks: true, + BTime: true, + Gen: true, + DataVersion: true, + } +} diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go new file mode 100644 index 00000000000..15d490cc618 --- /dev/null +++ b/plugin/plugins/filesystem/utils.go @@ -0,0 +1,30 @@ +package filesystem + +import ( + "errors" + "os" + "strings" +) + +const ( + EnvAddr = "IPFS_FS_ADDR" + + DefaultVersion = "9P2000.L" + DefaultProtocol = "tcp" + DefaultAddress = ":564" + DefaultMSize = 64 << 10 +) + +func getAddr() (proto, addr string, err error) { + if addr = os.Getenv(EnvAddr); addr == "" { + proto, addr = DefaultProtocol, DefaultAddress + } else { + pair := strings.Split(addr, ";") + if len(pair) != 2 { + err = errors.New("addr env-var not formated correctly") + return + } + proto, addr = pair[0], pair[1] + } + return +} From 2d9930208d6a4b3d3f41c98f33b4e395171a3236 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 22 Aug 2019 11:11:14 -0400 Subject: [PATCH 005/102] plugin/fs: Fix a bunch of metadata related issues --- go.mod | 1 + plugin/plugins/filesystem/nodes/ipfs.go | 13 ++- plugin/plugins/filesystem/nodes/pinfs.go | 10 +- plugin/plugins/filesystem/nodes/root.go | 6 +- plugin/plugins/filesystem/nodes/utils.go | 119 ++++++++++++----------- 5 files changed, 74 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index 5e6224a9f2c..7684f7d2f2c 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,7 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.3 + github.com/prometheus/common v0.4.0 github.com/prometheus/procfs v0.0.0-20190519111021-9935e8e0588d // indirect github.com/syndtr/goleveldb v1.0.0 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 74f1f75c29e..28ae75a45e7 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -42,18 +42,19 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { id.Logger.Debugf("ID GetAttr path: %v", id.Path) if id.Path.Namespace() == nRoot { + id.Logger.Errorf("impossible") _, mask := defaultRootAttr() return id.Qid, mask, id.meta, nil } - var attr p9.Attr var attrMask p9.AttrMask - qid := p9.QID{Path: cidToQPath(id.Path.Cid())} - - if err := coreGetAttr(id.Ctx, &attr, &attrMask, id.core, id.Path); err != nil { + if err := coreGetAttr(id.Ctx, &id.meta, &attrMask, id.core, id.Path); err != nil { return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err } - return qid, attrMask, attr, nil + timeStamp(&id.meta, &attrMask) + id.Qid.Type = p9.QIDType(id.meta.Mode.FileType()) //TODO [review]: conversion sanity check + + return id.Qid, attrMask, id.meta, nil } func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { @@ -68,6 +69,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { qids := make([]p9.QID, 0, len(names)) //TODO: [spec check] make sure we're not messing things up by doing this instead of mutating + // ^ Does internal library expect fid to mutate on success or does newfid clobber some external state anyway walkedNode := &IPFS{} // operate on a copy *walkedNode = *id @@ -90,6 +92,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { } walkedNode.Qid = dirEnt.QID + // qids = append(qids, walkedNode.Qid) } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 7ea979ad51e..48558dff66b 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -23,7 +23,7 @@ func initPinFS(ctx context.Context, core coreiface.CoreAPI, logger logging.Event Base: Base{ Logger: logger, Ctx: ctx, - Qid: p9.QID{Type: p9.TypeDir}}}} + Qid: p9.QID{Type: p9.TypeDir}}}} pd.Qid.Path = cidToQPath(pd.Path.Cid()) return pd @@ -37,15 +37,9 @@ func (pd *PinFS) Attach() (p9.File, error) { func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { pd.Logger.Debugf("PD GetAttr") - qid := p9.QID{ - Type: p9.TypeDir, - Version: 1, - Path: uint64(pPinRoot), - } - attr, attrMask := defaultRootAttr() - return qid, attrMask, attr, nil + return pd.Qid, attrMask, attr, nil } func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index b660355d0f1..04c2728a3a6 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -58,7 +58,7 @@ func NewRoot(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLo Base: Base{ Ctx: ctx, Logger: logger, - Qid: p9.QID{Type: p9.TypeDir}}}, + Qid: p9.QID{Type: p9.TypeDir}}}, subsystems: make([]p9.Dirent, 0, 1)} //TODO: [const]: dirent count ri.Qid.Path = cidToQPath(ri.Path.Cid()) @@ -76,8 +76,8 @@ func NewRoot(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLo pathUnion.Dirent.Offset = uint64(i + 1) pathUnion.Dirent.Name = pathUnion.string - pathNode := newRootPath("/"+pathUnion.string) - pathUnion.Dirent.QID.Path = cidToQPath(pathNode.Cid()) + pathNode := newRootPath("/" + pathUnion.string) + pathUnion.Dirent.QID.Path = cidToQPath(pathNode.Cid()) ri.subsystems = append(ri.subsystems, pathUnion.Dirent) } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 7027fdfc2be..3a170ed0f51 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -2,7 +2,6 @@ package fsnodes import ( "context" - "fmt" "hash/fnv" "time" @@ -22,6 +21,7 @@ func doClone(names []string) bool { return true } //TODO: double check the spec to make sure dot handling is correct + // we may only want to clone on ".." if we're a root if pc := names[0]; l == 1 && (pc == ".." || pc == "." || pc == "") { return true } @@ -50,17 +50,9 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask *p9.AttrMask, core if err != nil { return err } - attr.Mode = p9.Read | p9.Exec - switch t := ufsNode.Type(); t { - case unixfs.TFile: - attr.Mode |= p9.ModeRegular - case unixfs.TDirectory, unixfs.THAMTShard: - attr.Mode |= p9.ModeDirectory - case unixfs.TSymlink: - attr.Mode |= p9.ModeSymlink - default: - return fmt.Errorf("unexpected node type %d", t) - } + + attr.Mode = IRXA //TODO: this should probably be the callers responsability + attr.Mode |= unixfsTypeTo9Mode(ufsNode.Type()) attrMask.Mode = true if bs := ufsNode.BlockSizes(); len(bs) != 0 { @@ -89,11 +81,11 @@ func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { return err } - nodeType := unixfsTypeToQType(ufsNode.Type()) + nodeType := unixfsTypeTo9Mode(ufsNode.Type()).QIDType() dirEnt.Type = nodeType dirEnt.QID.Type = nodeType - dirEnt.QID.Version = 1 + //dirEnt.QID.Version = 1 dirEnt.QID.Path = cidToQPath(node.Cid()) return nil @@ -150,41 +142,40 @@ func coreLs(ctx context.Context, corePath corepath.Path, core coreiface.CoreAPI) return relayChan, err } -func coreTypeToQType(ct coreiface.FileType) p9.QIDType { +func coreTypeTo9Mode(ct coreiface.FileType) p9.FileMode { switch ct { // case coreiface.TDirectory, unixfs.THAMTShard // Should we account for this? case coreiface.TDirectory: - return p9.TypeDir + return p9.ModeDirectory case coreiface.TSymlink: - return p9.TypeSymlink + return p9.ModeSymlink default: //TODO: probably a bad assumption to make - return p9.TypeRegular + return p9.ModeRegular } } //TODO: see if we can remove the need for this; rely only on the core if we can -func unixfsTypeToQType(ut unixpb.Data_DataType) p9.QIDType { +func unixfsTypeTo9Mode(ut unixpb.Data_DataType) p9.FileMode { switch ut { // case unixpb.Data_DataDirectory, unixpb.Data_DataHAMTShard // Should we account for this? case unixpb.Data_Directory: - return p9.TypeDir + return p9.ModeDirectory case unixpb.Data_Symlink: - return p9.TypeSymlink + return p9.ModeSymlink default: //TODO: probably a bad assumption to make - return p9.TypeRegular + return p9.ModeRegular } } func coreEntTo9Ent(coreEnt coreiface.DirEntry) p9.Dirent { - entType := coreTypeToQType(coreEnt.Type) + entType := coreTypeTo9Mode(coreEnt.Type).QIDType() return p9.Dirent{ Name: coreEnt.Name, Type: entType, QID: p9.QID{ - Type: entType, - Version: 1, - Path: cidToQPath(coreEnt.Cid)}} + Type: entType, + Path: cidToQPath(coreEnt.Cid)}} } type timerContextActual struct { @@ -225,38 +216,48 @@ func deriveTimerContext(ctx context.Context, grace time.Duration) timerContext { return tctx } -func defaultRootAttr() (p9.Attr, p9.AttrMask) { - now := time.Now() +const ( // pedantic POSIX stuff + S_IROTH p9.FileMode = p9.Read + S_IWOTH = p9.Write + S_IXOTH = p9.Exec - return p9.Attr{ - //Mode: p9.ModeDirectory, - Mode: p9.ModeDirectory | p9.Read | p9.Exec, - //NLink: 1, - RDev: dMemory, - //UID: p9.NoUID, - //GID: p9.NoGID, - ATimeSeconds: uint64(now.Nanosecond()), - ATimeNanoSeconds: uint64(now.Second()), - MTimeSeconds: uint64(now.Nanosecond()), - MTimeNanoSeconds: uint64(now.Second()), - CTimeSeconds: uint64(now.Nanosecond()), - CTimeNanoSeconds: uint64(now.Second()), - BTimeSeconds: uint64(now.Nanosecond()), - BTimeNanoSeconds: uint64(now.Second()), - }, p9.AttrMask{ - Mode: true, - NLink: true, - //UID: true, - //GID: true, - RDev: true, - ATime: true, - MTime: true, - CTime: true, - INo: true, - Size: true, - Blocks: true, - BTime: true, - Gen: true, - DataVersion: true, - } + S_IRGRP = S_IROTH << 3 + S_IWGRP = S_IWOTH << 3 + S_IXGRP = S_IXOTH << 3 + + S_IRUSR = S_IRGRP << 3 + S_IWUSR = S_IWGRP << 3 + S_IXUSR = S_IXGRP << 3 + + S_IRWXO = S_IROTH | S_IWOTH | S_IXOTH + S_IRWXG = S_IRGRP | S_IWGRP | S_IXGRP + S_IRWXU = S_IRUSR | S_IWUSR | S_IXUSR + + IRWXA = S_IRWXU | S_IRWXG | S_IRWXO // 0777 + IRXA = IRWXA &^ (S_IWUSR | S_IWGRP | S_IWOTH) // 0555 +//03664 +) + +func defaultRootAttr() (attr p9.Attr, attrMask p9.AttrMask) { + attr.Mode = p9.ModeDirectory | IRXA + attr.RDev = dMemory + attrMask.Mode = true + attrMask.RDev = true + attrMask.Size = true + timeStamp(&attr, &attrMask) + return attr, attrMask +} + +func timeStamp(attr *p9.Attr, mask *p9.AttrMask) { + now := time.Now() + attr.ATimeSeconds = uint64(now.Unix()) + attr.ATimeNanoSeconds = uint64(now.UnixNano()) + attr.MTimeSeconds = uint64(now.Unix()) + attr.MTimeNanoSeconds = uint64(now.UnixNano()) + attr.CTimeSeconds = uint64(now.Unix()) + attr.CTimeNanoSeconds = uint64(now.UnixNano()) + + mask.ATime = true + mask.MTime = true + mask.CTime = true } From 75618a53b6cad3c660e6ad8abb8c14ad3c5c7c29 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 22 Aug 2019 18:28:55 -0400 Subject: [PATCH 006/102] plugin/fs: linting --- go.mod | 5 +- go.sum | 2 + plugin/plugins/filesystem/filesystem.go | 3 +- plugin/plugins/filesystem/nodes/base.go | 68 ++---------------------- plugin/plugins/filesystem/nodes/ipfs.go | 18 ++----- plugin/plugins/filesystem/nodes/pinfs.go | 4 +- plugin/plugins/filesystem/nodes/root.go | 26 ++------- plugin/plugins/filesystem/nodes/utils.go | 32 +++++++---- 8 files changed, 39 insertions(+), 119 deletions(-) diff --git a/go.mod b/go.mod index 7684f7d2f2c..5b114815155 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/golang-lru v0.5.3 github.com/hugelgupf/p9 v0.0.0-20190806022330-7ba0920fba11 - github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 // indirect github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.0 @@ -98,7 +97,6 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.3 - github.com/prometheus/common v0.4.0 github.com/prometheus/procfs v0.0.0-20190519111021-9935e8e0588d // indirect github.com/syndtr/goleveldb v1.0.0 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc @@ -112,10 +110,9 @@ require ( go.uber.org/goleak v0.10.0 // indirect go.uber.org/multierr v1.1.0 // indirect go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb + golang.org/x/sys v0.0.0-20190730183949-1393eb018365 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac // indirect golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect - google.golang.org/appengine v1.4.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 gotest.tools/gotestsum v0.3.4 ) diff --git a/go.sum b/go.sum index 4284d34d200..f16a88a9af6 100644 --- a/go.sum +++ b/go.sum @@ -890,6 +890,8 @@ golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 9e9a04ca7af..449b1f4d805 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -23,8 +23,7 @@ var _ plugin.PluginDaemon = (*FileSystemPlugin)(nil) type FileSystemPlugin struct { ctx context.Context cancel context.CancelFunc - root string - addr string + addr string //TODO: populate with value the server is listening on } func (*FileSystemPlugin) Name() string { diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 191c8a09aa7..a285c7e38bf 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -2,8 +2,6 @@ package fsnodes import ( "context" - "io" - "sync" "github.com/hugelgupf/p9/p9" "github.com/hugelgupf/p9/unimplfs" @@ -18,10 +16,6 @@ type FSNode interface { Stat() (p9.QID, error) } -const ( //type - tVirtual = iota - tIPFS -) const ( //device dMemory = iota dIPFS @@ -30,13 +24,6 @@ const ( //device const ( //FS namespaces nRoot = "root" ) -const ( //9P paths - pVirtualRoot uint64 = iota - //pIPFSRoot - //pIpnsRoot - pPinRoot - //pKeyRoot -) var _ p9.File = (*Base)(nil) @@ -47,8 +34,9 @@ var _ p9.File = (*Base)(nil) type Base struct { unimplfs.NoopFile p9.DefaultWalkGetAttr - Qid p9.QID - meta p9.Attr + Qid p9.QID + meta p9.Attr + metaMask p9.AttrMask Ctx context.Context Logger logging.EventLogger @@ -61,53 +49,3 @@ type IPFSBase struct { Path corepath.Resolved core coreiface.CoreAPI } - -type ResourceRef struct { - sync.Mutex - //meta p9p.Dir - closer io.Closer -} - -/* TODO: [current] master attach(with the name) { - global staticRoot = &root{} - reference := staticRoot - - } - - walk { - switch names[0] { - if "ipfs" { - return fs.roots[ipfs].walk(names[1:]) - } - } - } -} -*/ - -/* -func (bn *Base) Stat(ctx context.Context) (p9.QID, error) { - var ( - qid p9.QID - fi os.FileInfo - err error - ) - - // Stat the file. - if l.file != nil { - fi, err = l.file.Stat() - } else { - fi, err = os.Lstat(l.path) - } - if err != nil { - log.Printf("error stating %#v: %v", l, err) - return qid, nil, err - } - - // Construct the QID type. - qid.Type = p9.ModeFromOS(fi.Mode()).QIDType() - - // Save the path from the Ino. - qid.Path = fi.Sys().(*syscall.Stat_t).Ino - return qid, fi, nil -} -*/ diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 28ae75a45e7..cc19219dc02 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -18,16 +18,8 @@ type IPFS struct { //TODO: [review] check fields; better wrappers around inheritance init, etc. func initIPFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { - id := &IPFS{ - IPFSBase: IPFSBase{ - Path: newRootPath("/ipfs"), - core: core, - Base: Base{ - Logger: logger, - Ctx: ctx, - Qid: p9.QID{Type: p9.TypeDir}}}} - - id.Qid.Path = cidToQPath(id.Path.Cid()) + id := &IPFS{IPFSBase: newIPFSBase(ctx, newRootPath("/ipfs"), p9.TypeDir, core, logger)} + id.meta, id.metaMask = defaultRootAttr() return id } @@ -42,9 +34,7 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { id.Logger.Debugf("ID GetAttr path: %v", id.Path) if id.Path.Namespace() == nRoot { - id.Logger.Errorf("impossible") - _, mask := defaultRootAttr() - return id.Qid, mask, id.meta, nil + return id.Qid, id.metaMask, id.meta, nil } var attrMask p9.AttrMask @@ -52,7 +42,7 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err } timeStamp(&id.meta, &attrMask) - id.Qid.Type = p9.QIDType(id.meta.Mode.FileType()) //TODO [review]: conversion sanity check + id.Qid.Type = id.meta.Mode.QIDType() return id.Qid, attrMask, id.meta, nil } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 48558dff66b..1b440806048 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -26,6 +26,7 @@ func initPinFS(ctx context.Context, core coreiface.CoreAPI, logger logging.Event Qid: p9.QID{Type: p9.TypeDir}}}} pd.Qid.Path = cidToQPath(pd.Path.Cid()) + pd.meta, pd.metaMask = defaultRootAttr() return pd } @@ -37,9 +38,8 @@ func (pd *PinFS) Attach() (p9.File, error) { func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { pd.Logger.Debugf("PD GetAttr") - attr, attrMask := defaultRootAttr() - return pd.Qid, attrMask, attr, nil + return pd.Qid, pd.metaMask, pd.meta, nil } func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 04c2728a3a6..cac6417b8f2 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -23,7 +23,7 @@ type rootNode string func (rn rootNode) String() string { return string(rn) } func (rootNode) Namespace() string { return nRoot } func (rootNode) Mutable() bool { return true } -func (rootNode) IsValid() error { return nil } +func (rootNode) IsValid() error { return nil } //TODO: should return ENotImpl func (rn rootNode) Cid() cid.Cid { prefix := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN} c, err := prefix.Sum([]byte(rn)) @@ -42,26 +42,18 @@ func (rootNode) Root() cid.Cid { //TODO: this should probably reference a packag } func (rootNode) Remainder() string { return "" } -// - type RootIndex struct { IPFSBase subsystems []p9.Dirent } func NewRoot(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) (*RootIndex, error) { - //XXX: syntax abuse below, try not to look here - ri := &RootIndex{ - IPFSBase: IPFSBase{ - Path: newRootPath("/"), - core: core, - Base: Base{ - Ctx: ctx, - Logger: logger, - Qid: p9.QID{Type: p9.TypeDir}}}, + ri := &RootIndex{IPFSBase: newIPFSBase(ctx, newRootPath("/"), p9.TypeDir, core, logger), subsystems: make([]p9.Dirent, 0, 1)} //TODO: [const]: dirent count ri.Qid.Path = cidToQPath(ri.Path.Cid()) + ri.meta, ri.metaMask = defaultRootAttr() + rootDirTemplate := p9.Dirent{ Type: p9.TypeDir, QID: p9.QID{Type: p9.TypeDir}} @@ -91,17 +83,9 @@ func (ri *RootIndex) Attach() (p9.File, error) { func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { ri.Logger.Debugf("RI GetAttr") - qid := p9.QID{ - Type: p9.TypeDir, - Version: 1, - Path: uint64(pVirtualRoot), - } - ri.Logger.Debugf("RI mask: %v", req) - attr, attrMask := defaultRootAttr() - - return qid, attrMask, attr, nil + return ri.Qid, ri.metaMask, ri.meta, nil } func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { ri.Logger.Debugf("RI Walk names %v", names) diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 3a170ed0f51..f1c5ae25d53 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -8,6 +8,7 @@ import ( "github.com/hugelgupf/p9/p9" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log" "github.com/ipfs/go-unixfs" unixpb "github.com/ipfs/go-unixfs/pb" coreiface "github.com/ipfs/interface-go-ipfs-core" @@ -30,13 +31,12 @@ func doClone(names []string) bool { //TODO: rename this and/or extend // it only does some of the stat and not what people probably expect -func coreStat(ctx context.Context, dirEnt *p9.Dirent, core coreiface.CoreAPI, path corepath.Path) (err error) { - var ipldNode ipld.Node - if ipldNode, err = core.ResolveNode(ctx, path); err != nil { - return +func coreStat(ctx context.Context, dirEnt *p9.Dirent, core coreiface.CoreAPI, path corepath.Path) error { + if ipldNode, err := core.ResolveNode(ctx, path); err != nil { + return err + } else { + return ipldStat(dirEnt, ipldNode) } - err = ipldStat(dirEnt, ipldNode) - return } //TODO: consider how we want to use AttrMask @@ -51,7 +51,7 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask *p9.AttrMask, core return err } - attr.Mode = IRXA //TODO: this should probably be the callers responsability + attr.Mode = IRXA //TODO: this should probably be the callers responsability; just document that permissions should be set afterwards or something attr.Mode |= unixfsTypeTo9Mode(ufsNode.Type()) attrMask.Mode = true @@ -69,8 +69,6 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask *p9.AttrMask, core //etc. } - //TODO: rdev; switch off namespace => dIpfs, dIpns, etc. - //Blocks return nil } @@ -81,11 +79,10 @@ func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { return err } - nodeType := unixfsTypeTo9Mode(ufsNode.Type()).QIDType() + nodeType := unixfsTypeTo9Mode(ufsNode.Type()).QIDType() dirEnt.Type = nodeType dirEnt.QID.Type = nodeType - //dirEnt.QID.Version = 1 dirEnt.QID.Path = cidToQPath(node.Cid()) return nil @@ -261,3 +258,16 @@ func timeStamp(attr *p9.Attr, mask *p9.AttrMask) { mask.MTime = true mask.CTime = true } + +//TODO [name]: "new" implies pointer type; this is for embedded consturction +func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, core coreiface.CoreAPI, logger logging.EventLogger) IPFSBase { + return IPFSBase{ + Path: path, + core: core, + Base: Base{ + Logger: logger, + Ctx: ctx, + Qid: p9.QID{ + Type: kind, + Path: cidToQPath(path.Cid())}}} +} From 07534a3041208e39badee968c0939c06d019a693 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 26 Aug 2019 11:14:51 -0400 Subject: [PATCH 007/102] plugin/fs: use config --- go.mod | 8 +- go.sum | 8 +- plugin/plugins/filesystem/client/client.go | 178 +++++++++++++++++++ plugin/plugins/filesystem/client/cmd/main.go | 33 ++++ plugin/plugins/filesystem/client/options.go | 26 +++ plugin/plugins/filesystem/cmd/main.go | 114 ------------ plugin/plugins/filesystem/filesystem.go | 58 +++--- plugin/plugins/filesystem/utils.go | 130 ++++++++++++-- 8 files changed, 398 insertions(+), 157 deletions(-) create mode 100644 plugin/plugins/filesystem/client/client.go create mode 100644 plugin/plugins/filesystem/client/cmd/main.go create mode 100644 plugin/plugins/filesystem/client/options.go delete mode 100644 plugin/plugins/filesystem/cmd/main.go diff --git a/go.mod b/go.mod index 5b114815155..91ea6174079 100644 --- a/go.mod +++ b/go.mod @@ -119,6 +119,10 @@ require ( replace github.com/golangci/golangci-lint => github.com/mhutchinson/golangci-lint v1.17.2-0.20190819125825-d18f2136e32b -replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20190822034506-af913a6d8082 - go 1.12 + +// TODO: merge things and use proper repo references +replace ( + github.com/hugelgupf/p9 => github.com/djdv/p9 diff + github.com/ipfs/go-ipfs-config => github.com/djdv/go-ipfs-config experiment/filesystem +) diff --git a/go.sum b/go.sum index f16a88a9af6..501e235dbde 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,12 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/p9 v0.0.0-20190822034506-af913a6d8082 h1:Ml5E7tP7gL/A9CdCc7iRZf5cNIWFVDc9WeMYMP3F4ck= -github.com/djdv/p9 v0.0.0-20190822034506-af913a6d8082/go.mod h1:akLqomEjJslbyK2h+TEXmVabYavEatmOdB6iektRfEM= +github.com/djdv/go-ipfs-config v0.0.8-0.20190828150305-9817c5842355 h1:jDE4uGhU9c/fa01Mrkk9JnmVKcAPnQIYL5774iajBeI= +github.com/djdv/go-ipfs-config v0.0.8-0.20190828150305-9817c5842355/go.mod h1:ribou1kTrjMvMNeU7wQN4vjaFSaXPBKkSKx1dPxe3eM= +github.com/djdv/go-ipfs-config v0.0.8-0.20190828212302-95198d79d7c9 h1:SjcUvZEJ9VwMW58QJO0YRqAqr9CL4t4hixSnFpaJbDY= +github.com/djdv/go-ipfs-config v0.0.8-0.20190828212302-95198d79d7c9/go.mod h1:ribou1kTrjMvMNeU7wQN4vjaFSaXPBKkSKx1dPxe3eM= +github.com/djdv/p9 v0.0.0-20190828183744-eaad8e85066c h1:PhV3zAWDlk3FjvaF+vW3SVc5xg1eBWT/z6NZ+AHiD0Q= +github.com/djdv/p9 v0.0.0-20190828183744-eaad8e85066c/go.mod h1:akLqomEjJslbyK2h+TEXmVabYavEatmOdB6iektRfEM= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= diff --git a/plugin/plugins/filesystem/client/client.go b/plugin/plugins/filesystem/client/client.go new file mode 100644 index 00000000000..d4e0eafba1e --- /dev/null +++ b/plugin/plugins/filesystem/client/client.go @@ -0,0 +1,178 @@ +package p9client + +import ( + gopath "path" + "path/filepath" + "runtime" + "strings" + + "github.com/hugelgupf/p9/p9" + config "github.com/ipfs/go-ipfs-config" + "github.com/ipfs/go-ipfs/plugin/plugins/filesystem" + logging "github.com/ipfs/go-log" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" +) + +var logger logging.EventLogger = logging.Logger("9p") + +func Dial(options ...Option) (*p9.Client, error) { + ops := &Options{ + address: filesystem.DefaultListenAddress, + msize: filesystem.DefaultMSize, + version: filesystem.DefaultVersion} + + for _, op := range options { + op(ops) + } + + if ops.address == filesystem.DefaultListenAddress { + // expand $IPFS_PATH, using default if not exist + target, err := config.Path("", "filesystem.9p.sock") + if err != nil { + return nil, err + } + //TODO [manet]: doesn't like drive letters + //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively + if runtime.GOOS == "windows" { + if target, err = windowsToUnixFriendly(target); err != nil { + return nil, err + } + } + ops.address = gopath.Join("/unix", target) + } + + ma, err := multiaddr.NewMultiaddr(ops.address) + if err != nil { + return nil, err + } + + conn, err := manet.Dial(ma) + if err != nil { + return nil, err + } + + return p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) + /* + client, err := p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) + if err != nil { + return nil, err + } + logger.Infof("Connected to server supporting version:\n%v\n\n", client.Version()) + + rootRef, err := client.Attach("") + if err != nil { + return nil, err + } + logger.Debugf("Attached to root:\n%#v\n\n", rootRef) + */ +} + +func ReadDir(path string, fsRef p9.File, offset uint64) ([]p9.Dirent, error) { + components := strings.Split(strings.TrimPrefix(path, "/"), "/") + if len(components) == 1 && components[0] == "" { + components = nil + } + + qids, targetRef, err := fsRef.Walk(components) + if err != nil { + return nil, err + } + logger.Debugf("walked to %q :\nQIDs:%v, FID:%v\n\n", path, qids, targetRef) + + if _, _, err = targetRef.Open(0); err != nil { + return nil, err + } + + ents, err := targetRef.Readdir(offset, ^uint32(0)) + if err != nil { + return nil, err + } + + logger.Debugf("%q Readdir:\n[%d]%v\n\n", path, len(ents), ents) + if err = targetRef.Close(); err != nil { + return nil, err + } + logger.Debugf("%q closed:\n%#v\n\n", path, targetRef) + + return ents, nil +} + +/* TODO: rework +func Open(path string, fsRef p9.File) (p9.File, error) { + components := strings.Split(strings.TrimPrefix(path, "/"), "/") + if len(components) == 1 && components[0] == "" { + components = nil + } + + _, targetRef, err := fsRef.Walk(nil) + if err != nil { + return nil, err + } + + _, _, attr, err := targetRef.GetAttr(p9.AttrMask{Size: true}) + if err != nil { + return nil, err + } + logger.Debugf("Getattr for %q :\n%v\n\n", path, attr) + + refQid, ioUnit, err := targetRef.Open(0) + if err != nil { + return nil, err + } + +} + +func Read(path string, openedRef p9.File) { + components := strings.Split(strings.TrimPrefix(path, "/"), "/") + if len(components) == 1 && components[0] == "" { + components = nil + } + + _, targetRef, err := fsRef.Walk(components) + if err != nil { + return nil, err + } + + _, _, attr, err := targetRef.GetAttr(p9.AttrMask{Size: true}) + if err != nil { + return nil, err + } + logger.Debugf("Getattr for %q :\n%v\n\n", path, attr) + + refQid, ioUnit, err := targetRef.Open(0) + if err != nil { + return nil, err + } + logger.Debugf("%q Opened:\nQID:%v, iounit:%v\n\n", path, refQid, ioUnit) + + buf := make([]byte, attr.Size) + readBytes, err := targetRef.ReadAt(buf, 0) + if err != nil { + return nil, err + } + + logger.Debugf("%q Read:\n[%d bytes]\n%s\n\n", path, readBytes, buf) + if err = targetRef.Close(); err != nil { + return nil, err + } + + logger.Debugf("%q closed:\n%#v\n\n", path, targetRef) +} +*/ + +func windowsToUnixFriendly(target string) (string, error) { + if !filepath.IsAbs(target) { + var err error + if target, err = filepath.Abs(target); err != nil { + return target, err + } + } + + //TODO [manet]: doesn't like drive letters + //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively + if len(target) > 3 && target[1] == ':' { + target = target[3:] + } + return filepath.ToSlash(target), nil +} diff --git a/plugin/plugins/filesystem/client/cmd/main.go b/plugin/plugins/filesystem/client/cmd/main.go new file mode 100644 index 00000000000..69a73a735b0 --- /dev/null +++ b/plugin/plugins/filesystem/client/cmd/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "log" + + p9client "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/client" + logging "github.com/ipfs/go-log" +) + +func main() { + logger := logging.Logger("fs-client") + logging.SetLogLevel("fs-client", "info") + + client, err := p9client.Dial() + if err != nil { + log.Fatal(err) + } + + defer client.Close() + logger.Infof("Connected to server supporting version:\n%v\n\n", client.Version()) + + rootRef, err := client.Attach("") + if err != nil { + log.Fatal(err) + } + logger.Infof("Attached to root:\n%#v\n\n", rootRef) + + logger.Info(p9client.ReadDir("/", rootRef, 0)) + logger.Info(p9client.ReadDir("/ipfs", rootRef, 0)) + logger.Info(p9client.ReadDir("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", rootRef, 0)) + //readDBG("/ipfs/QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB", rootRef) + client.Close() +} diff --git a/plugin/plugins/filesystem/client/options.go b/plugin/plugins/filesystem/client/options.go new file mode 100644 index 00000000000..13789cf6c0a --- /dev/null +++ b/plugin/plugins/filesystem/client/options.go @@ -0,0 +1,26 @@ +package p9client + +type Options struct { + network, address, version string + msize int +} + +type Option func(*Options) + +func Address(address string) Option { + return func(ops *Options) { + ops.address = address + } +} + +func Version(version string) Option { + return func(ops *Options) { + ops.version = version + } +} + +func Msize(msize int) Option { + return func(ops *Options) { + ops.msize = msize + } +} diff --git a/plugin/plugins/filesystem/cmd/main.go b/plugin/plugins/filesystem/cmd/main.go deleted file mode 100644 index 2482825d163..00000000000 --- a/plugin/plugins/filesystem/cmd/main.go +++ /dev/null @@ -1,114 +0,0 @@ -package main - -import ( - "fmt" - "log" - "net" - "strings" - - "github.com/hugelgupf/p9/p9" - "github.com/ipfs/go-ipfs/plugin/plugins/filesystem" -) - -func main() { - - log.SetFlags(log.Flags() | log.Lshortfile) - - conn, err := net.Dial(filesystem.DefaultProtocol, filesystem.DefaultAddress) - if err != nil { - log.Fatal(err) - } - - client, err := p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) - if err != nil { - log.Fatal(err) - } - defer client.Close() - log.Printf("Connected to server supporting version:\n%v\n\n", client.Version()) - - rootRef, err := client.Attach("") - if err != nil { - log.Fatal(err) - } - log.Printf("Attached to root:\n%#v\n\n", rootRef) - - readDirDBG("/", rootRef) - readDirDBG("/ipfs", rootRef) - readDirDBG("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", rootRef) - readDBG("/ipfs/QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB", rootRef) - - client.Close() -} - -func readDirDBG(path string, fsRef p9.File) { - components := strings.Split(strings.TrimPrefix(path, "/"), "/") - if len(components) == 1 && components[0] == "" { - components = nil - } - - fmt.Printf("components: %#v\n", components) - - qids, targetRef, err := fsRef.Walk(components) - if err != nil { - log.Fatal(err) - } - log.Printf("walked to %q :\nQIDs:%v, FID:%v\n\n", path, qids, targetRef) - - refQid, ioUnit, err := targetRef.Open(0) - if err != nil { - log.Fatal(err) - } - log.Printf("%q Opened:\nQID:%v, iounit:%v\n\n", path, refQid, ioUnit) - - ents, err := targetRef.Readdir(0, ^uint32(0)) - if err != nil { - log.Fatal(err) - } - - log.Printf("%q Readdir:\n[%d]%v\n\n", path, len(ents), ents) - if err = targetRef.Close(); err != nil { - log.Fatal(err) - } - - log.Printf("%q closed:\n%#v\n\n", path, targetRef) -} - -func readDBG(path string, fsRef p9.File) { - components := strings.Split(strings.TrimPrefix(path, "/"), "/") - if len(components) == 1 && components[0] == "" { - components = nil - } - - fmt.Printf("components: %#v\n", components) - - qids, targetRef, err := fsRef.Walk(components) - if err != nil { - log.Fatal(err) - } - log.Printf("walked to %q :\nQIDs:%v, FID:%v\n\n", path, qids, targetRef) - - _, _, attr, err := targetRef.GetAttr(p9.AttrMask{Size: true}) - if err != nil { - log.Fatal(err) - } - log.Printf("Getattr for %q :\n%v\n\n", path, attr) - - refQid, ioUnit, err := targetRef.Open(0) - if err != nil { - log.Fatal(err) - } - log.Printf("%q Opened:\nQID:%v, iounit:%v\n\n", path, refQid, ioUnit) - - buf := make([]byte, attr.Size) - readBytes, err := targetRef.ReadAt(buf, 0) - if err != nil { - log.Fatal(err) - } - - log.Printf("%q Read:\n[%d bytes]\n%s\n\n", path, readBytes, buf) - if err = targetRef.Close(); err != nil { - log.Fatal(err) - } - - log.Printf("%q closed:\n%#v\n\n", path, targetRef) -} diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 449b1f4d805..6d1c822117e 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -2,28 +2,30 @@ package filesystem import ( "context" - "fmt" - "net" + "errors" "github.com/hugelgupf/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" + manet "github.com/multiformats/go-multiaddr-net" ) // Plugins is an exported list of plugins that will be loaded by go-ipfs. var Plugins = []plugin.Plugin{ - &FileSystemPlugin{}, + &FileSystemPlugin{}, //TODO: individually name implementations: &P9{} } // impl check var _ plugin.PluginDaemon = (*FileSystemPlugin)(nil) type FileSystemPlugin struct { - ctx context.Context - cancel context.CancelFunc - addr string //TODO: populate with value the server is listening on + ctx context.Context + cancel context.CancelFunc + listener manet.Listener + + disabled bool } func (*FileSystemPlugin) Name() string { @@ -39,50 +41,60 @@ func (fs *FileSystemPlugin) Init() error { return nil } +var ( + logger logging.EventLogger + errDisabled = errors.New("this experiment is disabled, enable with `ipfs config --json Experimental.FileSystemEnabled true`") +) + func init() { + logger = logging.Logger("plugin/filesystem") } func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { - logger := logging.Logger("plugin/filesystem") logger.Info("Initialising 9p resource server...") - // construct 9p resource server / config - proto, addr, err := getAddr() + la, err := GetListener() if err != nil { - logger.Errorf("9P server error: %s", err) + if err == errDisabled { + fs.disabled = true + logger.Info(err) + return nil + } + logger.Errorf("9P listen error: %s\n", err) return err } - p9pFSS, err := fsnodes.NewRoot(fs.ctx, core, logger) - if err != nil { - logger.Errorf("9P server error: %s", err) - return err - } + fs.listener = la - // Bind and listen on the socket. - serverSocket, err := net.Listen(proto, addr) + // construct 9p resource server + p9pFSS, err := fsnodes.NewRoot(fs.ctx, core, logger) if err != nil { - logger.Errorf("9P server error: %s", err) + logger.Errorf("9P server error: %s\n", err) return err } // Run the server. s := p9.NewServer(p9pFSS) go func() { - if err := s.Serve(serverSocket); err != nil { - logger.Errorf("9P server error: %s", err) + if err := s.Serve(manet.NetListener(la)); err != nil { + logger.Errorf("9P server error: %s\n", err) return } }() - //TODO: prettier print; mountpoints, socket/addr, confirm start actually succeeded, etc. - logger.Infof("9P service started on %s", addr) + //TODO: confirm start actually succeeded, etc. + logger.Infof("9P service started on %s\n", la.Addr()) return nil } func (fs *FileSystemPlugin) Close() error { + if fs.disabled { + return nil + } + //TODO: fmt.Println("Closing file system handles...") - fmt.Println("9P server requested to close") + logger.Info("9P server requested to close") fs.cancel() + fs.listener.Close() return nil } diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go index 15d490cc618..b072cbe85f3 100644 --- a/plugin/plugins/filesystem/utils.go +++ b/plugin/plugins/filesystem/utils.go @@ -1,30 +1,128 @@ package filesystem import ( - "errors" "os" - "strings" + gopath "path" + "path/filepath" + "runtime" + + config "github.com/ipfs/go-ipfs-config" + cserial "github.com/ipfs/go-ipfs-config/serialize" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" ) const ( - EnvAddr = "IPFS_FS_ADDR" + //TODO [config]: move elsewhere; related: https://github.com/ipfs/go-ipfs/issues/6526 + EnvAddr = "IPFS_FS_ADDR" // multiaddr string - DefaultVersion = "9P2000.L" - DefaultProtocol = "tcp" - DefaultAddress = ":564" - DefaultMSize = 64 << 10 + DefaultVersion = "9P2000.L" + DefaultListenAddress = "/unix/$IPFS_PATH/filesystem.9p.sock" + DefaultService = "9P" // (currently 9P2000.L) + DefaultMSize = 64 << 10 + // TODO: For platforms that don't support UDS (Windows < 17063, non-posix systems), fallback to TCP + //FallbackListenAddress = "/ip4/localhost/tcp/564" ) -func getAddr() (proto, addr string, err error) { - if addr = os.Getenv(EnvAddr); addr == "" { - proto, addr = DefaultProtocol, DefaultAddress +/* +func defaultConfig() (*config.FileSystem, error) { + fsConf := make(map[string]string) + target, err := config.Path("", "filesystem.9p.sock") + if err != nil { + return nil, err + } + fsConf["9p"] = gopath.Join("/unix", target) + return fsConf, nil +} +*/ + +func getConfig() (*config.Config, error) { + //TODO [plugin]: default config init+file should go somewhere else + // preferably we'd have plugin-scoped storage somehow + //$IPFS_PATH/plugins/filesystem/{sock, conf} or `plugin.Start(core, pluginCfg`) ? + cfgPath, err := config.Filename("") + if err != nil { + return nil, err + } + + return cserial.Load(cfgPath) +} + +func GetFSConf() (*config.FileSystem, error) { + cfg, err := getConfig() + if err != nil { + return nil, err + } + + /* We can't actually enable this yet 👀 + if !cfg.Experimental.FileSystemEnabled { + return nil, errDisabled + } + */ + + if addr := os.Getenv(EnvAddr); addr != "" { + cfg.FileSystem.Service[DefaultService] = addr + return &cfg.FileSystem, nil + } + + //TODO: after experiment, make sure this is populated from conf file, not initialised here + if cfg.FileSystem.Service == nil { + cfg.FileSystem.Service = make(map[string]string) + } + + if comp := cfg.FileSystem.Service[DefaultService]; comp == DefaultListenAddress || comp == "" { + // expand $IPFS_PATH, using default if not exist + target, err := config.Path("", "filesystem.9p.sock") + if err != nil { + return nil, err + } + + if runtime.GOOS == "windows" { + if target, err = windowsToUnixFriendly(target); err != nil { + return nil, err + } + } + cfg.FileSystem.Service[DefaultService] = gopath.Join("/unix", target) } else { - pair := strings.Split(addr, ";") - if len(pair) != 2 { - err = errors.New("addr env-var not formated correctly") - return + // assume user supplied env vars exists and expand them as-is + for service, target := range cfg.FileSystem.Service { + cfg.FileSystem.Service[service] = os.ExpandEnv(target) + } + } + + return &cfg.FileSystem, nil +} + +func windowsToUnixFriendly(target string) (string, error) { + if !filepath.IsAbs(target) { + var err error + if target, err = filepath.Abs(target); err != nil { + return target, err } - proto, addr = pair[0], pair[1] } - return + + //TODO [manet]: doesn't like drive letters + //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively + if len(target) > 3 && target[1] == ':' { + target = target[3:] + } + return filepath.ToSlash(target), nil +} + +func GetListener() (manet.Listener, error) { + cfg, err := GetFSConf() + if err != nil { + return nil, err + } + + ma, err := multiaddr.NewMultiaddr(cfg.Service[DefaultService]) + if err != nil { + return nil, err + } + + listenAddress, err := manet.Listen(ma) + if err != nil { + return nil, err + } + return listenAddress, nil } From 0d284894a88fea74107caa3e0fefa6b181d7f7c4 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 29 Aug 2019 16:33:37 -0400 Subject: [PATCH 008/102] plugin/fs: Better init/config --- go.mod | 5 +- plugin/plugins/filesystem/client/client.go | 50 ++------- plugin/plugins/filesystem/filesystem.go | 31 ++++-- plugin/plugins/filesystem/utils.go | 114 +++++++++------------ 4 files changed, 74 insertions(+), 126 deletions(-) diff --git a/go.mod b/go.mod index 91ea6174079..2c472f1dc74 100644 --- a/go.mod +++ b/go.mod @@ -122,7 +122,4 @@ replace github.com/golangci/golangci-lint => github.com/mhutchinson/golangci-lin go 1.12 // TODO: merge things and use proper repo references -replace ( - github.com/hugelgupf/p9 => github.com/djdv/p9 diff - github.com/ipfs/go-ipfs-config => github.com/djdv/go-ipfs-config experiment/filesystem -) +replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20190828183744-eaad8e85066c diff --git a/plugin/plugins/filesystem/client/client.go b/plugin/plugins/filesystem/client/client.go index d4e0eafba1e..3d8fc23e361 100644 --- a/plugin/plugins/filesystem/client/client.go +++ b/plugin/plugins/filesystem/client/client.go @@ -1,20 +1,16 @@ package p9client import ( - gopath "path" - "path/filepath" - "runtime" "strings" "github.com/hugelgupf/p9/p9" - config "github.com/ipfs/go-ipfs-config" "github.com/ipfs/go-ipfs/plugin/plugins/filesystem" logging "github.com/ipfs/go-log" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) -var logger logging.EventLogger = logging.Logger("9p") +var logger logging.EventLogger = logging.Logger("9P") func Dial(options ...Option) (*p9.Client, error) { ops := &Options{ @@ -27,19 +23,13 @@ func Dial(options ...Option) (*p9.Client, error) { } if ops.address == filesystem.DefaultListenAddress { - // expand $IPFS_PATH, using default if not exist - target, err := config.Path("", "filesystem.9p.sock") + // TODO: kludge + serviceConf, err := filesystem.XXX_GetFSConf() if err != nil { return nil, err } - //TODO [manet]: doesn't like drive letters - //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively - if runtime.GOOS == "windows" { - if target, err = windowsToUnixFriendly(target); err != nil { - return nil, err - } - } - ops.address = gopath.Join("/unix", target) + + ops.address = serviceConf.Service[filesystem.DefaultService] } ma, err := multiaddr.NewMultiaddr(ops.address) @@ -47,25 +37,13 @@ func Dial(options ...Option) (*p9.Client, error) { return nil, err } + //TODO [investigate;who's bug] on Windows, dialing a unix domain socket that doesn't exist will create it conn, err := manet.Dial(ma) if err != nil { return nil, err } return p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) - /* - client, err := p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) - if err != nil { - return nil, err - } - logger.Infof("Connected to server supporting version:\n%v\n\n", client.Version()) - - rootRef, err := client.Attach("") - if err != nil { - return nil, err - } - logger.Debugf("Attached to root:\n%#v\n\n", rootRef) - */ } func ReadDir(path string, fsRef p9.File, offset uint64) ([]p9.Dirent, error) { @@ -160,19 +138,3 @@ func Read(path string, openedRef p9.File) { logger.Debugf("%q closed:\n%#v\n\n", path, targetRef) } */ - -func windowsToUnixFriendly(target string) (string, error) { - if !filepath.IsAbs(target) { - var err error - if target, err = filepath.Abs(target); err != nil { - return target, err - } - } - - //TODO [manet]: doesn't like drive letters - //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively - if len(target) > 3 && target[1] == ':' { - target = target[3:] - } - return filepath.ToSlash(target), nil -} diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 6d1c822117e..63d66749cb2 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -9,6 +9,7 @@ import ( fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" + "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) @@ -29,11 +30,11 @@ type FileSystemPlugin struct { } func (*FileSystemPlugin) Name() string { - return "filesystem" + return PluginName } func (*FileSystemPlugin) Version() string { - return "0.0.1" + return PluginVersion } func (fs *FileSystemPlugin) Init() error { @@ -52,38 +53,46 @@ func init() { func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { logger.Info("Initialising 9p resource server...") + fs.disabled = true - la, err := GetListener() + serviceConfig, err := XXX_GetFSConf() if err != nil { if err == errDisabled { - fs.disabled = true - logger.Info(err) + logger.Warning(errDisabled) return nil } - logger.Errorf("9P listen error: %s\n", err) return err } - fs.listener = la + ma, err := multiaddr.NewMultiaddr(serviceConfig.Service[DefaultService]) + if err != nil { + logger.Errorf("9P multiaddr error: %s\n", err) + return err + } + + if fs.listener, err = manet.Listen(ma); err != nil { + logger.Errorf("9P listen error: %s\n", err) + return err + } // construct 9p resource server p9pFSS, err := fsnodes.NewRoot(fs.ctx, core, logger) if err != nil { - logger.Errorf("9P server error: %s\n", err) + logger.Errorf("9P construction error: %s\n", err) return err } // Run the server. s := p9.NewServer(p9pFSS) go func() { - if err := s.Serve(manet.NetListener(la)); err != nil { + if err := s.Serve(manet.NetListener(fs.listener)); err != nil { logger.Errorf("9P server error: %s\n", err) return } }() - //TODO: confirm start actually succeeded, etc. - logger.Infof("9P service started on %s\n", la.Addr()) + fs.disabled = false + logger.Infof("9P service started on %s\n", fs.listener.Addr()) return nil } diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go index b072cbe85f3..6b69db7a9ed 100644 --- a/plugin/plugins/filesystem/utils.go +++ b/plugin/plugins/filesystem/utils.go @@ -8,89 +8,51 @@ import ( config "github.com/ipfs/go-ipfs-config" cserial "github.com/ipfs/go-ipfs-config/serialize" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" ) const ( + PluginName = "filesystem" + PluginVersion = "0.0.1" + //TODO [config]: move elsewhere; related: https://github.com/ipfs/go-ipfs/issues/6526 EnvAddr = "IPFS_FS_ADDR" // multiaddr string DefaultVersion = "9P2000.L" - DefaultListenAddress = "/unix/$IPFS_PATH/filesystem.9p.sock" + DefaultListenAddress = "/unix/$IPFS_PATH/" + sockName DefaultService = "9P" // (currently 9P2000.L) DefaultMSize = 64 << 10 // TODO: For platforms that don't support UDS (Windows < 17063, non-posix systems), fallback to TCP //FallbackListenAddress = "/ip4/localhost/tcp/564" -) -/* -func defaultConfig() (*config.FileSystem, error) { - fsConf := make(map[string]string) - target, err := config.Path("", "filesystem.9p.sock") - if err != nil { - return nil, err - } - fsConf["9p"] = gopath.Join("/unix", target) - return fsConf, nil -} -*/ - -func getConfig() (*config.Config, error) { - //TODO [plugin]: default config init+file should go somewhere else - // preferably we'd have plugin-scoped storage somehow - //$IPFS_PATH/plugins/filesystem/{sock, conf} or `plugin.Start(core, pluginCfg`) ? - cfgPath, err := config.Filename("") - if err != nil { - return nil, err - } + sockName = "filesystem.9P.sock" +) - return cserial.Load(cfgPath) +type Config struct { // NOTE: unstable/experimental + // addresses for file system servers and clients + //e.g. "9P":"/ip4/localhost/tcp/564", "fuse":"/mountpoint", "🐇":"/rabbit-hutch/glenda", ... + Service map[string]string } -func GetFSConf() (*config.FileSystem, error) { - cfg, err := getConfig() +func defaultConfig() (*Config, error) { + serviceMap := make(map[string]string) + target, err := config.Path("", sockName) if err != nil { return nil, err } - /* We can't actually enable this yet 👀 - if !cfg.Experimental.FileSystemEnabled { - return nil, errDisabled - } - */ - - if addr := os.Getenv(EnvAddr); addr != "" { - cfg.FileSystem.Service[DefaultService] = addr - return &cfg.FileSystem, nil - } - - //TODO: after experiment, make sure this is populated from conf file, not initialised here - if cfg.FileSystem.Service == nil { - cfg.FileSystem.Service = make(map[string]string) - } - - if comp := cfg.FileSystem.Service[DefaultService]; comp == DefaultListenAddress || comp == "" { - // expand $IPFS_PATH, using default if not exist - target, err := config.Path("", "filesystem.9p.sock") - if err != nil { + if runtime.GOOS == "windows" { + if target, err = windowsToUnixFriendly(target); err != nil { return nil, err } - - if runtime.GOOS == "windows" { - if target, err = windowsToUnixFriendly(target); err != nil { - return nil, err - } - } - cfg.FileSystem.Service[DefaultService] = gopath.Join("/unix", target) - } else { - // assume user supplied env vars exists and expand them as-is - for service, target := range cfg.FileSystem.Service { - cfg.FileSystem.Service[service] = os.ExpandEnv(target) - } } - return &cfg.FileSystem, nil + serviceMap["9P"] = gopath.Join("/unix", target) + return &Config{serviceMap}, nil +} + +//TODO: better name +func XXX_GetFSConf() (*Config, error) { + return configFromPlugin(PluginName) } func windowsToUnixFriendly(target string) (string, error) { @@ -109,20 +71,38 @@ func windowsToUnixFriendly(target string) (string, error) { return filepath.ToSlash(target), nil } -func GetListener() (manet.Listener, error) { - cfg, err := GetFSConf() +func configFromPlugin(pluginName string) (*Config, error) { + //TODO: after experiment, make sure this is populated from conf file, not initialised here + cfgPath, err := config.Filename("") if err != nil { return nil, err } - ma, err := multiaddr.NewMultiaddr(cfg.Service[DefaultService]) + cfg, err := cserial.Load(cfgPath) if err != nil { return nil, err } - listenAddress, err := manet.Listen(ma) - if err != nil { - return nil, err + pluginCfg := cfg.Plugins.Plugins[PluginName] + if pluginCfg.Disabled { + return nil, errDisabled } - return listenAddress, nil + + serviceConfig, ok := pluginCfg.Config.(*Config) + if !ok { + if serviceConfig, err = defaultConfig(); err != nil { + return nil, err + } + } + + if addr := os.Getenv(EnvAddr); addr != "" { + serviceConfig.Service[DefaultService] = addr + } + + // assume user supplied env vars are set and expand them as-is + for service, target := range serviceConfig.Service { + serviceConfig.Service[service] = os.ExpandEnv(target) + } + + return serviceConfig, nil } From 7bb358f4f421c412d928ff4f3a1a7cf86ba7c6ca Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 30 Aug 2019 15:11:27 -0400 Subject: [PATCH 009/102] plugin/fs: Betterer init/config --- plugin/plugins/filesystem/client/client.go | 38 +++++++++-- plugin/plugins/filesystem/filesystem.go | 77 ++++++++++++---------- plugin/plugins/filesystem/utils.go | 75 +++------------------ 3 files changed, 85 insertions(+), 105 deletions(-) diff --git a/plugin/plugins/filesystem/client/client.go b/plugin/plugins/filesystem/client/client.go index 3d8fc23e361..0a8e737f84d 100644 --- a/plugin/plugins/filesystem/client/client.go +++ b/plugin/plugins/filesystem/client/client.go @@ -1,9 +1,13 @@ package p9client import ( + gopath "path" + "path/filepath" + "runtime" "strings" "github.com/hugelgupf/p9/p9" + config "github.com/ipfs/go-ipfs-config" "github.com/ipfs/go-ipfs/plugin/plugins/filesystem" logging "github.com/ipfs/go-log" "github.com/multiformats/go-multiaddr" @@ -22,14 +26,12 @@ func Dial(options ...Option) (*p9.Client, error) { op(ops) } + //TODO: this should probably be the callers responsibility if ops.address == filesystem.DefaultListenAddress { - // TODO: kludge - serviceConf, err := filesystem.XXX_GetFSConf() - if err != nil { + var err error + if ops.address, err = expandDefault(); err != nil { return nil, err } - - ops.address = serviceConf.Service[filesystem.DefaultService] } ma, err := multiaddr.NewMultiaddr(ops.address) @@ -46,6 +48,32 @@ func Dial(options ...Option) (*p9.Client, error) { return p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) } +func expandDefault() (string, error) { + _, sockName := gopath.Split(filesystem.DefaultListenAddress) + target, err := config.Path("", sockName) + if err != nil { + return target, err + } + + if !filepath.IsAbs(target) { + if target, err = filepath.Abs(target); err != nil { + return target, err + } + } + + if runtime.GOOS == "windows" { + //TODO [manet]: doesn't like drive letters + //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively + if len(target) > 2 && target[1] == ':' { + target = target[2:] + } + target = filepath.ToSlash(target) + } + + target = gopath.Join("/unix", target) + return target, nil +} + func ReadDir(path string, fsRef p9.File, offset uint64) ([]p9.Dirent, error) { components := strings.Split(strings.TrimPrefix(path, "/"), "/") if len(components) == 1 && components[0] == "" { diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 63d66749cb2..48a907a9055 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -2,7 +2,8 @@ package filesystem import ( "context" - "errors" + "encoding/json" + "path/filepath" "github.com/hugelgupf/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" @@ -22,11 +23,11 @@ var Plugins = []plugin.Plugin{ var _ plugin.PluginDaemon = (*FileSystemPlugin)(nil) type FileSystemPlugin struct { - ctx context.Context - cancel context.CancelFunc - listener manet.Listener + ctx context.Context + cancel context.CancelFunc - disabled bool + addr multiaddr.Multiaddr + listener manet.Listener } func (*FileSystemPlugin) Name() string { @@ -37,14 +38,42 @@ func (*FileSystemPlugin) Version() string { return PluginVersion } -func (fs *FileSystemPlugin) Init() error { +func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { + logger.Info("Initialising 9p resource server...") + if !filepath.IsAbs(env.Repo) { + absRepo, err := filepath.Abs(env.Repo) + if err != nil { + return err + } + env.Repo = absRepo + } + + cfg := &Config{} + if env.Config != nil { + byteRep, err := json.Marshal(env.Config) + if err != nil { + return err + } + if err = json.Unmarshal(byteRep, cfg); err != nil { + return err + } + } else { + cfg = defaultConfig(env.Repo) + } + + var err error + fs.addr, err = multiaddr.NewMultiaddr(cfg.Service[DefaultService]) + if err != nil { + return err + } + fs.ctx, fs.cancel = context.WithCancel(context.Background()) + logger.Info("9p resource server okay for launch") return nil } var ( - logger logging.EventLogger - errDisabled = errors.New("this experiment is disabled, enable with `ipfs config --json Experimental.FileSystemEnabled true`") + logger logging.EventLogger ) func init() { @@ -52,33 +81,18 @@ func init() { } func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { - logger.Info("Initialising 9p resource server...") - fs.disabled = true - - serviceConfig, err := XXX_GetFSConf() - if err != nil { - if err == errDisabled { - logger.Warning(errDisabled) - return nil - } - return err - } + logger.Info("Starting 9p resource server...") - ma, err := multiaddr.NewMultiaddr(serviceConfig.Service[DefaultService]) - if err != nil { - logger.Errorf("9P multiaddr error: %s\n", err) - return err - } - - if fs.listener, err = manet.Listen(ma); err != nil { + var err error + if fs.listener, err = manet.Listen(fs.addr); err != nil { logger.Errorf("9P listen error: %s\n", err) return err } - // construct 9p resource server + // construct 9P resource server p9pFSS, err := fsnodes.NewRoot(fs.ctx, core, logger) if err != nil { - logger.Errorf("9P construction error: %s\n", err) + logger.Errorf("9P root construction error: %s\n", err) return err } @@ -91,16 +105,11 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { } }() - fs.disabled = false - logger.Infof("9P service started on %s\n", fs.listener.Addr()) + logger.Infof("9P service is listening on %s\n", fs.listener.Addr()) return nil } func (fs *FileSystemPlugin) Close() error { - if fs.disabled { - return nil - } - //TODO: fmt.Println("Closing file system handles...") logger.Info("9P server requested to close") fs.cancel() diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go index 6b69db7a9ed..2b035079ee8 100644 --- a/plugin/plugins/filesystem/utils.go +++ b/plugin/plugins/filesystem/utils.go @@ -1,13 +1,9 @@ package filesystem import ( - "os" gopath "path" "path/filepath" "runtime" - - config "github.com/ipfs/go-ipfs-config" - cserial "github.com/ipfs/go-ipfs-config/serialize" ) const ( @@ -33,76 +29,23 @@ type Config struct { // NOTE: unstable/experimental Service map[string]string } -func defaultConfig() (*Config, error) { +func defaultConfig(storagePath string) *Config { serviceMap := make(map[string]string) - target, err := config.Path("", sockName) - if err != nil { - return nil, err - } + sockTarget := gopath.Join(storagePath, sockName) if runtime.GOOS == "windows" { - if target, err = windowsToUnixFriendly(target); err != nil { - return nil, err - } + sockTarget = windowsToUnixFriendly(sockTarget) } - serviceMap["9P"] = gopath.Join("/unix", target) - return &Config{serviceMap}, nil + serviceMap[DefaultService] = gopath.Join("/unix", sockTarget) + return &Config{serviceMap} } -//TODO: better name -func XXX_GetFSConf() (*Config, error) { - return configFromPlugin(PluginName) -} - -func windowsToUnixFriendly(target string) (string, error) { - if !filepath.IsAbs(target) { - var err error - if target, err = filepath.Abs(target); err != nil { - return target, err - } - } - +func windowsToUnixFriendly(target string) string { //TODO [manet]: doesn't like drive letters //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively - if len(target) > 3 && target[1] == ':' { - target = target[3:] - } - return filepath.ToSlash(target), nil -} - -func configFromPlugin(pluginName string) (*Config, error) { - //TODO: after experiment, make sure this is populated from conf file, not initialised here - cfgPath, err := config.Filename("") - if err != nil { - return nil, err - } - - cfg, err := cserial.Load(cfgPath) - if err != nil { - return nil, err - } - - pluginCfg := cfg.Plugins.Plugins[PluginName] - if pluginCfg.Disabled { - return nil, errDisabled - } - - serviceConfig, ok := pluginCfg.Config.(*Config) - if !ok { - if serviceConfig, err = defaultConfig(); err != nil { - return nil, err - } + if len(target) > 2 && target[1] == ':' { + target = target[2:] } - - if addr := os.Getenv(EnvAddr); addr != "" { - serviceConfig.Service[DefaultService] = addr - } - - // assume user supplied env vars are set and expand them as-is - for service, target := range serviceConfig.Service { - serviceConfig.Service[service] = os.ExpandEnv(target) - } - - return serviceConfig, nil + return filepath.ToSlash(target) } From 6a17332f1483d5e7777cd388c55d411982d9e60f Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 30 Aug 2019 16:43:55 -0400 Subject: [PATCH 010/102] plugin/fs: consistent protocol identifier --- plugin/plugins/filesystem/filesystem.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 48a907a9055..0799638bb45 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -39,7 +39,7 @@ func (*FileSystemPlugin) Version() string { } func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { - logger.Info("Initialising 9p resource server...") + logger.Info("Initialising 9P resource server...") if !filepath.IsAbs(env.Repo) { absRepo, err := filepath.Abs(env.Repo) if err != nil { @@ -68,7 +68,7 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } fs.ctx, fs.cancel = context.WithCancel(context.Background()) - logger.Info("9p resource server okay for launch") + logger.Info("9P resource server okay for launch") return nil } @@ -81,7 +81,7 @@ func init() { } func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { - logger.Info("Starting 9p resource server...") + logger.Info("Starting 9P resource server...") var err error if fs.listener, err = manet.Listen(fs.addr); err != nil { From 090f7cd55300bb660573beaac2aee77ac7b817e5 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 30 Aug 2019 16:44:51 -0400 Subject: [PATCH 011/102] plugin/fs: text secession --- plugin/plugins/filesystem/filesystem.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 0799638bb45..3c18b804971 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -39,7 +39,7 @@ func (*FileSystemPlugin) Version() string { } func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { - logger.Info("Initialising 9P resource server...") + logger.Info("Initializing 9P resource server...") if !filepath.IsAbs(env.Repo) { absRepo, err := filepath.Abs(env.Repo) if err != nil { From 19d5332f208ec45a1d9367362ec81ea0aceaa948 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 2 Sep 2019 12:41:08 -0400 Subject: [PATCH 012/102] squashme-deps --- go.mod | 7 ++----- go.sum | 13 ++++++------- plugin/plugins/filesystem/client/client.go | 2 +- plugin/plugins/filesystem/filesystem.go | 2 +- plugin/plugins/filesystem/nodes/base.go | 4 ++-- plugin/plugins/filesystem/nodes/ipfs.go | 2 +- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- plugin/plugins/filesystem/nodes/root.go | 2 +- plugin/plugins/filesystem/nodes/utils.go | 2 +- 9 files changed, 16 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 2c472f1dc74..f89073f40b5 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d + github.com/djdv/p9 v0.0.0-20190903035542-f10bd070d5fa github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fatih/color v1.7.0 // indirect @@ -15,7 +16,6 @@ require ( github.com/golangci/golangci-lint v1.17.1 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/golang-lru v0.5.3 - github.com/hugelgupf/p9 v0.0.0-20190806022330-7ba0920fba11 github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.0 @@ -110,7 +110,7 @@ require ( go.uber.org/goleak v0.10.0 // indirect go.uber.org/multierr v1.1.0 // indirect go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/sys v0.0.0-20190730183949-1393eb018365 + golang.org/x/sys v0.0.0-20190903213830-1f305c863dab golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac // indirect golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 @@ -120,6 +120,3 @@ require ( replace github.com/golangci/golangci-lint => github.com/mhutchinson/golangci-lint v1.17.2-0.20190819125825-d18f2136e32b go 1.12 - -// TODO: merge things and use proper repo references -replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20190828183744-eaad8e85066c diff --git a/go.sum b/go.sum index 501e235dbde..f75a35d298c 100644 --- a/go.sum +++ b/go.sum @@ -72,12 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/go-ipfs-config v0.0.8-0.20190828150305-9817c5842355 h1:jDE4uGhU9c/fa01Mrkk9JnmVKcAPnQIYL5774iajBeI= -github.com/djdv/go-ipfs-config v0.0.8-0.20190828150305-9817c5842355/go.mod h1:ribou1kTrjMvMNeU7wQN4vjaFSaXPBKkSKx1dPxe3eM= -github.com/djdv/go-ipfs-config v0.0.8-0.20190828212302-95198d79d7c9 h1:SjcUvZEJ9VwMW58QJO0YRqAqr9CL4t4hixSnFpaJbDY= -github.com/djdv/go-ipfs-config v0.0.8-0.20190828212302-95198d79d7c9/go.mod h1:ribou1kTrjMvMNeU7wQN4vjaFSaXPBKkSKx1dPxe3eM= -github.com/djdv/p9 v0.0.0-20190828183744-eaad8e85066c h1:PhV3zAWDlk3FjvaF+vW3SVc5xg1eBWT/z6NZ+AHiD0Q= -github.com/djdv/p9 v0.0.0-20190828183744-eaad8e85066c/go.mod h1:akLqomEjJslbyK2h+TEXmVabYavEatmOdB6iektRfEM= +github.com/djdv/p9 v0.0.0-20190903035542-f10bd070d5fa h1:io9eEeeCTX2C/8X9B+bqjAC4tfTdWnnP/xGvavjGsU8= +github.com/djdv/p9 v0.0.0-20190903035542-f10bd070d5fa/go.mod h1:zkE6LlYAu33C3BZhXmX0eTFyHDwM7cAnRvqd+JcEf4M= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= @@ -201,6 +197,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hugelgupf/p9 v0.0.0-20190902012917-f08d8f7f979b h1:8IouwKZVKhBvn3+SjnR7OFPIdCsDaN8rGOvbqI7zV5k= +github.com/hugelgupf/p9 v0.0.0-20190902012917-f08d8f7f979b/go.mod h1:lQha5pHJCBOvLDb0yaewOSud/TGvhlp9jtfQuIbgzJU= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= @@ -871,8 +869,9 @@ golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190730183949-1393eb018365 h1:SaXEMXhWzMJThc05vu6uh61Q245r4KaWMrsTedk0FDc= golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190903213830-1f305c863dab h1:2WmrFWBmUPXp+o/5X0nS66SLRS6DKwZlgFD76BKThvc= +golang.org/x/sys v0.0.0-20190903213830-1f305c863dab/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= diff --git a/plugin/plugins/filesystem/client/client.go b/plugin/plugins/filesystem/client/client.go index 0a8e737f84d..e62a8c4401f 100644 --- a/plugin/plugins/filesystem/client/client.go +++ b/plugin/plugins/filesystem/client/client.go @@ -6,7 +6,7 @@ import ( "runtime" "strings" - "github.com/hugelgupf/p9/p9" + "github.com/djdv/p9/p9" config "github.com/ipfs/go-ipfs-config" "github.com/ipfs/go-ipfs/plugin/plugins/filesystem" logging "github.com/ipfs/go-log" diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 3c18b804971..51026743097 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -5,7 +5,7 @@ import ( "encoding/json" "path/filepath" - "github.com/hugelgupf/p9/p9" + "github.com/djdv/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" logging "github.com/ipfs/go-log" diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index a285c7e38bf..87080dc25cb 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -3,8 +3,8 @@ package fsnodes import ( "context" - "github.com/hugelgupf/p9/p9" - "github.com/hugelgupf/p9/unimplfs" + "github.com/djdv/p9/p9" + "github.com/djdv/p9/unimplfs" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index cc19219dc02..da1b01986ca 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - "github.com/hugelgupf/p9/p9" + "github.com/djdv/p9/p9" files "github.com/ipfs/go-ipfs-files" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 1b440806048..331cc068c6d 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -4,7 +4,7 @@ import ( "context" gopath "path" - "github.com/hugelgupf/p9/p9" + "github.com/djdv/p9/p9" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" coreoptions "github.com/ipfs/interface-go-ipfs-core/options" diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index cac6417b8f2..daa41ee9bf3 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/hugelgupf/p9/p9" + "github.com/djdv/p9/p9" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index f1c5ae25d53..bf955405bae 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -5,7 +5,7 @@ import ( "hash/fnv" "time" - "github.com/hugelgupf/p9/p9" + "github.com/djdv/p9/p9" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" From 353007ee99e712cfb4ace6c1db557f9eaccd1af0 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 4 Sep 2019 08:22:07 -0400 Subject: [PATCH 013/102] plugin/fs: client fixes --- plugin/plugins/filesystem/client/client.go | 2 +- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- plugin/plugins/filesystem/nodes/root.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/plugins/filesystem/client/client.go b/plugin/plugins/filesystem/client/client.go index e62a8c4401f..25a73d384c6 100644 --- a/plugin/plugins/filesystem/client/client.go +++ b/plugin/plugins/filesystem/client/client.go @@ -86,7 +86,7 @@ func ReadDir(path string, fsRef p9.File, offset uint64) ([]p9.Dirent, error) { } logger.Debugf("walked to %q :\nQIDs:%v, FID:%v\n\n", path, qids, targetRef) - if _, _, err = targetRef.Open(0); err != nil { + if _, _, err = targetRef.Open(p9.ReadOnly); err != nil { return nil, err } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 331cc068c6d..a847c6d6be8 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -15,7 +15,7 @@ type PinFS struct { } //TODO: [review] check fields -func initPinFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { +func InitPinFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { pd := &PinFS{ IPFSBase: IPFSBase{ Path: newRootPath("/ipfs"), diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index daa41ee9bf3..d87a5c4c117 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -99,7 +99,7 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { //NOTE: if doClone is false, it implies len(names) > 0 switch names[0] { case "ipfs": - pinDir, err := initPinFS(ri.Ctx, ri.core, ri.Logger).Attach() + pinDir, err := InitPinFS(ri.Ctx, ri.core, ri.Logger).Attach() if err != nil { return nil, nil, err } From 6e7ea19bea31a27c5640b3982768a6ff4e6c7735 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 4 Sep 2019 08:12:33 -0400 Subject: [PATCH 014/102] plugin/fs: add PinFS test --- plugin/plugins/filesystem/filesystem_test.go | 248 +++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 plugin/plugins/filesystem/filesystem_test.go diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go new file mode 100644 index 00000000000..8a905ca4bad --- /dev/null +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -0,0 +1,248 @@ +package filesystem + +import ( + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + gopath "path" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/djdv/p9/p9" + files "github.com/ipfs/go-ipfs-files" + + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/core/coreapi" + fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" + coreoptions "github.com/ipfs/interface-go-ipfs-core/options" + corepath "github.com/ipfs/interface-go-ipfs-core/path" +) + +func TestRoot(t *testing.T) {} +func TestPinFS(t *testing.T) { //TODO: breakup + //init + ctx := context.TODO() + core, err := initCore(ctx) + if err != nil { + t.Fatalf("Failed to construct IPFS node: %s\n", err) + } + + logger := logging.Logger("plugin/filesystem") + pinAttacher := fsnodes.InitPinFS(ctx, core, logger) // like a sewist? + + root, err := pinAttacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) + } + + same := func(base, target []string) bool { + if len(base) != len(target) { + return false + } + sort.Strings(base) + sort.Strings(target) + + for i := len(base) - 1; i >= 0; i-- { + if target[i] != base[i] { + return false + } + } + return true + } + + //test default (likely empty) test repo pins + basePins, err := pinNames(ctx, core) + if err != nil { + t.Fatalf("Failed to list IPFS pins: %s\n", err) + } + p9Pins, err := p9PinNames(root) + if err != nil { + t.Fatalf("Failed to list 9P pins: %s\n", err) + } + + if !same(basePins, p9Pins) { + t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + } + + // test modifying pinset +1; initEnv pins its IPFS envrionment + env, iEnv, err := initEnv(ctx, core) + if err != nil { + t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + } + defer os.RemoveAll(env) + basePins = append(basePins, gopath.Base(iEnv.String())) + p9Pins, err = p9PinNames(root) + if err != nil { + t.Fatalf("Failed to list 9P pins: %s\n", err) + } + + if !same(basePins, p9Pins) { + t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + } + + // test modifying pinset +1 again; generate garbage and pin it + { + if err := generateGarbage(env); err != nil { + t.Fatalf("Failed to generate test data: %s\n", err) + } + + iPath, err := pinAddDir(ctx, core, env) + if err != nil { + t.Fatalf("Failed to add directory to IPFS: %s\n", err) + } + basePins = append(basePins, gopath.Base(iPath.String())) + } + + p9Pins, err = p9PinNames(root) + if err != nil { + t.Fatalf("Failed to list 9P pins: %s\n", err) + } + + if !same(basePins, p9Pins) { + t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + } + + t.Logf("pinroot contains: %v\n", p9Pins) +} +func TestIPFS(t *testing.T) { + /* TODO + ctx := context.TODO() + core, err := initCore(ctx) + if err != nil { + t.Fatalf("Failed to construct IPFS node: %s\n", err) + } + + env, iEnv, err := initEnv(ctx, core) + if err != nil { + t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + } + + t.Logf("env:%v\niEnv:%v\nerr:%s\n", env, iEnv, err) + defer os.RemoveAll(env) + */ +} + +func initCore(ctx context.Context) (coreiface.CoreAPI, error) { + node, err := core.NewNode(ctx, &core.BuildCfg{ + Online: false, + Permanent: false, + DisableEncryptedConnections: true, + }) + if err != nil { + return nil, err + } + + return coreapi.NewCoreAPI(node) +} + +const incantation = "May the bits passing through this device somehow help bring peace to this world" + +func initEnv(ctx context.Context, core coreiface.CoreAPI) (string, corepath.Resolved, error) { + tempDir, err := ioutil.TempDir("", "ipfs-") + if err != nil { + return "", nil, err + } + + if err = ioutil.WriteFile(filepath.Join(tempDir, "empty"), + []byte(nil), + 0644); err != nil { + return "", nil, err + } + + if err = ioutil.WriteFile(filepath.Join(tempDir, "small"), + []byte(incantation), + 0644); err != nil { + return "", nil, err + } + + if err := generateGarbage(tempDir); err != nil { + return "", nil, err + } + + iPath, err := pinAddDir(ctx, core, tempDir) + if err != nil { + return "", nil, err + } + + return tempDir, iPath, err +} + +func pinAddDir(ctx context.Context, core coreiface.CoreAPI, path string) (corepath.Resolved, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + + node, err := files.NewSerialFile(path, false, fi) + if err != nil { + return nil, err + } + + iPath, err := core.Unixfs().Add(ctx, node.(files.Directory), coreoptions.Unixfs.Pin(true)) + if err != nil { + return nil, err + } + return iPath, nil +} + +func generateGarbage(tempDir string) error { + randDev := rand.New(rand.NewSource(time.Now().UnixNano())) + + for _, size := range []int{4, 8, 16, 32} { + buf := make([]byte, size<<(10*2)) + if _, err := randDev.Read(buf); err != nil { + return err + } + + name := fmt.Sprintf("%dMiB", size) + if err := ioutil.WriteFile(filepath.Join(tempDir, name), + buf, + 0644); err != nil { + return err + } + } + + return nil +} + +func pinNames(ctx context.Context, core coreiface.CoreAPI) ([]string, error) { + pins, err := core.Pin().Ls(ctx, coreoptions.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + names := make([]string, 0, len(pins)) + for _, pin := range pins { + names = append(names, gopath.Base(pin.Path().String())) + } + return names, nil +} + +func p9PinNames(root p9.File) ([]string, error) { + _, rootDir, err := root.Walk(nil) + if err != nil { + return nil, err + } + + _, _, err = rootDir.Open(p9.ReadOnly) + if err != nil { + return nil, err + } + ents, err := rootDir.Readdir(0, ^uint32(0)) + if err != nil { + return nil, err + } + + names := make([]string, 0, len(ents)) + + for _, ent := range ents { + names = append(names, ent.Name) + } + + return names, rootDir.Close() +} From 28c5cffb88a5d2856ee537f6acfe75a638d87435 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 4 Sep 2019 08:50:05 -0400 Subject: [PATCH 015/102] plugin/fs: add root test + change pin test --- plugin/plugins/filesystem/filesystem_test.go | 96 ++++++++++++-------- 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 8a905ca4bad..0b83198d112 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -24,9 +24,7 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) -func TestRoot(t *testing.T) {} -func TestPinFS(t *testing.T) { //TODO: breakup - //init +func TestAll(t *testing.T) { ctx := context.TODO() core, err := initCore(ctx) if err != nil { @@ -34,9 +32,42 @@ func TestPinFS(t *testing.T) { //TODO: breakup } logger := logging.Logger("plugin/filesystem") - pinAttacher := fsnodes.InitPinFS(ctx, core, logger) // like a sewist? - root, err := pinAttacher.Attach() + t.Run("RootFS", func(t *testing.T) { testRootFS(t, ctx, core, logger) }) + t.Run("PinFS", func(t *testing.T) { testPinFS(t, ctx, core, logger) }) +} + +func testRootFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) { + ri, err := fsnodes.NewRoot(ctx, core, logger) + if err != nil { + t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + } + + nineRoot, err := ri.Attach() + _, nineRef, err := nineRoot.Walk(nil) + if err != nil { + t.Fatalf("Failed to walk root: %s\n", err) + } + if _, _, err = nineRef.Open(p9.ReadOnly); err != nil { + t.Fatalf("Failed to open root: %s\n", err) + } + + ents, err := nineRef.Readdir(0, ^uint32(0)) + if err != nil { + t.Fatalf("Failed to read root: %s\n", err) + } + + //TODO: currently magic, as subsystems are implemented, rework this part of the test + lib + if len(ents) != 1 || ents[0].Name != "ipfs" { + t.Fatalf("Failed, root has bad entries:: %v\n", ents) + } + + //TODO: type checking +} + +func testPinFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) { + //init + pinRoot, err := fsnodes.InitPinFS(ctx, core, logger).Attach() if err != nil { t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) } @@ -56,35 +87,31 @@ func TestPinFS(t *testing.T) { //TODO: breakup return true } - //test default (likely empty) test repo pins - basePins, err := pinNames(ctx, core) - if err != nil { - t.Fatalf("Failed to list IPFS pins: %s\n", err) - } - p9Pins, err := p9PinNames(root) - if err != nil { - t.Fatalf("Failed to list 9P pins: %s\n", err) - } + shallowCompare := func() { + basePins, err := pinNames(ctx, core) + if err != nil { + t.Fatalf("Failed to list IPFS pins: %s\n", err) + } + p9Pins, err := p9PinNames(pinRoot) + if err != nil { + t.Fatalf("Failed to list 9P pins: %s\n", err) + } - if !same(basePins, p9Pins) { - t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + if !same(basePins, p9Pins) { + t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + } } + //test default (likely empty) test repo pins + shallowCompare() + // test modifying pinset +1; initEnv pins its IPFS envrionment - env, iEnv, err := initEnv(ctx, core) + env, _, err := initEnv(ctx, core) if err != nil { t.Fatalf("Failed to construct IPFS test environment: %s\n", err) } defer os.RemoveAll(env) - basePins = append(basePins, gopath.Base(iEnv.String())) - p9Pins, err = p9PinNames(root) - if err != nil { - t.Fatalf("Failed to list 9P pins: %s\n", err) - } - - if !same(basePins, p9Pins) { - t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) - } + shallowCompare() // test modifying pinset +1 again; generate garbage and pin it { @@ -92,26 +119,16 @@ func TestPinFS(t *testing.T) { //TODO: breakup t.Fatalf("Failed to generate test data: %s\n", err) } - iPath, err := pinAddDir(ctx, core, env) + _, err := pinAddDir(ctx, core, env) if err != nil { t.Fatalf("Failed to add directory to IPFS: %s\n", err) } - basePins = append(basePins, gopath.Base(iPath.String())) - } - - p9Pins, err = p9PinNames(root) - if err != nil { - t.Fatalf("Failed to list 9P pins: %s\n", err) - } - - if !same(basePins, p9Pins) { - t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) } + shallowCompare() - t.Logf("pinroot contains: %v\n", p9Pins) + //TODO: type checking } func TestIPFS(t *testing.T) { - /* TODO ctx := context.TODO() core, err := initCore(ctx) if err != nil { @@ -125,7 +142,6 @@ func TestIPFS(t *testing.T) { t.Logf("env:%v\niEnv:%v\nerr:%s\n", env, iEnv, err) defer os.RemoveAll(env) - */ } func initCore(ctx context.Context) (coreiface.CoreAPI, error) { From 81b9a068e5cf79ac616b5dc8e3695619ace30c4e Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 4 Sep 2019 15:53:31 -0400 Subject: [PATCH 016/102] IPFS-test WIP --- go.mod | 2 +- go.sum | 7 +- plugin/plugins/filesystem/filesystem_test.go | 190 +++++++++++++++---- plugin/plugins/filesystem/nodes/ipfs.go | 11 +- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- plugin/plugins/filesystem/nodes/utils.go | 45 +++-- 6 files changed, 193 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index f89073f40b5..64456c5455f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d - github.com/djdv/p9 v0.0.0-20190903035542-f10bd070d5fa + github.com/djdv/p9 v0.0.0-20190904195120-79d6235e5781 github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fatih/color v1.7.0 // indirect diff --git a/go.sum b/go.sum index f75a35d298c..f57d01a5a8e 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/p9 v0.0.0-20190903035542-f10bd070d5fa h1:io9eEeeCTX2C/8X9B+bqjAC4tfTdWnnP/xGvavjGsU8= -github.com/djdv/p9 v0.0.0-20190903035542-f10bd070d5fa/go.mod h1:zkE6LlYAu33C3BZhXmX0eTFyHDwM7cAnRvqd+JcEf4M= +github.com/djdv/p9 v0.0.0-20190904195120-79d6235e5781 h1:S4uzVqSytBLRlOp444yimbtrIqm/A31BC7KbyTpgAm8= +github.com/djdv/p9 v0.0.0-20190904195120-79d6235e5781/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= @@ -197,8 +197,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hugelgupf/p9 v0.0.0-20190902012917-f08d8f7f979b h1:8IouwKZVKhBvn3+SjnR7OFPIdCsDaN8rGOvbqI7zV5k= -github.com/hugelgupf/p9 v0.0.0-20190902012917-f08d8f7f979b/go.mod h1:lQha5pHJCBOvLDb0yaewOSud/TGvhlp9jtfQuIbgzJU= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= @@ -869,7 +867,6 @@ golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190903213830-1f305c863dab h1:2WmrFWBmUPXp+o/5X0nS66SLRS6DKwZlgFD76BKThvc= golang.org/x/sys v0.0.0-20190903213830-1f305c863dab/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 0b83198d112..568690aa7cb 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/djdv/p9/localfs" "github.com/djdv/p9/p9" files "github.com/ipfs/go-ipfs-files" @@ -24,6 +25,11 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) +var attrMaskIPFSTest = p9.AttrMask{ + Mode: true, + Size: true, +} + func TestAll(t *testing.T) { ctx := context.TODO() core, err := initCore(ctx) @@ -35,6 +41,7 @@ func TestAll(t *testing.T) { t.Run("RootFS", func(t *testing.T) { testRootFS(t, ctx, core, logger) }) t.Run("PinFS", func(t *testing.T) { testPinFS(t, ctx, core, logger) }) + t.Run("IPFS", func(t *testing.T) { testIPFS(t, ctx, core, logger) }) } func testRootFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) { @@ -114,34 +121,125 @@ func testPinFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger shallowCompare() // test modifying pinset +1 again; generate garbage and pin it - { - if err := generateGarbage(env); err != nil { - t.Fatalf("Failed to generate test data: %s\n", err) - } - - _, err := pinAddDir(ctx, core, env) - if err != nil { - t.Fatalf("Failed to add directory to IPFS: %s\n", err) - } + if err := generateGarbage(env); err != nil { + t.Fatalf("Failed to generate test data: %s\n", err) + } + if _, err = pinAddDir(ctx, core, env); err != nil { + t.Fatalf("Failed to add directory to IPFS: %s\n", err) } shallowCompare() //TODO: type checking } -func TestIPFS(t *testing.T) { - ctx := context.TODO() - core, err := initCore(ctx) +func testIPFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) { + env, iEnv, err := initEnv(ctx, core) if err != nil { - t.Fatalf("Failed to construct IPFS node: %s\n", err) + t.Fatalf("Failed to construct IPFS test environment: %s\n", err) } + defer os.RemoveAll(env) - env, iEnv, err := initEnv(ctx, core) + localEnv, err := localfs.Attacher(env).Attach() if err != nil { - t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + t.Fatalf("Failed to attach to local resource %q: %s\n", env, err) } - t.Logf("env:%v\niEnv:%v\nerr:%s\n", env, iEnv, err) - defer os.RemoveAll(env) + ipfsRoot, err := fsnodes.InitIPFS(ctx, core, logger).Attach() + if err != nil { + t.Fatalf("Failed to attach to IPFS resource: %s\n", err) + } + _, ipfsEnv, err := ipfsRoot.Walk([]string{gopath.Base(iEnv.String())}) + if err != nil { + t.Fatalf("Failed to walk to IPFS test envrionment: %s\n", err) + } + + recursiveCompare(t, localEnv, ipfsEnv) +} + +//TODO: rename +func recursiveCompare(t *testing.T, f1, f2 p9.File) { + var expand func(p9.File) (map[string]p9.Attr, error) + expand = func(nineRef p9.File) (map[string]p9.Attr, error) { + ents, err := p9Readdir(nineRef) + if err != nil { + return nil, err + } + + //TODO: current + // map[string]Stat; ["/sub/incantation"]{...} + ///from root, walk(name); stat; if dir; recurse + + res := make(map[string]p9.Attr) + for _, ent := range ents { + _, child, err := nineRef.Walk([]string{ent.Name}) + if err != nil { + return nil, err + } + + _, _, attr, err := child.GetAttr(attrMaskIPFSTest) + if err != nil { + return nil, err + } + res[ent.Name] = attr + //p9.AttrMaskAll + + if ent.Type == p9.TypeDir { + subRes, err := expand(child) + if err != nil { + return nil, err + } + for name, attr := range subRes { + res[gopath.Join(ent.Name, name)] = attr + } + } + } + return res, nil + } + + f1Map, err := expand(f1) + if err != nil { + t.Fatal(err) + } + + f2Map, err := expand(f2) + if err != nil { + t.Fatal(err) + } + + same := func(permissionContains p9.FileMode, base, target map[string]p9.Attr) bool { + if len(base) != len(target) { + + var baseNames []string + var targetNames []string + for name, _ := range base { + baseNames = append(baseNames, name) + } + for name, _ := range target { + targetNames = append(targetNames, name) + } + + t.Fatalf("map lengths don't match:\nbase:%v\ntarget:%v\n", baseNames, targetNames) + return false + } + + for path, baseAttr := range base { + bMode := baseAttr.Mode + tMode := target[path].Mode + + if bMode.FileType() != tMode.FileType() { + t.Fatalf("type for %q don't match:\nbase:%v\ntarget:%v\n", path, bMode, tMode) + return false + } + + if ((bMode.Permissions() & permissionContains) & (tMode.Permissions() & permissionContains)) == 0 { + t.Fatalf("permissions for %q don't match (unfiltered):\nbase:%v\ntarget:%v\n", path, bMode.Permissions(), tMode.Permissions()) + return false + } + } + return true + } + if !same(p9.Read, f1Map, f2Map) { + t.Fatalf("contents don't match \nf1:%v\nf2:%v\n", f1Map, f2Map) + } } func initCore(ctx context.Context) (coreiface.CoreAPI, error) { @@ -160,33 +258,41 @@ func initCore(ctx context.Context) (coreiface.CoreAPI, error) { const incantation = "May the bits passing through this device somehow help bring peace to this world" func initEnv(ctx context.Context, core coreiface.CoreAPI) (string, corepath.Resolved, error) { - tempDir, err := ioutil.TempDir("", "ipfs-") + testDir, err := ioutil.TempDir("", "ipfs-") if err != nil { return "", nil, err } - if err = ioutil.WriteFile(filepath.Join(tempDir, "empty"), + if err = ioutil.WriteFile(filepath.Join(testDir, "empty"), []byte(nil), 0644); err != nil { return "", nil, err } - if err = ioutil.WriteFile(filepath.Join(tempDir, "small"), + if err = ioutil.WriteFile(filepath.Join(testDir, "small"), []byte(incantation), 0644); err != nil { return "", nil, err } - if err := generateGarbage(tempDir); err != nil { + if err := generateGarbage(testDir); err != nil { return "", nil, err } - iPath, err := pinAddDir(ctx, core, tempDir) + testSubDir, err := ioutil.TempDir(testDir, "ipfs-") if err != nil { return "", nil, err } + if err := generateGarbage(testSubDir); err != nil { + return "", nil, err + } - return tempDir, iPath, err + iPath, err := pinAddDir(ctx, core, testDir) + if err != nil { + return "", nil, err + } + + return testDir, iPath, err } func pinAddDir(ctx context.Context, core coreiface.CoreAPI, path string) (corepath.Resolved, error) { @@ -240,25 +346,45 @@ func pinNames(ctx context.Context, core coreiface.CoreAPI) ([]string, error) { } func p9PinNames(root p9.File) ([]string, error) { - _, rootDir, err := root.Walk(nil) + ents, err := p9Readdir(root) if err != nil { return nil, err } - _, _, err = rootDir.Open(p9.ReadOnly) + names := make([]string, 0, len(ents)) + + for _, ent := range ents { + names = append(names, ent.Name) + } + + return names, root.Close() +} + +func p9Readdir(dir p9.File) ([]p9.Dirent, error) { + _, dir, err := dir.Walk(nil) if err != nil { return nil, err } - ents, err := rootDir.Readdir(0, ^uint32(0)) + + _, _, err = dir.Open(p9.ReadOnly) if err != nil { return nil, err } + defer dir.Close() + return dir.Readdir(0, ^uint32(0)) +} - names := make([]string, 0, len(ents)) - - for _, ent := range ents { - names = append(names, ent.Name) +// NOTE: compares a subset of attributes, matching those of IPFS +func testIPFSCompare(t *testing.T, f1, f2 p9.File) { + _, _, f1Attr, err := f1.GetAttr(attrMaskIPFSTest) + if err != nil { + t.Errorf("Attr(%v) = %v, want nil", f1, err) + } + _, _, f2Attr, err := f2.GetAttr(attrMaskIPFSTest) + if err != nil { + t.Errorf("Attr(%v) = %v, want nil", f2, err) + } + if f1Attr != f2Attr { + t.Errorf("Attributes of same files do not match: %v and %v", f1Attr, f2Attr) } - - return names, rootDir.Close() } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index da1b01986ca..ce770152f2a 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -17,7 +17,7 @@ type IPFS struct { } //TODO: [review] check fields; better wrappers around inheritance init, etc. -func initIPFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { +func InitIPFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { id := &IPFS{IPFSBase: newIPFSBase(ctx, newRootPath("/ipfs"), p9.TypeDir, core, logger)} id.meta, id.metaMask = defaultRootAttr() return id @@ -37,14 +37,15 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { return id.Qid, id.metaMask, id.meta, nil } - var attrMask p9.AttrMask - if err := coreGetAttr(id.Ctx, &id.meta, &attrMask, id.core, id.Path); err != nil { + if err := coreGetAttr(id.Ctx, &id.meta, req, id.core, id.Path); err != nil { return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err } - timeStamp(&id.meta, &attrMask) id.Qid.Type = id.meta.Mode.QIDType() - return id.Qid, attrMask, id.meta, nil + metaClone := id.meta + metaClone.Filter(req) + + return id.Qid, req, metaClone, nil } func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index a847c6d6be8..3f7c6f1e4f8 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -51,7 +51,7 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { return []p9.QID{pd.Qid}, pd, nil } - ipfsDir, err := initIPFS(pd.Ctx, pd.core, pd.Logger).Attach() + ipfsDir, err := InitIPFS(pd.Ctx, pd.core, pd.Logger).Attach() if err != nil { return nil, nil, err } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index bf955405bae..e605e6c17ff 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -12,7 +12,6 @@ import ( "github.com/ipfs/go-unixfs" unixpb "github.com/ipfs/go-unixfs/pb" coreiface "github.com/ipfs/interface-go-ipfs-core" - coreoptions "github.com/ipfs/interface-go-ipfs-core/options" corepath "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -39,9 +38,7 @@ func coreStat(ctx context.Context, dirEnt *p9.Dirent, core coreiface.CoreAPI, pa } } -//TODO: consider how we want to use AttrMask -// instead of filling it we can use it to only populate requested fields (as is intended) -func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask *p9.AttrMask, core coreiface.CoreAPI, path corepath.Path) (err error) { +func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask p9.AttrMask, core coreiface.CoreAPI, path corepath.Path) (err error) { ipldNode, err := core.ResolveNode(ctx, path) if err != nil { return err @@ -51,16 +48,25 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask *p9.AttrMask, core return err } + if attrMask.Mode { attr.Mode = IRXA //TODO: this should probably be the callers responsability; just document that permissions should be set afterwards or something attr.Mode |= unixfsTypeTo9Mode(ufsNode.Type()) - attrMask.Mode = true + } + if attrMask.Blocks{ if bs := ufsNode.BlockSizes(); len(bs) != 0 { attr.BlockSize = bs[0] //NOTE: this value is to be used as a hint only; subsequent child block size may differ } + //TODO [eventualy]: switch off here for handling of time metadata in new format standard + timeStamp(attr, attrMask) +} + +if attrMask.Size { attr.Size, attrMask.Size = ufsNode.FileSize(), true +} +if attrMask.RDev { switch path.Namespace() { case "ipfs": attr.RDev, attrMask.RDev = dIPFS, true @@ -68,6 +74,7 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask *p9.AttrMask, core //attr.RDev, attrMask.RDev = dIPNS, true //etc. } +} return nil } @@ -102,7 +109,7 @@ func coreLs(ctx context.Context, corePath corepath.Path, core coreiface.CoreAPI) //asyncContext := deriveTimerContext(ctx, 10*time.Second) asyncContext := ctx - coreChan, err := core.Unixfs().Ls(asyncContext, corePath, coreoptions.Unixfs.ResolveChildren(false)) + coreChan, err := core.Unixfs().Ls(asyncContext, corePath) if err != nil { //asyncContext.Cancel() return nil, err @@ -232,7 +239,6 @@ const ( // pedantic POSIX stuff IRWXA = S_IRWXU | S_IRWXG | S_IRWXO // 0777 IRXA = IRWXA &^ (S_IWUSR | S_IWGRP | S_IWOTH) // 0555 -//03664 ) func defaultRootAttr() (attr p9.Attr, attrMask p9.AttrMask) { @@ -241,23 +247,22 @@ func defaultRootAttr() (attr p9.Attr, attrMask p9.AttrMask) { attrMask.Mode = true attrMask.RDev = true attrMask.Size = true - timeStamp(&attr, &attrMask) + //timeStamp(&attr, attrMask) return attr, attrMask } -func timeStamp(attr *p9.Attr, mask *p9.AttrMask) { +func timeStamp(attr *p9.Attr, mask p9.AttrMask) { now := time.Now() - attr.ATimeSeconds = uint64(now.Unix()) - attr.ATimeNanoSeconds = uint64(now.UnixNano()) - attr.MTimeSeconds = uint64(now.Unix()) - attr.MTimeNanoSeconds = uint64(now.UnixNano()) - attr.CTimeSeconds = uint64(now.Unix()) - attr.CTimeNanoSeconds = uint64(now.UnixNano()) - - mask.ATime = true - mask.MTime = true - mask.CTime = true -} +if mask.ATime { +attr.ATimeSeconds , attr.ATimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) + } + if mask.MTime { +attr.MTimeSeconds , attr.MTimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) + } + if mask.CTime { +attr.CTimeSeconds , attr.CTimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) + } + } //TODO [name]: "new" implies pointer type; this is for embedded consturction func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, core coreiface.CoreAPI, logger logging.EventLogger) IPFSBase { From 864dbfe2ab34cc281ef3f1f8555bf1244f12ab66 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 5 Sep 2019 09:51:52 -0400 Subject: [PATCH 017/102] deps --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 64456c5455f..103c89a65a6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d - github.com/djdv/p9 v0.0.0-20190904195120-79d6235e5781 + github.com/djdv/p9 v0.0.0-20190906032509-18ce8c2eba44 github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fatih/color v1.7.0 // indirect diff --git a/go.sum b/go.sum index f57d01a5a8e..57117759b4d 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/p9 v0.0.0-20190904195120-79d6235e5781 h1:S4uzVqSytBLRlOp444yimbtrIqm/A31BC7KbyTpgAm8= -github.com/djdv/p9 v0.0.0-20190904195120-79d6235e5781/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= +github.com/djdv/p9 v0.0.0-20190906032509-18ce8c2eba44 h1:y9wbNe6laqcn1PgVJXKhQlJqMZ3M4kYb0MxPDbZnVmA= +github.com/djdv/p9 v0.0.0-20190906032509-18ce8c2eba44/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= From 35eee218f3f177ea95295ad2dcb2f824a4744937 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 6 Sep 2019 16:45:22 -0400 Subject: [PATCH 018/102] fix fs test on linux --- plugin/plugins/filesystem/filesystem_test.go | 13 ++++- plugin/plugins/filesystem/nodes/utils.go | 50 ++++++++++---------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 568690aa7cb..ac7e3051d0a 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -231,7 +231,11 @@ func recursiveCompare(t *testing.T, f1, f2 p9.File) { } if ((bMode.Permissions() & permissionContains) & (tMode.Permissions() & permissionContains)) == 0 { - t.Fatalf("permissions for %q don't match (unfiltered):\nbase:%v\ntarget:%v\n", path, bMode.Permissions(), tMode.Permissions()) + t.Fatalf("permissions for %q don't match\n(unfiltered)\nbase:%v\ntarget:%v\n(filtered)\nbase:%v\ntarget:%v\n", + path, + bMode.Permissions(), tMode.Permissions(), + bMode.Permissions()&permissionContains, tMode.Permissions()&permissionContains, + ) return false } } @@ -262,6 +266,9 @@ func initEnv(ctx context.Context, core coreiface.CoreAPI) (string, corepath.Reso if err != nil { return "", nil, err } + if err := os.Chmod(testDir, 0775); err != nil { + return "", nil, err + } if err = ioutil.WriteFile(filepath.Join(testDir, "empty"), []byte(nil), @@ -283,6 +290,10 @@ func initEnv(ctx context.Context, core coreiface.CoreAPI) (string, corepath.Reso if err != nil { return "", nil, err } + if err := os.Chmod(testSubDir, 0775); err != nil { + return "", nil, err + } + if err := generateGarbage(testSubDir); err != nil { return "", nil, err } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index e605e6c17ff..be784ca0326 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -49,32 +49,32 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask p9.AttrMask, core } if attrMask.Mode { - attr.Mode = IRXA //TODO: this should probably be the callers responsability; just document that permissions should be set afterwards or something - attr.Mode |= unixfsTypeTo9Mode(ufsNode.Type()) + attr.Mode = IRXA //TODO: this should probably be the callers responsability; just document that permissions should be set afterwards or something + attr.Mode |= unixfsTypeTo9Mode(ufsNode.Type()) } - if attrMask.Blocks{ - if bs := ufsNode.BlockSizes(); len(bs) != 0 { - attr.BlockSize = bs[0] //NOTE: this value is to be used as a hint only; subsequent child block size may differ - } + if attrMask.Blocks { + if bs := ufsNode.BlockSizes(); len(bs) != 0 { + attr.BlockSize = bs[0] //NOTE: this value is to be used as a hint only; subsequent child block size may differ + } - //TODO [eventualy]: switch off here for handling of time metadata in new format standard - timeStamp(attr, attrMask) -} + //TODO [eventualy]: switch off here for handling of time metadata in new format standard + timeStamp(attr, attrMask) + } -if attrMask.Size { - attr.Size, attrMask.Size = ufsNode.FileSize(), true -} + if attrMask.Size { + attr.Size, attrMask.Size = ufsNode.FileSize(), true + } -if attrMask.RDev { - switch path.Namespace() { - case "ipfs": - attr.RDev, attrMask.RDev = dIPFS, true - //case "ipns": - //attr.RDev, attrMask.RDev = dIPNS, true - //etc. + if attrMask.RDev { + switch path.Namespace() { + case "ipfs": + attr.RDev, attrMask.RDev = dIPFS, true + //case "ipns": + //attr.RDev, attrMask.RDev = dIPNS, true + //etc. + } } -} return nil } @@ -253,16 +253,16 @@ func defaultRootAttr() (attr p9.Attr, attrMask p9.AttrMask) { func timeStamp(attr *p9.Attr, mask p9.AttrMask) { now := time.Now() -if mask.ATime { -attr.ATimeSeconds , attr.ATimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) + if mask.ATime { + attr.ATimeSeconds, attr.ATimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) } if mask.MTime { -attr.MTimeSeconds , attr.MTimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) + attr.MTimeSeconds, attr.MTimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) } if mask.CTime { -attr.CTimeSeconds , attr.CTimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) + attr.CTimeSeconds, attr.CTimeNanoSeconds = uint64(now.Unix()), uint64(now.UnixNano()) } - } +} //TODO [name]: "new" implies pointer type; this is for embedded consturction func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, core coreiface.CoreAPI, logger logging.EventLogger) IPFSBase { From 483845c7180984ebc13525b0aa24b1bd13f90589 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 6 Sep 2019 22:30:08 -0400 Subject: [PATCH 019/102] CR feedback + docs WIP, linting, fix freebsd builds --- go.mod | 2 +- go.sum | 4 +- plugin/plugins/filesystem/doc.go | 10 ++++ plugin/plugins/filesystem/filesystem.go | 40 ++++++-------- plugin/plugins/filesystem/filesystem_test.go | 56 ++++++++------------ plugin/plugins/filesystem/nodes/doc.go | 7 +++ plugin/plugins/filesystem/nodes/doc_test.go | 31 +++++++++++ plugin/plugins/filesystem/nodes/ipfs.go | 9 ++-- plugin/plugins/filesystem/nodes/pinfs.go | 16 ++---- plugin/plugins/filesystem/nodes/root.go | 16 +++--- plugin/plugins/filesystem/nodes/utils.go | 2 +- plugin/plugins/filesystem/utils.go | 12 ++--- 12 files changed, 114 insertions(+), 91 deletions(-) create mode 100644 plugin/plugins/filesystem/doc.go create mode 100644 plugin/plugins/filesystem/nodes/doc.go create mode 100644 plugin/plugins/filesystem/nodes/doc_test.go diff --git a/go.mod b/go.mod index 103c89a65a6..6e08766e5e8 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d - github.com/djdv/p9 v0.0.0-20190906032509-18ce8c2eba44 + github.com/djdv/p9 v0.0.0-20190907022512-79c201cd4ecc github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fatih/color v1.7.0 // indirect diff --git a/go.sum b/go.sum index 57117759b4d..e5183c18dd6 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/p9 v0.0.0-20190906032509-18ce8c2eba44 h1:y9wbNe6laqcn1PgVJXKhQlJqMZ3M4kYb0MxPDbZnVmA= -github.com/djdv/p9 v0.0.0-20190906032509-18ce8c2eba44/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= +github.com/djdv/p9 v0.0.0-20190907022512-79c201cd4ecc h1:v3PQvBZcOFgmwlNYeyFBlT23/x1ec9gZZTM6GQ/9Sbs= +github.com/djdv/p9 v0.0.0-20190907022512-79c201cd4ecc/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= diff --git a/plugin/plugins/filesystem/doc.go b/plugin/plugins/filesystem/doc.go new file mode 100644 index 00000000000..d35b72593ce --- /dev/null +++ b/plugin/plugins/filesystem/doc.go @@ -0,0 +1,10 @@ +/*Package filesystem is an experimental package, that implements the go-ipfs daemon plugin interface + and defines the plugins config structure. + + To set the multiaddr listen address, you may use the environment variable $IPFS_FS_ADDR, or set the option in a config file + via `ipfs config --json 'Plugins.Plugins.filesystem.Config "Config":{"Service":{"9P":"/ip4/127.0.0.1/tcp/567"}}'` + To disable this plugin entirely, use: `ipfs config --json Plugins.Plugins.filesystem.Disabled true` + + By default, we try to expose the IPFS namespace using the 9P2000.L protocol, over a unix domain socket + (located at $IPFS_PATH/filesystem.9P.sock)*/ +package filesystem diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 51026743097..88b368ba576 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -14,13 +14,20 @@ import ( manet "github.com/multiformats/go-multiaddr-net" ) -// Plugins is an exported list of plugins that will be loaded by go-ipfs. -var Plugins = []plugin.Plugin{ - &FileSystemPlugin{}, //TODO: individually name implementations: &P9{} -} +var ( + _ plugin.PluginDaemon = (*FileSystemPlugin)(nil) // impl check + + // Plugins is an exported list of plugins that will be loaded by go-ipfs. + Plugins = []plugin.Plugin{ + &FileSystemPlugin{}, //TODO: individually name implementations: &P9{} + } -// impl check -var _ plugin.PluginDaemon = (*FileSystemPlugin)(nil) + logger logging.EventLogger +) + +func init() { + logger = logging.Logger("plugin/filesystem") +} type FileSystemPlugin struct { ctx context.Context @@ -62,7 +69,7 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } var err error - fs.addr, err = multiaddr.NewMultiaddr(cfg.Service[DefaultService]) + fs.addr, err = multiaddr.NewMultiaddr(cfg.Service[defaultService]) if err != nil { return err } @@ -72,14 +79,6 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { return nil } -var ( - logger logging.EventLogger -) - -func init() { - logger = logging.Logger("plugin/filesystem") -} - func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { logger.Info("Starting 9P resource server...") @@ -89,15 +88,8 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { return err } - // construct 9P resource server - p9pFSS, err := fsnodes.NewRoot(fs.ctx, core, logger) - if err != nil { - logger.Errorf("9P root construction error: %s\n", err) - return err - } - - // Run the server. - s := p9.NewServer(p9pFSS) + // construct and run the 9P resource server + s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core)) go func() { if err := s.Serve(manet.NetListener(fs.listener)); err != nil { logger.Errorf("9P server error: %s\n", err) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index ac7e3051d0a..ef5a5fb8ffb 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -19,7 +19,6 @@ import ( "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreapi" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" - logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" coreoptions "github.com/ipfs/interface-go-ipfs-core/options" corepath "github.com/ipfs/interface-go-ipfs-core/path" @@ -37,20 +36,17 @@ func TestAll(t *testing.T) { t.Fatalf("Failed to construct IPFS node: %s\n", err) } - logger := logging.Logger("plugin/filesystem") - - t.Run("RootFS", func(t *testing.T) { testRootFS(t, ctx, core, logger) }) - t.Run("PinFS", func(t *testing.T) { testPinFS(t, ctx, core, logger) }) - t.Run("IPFS", func(t *testing.T) { testIPFS(t, ctx, core, logger) }) + t.Run("RootFS", func(t *testing.T) { testRootFS(ctx, t, core) }) + t.Run("PinFS", func(t *testing.T) { testPinFS(ctx, t, core) }) + t.Run("IPFS", func(t *testing.T) { testIPFS(ctx, t, core) }) } -func testRootFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) { - ri, err := fsnodes.NewRoot(ctx, core, logger) +func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + nineRoot, err := fsnodes.RootAttacher(ctx, core).Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource: %s\n", err) } - nineRoot, err := ri.Attach() _, nineRef, err := nineRoot.Walk(nil) if err != nil { t.Fatalf("Failed to walk root: %s\n", err) @@ -64,7 +60,7 @@ func testRootFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logge t.Fatalf("Failed to read root: %s\n", err) } - //TODO: currently magic, as subsystems are implemented, rework this part of the test + lib + //TODO: currently magic. As subsystems are implemented, rework this part of the test + lib to contain some list if len(ents) != 1 || ents[0].Name != "ipfs" { t.Fatalf("Failed, root has bad entries:: %v\n", ents) } @@ -72,9 +68,8 @@ func testRootFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logge //TODO: type checking } -func testPinFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) { - //init - pinRoot, err := fsnodes.InitPinFS(ctx, core, logger).Attach() +func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + pinRoot, err := fsnodes.PinFSAttacher(ctx, core).Attach() if err != nil { t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) } @@ -112,7 +107,7 @@ func testPinFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger //test default (likely empty) test repo pins shallowCompare() - // test modifying pinset +1; initEnv pins its IPFS envrionment + // test modifying pinset +1; initEnv pins its IPFS environment env, _, err := initEnv(ctx, core) if err != nil { t.Fatalf("Failed to construct IPFS test environment: %s\n", err) @@ -128,10 +123,9 @@ func testPinFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger t.Fatalf("Failed to add directory to IPFS: %s\n", err) } shallowCompare() - - //TODO: type checking } -func testIPFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) { + +func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { env, iEnv, err := initEnv(ctx, core) if err != nil { t.Fatalf("Failed to construct IPFS test environment: %s\n", err) @@ -143,20 +137,19 @@ func testIPFS(t *testing.T, ctx context.Context, core coreiface.CoreAPI, logger t.Fatalf("Failed to attach to local resource %q: %s\n", env, err) } - ipfsRoot, err := fsnodes.InitIPFS(ctx, core, logger).Attach() + ipfsRoot, err := fsnodes.IPFSAttacher(ctx, core).Attach() if err != nil { t.Fatalf("Failed to attach to IPFS resource: %s\n", err) } _, ipfsEnv, err := ipfsRoot.Walk([]string{gopath.Base(iEnv.String())}) if err != nil { - t.Fatalf("Failed to walk to IPFS test envrionment: %s\n", err) + t.Fatalf("Failed to walk to IPFS test environment: %s\n", err) } - recursiveCompare(t, localEnv, ipfsEnv) + testCompareTreeModes(t, localEnv, ipfsEnv) } -//TODO: rename -func recursiveCompare(t *testing.T, f1, f2 p9.File) { +func testCompareTreeModes(t *testing.T, f1, f2 p9.File) { var expand func(p9.File) (map[string]p9.Attr, error) expand = func(nineRef p9.File) (map[string]p9.Attr, error) { ents, err := p9Readdir(nineRef) @@ -164,10 +157,6 @@ func recursiveCompare(t *testing.T, f1, f2 p9.File) { return nil, err } - //TODO: current - // map[string]Stat; ["/sub/incantation"]{...} - ///from root, walk(name); stat; if dir; recurse - res := make(map[string]p9.Attr) for _, ent := range ents { _, child, err := nineRef.Walk([]string{ent.Name}) @@ -210,10 +199,10 @@ func recursiveCompare(t *testing.T, f1, f2 p9.File) { var baseNames []string var targetNames []string - for name, _ := range base { + for name := range base { baseNames = append(baseNames, name) } - for name, _ := range target { + for name := range target { targetNames = append(targetNames, name) } @@ -368,23 +357,24 @@ func p9PinNames(root p9.File) ([]string, error) { names = append(names, ent.Name) } - return names, root.Close() + return names, nil } func p9Readdir(dir p9.File) ([]p9.Dirent, error) { - _, dir, err := dir.Walk(nil) + _, dirClone, err := dir.Walk(nil) if err != nil { return nil, err } - _, _, err = dir.Open(p9.ReadOnly) + _, _, err = dirClone.Open(p9.ReadOnly) if err != nil { return nil, err } - defer dir.Close() - return dir.Readdir(0, ^uint32(0)) + defer dirClone.Close() + return dirClone.Readdir(0, ^uint32(0)) } +//TODO: // NOTE: compares a subset of attributes, matching those of IPFS func testIPFSCompare(t *testing.T, f1, f2 p9.File) { _, _, f1Attr, err := f1.GetAttr(attrMaskIPFSTest) diff --git a/plugin/plugins/filesystem/nodes/doc.go b/plugin/plugins/filesystem/nodes/doc.go new file mode 100644 index 00000000000..346b4e06d8f --- /dev/null +++ b/plugin/plugins/filesystem/nodes/doc.go @@ -0,0 +1,7 @@ +/*Package fsnodes provides constructors and interfaces, for composing various 9P + file systems implementations and wrappers. + + The default RootIndex provided by RootAttacher, is a file system of itself + which relays request to various IPFS subsystems. + Utilizing the subsystem implementations itself, in the same way a client program would.*/ +package fsnodes diff --git a/plugin/plugins/filesystem/nodes/doc_test.go b/plugin/plugins/filesystem/nodes/doc_test.go new file mode 100644 index 00000000000..8badc2e5723 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/doc_test.go @@ -0,0 +1,31 @@ +package fsnodes + +import ( + "bytes" + "strings" + + "github.com/djdv/p9/p9" +) + +func ExampleRootIndex() { + root, err := fsnodes.RootAttacher(ctx, coreAPI).Attach() + _, file, err := root.Walk(strings.Split("ipfs/Qm.../subdir/file", "/")) + _, _, err := file.Open(p9.ReadOnly) + defer file.Close() + _, err := file.ReadAt(byteBuffer, offset) +} + +func ExampleIPFS() { + ipfs, err := fsnodes.IPFSAttacher(ctx, coreAPI).Attach() + _, file, err := ipfs.Walk(strings.Split("Qm.../subdir/file", "/")) + _, _, err := file.Open(p9.ReadOnly) + defer file.Close() + _, err := file.ReadAt(byteBuffer, offset) +} + +func ExamplePinFS() { + ipfs, err := fsnodes.PinFSAttacher(ctx, coreAPI).Attach() + _, dir, err := ipfs.Walk(nil) + _, _, err := dir.Open(p9.ReadOnly) + entries, err := dirClone.Readdir(offset, entryReturnCount) +} diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index ce770152f2a..e4fb0dd1078 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -12,13 +12,16 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) +// IPFS exposes the IPFS API over a p9.File interface +// Walk does not expect a namespace, only its path argument +// e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { IPFSBase } -//TODO: [review] check fields; better wrappers around inheritance init, etc. -func InitIPFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { - id := &IPFS{IPFSBase: newIPFSBase(ctx, newRootPath("/ipfs"), p9.TypeDir, core, logger)} +func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { + id := &IPFS{IPFSBase: newIPFSBase(ctx, newRootPath("/ipfs"), p9.TypeDir, + core, logging.Logger("IPFS"))} id.meta, id.metaMask = defaultRootAttr() return id } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 3f7c6f1e4f8..a9a9f296c8c 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -15,17 +15,9 @@ type PinFS struct { } //TODO: [review] check fields -func InitPinFS(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) p9.Attacher { - pd := &PinFS{ - IPFSBase: IPFSBase{ - Path: newRootPath("/ipfs"), - core: core, - Base: Base{ - Logger: logger, - Ctx: ctx, - Qid: p9.QID{Type: p9.TypeDir}}}} - - pd.Qid.Path = cidToQPath(pd.Path.Cid()) +func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI) *PinFS { + pd := &PinFS{IPFSBase: newIPFSBase(ctx, newRootPath("/ipfs"), p9.TypeDir, + core, logging.Logger("PinFS"))} pd.meta, pd.metaMask = defaultRootAttr() return pd } @@ -51,7 +43,7 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { return []p9.QID{pd.Qid}, pd, nil } - ipfsDir, err := InitIPFS(pd.Ctx, pd.core, pd.Logger).Attach() + ipfsDir, err := IPFSAttacher(pd.Ctx, pd.core).Attach() if err != nil { return nil, nil, err } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index d87a5c4c117..10c6b68ea5b 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -42,13 +42,18 @@ func (rootNode) Root() cid.Cid { //TODO: this should probably reference a packag } func (rootNode) Remainder() string { return "" } +// RootIndex is a virtual directory file system, that maps a set of filesystem implementations to a hierarchy +// Currently: "/":RootIndex, "/ipfs":PinFS, "/ipfs/*:IPFS type RootIndex struct { IPFSBase subsystems []p9.Dirent } -func NewRoot(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLogger) (*RootIndex, error) { - ri := &RootIndex{IPFSBase: newIPFSBase(ctx, newRootPath("/"), p9.TypeDir, core, logger), +// RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it +func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { + ri := &RootIndex{ + IPFSBase: newIPFSBase(ctx, newRootPath("/"), p9.TypeDir, + core, logging.Logger("RootFS")), subsystems: make([]p9.Dirent, 0, 1)} //TODO: [const]: dirent count ri.Qid.Path = cidToQPath(ri.Path.Cid()) @@ -73,7 +78,7 @@ func NewRoot(ctx context.Context, core coreiface.CoreAPI, logger logging.EventLo ri.subsystems = append(ri.subsystems, pathUnion.Dirent) } - return ri, nil + return ri } func (ri *RootIndex) Attach() (p9.File, error) { @@ -87,6 +92,7 @@ func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, err return ri.Qid, ri.metaMask, ri.meta, nil } + func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { ri.Logger.Debugf("RI Walk names %v", names) ri.Logger.Debugf("RI Walk myself: %v", ri.Qid) @@ -99,7 +105,7 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { //NOTE: if doClone is false, it implies len(names) > 0 switch names[0] { case "ipfs": - pinDir, err := InitPinFS(ri.Ctx, ri.core, ri.Logger).Attach() + pinDir, err := PinFSAttacher(ri.Ctx, ri.core).Attach() if err != nil { return nil, nil, err } @@ -109,8 +115,6 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { } } -// TODO: check specs for directory iounit size, -// if it's undefined we should repurpose it to return the count to the client func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { ri.Logger.Debugf("RI Open") return ri.Qid, 0, nil diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index be784ca0326..60f59484998 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -264,7 +264,7 @@ func timeStamp(attr *p9.Attr, mask p9.AttrMask) { } } -//TODO [name]: "new" implies pointer type; this is for embedded consturction +//TODO [name]: "new" implies pointer type; this is for embedded construction func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, core coreiface.CoreAPI, logger logging.EventLogger) IPFSBase { return IPFSBase{ Path: path, diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go index 2b035079ee8..81a197ff6c4 100644 --- a/plugin/plugins/filesystem/utils.go +++ b/plugin/plugins/filesystem/utils.go @@ -13,14 +13,8 @@ const ( //TODO [config]: move elsewhere; related: https://github.com/ipfs/go-ipfs/issues/6526 EnvAddr = "IPFS_FS_ADDR" // multiaddr string - DefaultVersion = "9P2000.L" - DefaultListenAddress = "/unix/$IPFS_PATH/" + sockName - DefaultService = "9P" // (currently 9P2000.L) - DefaultMSize = 64 << 10 - // TODO: For platforms that don't support UDS (Windows < 17063, non-posix systems), fallback to TCP - //FallbackListenAddress = "/ip4/localhost/tcp/564" - - sockName = "filesystem.9P.sock" + sockName = "filesystem.9P.sock" + defaultService = "9P" // (currently 9P2000.L) ) type Config struct { // NOTE: unstable/experimental @@ -37,7 +31,7 @@ func defaultConfig(storagePath string) *Config { sockTarget = windowsToUnixFriendly(sockTarget) } - serviceMap[DefaultService] = gopath.Join("/unix", sockTarget) + serviceMap[defaultService] = gopath.Join("/unix", sockTarget) return &Config{serviceMap} } From 38763021b9065efca9f747a44e0ef6074e7e3a3c Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 7 Sep 2019 11:13:09 -0400 Subject: [PATCH 020/102] openbsd builds --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6e08766e5e8..fab1be06945 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d - github.com/djdv/p9 v0.0.0-20190907022512-79c201cd4ecc + github.com/djdv/p9 v0.0.0-20190907154417-3fdf0632585f github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fatih/color v1.7.0 // indirect diff --git a/go.sum b/go.sum index e5183c18dd6..431224ae038 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/p9 v0.0.0-20190907022512-79c201cd4ecc h1:v3PQvBZcOFgmwlNYeyFBlT23/x1ec9gZZTM6GQ/9Sbs= -github.com/djdv/p9 v0.0.0-20190907022512-79c201cd4ecc/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= +github.com/djdv/p9 v0.0.0-20190907154417-3fdf0632585f h1:CUjmKarvWU1sBD/hUbKDes68c9ApZcmHnYaxRL9wvgc= +github.com/djdv/p9 v0.0.0-20190907154417-3fdf0632585f/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= From d9fd656e694489a5d8bb452537dcba3690af9cf2 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 7 Sep 2019 12:04:42 -0400 Subject: [PATCH 021/102] fix doc format --- plugin/plugins/filesystem/doc.go | 16 +++++++++------- plugin/plugins/filesystem/nodes/base.go | 16 +++++----------- plugin/plugins/filesystem/nodes/doc.go | 12 +++++++----- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/plugin/plugins/filesystem/doc.go b/plugin/plugins/filesystem/doc.go index d35b72593ce..1041093571d 100644 --- a/plugin/plugins/filesystem/doc.go +++ b/plugin/plugins/filesystem/doc.go @@ -1,10 +1,12 @@ -/*Package filesystem is an experimental package, that implements the go-ipfs daemon plugin interface - and defines the plugins config structure. +/* +Package filesystem is an experimental package that implements the go-ipfs daemon plugin interface +and defines the plugin's config structure. - To set the multiaddr listen address, you may use the environment variable $IPFS_FS_ADDR, or set the option in a config file - via `ipfs config --json 'Plugins.Plugins.filesystem.Config "Config":{"Service":{"9P":"/ip4/127.0.0.1/tcp/567"}}'` - To disable this plugin entirely, use: `ipfs config --json Plugins.Plugins.filesystem.Disabled true` +To set the multiaddr listen address, you may use the environment variable $IPFS_FS_ADDR, or set the option in a config file +via `ipfs config --json 'Plugins.Plugins.filesystem.Config "Config":{"Service":{"9P":"/ip4/127.0.0.1/tcp/567"}}'` +To disable this plugin entirely, use: `ipfs config --json Plugins.Plugins.filesystem.Disabled true` - By default, we try to expose the IPFS namespace using the 9P2000.L protocol, over a unix domain socket - (located at $IPFS_PATH/filesystem.9P.sock)*/ +By default, we try to expose the IPFS namespace using the 9P2000.L protocol, over a unix domain socket +(located at $IPFS_PATH/filesystem.9P.sock) +*/ package filesystem diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 87080dc25cb..a8eb382d28b 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -10,12 +10,6 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) -type FSNode interface { - corepath.Path - //RWLocker - Stat() (p9.QID, error) -} - const ( //device dMemory = iota dIPFS @@ -27,13 +21,13 @@ const ( //FS namespaces var _ p9.File = (*Base)(nil) -//var _ FSNode = (*Base)(nil) - -//TODO: docs -// Base is a foundational node, intended to be embedded/extended +// Base is a foundational file system node that provides common file meta data as well as stubs for unimplemented methods type Base struct { + // Provide stubs for unimplemented methods unimplfs.NoopFile p9.DefaultWalkGetAttr + + // Storage for file's metadata Qid p9.QID meta p9.Attr metaMask p9.AttrMask @@ -42,10 +36,10 @@ type Base struct { Logger logging.EventLogger } +// IPFSBase is much like Base but extends it to hold IPFS specific metadata type IPFSBase struct { Base - //Path corepath.Path Path corepath.Resolved core coreiface.CoreAPI } diff --git a/plugin/plugins/filesystem/nodes/doc.go b/plugin/plugins/filesystem/nodes/doc.go index 346b4e06d8f..f696a27b9cb 100644 --- a/plugin/plugins/filesystem/nodes/doc.go +++ b/plugin/plugins/filesystem/nodes/doc.go @@ -1,7 +1,9 @@ -/*Package fsnodes provides constructors and interfaces, for composing various 9P - file systems implementations and wrappers. +/* +Package fsnodes provides constructors and interfaces, for composing various 9P +file systems implementations and wrappers. - The default RootIndex provided by RootAttacher, is a file system of itself - which relays request to various IPFS subsystems. - Utilizing the subsystem implementations itself, in the same way a client program would.*/ +The default RootIndex provided by RootAttacher, is a file system of itself +which relays request to various IPFS subsystems. +Utilizing the subsystem implementations itself, in the same way a client program would. +*/ package fsnodes From f48f4a87ab37b0a967b645a21b5613c478990979 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 7 Sep 2019 12:33:01 -0400 Subject: [PATCH 022/102] stop tracking client-lib --- plugin/plugins/filesystem/client/client.go | 168 ------------------- plugin/plugins/filesystem/client/cmd/main.go | 33 ---- plugin/plugins/filesystem/client/options.go | 26 --- 3 files changed, 227 deletions(-) delete mode 100644 plugin/plugins/filesystem/client/client.go delete mode 100644 plugin/plugins/filesystem/client/cmd/main.go delete mode 100644 plugin/plugins/filesystem/client/options.go diff --git a/plugin/plugins/filesystem/client/client.go b/plugin/plugins/filesystem/client/client.go deleted file mode 100644 index 25a73d384c6..00000000000 --- a/plugin/plugins/filesystem/client/client.go +++ /dev/null @@ -1,168 +0,0 @@ -package p9client - -import ( - gopath "path" - "path/filepath" - "runtime" - "strings" - - "github.com/djdv/p9/p9" - config "github.com/ipfs/go-ipfs-config" - "github.com/ipfs/go-ipfs/plugin/plugins/filesystem" - logging "github.com/ipfs/go-log" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" -) - -var logger logging.EventLogger = logging.Logger("9P") - -func Dial(options ...Option) (*p9.Client, error) { - ops := &Options{ - address: filesystem.DefaultListenAddress, - msize: filesystem.DefaultMSize, - version: filesystem.DefaultVersion} - - for _, op := range options { - op(ops) - } - - //TODO: this should probably be the callers responsibility - if ops.address == filesystem.DefaultListenAddress { - var err error - if ops.address, err = expandDefault(); err != nil { - return nil, err - } - } - - ma, err := multiaddr.NewMultiaddr(ops.address) - if err != nil { - return nil, err - } - - //TODO [investigate;who's bug] on Windows, dialing a unix domain socket that doesn't exist will create it - conn, err := manet.Dial(ma) - if err != nil { - return nil, err - } - - return p9.NewClient(conn, filesystem.DefaultMSize, filesystem.DefaultVersion) -} - -func expandDefault() (string, error) { - _, sockName := gopath.Split(filesystem.DefaultListenAddress) - target, err := config.Path("", sockName) - if err != nil { - return target, err - } - - if !filepath.IsAbs(target) { - if target, err = filepath.Abs(target); err != nil { - return target, err - } - } - - if runtime.GOOS == "windows" { - //TODO [manet]: doesn't like drive letters - //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively - if len(target) > 2 && target[1] == ':' { - target = target[2:] - } - target = filepath.ToSlash(target) - } - - target = gopath.Join("/unix", target) - return target, nil -} - -func ReadDir(path string, fsRef p9.File, offset uint64) ([]p9.Dirent, error) { - components := strings.Split(strings.TrimPrefix(path, "/"), "/") - if len(components) == 1 && components[0] == "" { - components = nil - } - - qids, targetRef, err := fsRef.Walk(components) - if err != nil { - return nil, err - } - logger.Debugf("walked to %q :\nQIDs:%v, FID:%v\n\n", path, qids, targetRef) - - if _, _, err = targetRef.Open(p9.ReadOnly); err != nil { - return nil, err - } - - ents, err := targetRef.Readdir(offset, ^uint32(0)) - if err != nil { - return nil, err - } - - logger.Debugf("%q Readdir:\n[%d]%v\n\n", path, len(ents), ents) - if err = targetRef.Close(); err != nil { - return nil, err - } - logger.Debugf("%q closed:\n%#v\n\n", path, targetRef) - - return ents, nil -} - -/* TODO: rework -func Open(path string, fsRef p9.File) (p9.File, error) { - components := strings.Split(strings.TrimPrefix(path, "/"), "/") - if len(components) == 1 && components[0] == "" { - components = nil - } - - _, targetRef, err := fsRef.Walk(nil) - if err != nil { - return nil, err - } - - _, _, attr, err := targetRef.GetAttr(p9.AttrMask{Size: true}) - if err != nil { - return nil, err - } - logger.Debugf("Getattr for %q :\n%v\n\n", path, attr) - - refQid, ioUnit, err := targetRef.Open(0) - if err != nil { - return nil, err - } - -} - -func Read(path string, openedRef p9.File) { - components := strings.Split(strings.TrimPrefix(path, "/"), "/") - if len(components) == 1 && components[0] == "" { - components = nil - } - - _, targetRef, err := fsRef.Walk(components) - if err != nil { - return nil, err - } - - _, _, attr, err := targetRef.GetAttr(p9.AttrMask{Size: true}) - if err != nil { - return nil, err - } - logger.Debugf("Getattr for %q :\n%v\n\n", path, attr) - - refQid, ioUnit, err := targetRef.Open(0) - if err != nil { - return nil, err - } - logger.Debugf("%q Opened:\nQID:%v, iounit:%v\n\n", path, refQid, ioUnit) - - buf := make([]byte, attr.Size) - readBytes, err := targetRef.ReadAt(buf, 0) - if err != nil { - return nil, err - } - - logger.Debugf("%q Read:\n[%d bytes]\n%s\n\n", path, readBytes, buf) - if err = targetRef.Close(); err != nil { - return nil, err - } - - logger.Debugf("%q closed:\n%#v\n\n", path, targetRef) -} -*/ diff --git a/plugin/plugins/filesystem/client/cmd/main.go b/plugin/plugins/filesystem/client/cmd/main.go deleted file mode 100644 index 69a73a735b0..00000000000 --- a/plugin/plugins/filesystem/client/cmd/main.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "log" - - p9client "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/client" - logging "github.com/ipfs/go-log" -) - -func main() { - logger := logging.Logger("fs-client") - logging.SetLogLevel("fs-client", "info") - - client, err := p9client.Dial() - if err != nil { - log.Fatal(err) - } - - defer client.Close() - logger.Infof("Connected to server supporting version:\n%v\n\n", client.Version()) - - rootRef, err := client.Attach("") - if err != nil { - log.Fatal(err) - } - logger.Infof("Attached to root:\n%#v\n\n", rootRef) - - logger.Info(p9client.ReadDir("/", rootRef, 0)) - logger.Info(p9client.ReadDir("/ipfs", rootRef, 0)) - logger.Info(p9client.ReadDir("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", rootRef, 0)) - //readDBG("/ipfs/QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB", rootRef) - client.Close() -} diff --git a/plugin/plugins/filesystem/client/options.go b/plugin/plugins/filesystem/client/options.go deleted file mode 100644 index 13789cf6c0a..00000000000 --- a/plugin/plugins/filesystem/client/options.go +++ /dev/null @@ -1,26 +0,0 @@ -package p9client - -type Options struct { - network, address, version string - msize int -} - -type Option func(*Options) - -func Address(address string) Option { - return func(ops *Options) { - ops.address = address - } -} - -func Version(version string) Option { - return func(ops *Options) { - ops.version = version - } -} - -func Msize(msize int) Option { - return func(ops *Options) { - ops.msize = msize - } -} From 4bb5712adbe4d181acc00c3065b9818dccb33b2c Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 7 Sep 2019 18:54:00 -0400 Subject: [PATCH 023/102] docfix --- plugin/plugins/filesystem/doc.go | 10 +++++----- plugin/plugins/filesystem/nodes/base.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugin/plugins/filesystem/doc.go b/plugin/plugins/filesystem/doc.go index 1041093571d..38dbf195bd9 100644 --- a/plugin/plugins/filesystem/doc.go +++ b/plugin/plugins/filesystem/doc.go @@ -1,12 +1,12 @@ /* Package filesystem is an experimental package that implements the go-ipfs daemon plugin interface -and defines the plugin's config structure. - -To set the multiaddr listen address, you may use the environment variable $IPFS_FS_ADDR, or set the option in a config file -via `ipfs config --json 'Plugins.Plugins.filesystem.Config "Config":{"Service":{"9P":"/ip4/127.0.0.1/tcp/567"}}'` -To disable this plugin entirely, use: `ipfs config --json Plugins.Plugins.filesystem.Disabled true` +and defines the plugin's config structure. The plugin itself exposes file system services over a multiaddr listener. By default, we try to expose the IPFS namespace using the 9P2000.L protocol, over a unix domain socket (located at $IPFS_PATH/filesystem.9P.sock) + +To set the multiaddr listen address, you may use the environment variable $IPFS_FS_ADDR, or set the option in the node's config file +via `ipfs config --json 'Plugins.Plugins.filesystem.Config "Config":{"Service":{"9P":"/ip4/127.0.0.1/tcp/567"}}'` +To disable this plugin entirely, use: `ipfs config --json Plugins.Plugins.filesystem.Disabled true` */ package filesystem diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index a8eb382d28b..6bf6abfb1e7 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -21,7 +21,7 @@ const ( //FS namespaces var _ p9.File = (*Base)(nil) -// Base is a foundational file system node that provides common file meta data as well as stubs for unimplemented methods +// Base is a foundational file system node that provides common file metadata as well as stubs for unimplemented methods type Base struct { // Provide stubs for unimplemented methods unimplfs.NoopFile From 760d3c6078b1de8988b80e8914f5f4e668148511 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 7 Sep 2019 18:54:16 -0400 Subject: [PATCH 024/102] wait for fs service to return on close + remove listener on Windows if exist --- plugin/plugins/filesystem/filesystem.go | 21 +++++++++++++-------- plugin/plugins/filesystem/utils.go | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 88b368ba576..c2e9aeb961e 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "path/filepath" + "runtime" "github.com/djdv/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" @@ -33,8 +34,9 @@ type FileSystemPlugin struct { ctx context.Context cancel context.CancelFunc - addr multiaddr.Multiaddr - listener manet.Listener + addr multiaddr.Multiaddr + listener manet.Listener + errorChan chan error } func (*FileSystemPlugin) Name() string { @@ -73,8 +75,15 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { if err != nil { return err } + //TODO [manet]: unix sockets are not removed on process death (on Windows) + // so for now we just try to remove it before listening on it + if runtime.GOOS == "windows" { + removeUnixSockets(fs.addr) + } fs.ctx, fs.cancel = context.WithCancel(context.Background()) + fs.errorChan = make(chan error) + logger.Info("9P resource server okay for launch") return nil } @@ -91,10 +100,7 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { // construct and run the 9P resource server s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core)) go func() { - if err := s.Serve(manet.NetListener(fs.listener)); err != nil { - logger.Errorf("9P server error: %s\n", err) - return - } + fs.errorChan <- s.Serve(manet.NetListener(fs.listener)) }() logger.Infof("9P service is listening on %s\n", fs.listener.Addr()) @@ -102,9 +108,8 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { } func (fs *FileSystemPlugin) Close() error { - //TODO: fmt.Println("Closing file system handles...") logger.Info("9P server requested to close") fs.cancel() fs.listener.Close() - return nil + return <-fs.errorChan } diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go index 81a197ff6c4..a22bd9f33be 100644 --- a/plugin/plugins/filesystem/utils.go +++ b/plugin/plugins/filesystem/utils.go @@ -1,9 +1,13 @@ package filesystem import ( + "os" gopath "path" "path/filepath" "runtime" + "strings" + + "github.com/multiformats/go-multiaddr" ) const ( @@ -43,3 +47,18 @@ func windowsToUnixFriendly(target string) string { } return filepath.ToSlash(target) } + +// removeUnixSockets attempts to remove all unix domain paths from a multiaddr +// Does not stop on error, returns last encountered error +func removeUnixSockets(ma multiaddr.Multiaddr) error { + var retErr error + multiaddr.ForEach(ma, func(comp multiaddr.Component) bool { + if comp.Protocol().Code == multiaddr.P_UNIX { + if err := os.Remove(strings.TrimPrefix(comp.String(), "/unix")); err != nil { + retErr = err + } + } + return false + }) + return retErr +} From 6e87be8c769addb234f9dcd8a6ae73230af1de94 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 09:30:17 -0400 Subject: [PATCH 025/102] change how IPFS reads directory streams --- plugin/plugins/filesystem/nodes/ipfs.go | 83 +++++++++++++++--------- plugin/plugins/filesystem/nodes/utils.go | 81 ----------------------- 2 files changed, 53 insertions(+), 111 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index e4fb0dd1078..b3977cb5f4f 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -17,6 +17,12 @@ import ( // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { IPFSBase + directory *directoryStream +} + +type directoryStream struct { + entryChan <-chan coreiface.DirEntry + cursor uint64 } func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { @@ -96,51 +102,68 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { id.Logger.Debugf("ID Open") + if id.meta.Mode.IsDir() { + c, err := id.core.Unixfs().Ls(id.Ctx, id.Path) + if err != nil { + return id.Qid, 0, err + } + id.directory = &directoryStream{ + entryChan: c, + } + } + return id.Qid, 0, nil } func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { id.Logger.Debugf("ID Readdir") - //FIXME: we read the entire directory for each readdir call; this is very wasteful with small requests (Unix `ls`) - //TODO [quick hack]: append only entry list stored on id; likely going to be problematic for large directories (test: Wikipedia) - entChan, err := coreLs(id.Ctx, id.Path, id.core) - if err != nil { - return nil, err + if id.directory == nil { + return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) } - var ents []p9.Dirent - - var off uint64 = 1 - for ent := range entChan { - if ent.Err != nil { - return nil, err - } - off++ - - nineEnt := coreEntTo9Ent(ent) - nineEnt.Offset = off - ents = append(ents, nineEnt) + if count == 0 { + return nil, nil + } - if uint32(len(ents)) == count { - break - } + if offset < id.directory.cursor { + return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) } - //FIXME: I don't think order is gauranteed from Ls - eLen := uint64(len(ents)) - if offset >= eLen { - return nil, nil + ents := make([]p9.Dirent, 0) + +out: + for { + select { + case entry, open := <-id.directory.entryChan: + if !open { + break out + } + if entry.Err != nil { + return nil, entry.Err + } + + id.directory.cursor++ + + nineEnt := coreEntTo9Ent(entry) + nineEnt.Offset = id.directory.cursor + ents = append(ents, nineEnt) + count-- + + if count == 0 { + break out + } + case <-id.Ctx.Done(): + return ents, id.Ctx.Err() + } } - offsetIndex := ents[offset:] - if len(offsetIndex) > int(count) { - id.Logger.Debugf("ID Readdir returning [%d]%v\n", count, offsetIndex[:count]) - return offsetIndex[:count], nil + if offset > uint64(len(ents)) { + return nil, nil //cursor is at end of stream, nothing to return } - id.Logger.Debugf("ID Readdir returning [%d]%v\n", len(offsetIndex), offsetIndex) - return offsetIndex, nil + id.Logger.Debugf("ID Readdir returning [%d]%v\n", len(ents), ents) + return ents, nil } func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 60f59484998..5b4c9d5330f 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -103,49 +103,6 @@ func cidToQPath(cid cid.Cid) uint64 { return hasher.Sum64() } -func coreLs(ctx context.Context, corePath corepath.Path, core coreiface.CoreAPI) (<-chan coreiface.DirEntry, error) { - - //FIXME: asyncContext hangs on reset - //asyncContext := deriveTimerContext(ctx, 10*time.Second) - asyncContext := ctx - - coreChan, err := core.Unixfs().Ls(asyncContext, corePath) - if err != nil { - //asyncContext.Cancel() - return nil, err - } - - oStat, err := core.Object().Stat(asyncContext, corePath) - if err != nil { - return nil, err - } - - relayChan := make(chan coreiface.DirEntry) - go func() { - //defer asyncContext.Cancel() - defer close(relayChan) - - for i := 0; i != oStat.NumLinks; i++ { - select { - case <-asyncContext.Done(): - return - case msg, ok := <-coreChan: - if !ok { - return - } - if msg.Err != nil { - relayChan <- msg - return - } - relayChan <- msg - //asyncContext.Reset() //reset timeout for each entry we receive successfully - } - } - }() - - return relayChan, err -} - func coreTypeTo9Mode(ct coreiface.FileType) p9.FileMode { switch ct { // case coreiface.TDirectory, unixfs.THAMTShard // Should we account for this? @@ -182,44 +139,6 @@ func coreEntTo9Ent(coreEnt coreiface.DirEntry) p9.Dirent { Path: cidToQPath(coreEnt.Cid)}} } -type timerContextActual struct { - context.Context - cancel context.CancelFunc - timer time.Timer - grace time.Duration -} - -func (tctx timerContextActual) Reset() { - if !tctx.timer.Stop() { - <-tctx.timer.C - } - tctx.timer.Reset(tctx.grace) -} - -func (tctx timerContextActual) Cancel() { - tctx.cancel() - if !tctx.timer.Stop() { - <-tctx.timer.C - } -} - -type timerContext interface { - context.Context - Reset() - Cancel() -} - -func deriveTimerContext(ctx context.Context, grace time.Duration) timerContext { - asyncContext, cancel := context.WithCancel(ctx) - timer := time.AfterFunc(grace, cancel) - tctx := timerContextActual{Context: asyncContext, - cancel: cancel, - grace: grace, - timer: *timer} - - return tctx -} - const ( // pedantic POSIX stuff S_IROTH p9.FileMode = p9.Read S_IWOTH = p9.Write From 7ff0d1a793b074709ddf6dadeaa00aea05d253ec Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 10:07:33 -0400 Subject: [PATCH 026/102] fix readdir offset + add test --- plugin/plugins/filesystem/filesystem_test.go | 18 ++++++++++++++++++ plugin/plugins/filesystem/nodes/ipfs.go | 12 +++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index ef5a5fb8ffb..0e5ec2a03ab 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -145,8 +145,26 @@ func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { if err != nil { t.Fatalf("Failed to walk to IPFS test environment: %s\n", err) } + _, envClone, err := ipfsEnv.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone IPFS environment handle: %s\n", err) + } testCompareTreeModes(t, localEnv, ipfsEnv) + + // test readdir bounds + //TODO: compare against a table, not just lengths + _, _, err = envClone.Open(p9.ReadOnly) + if err != nil { + t.Fatalf("Failed to open IPFS test directory: %s\n", err) + } + ents, err := envClone.Readdir(2, 2) // start at ent 2, return max 2 + if err != nil { + t.Fatalf("Failed to read IPFS test directory: %s\n", err) + } + if l := len(ents); l == 0 || l > 2 { + t.Fatalf("IPFS test directory contents don't match read request: %v\n", ents) + } } func testCompareTreeModes(t *testing.T, f1, f2 p9.File) { diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index b3977cb5f4f..5b37e873611 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -143,12 +143,14 @@ out: return nil, entry.Err } - id.directory.cursor++ + if offset <= id.directory.cursor { + nineEnt := coreEntTo9Ent(entry) + nineEnt.Offset = id.directory.cursor + ents = append(ents, nineEnt) + count-- + } - nineEnt := coreEntTo9Ent(entry) - nineEnt.Offset = id.directory.cursor - ents = append(ents, nineEnt) - count-- + id.directory.cursor++ if count == 0 { break out From 99df4100a7b09f067b15eb4dae4d508f28e87e49 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 11:21:05 -0400 Subject: [PATCH 027/102] fix: use environment variable when set --- plugin/plugins/filesystem/filesystem.go | 8 +++++++- plugin/plugins/filesystem/utils.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index c2e9aeb961e..b2f5ae8594c 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -3,6 +3,7 @@ package filesystem import ( "context" "encoding/json" + "os" "path/filepath" "runtime" @@ -71,10 +72,15 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } var err error - fs.addr, err = multiaddr.NewMultiaddr(cfg.Service[defaultService]) + if envAddr := os.ExpandEnv(EnvAddr); envAddr == "" { + fs.addr, err = multiaddr.NewMultiaddr(cfg.Service[defaultService]) + } else { + fs.addr, err = multiaddr.NewMultiaddr(envAddr) + } if err != nil { return err } + //TODO [manet]: unix sockets are not removed on process death (on Windows) // so for now we just try to remove it before listening on it if runtime.GOOS == "windows" { diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go index a22bd9f33be..fe5a9302d25 100644 --- a/plugin/plugins/filesystem/utils.go +++ b/plugin/plugins/filesystem/utils.go @@ -15,7 +15,7 @@ const ( PluginVersion = "0.0.1" //TODO [config]: move elsewhere; related: https://github.com/ipfs/go-ipfs/issues/6526 - EnvAddr = "IPFS_FS_ADDR" // multiaddr string + EnvAddr = "$IPFS_FS_ADDR" // multiaddr string sockName = "filesystem.9P.sock" defaultService = "9P" // (currently 9P2000.L) From dc6287d01ad36bec2efcdaf90c86bb4addd5eef8 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 11:37:56 -0400 Subject: [PATCH 028/102] docs: wrong port + add more to the client example --- docs/experimental-features.md | 11 +++++++++++ plugin/plugins/filesystem/doc.go | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 98a3a02a10d..14ab29bba7f 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -716,8 +716,19 @@ Experimental, not included by default. This daemon plugin wraps the IPFS node and exposes file system services over a multiaddr listener. Currently we use the 9P2000.L protocol, and offer read-only support for `/ipfs` requests. With support for other (writable) systems coming next. You may connect to this service using the `v9fs` client used in the Linux kernel. +By default we listen locally on the machine, using a Unix domain socket. `mount -t 9p -o trans=unix $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` To configure the listening address and more, see the [package documentation](https://godoc.org/github.com/ipfs/go-ipfs/plugin/plugins/filesystem) +See the [v9fs documentation](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/filesystems/9p.txt) for instructions on using transports, such as TCP. +i.e. +``` +> export IPFS_FS_ADDR=/ip4/127.0.0.1/tcp/564 +> ipfs daemon & +> mount -t 9p 127.0.0.1 ~/ipfs-mount +> ... +> umount ~/ipfs-mount +> ipfs shutdown +``` At the moment, [a modified 9P library](https://github.com/djdv/p9/tree/diff) is being used to connect golang clients to the service. diff --git a/plugin/plugins/filesystem/doc.go b/plugin/plugins/filesystem/doc.go index 38dbf195bd9..b5965b3f00a 100644 --- a/plugin/plugins/filesystem/doc.go +++ b/plugin/plugins/filesystem/doc.go @@ -6,7 +6,7 @@ By default, we try to expose the IPFS namespace using the 9P2000.L protocol, ove (located at $IPFS_PATH/filesystem.9P.sock) To set the multiaddr listen address, you may use the environment variable $IPFS_FS_ADDR, or set the option in the node's config file -via `ipfs config --json 'Plugins.Plugins.filesystem.Config "Config":{"Service":{"9P":"/ip4/127.0.0.1/tcp/567"}}'` +via `ipfs config --json 'Plugins.Plugins.filesystem.Config "Config":{"Service":{"9P":"/ip4/127.0.0.1/tcp/564"}}'` To disable this plugin entirely, use: `ipfs config --json Plugins.Plugins.filesystem.Disabled true` */ package filesystem From b8c4a3c75b104f6d56cf006b4527116066923759 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 12:51:13 -0400 Subject: [PATCH 029/102] markdown fixes --- docs/experimental-features.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 14ab29bba7f..ac220de729d 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -30,7 +30,7 @@ the above issue. - [AutoRelay](#autorelay) - [TLS 1.3 Handshake](#tls-13-as-default-handshake-protocol) - [Strategic Providing](#strategic-providing) -- [IPFS filesystem API plugin](#filesystem-plugin) +- [IPFS filesystem API plugin](#filesystem-api-plugin) --- @@ -736,8 +736,8 @@ In the future, we plan to add a client library that wraps this and provides a st ### How to enable -See the Plugin documentation [here]https://github.com/ipfs/go-ipfs/blob/master/docs/plugins.md#installing-plugins -You will likely want to add the plugin to the `go-ipfs` preload list +See the Plugin documentation [here](https://github.com/ipfs/go-ipfs/blob/master/docs/plugins.md#installing-plugins). +You will likely want to add the plugin to the `go-ipfs` plugin preload list `filesystem github.com/ipfs/go-ipfs/plugin/plugins/filesystem *` ### Road to being a real feature From 099745476164ac4b6c55a8d0d75d6b4aa1c19ede Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 18:15:59 -0400 Subject: [PATCH 030/102] the linebreaks too --- docs/experimental-features.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index ac220de729d..dcd2b2d508e 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -713,11 +713,13 @@ ipfs config --json Experimental.StrategicProviding true Experimental, not included by default. -This daemon plugin wraps the IPFS node and exposes file system services over a multiaddr listener. -Currently we use the 9P2000.L protocol, and offer read-only support for `/ipfs` requests. With support for other (writable) systems coming next. +This daemon plugin wraps the IPFS node and exposes file system services over a multiaddr listener. +Currently we use the 9P2000.L protocol, and offer read-only support for `/ipfs` requests. With support for other (writable) systems coming next. + You may connect to this service using the `v9fs` client used in the Linux kernel. -By default we listen locally on the machine, using a Unix domain socket. -`mount -t 9p -o trans=unix $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` +By default we listen locally on the machine, using a Unix domain socket. +`mount -t 9p -o trans=unix $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` + To configure the listening address and more, see the [package documentation](https://godoc.org/github.com/ipfs/go-ipfs/plugin/plugins/filesystem) See the [v9fs documentation](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/filesystems/9p.txt) for instructions on using transports, such as TCP. i.e. @@ -730,7 +732,6 @@ i.e. > ipfs shutdown ``` - At the moment, [a modified 9P library](https://github.com/djdv/p9/tree/diff) is being used to connect golang clients to the service. In the future, we plan to add a client library that wraps this and provides a standard client interface. From 3ee383a375b17072183df728100b82e032bd0461 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 9 Sep 2019 18:42:41 -0400 Subject: [PATCH 031/102] doc: add a table of the root index mapping --- plugin/plugins/filesystem/nodes/doc.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/nodes/doc.go b/plugin/plugins/filesystem/nodes/doc.go index f696a27b9cb..a1c2fd67e83 100644 --- a/plugin/plugins/filesystem/nodes/doc.go +++ b/plugin/plugins/filesystem/nodes/doc.go @@ -4,6 +4,12 @@ file systems implementations and wrappers. The default RootIndex provided by RootAttacher, is a file system of itself which relays request to various IPFS subsystems. -Utilizing the subsystem implementations itself, in the same way a client program would. +Attaching to and utilizing P9.File implementations themselves, in the same way a client program could use them independently. + +The current mapping looks like this: + - mountpoint - associated filesystem + - / - points back to itself, returning the root index + - /ipfs - PinFS + - /ipfs/* - IPFS */ package fsnodes From 15e894a9772923822004e1040b21f1abd4267c4e Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 11 Sep 2019 17:02:18 -0400 Subject: [PATCH 032/102] CR bundle (split later) --- docs/experimental-features.md | 3 - plugin/plugins/filesystem/filesystem.go | 2 +- plugin/plugins/filesystem/nodes/doc_test.go | 4 +- plugin/plugins/filesystem/nodes/ipfs.go | 7 +- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- plugin/plugins/filesystem/nodes/root.go | 2 +- plugin/plugins/filesystem/nodes/utils.go | 97 ++++++++++++++------- 7 files changed, 77 insertions(+), 40 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index dcd2b2d508e..9b29c2f4920 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -732,9 +732,6 @@ i.e. > ipfs shutdown ``` -At the moment, [a modified 9P library](https://github.com/djdv/p9/tree/diff) is being used to connect golang clients to the service. -In the future, we plan to add a client library that wraps this and provides a standard client interface. - ### How to enable See the Plugin documentation [here](https://github.com/ipfs/go-ipfs/blob/master/docs/plugins.md#installing-plugins). diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index b2f5ae8594c..258d661a876 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -88,7 +88,7 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } fs.ctx, fs.cancel = context.WithCancel(context.Background()) - fs.errorChan = make(chan error) + fs.errorChan = make(chan error, 1) logger.Info("9P resource server okay for launch") return nil diff --git a/plugin/plugins/filesystem/nodes/doc_test.go b/plugin/plugins/filesystem/nodes/doc_test.go index 8badc2e5723..de9e03da879 100644 --- a/plugin/plugins/filesystem/nodes/doc_test.go +++ b/plugin/plugins/filesystem/nodes/doc_test.go @@ -1,12 +1,14 @@ package fsnodes import ( - "bytes" "strings" "github.com/djdv/p9/p9" ) +//TODO: compiler legal examples so Go tools don't get mad; needs client wrappers to look nice +// `Output:` comments too + func ExampleRootIndex() { root, err := fsnodes.RootAttacher(ctx, coreAPI).Attach() _, file, err := root.Walk(strings.Split("ipfs/Qm.../subdir/file", "/")) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 5b37e873611..9a02b9f2cda 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -61,7 +61,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { id.Logger.Debugf("ID Walk names %v", names) id.Logger.Debugf("ID Walk myself: %s:%v", id.Path, id.Qid) - if doClone(names) { + if shouldClone(names) { id.Logger.Debugf("ID Walk cloned") return []p9.QID{id.Qid}, id, nil } @@ -144,7 +144,10 @@ out: } if offset <= id.directory.cursor { - nineEnt := coreEntTo9Ent(entry) + nineEnt, err := coreEntTo9Ent(entry) + if err != nil { + return nil, err + } nineEnt.Offset = id.directory.cursor ents = append(ents, nineEnt) count-- diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index a9a9f296c8c..ab9e5f16223 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -38,7 +38,7 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { pd.Logger.Debugf("PD Walk names %v", names) pd.Logger.Debugf("PD Walk myself: %v", pd.Qid) - if doClone(names) { + if shouldClone(names) { pd.Logger.Debugf("PD Walk cloned") return []p9.QID{pd.Qid}, pd, nil } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 10c6b68ea5b..445ffc7b3f5 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -97,7 +97,7 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { ri.Logger.Debugf("RI Walk names %v", names) ri.Logger.Debugf("RI Walk myself: %v", ri.Qid) - if doClone(names) { + if shouldClone(names) { ri.Logger.Debugf("RI Walk cloned") return []p9.QID{ri.Qid}, ri, nil } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 5b4c9d5330f..f2388580d29 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -2,7 +2,10 @@ package fsnodes import ( "context" + "crypto/rand" + "fmt" "hash/fnv" + "io" "time" "github.com/djdv/p9/p9" @@ -15,17 +18,26 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) -func doClone(names []string) bool { - l := len(names) - if l < 1 { - return true +var salt []byte + +func init() { + salt = make([]byte, 2048) //XXX: size is arbitrary + _, err := io.ReadFull(rand.Reader, salt) + if err != nil { + panic(err) } - //TODO: double check the spec to make sure dot handling is correct - // we may only want to clone on ".." if we're a root - if pc := names[0]; l == 1 && (pc == ".." || pc == "." || pc == "") { +} + +func shouldClone(names []string) bool { + switch len(names) { + case 0: // empty path return true + case 1: // self? + pc := names[0] + return pc == ".." || pc == "." || pc == "." + default: + return false } - return false } //TODO: rename this and/or extend @@ -50,7 +62,11 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask p9.AttrMask, core if attrMask.Mode { attr.Mode = IRXA //TODO: this should probably be the callers responsability; just document that permissions should be set afterwards or something - attr.Mode |= unixfsTypeTo9Mode(ufsNode.Type()) + tBits, err := unixfsTypeTo9Type(ufsNode.Type()) + if err != nil { + return err + } + attr.Mode |= tBits } if attrMask.Blocks { @@ -86,10 +102,13 @@ func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { return err } - nodeType := unixfsTypeTo9Mode(ufsNode.Type()).QIDType() + nineType, err := unixfsTypeTo9Type(ufsNode.Type()) + if err != nil { + return err + } - dirEnt.Type = nodeType - dirEnt.QID.Type = nodeType + dirEnt.Type = nineType.QIDType() + dirEnt.QID.Type = nineType.QIDType() dirEnt.QID.Path = cidToQPath(node.Cid()) return nil @@ -97,46 +116,59 @@ func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { func cidToQPath(cid cid.Cid) uint64 { hasher := fnv.New64a() + if _, err := hasher.Write(salt); err != nil { + panic(err) + } if _, err := hasher.Write(cid.Bytes()); err != nil { panic(err) } return hasher.Sum64() } -func coreTypeTo9Mode(ct coreiface.FileType) p9.FileMode { +//NOTE [2019.09.11]: IPFS CoreAPI abstracts over HAMT structures; Unixfs returns raw type + +func coreTypeTo9Type(ct coreiface.FileType) (p9.FileMode, error) { switch ct { - // case coreiface.TDirectory, unixfs.THAMTShard // Should we account for this? case coreiface.TDirectory: - return p9.ModeDirectory + return p9.ModeDirectory, nil case coreiface.TSymlink: - return p9.ModeSymlink - default: //TODO: probably a bad assumption to make - return p9.ModeRegular + return p9.ModeSymlink, nil + case coreiface.TFile: + return p9.ModeRegular, nil + default: + return p9.ModeRegular, fmt.Errorf("CoreAPI data type %q was not expected, treating as regular file", ct) } } //TODO: see if we can remove the need for this; rely only on the core if we can -func unixfsTypeTo9Mode(ut unixpb.Data_DataType) p9.FileMode { +func unixfsTypeTo9Type(ut unixpb.Data_DataType) (p9.FileMode, error) { switch ut { - // case unixpb.Data_DataDirectory, unixpb.Data_DataHAMTShard // Should we account for this? - case unixpb.Data_Directory: - return p9.ModeDirectory + //TODO: directories and hamt shards are not synonymous; HAMTs may need special handling + case unixpb.Data_Directory, unixpb.Data_HAMTShard: + return p9.ModeDirectory, nil case unixpb.Data_Symlink: - return p9.ModeSymlink - default: //TODO: probably a bad assumption to make - return p9.ModeRegular + return p9.ModeSymlink, nil + case unixpb.Data_File: + return p9.ModeRegular, nil + default: + return p9.ModeRegular, fmt.Errorf("UFS data type %q was not expected, treating as regular file", ut) } } -func coreEntTo9Ent(coreEnt coreiface.DirEntry) p9.Dirent { - entType := coreTypeTo9Mode(coreEnt.Type).QIDType() +func coreEntTo9Ent(coreEnt coreiface.DirEntry) (p9.Dirent, error) { + entType, err := coreTypeTo9Type(coreEnt.Type) + if err != nil { + return p9.Dirent{}, err + } return p9.Dirent{ Name: coreEnt.Name, - Type: entType, + Type: entType.QIDType(), QID: p9.QID{ - Type: entType, - Path: cidToQPath(coreEnt.Cid)}} + Type: entType.QIDType(), + Path: cidToQPath(coreEnt.Cid), + }, + }, nil } const ( // pedantic POSIX stuff @@ -193,5 +225,8 @@ func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, c Ctx: ctx, Qid: p9.QID{ Type: kind, - Path: cidToQPath(path.Cid())}}} + Path: cidToQPath(path.Cid()), + }, + }, + } } From e5ec67218d3a297f9f9d642251c7caae2ee6712a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 09:31:02 -0400 Subject: [PATCH 033/102] qid-fix --- plugin/plugins/filesystem/nodes/utils.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index f2388580d29..2221ebd0dfb 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -18,10 +18,15 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) +// NOTE [2019.09.12]: QID's have a high collision probability +// as a result we add a salt to hashes to attempt to mitigate this +// for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 var salt []byte +const saltSize = 32 + func init() { - salt = make([]byte, 2048) //XXX: size is arbitrary + salt = make([]byte, saltSize) _, err := io.ReadFull(rand.Reader, salt) if err != nil { panic(err) From c653fc5fc71918b60872188bccb15c07e10ed9f5 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 09:31:12 -0400 Subject: [PATCH 034/102] clone-fix --- plugin/plugins/filesystem/nodes/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 2221ebd0dfb..d8cf83af652 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -39,7 +39,7 @@ func shouldClone(names []string) bool { return true case 1: // self? pc := names[0] - return pc == ".." || pc == "." || pc == "." + return pc == ".." || pc == "." || pc == "" default: return false } From b6d150654c2609f90df8b1c232cb06396fe688c7 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 11:04:39 -0400 Subject: [PATCH 035/102] change config handling (needs tests) --- plugin/plugins/filesystem/filesystem.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 258d661a876..0454fdb9d90 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -3,6 +3,7 @@ package filesystem import ( "context" "encoding/json" + "fmt" "os" "path/filepath" "runtime" @@ -59,15 +60,17 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } cfg := &Config{} - if env.Config != nil { - byteRep, err := json.Marshal(env.Config) - if err != nil { - return err - } - if err = json.Unmarshal(byteRep, cfg); err != nil { + // config not being set is okay and will load defalts + // config being set with malformed data is not okay and will instruct the daemon to halt-and-catch-fire + rawConf, ok := (env.Config).(json.RawMessage) + if ok { + if err := json.Unmarshal(rawConf, cfg); err != nil { return err } } else { + if env.Config != nil { + return fmt.Errorf("plugin config does not appear to be correctly formatted: %#v", env.Config) + } cfg = defaultConfig(env.Repo) } From fbf5bca991c381b1affa7de7dc5b7d9597bd123a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 11:10:13 -0400 Subject: [PATCH 036/102] delay listener init --- plugin/plugins/filesystem/filesystem.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 0454fdb9d90..359753aa446 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -84,6 +84,13 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { return err } + logger.Info("9P resource server okay for launch") + return nil +} + +func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { + logger.Info("Starting 9P resource server...") + //TODO [manet]: unix sockets are not removed on process death (on Windows) // so for now we just try to remove it before listening on it if runtime.GOOS == "windows" { @@ -93,13 +100,6 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { fs.ctx, fs.cancel = context.WithCancel(context.Background()) fs.errorChan = make(chan error, 1) - logger.Info("9P resource server okay for launch") - return nil -} - -func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { - logger.Info("Starting 9P resource server...") - var err error if fs.listener, err = manet.Listen(fs.addr); err != nil { logger.Errorf("9P listen error: %s\n", err) From 888c33d0d217a65fbd3777a1ea63ad022e476027 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 11:34:06 -0400 Subject: [PATCH 037/102] use multicodec standard --- plugin/plugins/filesystem/nodes/base.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 6bf6abfb1e7..9dc58d2dcef 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -10,12 +10,14 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) -const ( //device - dMemory = iota - dIPFS -) - -const ( //FS namespaces +const ( //device - attempts to comply with standard multicodec table + dMemory = 0x2f + dIPFS = 0xe4 + dIPNS = 0xe5 + //TODO: decide what MFS/Files API should be; IPLD? + //dFiles = 0xe2 + + //TODO: obviate this nRoot = "root" ) From 30175865ad78fbbdeba21d3c9655004e077d19ee Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 11:44:51 -0400 Subject: [PATCH 038/102] don't implicitly timestamp core objects --- plugin/plugins/filesystem/nodes/utils.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index d8cf83af652..87a0ee9c5c5 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -78,9 +78,6 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask p9.AttrMask, core if bs := ufsNode.BlockSizes(); len(bs) != 0 { attr.BlockSize = bs[0] //NOTE: this value is to be used as a hint only; subsequent child block size may differ } - - //TODO [eventualy]: switch off here for handling of time metadata in new format standard - timeStamp(attr, attrMask) } if attrMask.Size { @@ -97,6 +94,8 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask p9.AttrMask, core } } + //TODO [eventually]: switch off here for handling of time metadata in new UFS format standard + return nil } From 6507b344a7718593fc80e61ff5dcf7e43d5f1099 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 12:01:02 -0400 Subject: [PATCH 039/102] use a standard blocksize --- plugin/plugins/filesystem/nodes/ipfs.go | 2 +- plugin/plugins/filesystem/nodes/utils.go | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 9a02b9f2cda..b216da4beca 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -112,7 +112,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } } - return id.Qid, 0, nil + return id.Qid, ipfsBlockSize, nil } func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 87a0ee9c5c5..170f454186f 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -23,7 +23,14 @@ import ( // for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 var salt []byte -const saltSize = 32 +const ( + // TODO [2019.09.12; anyone] + // Start a discussion around block sizes + // should we use the de-facto standard of 4KiB or use our own of 256KiB? + // context: https://github.com/ipfs/go-ipfs/pull/6612/files#r322989041 + ipfsBlockSize = 256 << 10 + saltSize = 32 +) func init() { salt = make([]byte, saltSize) @@ -75,9 +82,8 @@ func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask p9.AttrMask, core } if attrMask.Blocks { - if bs := ufsNode.BlockSizes(); len(bs) != 0 { - attr.BlockSize = bs[0] //NOTE: this value is to be used as a hint only; subsequent child block size may differ - } + //TODO: when/if UFS supports this metadata field, use it instead + attr.BlockSize = ipfsBlockSize } if attrMask.Size { From 4a471d646d4b7b6f81bbaea166b5489696f62fb4 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 12 Sep 2019 12:35:36 -0400 Subject: [PATCH 040/102] stop listening sooner --- plugin/plugins/filesystem/filesystem.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 359753aa446..bc4c1a4da76 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -118,7 +118,7 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { func (fs *FileSystemPlugin) Close() error { logger.Info("9P server requested to close") - fs.cancel() fs.listener.Close() + fs.cancel() return <-fs.errorChan } From afd9cb10c72eb7f2931b2186f58f55e7c8e2e9f8 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 13 Sep 2019 09:06:20 -0400 Subject: [PATCH 041/102] ipfs read fixes --- plugin/plugins/filesystem/nodes/ipfs.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index b216da4beca..a223ac4ad71 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -133,7 +133,7 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { ents := make([]p9.Dirent, 0) out: - for { + for len(ents) < int(count) { select { case entry, open := <-id.directory.entryChan: if !open { @@ -150,30 +150,22 @@ out: } nineEnt.Offset = id.directory.cursor ents = append(ents, nineEnt) - count-- } id.directory.cursor++ - if count == 0 { - break out - } case <-id.Ctx.Done(): return ents, id.Ctx.Err() } } - if offset > uint64(len(ents)) { - return nil, nil //cursor is at end of stream, nothing to return - } - id.Logger.Debugf("ID Readdir returning [%d]%v\n", len(ents), ents) return ents, nil } func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { id.Logger.Debugf("ID ReadAt") - const replaceMe = -1 //TODO: proper error codes + const replaceMe = -1 //TODO [upstream/p9]: add constants for error 9P2000.L/Linux error values apiNode, err := id.core.Unixfs().Get(id.Ctx, id.Path) if err != nil { @@ -188,7 +180,7 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { if fileBound, err := fIo.Size(); err == nil { if int64(offset) >= fileBound { //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. - return 0, nil + return 0, io.EOF } } From e8c1a5e78819b84a45caae1921cf3b78be9b300a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 13 Sep 2019 15:14:53 -0400 Subject: [PATCH 042/102] docs: experimental stability before feature reality --- docs/experimental-features.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 9b29c2f4920..913aa1e9c0e 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -740,6 +740,8 @@ You will likely want to add the plugin to the `go-ipfs` plugin preload list ### Road to being a real feature -- [ ] needs testing +- [ ] Needs testing - Technically correct (`go test`, sharness, etc.) - - Practically/API correct (does this service provide clients with what they need) +- [ ] Needs finalizing + - Config structure and environment variable names become stable/frozen + - Practically/API correct (does this service fulfill the practical needs of users/clients?) From a2a9e8b6c57dcb783be95ef84329471ab913dc1a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 13 Sep 2019 16:03:44 -0400 Subject: [PATCH 043/102] readdir improvments --- plugin/plugins/filesystem/nodes/ipfs.go | 4 +- plugin/plugins/filesystem/nodes/pinfs.go | 62 ++++++++++++++++-------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index a223ac4ad71..87bba3cd7e9 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -122,8 +122,8 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) } - if count == 0 { - return nil, nil + if offset < 0 { + return nil, fmt.Errorf("offset %d can't be negative", offset) } if offset < id.directory.cursor { diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index ab9e5f16223..2fc9037bd2f 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -2,6 +2,7 @@ package fsnodes import ( "context" + "fmt" gopath "path" "github.com/djdv/p9/p9" @@ -12,6 +13,12 @@ import ( type PinFS struct { IPFSBase + directory *directoryStorage +} + +type directoryStorage struct { + ents []p9.Dirent + cursor uint64 } //TODO: [review] check fields @@ -53,41 +60,54 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("PD Open") - return pd.Qid, 0, nil -} - -func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - pd.Logger.Debugf("PD Readdir") + // IPFS core representation pins, err := pd.core.Pin().Ls(pd.Ctx, coreoptions.Pin.Type.Recursive()) if err != nil { - return nil, err + return pd.Qid, ipfsBlockSize, err } - pLen := uint64(len(pins)) - if offset >= pLen { - return nil, nil + // 9P representation + pd.directory = &directoryStorage{ + ents: make([]p9.Dirent, 0, len(pins)), } - //TODO: we should initialize and store entries during Open() to assure order is maintained through read calls - offsetIndex := pins[offset:] - if len(offsetIndex) > int(count) { - offsetIndex = offsetIndex[:count] - } - - ents := make([]p9.Dirent, 0, len(offsetIndex)) - for i, pin := range offsetIndex { + // actual conversion + for i, pin := range pins { dirEnt := &p9.Dirent{ Name: gopath.Base(pin.Path().String()), Offset: uint64(i + 1), } if err = coreStat(pd.Ctx, dirEnt, pd.core, pin.Path()); err != nil { - return nil, err + return pd.Qid, 0, err } - ents = append(ents, *dirEnt) + pd.directory.ents = append(pd.directory.ents, *dirEnt) + } + + return pd.Qid, ipfsBlockSize, nil +} + +func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + pd.Logger.Debugf("PD Readdir") + + if pd.directory == nil { + return nil, fmt.Errorf("directory %q is not open for reading", pd.Path.String()) + } + + if offset < 0 { + return nil, fmt.Errorf("offset %d can't be negative", offset) + } + + if entCount := uint64(len(pd.directory.ents)); offset > entCount { + return nil, fmt.Errorf("offset %d extends beyond directory bound %d", offset, entCount) + } + + subSlice := pd.directory.ents[offset:] + if len(subSlice) > int(count) { + subSlice = subSlice[:count] } - pd.Logger.Debugf("PD Readdir returning ents:%v", ents) - return ents, err + pd.Logger.Debugf("PD Readdir returning ents: %v", subSlice) + return subSlice, nil } From 0a6110dfe201249784646c98ee371a5caee7c72a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 13 Sep 2019 17:45:03 -0400 Subject: [PATCH 044/102] readdir end of stream fix --- plugin/plugins/filesystem/nodes/ipfs.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 87bba3cd7e9..f84599737df 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -23,6 +23,7 @@ type IPFS struct { type directoryStream struct { entryChan <-chan coreiface.DirEntry cursor uint64 + eos bool // have seen end of stream? } func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { @@ -116,7 +117,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - id.Logger.Debugf("ID Readdir") + id.Logger.Debugf("ID Readdir %d %d", offset, count) if id.directory == nil { return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) @@ -126,26 +127,38 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, fmt.Errorf("offset %d can't be negative", offset) } + if id.directory.eos { + if offset == id.directory.cursor-1 { + return nil, nil // valid end of stream request + } + } + + if id.directory.eos && offset == id.directory.cursor-1 { + return nil, nil // valid end of stream request + } + if offset < id.directory.cursor { return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) } ents := make([]p9.Dirent, 0) - out: for len(ents) < int(count) { select { case entry, open := <-id.directory.entryChan: if !open { + id.directory.eos = true break out } if entry.Err != nil { + id.directory = nil return nil, entry.Err } if offset <= id.directory.cursor { nineEnt, err := coreEntTo9Ent(entry) if err != nil { + id.directory = nil return nil, err } nineEnt.Offset = id.directory.cursor @@ -155,6 +168,7 @@ out: id.directory.cursor++ case <-id.Ctx.Done(): + id.directory = nil return ents, id.Ctx.Err() } } From 1dd3125637afa2c45455b938bd3258a3cae7b10d Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 13 Sep 2019 17:45:25 -0400 Subject: [PATCH 045/102] store ipfs reference on open for use in read --- plugin/plugins/filesystem/nodes/ipfs.go | 38 ++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index f84599737df..97b6a0ca21a 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -17,6 +17,7 @@ import ( // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { IPFSBase + file files.File directory *directoryStream } @@ -108,9 +109,22 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { if err != nil { return id.Qid, 0, err } + id.directory = &directoryStream{ entryChan: c, } + return id.Qid, 0, nil + } + + // else treat as a file + apiNode, err := id.core.Unixfs().Get(id.Ctx, id.Path) + if err != nil { + return id.Qid, 0, err + } + + var ok bool + if id.file, ok = apiNode.(files.File); !ok { + return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) } return id.Qid, ipfsBlockSize, nil @@ -179,19 +193,12 @@ out: func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { id.Logger.Debugf("ID ReadAt") - const replaceMe = -1 //TODO [upstream/p9]: add constants for error 9P2000.L/Linux error values - - apiNode, err := id.core.Unixfs().Get(id.Ctx, id.Path) - if err != nil { - return replaceMe, err - } - fIo, ok := apiNode.(files.File) - if !ok { - return replaceMe, fmt.Errorf("%q is not a file", id.Path.String()) + if id.file == nil { + return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) } - if fileBound, err := fIo.Size(); err == nil { + if fileBound, err := id.file.Size(); err == nil { if int64(offset) >= fileBound { //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. return 0, io.EOF @@ -199,15 +206,14 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { } if offset != 0 { - _, err = fIo.Seek(int64(offset), io.SeekStart) - if err != nil { - return replaceMe, fmt.Errorf("Read - seek error: %s", err) + if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { + return -1, fmt.Errorf("Read - seek error: %s", err) } } - readBytes, err := fIo.Read(p) + readBytes, err := id.file.Read(p) if err != nil && err != io.EOF { - return replaceMe, err + return -1, err } - return readBytes, nil + return readBytes, err } From 7b10b5f6e6c74588b0fffd9e146da3c8ce59945a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 13 Sep 2019 18:29:17 -0400 Subject: [PATCH 046/102] better constrain/contextualize operation calls --- plugin/plugins/filesystem/nodes/base.go | 15 +++++++++++++++ plugin/plugins/filesystem/nodes/ipfs.go | 17 ++++++++++++++--- plugin/plugins/filesystem/nodes/pinfs.go | 9 +++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 9dc58d2dcef..917333180e6 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -34,6 +34,8 @@ type Base struct { meta p9.Attr metaMask p9.AttrMask + // The base context should be derived from a long running context (like Background) + // and used to derive call specific contexts from Ctx context.Context Logger logging.EventLogger } @@ -44,4 +46,17 @@ type IPFSBase struct { Path corepath.Resolved core coreiface.CoreAPI + + // you will typically want to derive a context from the base context within one operation (like Open) + // use it with the CoreAPI for something + // and cancel it in another operation (like Close) + // that pointer should be stored here between calls + cancel context.CancelFunc +} + +func (ib *IPFSBase) Close() error { + if ib.cancel != nil { + ib.cancel() + } + return nil } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 97b6a0ca21a..ef2b97bdd4b 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -17,6 +17,7 @@ import ( // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { IPFSBase + file files.File directory *directoryStream } @@ -104,9 +105,17 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { id.Logger.Debugf("ID Open") + + // set up handle amenities + var handleContext context.Context + handleContext, cancel := context.WithCancel(id.Ctx) + id.cancel = cancel + + // handle directories if id.meta.Mode.IsDir() { - c, err := id.core.Unixfs().Ls(id.Ctx, id.Path) + c, err := id.core.Unixfs().Ls(handleContext, id.Path) if err != nil { + cancel() return id.Qid, 0, err } @@ -116,14 +125,16 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return id.Qid, 0, nil } - // else treat as a file - apiNode, err := id.core.Unixfs().Get(id.Ctx, id.Path) + // handle files + apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) if err != nil { + cancel() return id.Qid, 0, err } var ok bool if id.file, ok = apiNode.(files.File); !ok { + cancel() return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 2fc9037bd2f..7606fd68311 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -61,9 +61,13 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("PD Open") + handleContext, cancel := context.WithCancel(pd.Ctx) + pd.cancel = cancel + // IPFS core representation - pins, err := pd.core.Pin().Ls(pd.Ctx, coreoptions.Pin.Type.Recursive()) + pins, err := pd.core.Pin().Ls(handleContext, coreoptions.Pin.Type.Recursive()) if err != nil { + cancel() return pd.Qid, ipfsBlockSize, err } @@ -79,7 +83,8 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { Offset: uint64(i + 1), } - if err = coreStat(pd.Ctx, dirEnt, pd.core, pin.Path()); err != nil { + if err = coreStat(handleContext, dirEnt, pd.core, pin.Path()); err != nil { + cancel() return pd.Qid, 0, err } pd.directory.ents = append(pd.directory.ents, *dirEnt) From 82780a166db3dda23f6d1d4c839b1d288591de01 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 14 Sep 2019 14:25:57 -0400 Subject: [PATCH 047/102] change root construction --- plugin/plugins/filesystem/nodes/base.go | 3 -- plugin/plugins/filesystem/nodes/ipfs.go | 4 +- plugin/plugins/filesystem/nodes/pinfs.go | 11 ++--- plugin/plugins/filesystem/nodes/root.go | 63 +++++++++++------------- plugin/plugins/filesystem/nodes/utils.go | 25 +++++++--- 5 files changed, 55 insertions(+), 51 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 917333180e6..2fd1dff371d 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -16,9 +16,6 @@ const ( //device - attempts to comply with standard multicodec table dIPNS = 0xe5 //TODO: decide what MFS/Files API should be; IPLD? //dFiles = 0xe2 - - //TODO: obviate this - nRoot = "root" ) var _ p9.File = (*Base)(nil) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index ef2b97bdd4b..9b97b1c65c4 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -29,7 +29,7 @@ type directoryStream struct { } func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { - id := &IPFS{IPFSBase: newIPFSBase(ctx, newRootPath("/ipfs"), p9.TypeDir, + id := &IPFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, core, logging.Logger("IPFS"))} id.meta, id.metaMask = defaultRootAttr() return id @@ -45,7 +45,7 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { id.Logger.Debugf("ID GetAttr") id.Logger.Debugf("ID GetAttr path: %v", id.Path) - if id.Path.Namespace() == nRoot { + if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI return id.Qid, id.metaMask, id.meta, nil } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 7606fd68311..b49258a0c7f 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -23,7 +23,7 @@ type directoryStorage struct { //TODO: [review] check fields func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI) *PinFS { - pd := &PinFS{IPFSBase: newIPFSBase(ctx, newRootPath("/ipfs"), p9.TypeDir, + pd := &PinFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, core, logging.Logger("PinFS"))} pd.meta, pd.metaMask = defaultRootAttr() return pd @@ -100,12 +100,9 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, fmt.Errorf("directory %q is not open for reading", pd.Path.String()) } - if offset < 0 { - return nil, fmt.Errorf("offset %d can't be negative", offset) - } - - if entCount := uint64(len(pd.directory.ents)); offset > entCount { - return nil, fmt.Errorf("offset %d extends beyond directory bound %d", offset, entCount) + shouldExit, err := boundCheck(offset, len(pd.directory.ents)) + if shouldExit { + return nil, err } subSlice := pd.directory.ents[offset:] diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 445ffc7b3f5..b8154cf7df2 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -8,55 +8,53 @@ import ( cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" - corepath "github.com/ipfs/interface-go-ipfs-core/path" "github.com/multiformats/go-multihash" ) +const nRoot = "" // root namespace is intentionally left blank var _ p9.File = (*RootIndex)(nil) -func newRootPath(path string) corepath.Resolved { - return rootNode(path) -} - -type rootNode string +type rootPath string -func (rn rootNode) String() string { return string(rn) } -func (rootNode) Namespace() string { return nRoot } -func (rootNode) Mutable() bool { return true } -func (rootNode) IsValid() error { return nil } //TODO: should return ENotImpl -func (rn rootNode) Cid() cid.Cid { - prefix := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN} - c, err := prefix.Sum([]byte(rn)) - if err != nil { - panic(err) //invalid root - } - return c -} -func (rootNode) Root() cid.Cid { //TODO: this should probably reference a package variable set during init `rootCid` +func (rp rootPath) String() string { return string(rp) } +func (rootPath) Namespace() string { return nRoot } +func (rp rootPath) Mutable() bool { return true } +func (rp rootPath) IsValid() error { return nil } +func (rootPath) Root() cid.Cid { return cid.Cid{} } //TODO: reference the root CID when overlay is finished +func (rootPath) Remainder() string { return "" } +func (rp rootPath) Cid() cid.Cid { prefix := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN} - c, err := prefix.Sum([]byte("/")) + c, err := prefix.Sum([]byte(rp)) if err != nil { panic(err) //invalid root } return c } -func (rootNode) Remainder() string { return "" } -// RootIndex is a virtual directory file system, that maps a set of filesystem implementations to a hierarchy +// RootIndex is a virtual directory file system, that maps a set of file system implementations to a hierarchy // Currently: "/":RootIndex, "/ipfs":PinFS, "/ipfs/*:IPFS type RootIndex struct { - IPFSBase + Base + subsystems []p9.Dirent + + core coreiface.CoreAPI } // RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { ri := &RootIndex{ - IPFSBase: newIPFSBase(ctx, newRootPath("/"), p9.TypeDir, - core, logging.Logger("RootFS")), - subsystems: make([]p9.Dirent, 0, 1)} //TODO: [const]: dirent count - ri.Qid.Path = cidToQPath(ri.Path.Cid()) - + core: core, + subsystems: make([]p9.Dirent, 0, 1), //NOTE: capacity should be tied to root child count + Base: Base{ + Logger: logging.Logger("RootFS"), + Ctx: ctx, + Qid: p9.QID{ + Type: p9.TypeDir, + Path: cidToQPath(rootPath("/").Cid()), + }, + }, + } ri.meta, ri.metaMask = defaultRootAttr() rootDirTemplate := p9.Dirent{ @@ -73,7 +71,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { pathUnion.Dirent.Offset = uint64(i + 1) pathUnion.Dirent.Name = pathUnion.string - pathNode := newRootPath("/" + pathUnion.string) + pathNode := rootPath("/" + pathUnion.string) pathUnion.Dirent.QID.Path = cidToQPath(pathNode.Cid()) ri.subsystems = append(ri.subsystems, pathUnion.Dirent) } @@ -102,7 +100,6 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { return []p9.QID{ri.Qid}, ri, nil } - //NOTE: if doClone is false, it implies len(names) > 0 switch names[0] { case "ipfs": pinDir, err := PinFSAttacher(ri.Ctx, ri.core).Attach() @@ -122,10 +119,10 @@ func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { ri.Logger.Debugf("RI Readdir {%d}", count) - sLen := uint64(len(ri.subsystems)) - if offset >= sLen { - return nil, nil //TODO: [spec] should we error here? + shouldExit, err := boundCheck(offset, len(ri.subsystems)) + if shouldExit { + return nil, err } offsetIndex := ri.subsystems[offset:] diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 170f454186f..170ad40b12d 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -18,11 +18,6 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) -// NOTE [2019.09.12]: QID's have a high collision probability -// as a result we add a salt to hashes to attempt to mitigate this -// for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 -var salt []byte - const ( // TODO [2019.09.12; anyone] // Start a discussion around block sizes @@ -32,6 +27,11 @@ const ( saltSize = 32 ) +// NOTE [2019.09.12]: QID's have a high collision probability +// as a result we add a salt to hashes to attempt to mitigate this +// for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 +var salt []byte + func init() { salt = make([]byte, saltSize) _, err := io.ReadFull(rand.Reader, salt) @@ -225,7 +225,6 @@ func timeStamp(attr *p9.Attr, mask p9.AttrMask) { } } -//TODO [name]: "new" implies pointer type; this is for embedded construction func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, core coreiface.CoreAPI, logger logging.EventLogger) IPFSBase { return IPFSBase{ Path: path, @@ -240,3 +239,17 @@ func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, c }, } } + +func boundCheck(offset uint64, length int) (shouldReturn bool, err error) { + switch { + case offset < 0: + return true, fmt.Errorf("offset %d can't be negative", offset) + case offset == uint64(length): + return true, nil // EOS + case offset > uint64(length): + return true, fmt.Errorf("offset %d extends beyond directory bound %d", offset, length) + default: + // not at end of stream and okay to continue + return false, nil + } +} From 4c9a79070a186df39718612c690b11b1fbcb1dd5 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 14 Sep 2019 21:59:54 -0400 Subject: [PATCH 048/102] better walk logic (and more; todo:split commit) --- plugin/plugins/filesystem/filesystem_test.go | 2 +- plugin/plugins/filesystem/nodes/base.go | 38 ++++ plugin/plugins/filesystem/nodes/ipfs.go | 16 +- plugin/plugins/filesystem/nodes/ipns.go | 223 +++++++++++++++++++ plugin/plugins/filesystem/nodes/keyfs.go | 97 ++++++++ plugin/plugins/filesystem/nodes/pinfs.go | 46 ++-- plugin/plugins/filesystem/nodes/root.go | 32 ++- plugin/plugins/filesystem/nodes/utils.go | 80 ++++++- 8 files changed, 487 insertions(+), 47 deletions(-) create mode 100644 plugin/plugins/filesystem/nodes/ipns.go create mode 100644 plugin/plugins/filesystem/nodes/keyfs.go diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 0e5ec2a03ab..54a995e0dac 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -61,7 +61,7 @@ func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { } //TODO: currently magic. As subsystems are implemented, rework this part of the test + lib to contain some list - if len(ents) != 1 || ents[0].Name != "ipfs" { + if len(ents) != 2 || ents[0].Name != "ipfs" || ents[1].Name != "ipns" { t.Fatalf("Failed, root has bad entries:: %v\n", ents) } diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 2fd1dff371d..59f09cffcd2 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -5,6 +5,7 @@ import ( "github.com/djdv/p9/p9" "github.com/djdv/p9/unimplfs" + files "github.com/ipfs/go-ipfs-files" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" @@ -35,6 +36,9 @@ type Base struct { // and used to derive call specific contexts from Ctx context.Context Logger logging.EventLogger + + parent walkRef // parent should be set and used to handle ".." requests + child walkRef // child is an optional field, used to hand off child requests to another file system } // IPFSBase is much like Base but extends it to hold IPFS specific metadata @@ -57,3 +61,37 @@ func (ib *IPFSBase) Close() error { } return nil } + +type ipfsHandle struct { + file files.File + directory *directoryStream +} + +type directoryStream struct { + entryChan <-chan coreiface.DirEntry + cursor uint64 + eos bool // have seen end of stream? +} + +type walkRef interface { + p9.File + QID() p9.QID + Parent() walkRef + Child() walkRef +} + +func (b Base) Self() walkRef { + return b +} + +func (b Base) Parent() walkRef { + return b.parent +} + +func (b Base) Child() walkRef { + return b.child +} + +func (b Base) QID() p9.QID { + return b.Qid +} diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 9b97b1c65c4..17d31a4c37f 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -17,15 +17,7 @@ import ( // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { IPFSBase - - file files.File - directory *directoryStream -} - -type directoryStream struct { - entryChan <-chan coreiface.DirEntry - cursor uint64 - eos bool // have seen end of stream? + ipfsHandle } func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { @@ -71,8 +63,6 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { qids := make([]p9.QID, 0, len(names)) - //TODO: [spec check] make sure we're not messing things up by doing this instead of mutating - // ^ Does internal library expect fid to mutate on success or does newfid clobber some external state anyway walkedNode := &IPFS{} // operate on a copy *walkedNode = *id @@ -104,7 +94,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { } func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - id.Logger.Debugf("ID Open") + id.Logger.Debugf("ID Open %q", id.Path) // set up handle amenities var handleContext context.Context @@ -115,6 +105,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { if id.meta.Mode.IsDir() { c, err := id.core.Unixfs().Ls(handleContext, id.Path) if err != nil { + id.Logger.Errorf("hit\n%q\n%#v\n", id.Path, err) cancel() return id.Qid, 0, err } @@ -128,6 +119,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // handle files apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) if err != nil { + id.Logger.Errorf("hit\n%q\n%#v\n", id.Path, err) cancel() return id.Qid, 0, err } diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go new file mode 100644 index 00000000000..b94540d9267 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -0,0 +1,223 @@ +package fsnodes + +import ( + "context" + "fmt" + "io" + + "github.com/djdv/p9/p9" + files "github.com/ipfs/go-ipfs-files" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" + corepath "github.com/ipfs/interface-go-ipfs-core/path" +) + +// IPNS exposes the IPNS API over a p9.File interface +// Walk does not expect a namespace, only its path argument +// e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipns", "Qm...", "subdir")` +type IPNS struct { + IPFSBase + ipfsHandle + key coreiface.Key +} + +func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPNS { + id := &IPNS{IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, + core, logging.Logger("IPNS"))} + id.meta, id.metaMask = defaultRootAttr() + return id +} + +func (id *IPNS) Attach() (p9.File, error) { + id.Logger.Debugf("ID Attach") + //TODO: check core connection here + return id, nil +} + +func (id *IPNS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + id.Logger.Debugf("ID GetAttr") + id.Logger.Debugf("ID GetAttr path: %v", id.Path) + + if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI + return id.Qid, id.metaMask, id.meta, nil + } + + if err := coreGetAttr(id.Ctx, &id.meta, req, id.core, id.Path); err != nil { + return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err + } + id.Qid.Type = id.meta.Mode.QIDType() + + metaClone := id.meta + metaClone.Filter(req) + + return id.Qid, req, metaClone, nil +} + +func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { + id.Logger.Debugf("ID Walk names %v", names) + id.Logger.Debugf("ID Walk myself: %s:%v", id.Path, id.Qid) + + if shouldClone(names) { + id.Logger.Debugf("ID Walk cloned") + return []p9.QID{id.Qid}, id, nil + } + + qids := make([]p9.QID, 0, len(names)) + + //TODO: [spec check] make sure we're not messing things up by doing this instead of mutating + // ^ Does internal library expect fid to mutate on success or does newfid clobber some external state anyway + walkedNode := &IPNS{} // operate on a copy + *walkedNode = *id + + var err error + for _, name := range names { + //TODO: timerctx; don't want to hang forever + if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, corepath.Join(walkedNode.Path, name)); err != nil { + return qids, nil, err + } + + ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) + if err != nil { + return qids, nil, err + } + + //TODO: this is too opague; we want core path => qid, dirent isn't necessary + dirEnt := &p9.Dirent{} + if err = ipldStat(dirEnt, ipldNode); err != nil { + return qids, nil, err + } + + walkedNode.Qid = dirEnt.QID + // + qids = append(qids, walkedNode.Qid) + } + + id.Logger.Debugf("ID Walk ret %v, %v", qids, walkedNode) + return qids, walkedNode, err +} + +func (id *IPNS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + id.Logger.Debugf("ID Open") + + // set up handle amenities + var handleContext context.Context + handleContext, cancel := context.WithCancel(id.Ctx) + id.cancel = cancel + + // handle directories + if id.meta.Mode.IsDir() { + c, err := id.core.Unixfs().Ls(handleContext, id.Path) + if err != nil { + cancel() + return id.Qid, 0, err + } + + id.directory = &directoryStream{ + entryChan: c, + } + return id.Qid, 0, nil + } + + // handle files + apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) + if err != nil { + cancel() + return id.Qid, 0, err + } + + var ok bool + if id.file, ok = apiNode.(files.File); !ok { + cancel() + return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) + } + + return id.Qid, ipfsBlockSize, nil +} + +func (id *IPNS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + id.Logger.Debugf("ID Readdir %d %d", offset, count) + + if id.directory == nil { + return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) + } + + if offset < 0 { + return nil, fmt.Errorf("offset %d can't be negative", offset) + } + + if id.directory.eos { + if offset == id.directory.cursor-1 { + return nil, nil // valid end of stream request + } + } + + if id.directory.eos && offset == id.directory.cursor-1 { + return nil, nil // valid end of stream request + } + + if offset < id.directory.cursor { + return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) + } + + ents := make([]p9.Dirent, 0) +out: + for len(ents) < int(count) { + select { + case entry, open := <-id.directory.entryChan: + if !open { + id.directory.eos = true + break out + } + if entry.Err != nil { + id.directory = nil + return nil, entry.Err + } + + if offset <= id.directory.cursor { + nineEnt, err := coreEntTo9Ent(entry) + if err != nil { + id.directory = nil + return nil, err + } + nineEnt.Offset = id.directory.cursor + ents = append(ents, nineEnt) + } + + id.directory.cursor++ + + case <-id.Ctx.Done(): + id.directory = nil + return ents, id.Ctx.Err() + } + } + + id.Logger.Debugf("ID Readdir returning [%d]%v\n", len(ents), ents) + return ents, nil +} + +func (id *IPNS) ReadAt(p []byte, offset uint64) (int, error) { + id.Logger.Debugf("ID ReadAt") + + if id.file == nil { + return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) + } + + if fileBound, err := id.file.Size(); err == nil { + if int64(offset) >= fileBound { + //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. + return 0, io.EOF + } + } + + if offset != 0 { + if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { + return -1, fmt.Errorf("Read - seek error: %s", err) + } + } + + readBytes, err := id.file.Read(p) + if err != nil && err != io.EOF { + return -1, err + } + return readBytes, err +} diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go new file mode 100644 index 00000000000..d30266fe726 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -0,0 +1,97 @@ +package fsnodes + +import ( + "context" + "fmt" + + "github.com/djdv/p9/p9" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +type KeyFS struct { + IPFSBase + ents []p9.Dirent +} + +func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI) *KeyFS { + kd := &KeyFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, + core, logging.Logger("KeyFS"))} + kd.meta, kd.metaMask = defaultRootAttr() + kd.meta.Mode |= 0220 + return kd +} + +func (kd *KeyFS) Attach() (p9.File, error) { + kd.Logger.Debugf("KD Attach") + //TODO: check core connection here + return kd, nil +} + +func (kd *KeyFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + kd.Logger.Debugf("KD GetAttr") + + return kd.Qid, kd.metaMask, kd.meta, nil +} + +func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { + kd.Logger.Debugf("KD Walk names %v", names) + kd.Logger.Debugf("KD Walk myself: %v", kd.Qid) + + if shouldClone(names) { + kd.Logger.Debugf("KD Walk cloned") + return []p9.QID{kd.Qid}, kd, nil + } + + return walker(kd, names) + + if kd.ents == nil { + var err error + if kd.ents, err = getKeys(kd.Ctx, kd.core); err != nil { + return nil, nil, err + } + } + + ipfsDir, err := IPFSAttacher(kd.Ctx, kd.core).Attach() + if err != nil { + return nil, nil, err + } + + return ipfsDir.Walk(names) +} + +func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + kd.Logger.Debugf("KD Open") + + /* + handleContext, cancel := context.WithCancel(kd.Ctx) + kd.cancel = cancel + */ + + // IPFS core representation + + kd.Logger.Errorf("key ents:%#v", kd.ents) + + return kd.Qid, ipfsBlockSize, nil +} + +func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + kd.Logger.Debugf("KD Readdir") + + if kd.ents == nil { + return nil, fmt.Errorf("directory %q is not open for reading", kd.Path.String()) + } + + shouldExit, err := boundCheck(offset, len(kd.ents)) + if shouldExit { + return nil, err + } + + subSlice := kd.ents[offset:] + if len(subSlice) > int(count) { + subSlice = subSlice[:count] + } + + kd.Logger.Debugf("KD Readdir returning ents: %v", subSlice) + return subSlice, nil +} diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index b49258a0c7f..d385b929a1a 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -13,25 +13,31 @@ import ( type PinFS struct { IPFSBase - directory *directoryStorage + ents []p9.Dirent } -type directoryStorage struct { - ents []p9.Dirent - cursor uint64 -} - -//TODO: [review] check fields func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI) *PinFS { pd := &PinFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, core, logging.Logger("PinFS"))} pd.meta, pd.metaMask = defaultRootAttr() + return pd } func (pd *PinFS) Attach() (p9.File, error) { pd.Logger.Debugf("PD Attach") - //TODO: check core connection here + + var subSystem walkRef = IPFSAttacher(pd.Ctx, pd.core) + attacher, ok := subSystem.(p9.Attacher) + if !ok { + return nil, fmt.Errorf("subsystem %T is not a valid file system", subSystem) + } + + if _, err := attacher.Attach(); err != nil { + return nil, err + } + pd.child = subSystem + return pd, nil } @@ -45,17 +51,7 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { pd.Logger.Debugf("PD Walk names %v", names) pd.Logger.Debugf("PD Walk myself: %v", pd.Qid) - if shouldClone(names) { - pd.Logger.Debugf("PD Walk cloned") - return []p9.QID{pd.Qid}, pd, nil - } - - ipfsDir, err := IPFSAttacher(pd.Ctx, pd.core).Attach() - if err != nil { - return nil, nil, err - } - - return ipfsDir.Walk(names) + return walker(pd, names) } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { @@ -72,9 +68,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } // 9P representation - pd.directory = &directoryStorage{ - ents: make([]p9.Dirent, 0, len(pins)), - } + pd.ents = make([]p9.Dirent, 0, len(pins)) // actual conversion for i, pin := range pins { @@ -87,7 +81,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { cancel() return pd.Qid, 0, err } - pd.directory.ents = append(pd.directory.ents, *dirEnt) + pd.ents = append(pd.ents, *dirEnt) } return pd.Qid, ipfsBlockSize, nil @@ -96,16 +90,16 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { pd.Logger.Debugf("PD Readdir") - if pd.directory == nil { + if pd.ents == nil { return nil, fmt.Errorf("directory %q is not open for reading", pd.Path.String()) } - shouldExit, err := boundCheck(offset, len(pd.directory.ents)) + shouldExit, err := boundCheck(offset, len(pd.ents)) if shouldExit { return nil, err } - subSlice := pd.directory.ents[offset:] + subSlice := pd.ents[offset:] if len(subSlice) > int(count) { subSlice = subSlice[:count] } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index b8154cf7df2..f63fc5c0ab9 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -66,7 +66,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { p9.Dirent }{ {"ipfs", rootDirTemplate}, - //{"ipns", rootDirTemplate}, + {"ipns", rootDirTemplate}, } { pathUnion.Dirent.Offset = uint64(i + 1) pathUnion.Dirent.Name = pathUnion.string @@ -81,6 +81,8 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { func (ri *RootIndex) Attach() (p9.File, error) { ri.Logger.Debugf("RI Attach") + + ri.parent = ri return ri, nil } @@ -100,16 +102,32 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { return []p9.QID{ri.Qid}, ri, nil } + var ( + subSystem walkRef + err error + ) + switch names[0] { case "ipfs": - pinDir, err := PinFSAttacher(ri.Ctx, ri.core).Attach() - if err != nil { - return nil, nil, err - } - return pinDir.Walk(names[1:]) + subSystem = PinFSAttacher(ri.Ctx, ri.core) + case "ipns": + subSystem = KeyFSAttacher(ri.Ctx, ri.core) default: - return nil, nil, fmt.Errorf("%q is not provided by us", names[0]) //TODO: Err vars + return nil, nil, fmt.Errorf("%q is not provided by us", names[0]) + } + + attacher, ok := subSystem.(p9.Attacher) + if !ok { + return nil, nil, fmt.Errorf("%q is not a valid file system", names[0]) } + + if _, err = attacher.Attach(); err != nil { + return nil, nil, err + } + + ri.child = subSystem + + return walker(subSystem, names[1:]) } func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 170ad40b12d..08714da451b 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -3,6 +3,7 @@ package fsnodes import ( "context" "crypto/rand" + "errors" "fmt" "hash/fnv" "io" @@ -240,7 +241,9 @@ func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, c } } -func boundCheck(offset uint64, length int) (shouldReturn bool, err error) { +// boundCheck assures operation arguments are valid +// returns true if the caller should return immediately with our values +func boundCheck(offset uint64, length int) (bool, error) { switch { case offset < 0: return true, fmt.Errorf("offset %d can't be negative", offset) @@ -253,3 +256,78 @@ func boundCheck(offset uint64, length int) (shouldReturn bool, err error) { return false, nil } } + +// walker acts as a dispatcher for intermediate filesystems +// sending path component requests to their appropriate system +func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { + // clone requests go right back to the caller + if shouldClone(names) { + return []p9.QID{ref.QID()}, ref, nil + } + + var ( + qids, subQids []p9.QID + lastFile, curFile p9.File + err error + ) + + var i int + for len(names) != 0 { + if names[0] == ".." { // climb to parent / leftwards requests + p := ref.Parent() + if p == nil { + return []p9.QID{ref.QID()}, ref, errors.New("parent not assigned") + } + + subQids, curFile, err = p.Walk(names) + } else { // handle remainder / rightward requests + c := ref.Child() + if c == nil { + return []p9.QID{ref.QID()}, ref, errors.New("child not assigned") + } + + subQids, curFile, err = c.Walk(names) + } + + if err != nil { + return qids, lastFile, err + } + + qids = append(qids, subQids...) + lastFile = curFile + names = names[1:] + i++ + } + + return qids, lastFile, nil +} + +func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]p9.Dirent, error) { + keys, err := core.Key().List(ctx) + if err != nil { + return nil, err + } + + ents := make([]p9.Dirent, 0, len(keys)) + + var offset = 1 + for _, key := range keys { + dirEnt := &p9.Dirent{ + Name: key.Name(), + Offset: uint64(offset), + } + + if err = coreStat(ctx, dirEnt, core, key.Path()); err != nil { + //FIXME: bug in either the CoreAPI, http client, or somewhere else + //if err == coreiface.ErrResolveFailed { + //HACK: + if err.Error() == coreiface.ErrResolveFailed.Error() { + continue // skip unresolvable + } + return nil, err + } + ents = append(ents, *dirEnt) + offset++ + } + return ents, nil +} From fb20a4487147451fa8d34b966bcbc954746c869b Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sun, 15 Sep 2019 10:46:26 -0400 Subject: [PATCH 049/102] ipns initial --- plugin/plugins/filesystem/nodes/ipns.go | 37 +++++++++++- plugin/plugins/filesystem/nodes/keyfs.go | 75 +++++++++++++++++------- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- plugin/plugins/filesystem/nodes/utils.go | 12 ++-- 4 files changed, 96 insertions(+), 30 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go index b94540d9267..b7ded00f2df 100644 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -21,9 +21,12 @@ type IPNS struct { key coreiface.Key } -func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPNS { - id := &IPNS{IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, - core, logging.Logger("IPNS"))} +func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, key coreiface.Key) *IPNS { + id := &IPNS{ + key: key, + IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, + core, logging.Logger("IPNS")), + } id.meta, id.metaMask = defaultRootAttr() return id } @@ -69,6 +72,34 @@ func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { walkedNode := &IPNS{} // operate on a copy *walkedNode = *id + id.Logger.Errorf("self:\n%#v\nkey path:%q", id, id.key.Path()) + + // first child requests will take care to translate the key's string name into a valid global path + // this path will act as a subroot for further child requests + if id.Path.Namespace() == "" && id.key != nil { + //TODO [refactor]: repetition + var err error + if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, id.key.Path()); err != nil { + return qids, nil, err + } + + ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) + if err != nil { + return qids, nil, err + } + + //TODO: this is too opague; we want core path => qid, dirent isn't necessary + dirEnt := &p9.Dirent{} + if err = ipldStat(dirEnt, ipldNode); err != nil { + return qids, nil, err + } + + walkedNode.Qid = dirEnt.QID + // + qids = append(qids, walkedNode.Qid) + names = names[1:] + } + var err error for _, name := range names { //TODO: timerctx; don't want to hang forever diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index d30266fe726..f3dff52ba49 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -9,9 +9,14 @@ import ( coreiface "github.com/ipfs/interface-go-ipfs-core" ) +type entPair struct { + ent p9.Dirent + key coreiface.Key +} + type KeyFS struct { IPFSBase - ents []p9.Dirent + keyEnts []entPair } func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI) *KeyFS { @@ -24,7 +29,18 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI) *KeyFS { func (kd *KeyFS) Attach() (p9.File, error) { kd.Logger.Debugf("KD Attach") - //TODO: check core connection here + + var subSystem walkRef = IPFSAttacher(kd.Ctx, kd.core) + attacher, ok := subSystem.(p9.Attacher) + if !ok { + return nil, fmt.Errorf("subsystem %T is not a valid file system", subSystem) + } + + if _, err := attacher.Attach(); err != nil { + return nil, err + } + kd.child = subSystem + return kd, nil } @@ -43,34 +59,44 @@ func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { return []p9.QID{kd.Qid}, kd, nil } - return walker(kd, names) + keys, err := kd.core.Key().List(kd.Ctx) + if err != nil { + return nil, nil, err + } - if kd.ents == nil { - var err error - if kd.ents, err = getKeys(kd.Ctx, kd.core); err != nil { - return nil, nil, err + var coreKey coreiface.Key + for _, key := range keys { + if names[0] == key.Name() { + coreKey = key + break } } - ipfsDir, err := IPFSAttacher(kd.Ctx, kd.core).Attach() - if err != nil { - return nil, nil, err + if coreKey != nil { // use IPNS FS to allow write support to the key + ipns, err := IPNSAttacher(kd.Ctx, kd.core, coreKey).Attach() + if err != nil { + return nil, nil, err + } + return ipns.Walk(names) } - return ipfsDir.Walk(names) + // if we don't own the key, treat this as a typical IPFS core-request + return walker(kd, names) } func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { kd.Logger.Debugf("KD Open") - /* - handleContext, cancel := context.WithCancel(kd.Ctx) - kd.cancel = cancel - */ + handleContext, cancel := context.WithCancel(kd.Ctx) + kd.cancel = cancel - // IPFS core representation + var err error + if kd.keyEnts, err = getKeys(handleContext, kd.core); err != nil { + cancel() + return kd.Qid, 0, err + } - kd.Logger.Errorf("key ents:%#v", kd.ents) + kd.Logger.Errorf("key ents:%#v", kd.keyEnts) return kd.Qid, ipfsBlockSize, nil } @@ -78,20 +104,25 @@ func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { kd.Logger.Debugf("KD Readdir") - if kd.ents == nil { + if kd.keyEnts == nil { return nil, fmt.Errorf("directory %q is not open for reading", kd.Path.String()) } - shouldExit, err := boundCheck(offset, len(kd.ents)) + shouldExit, err := boundCheck(offset, len(kd.keyEnts)) if shouldExit { return nil, err } - subSlice := kd.ents[offset:] + subSlice := kd.keyEnts[offset:] if len(subSlice) > int(count) { subSlice = subSlice[:count] } - kd.Logger.Debugf("KD Readdir returning ents: %v", subSlice) - return subSlice, nil + nineEnts := make([]p9.Dirent, 0, len(subSlice)) + for _, pair := range subSlice { + nineEnts = append(nineEnts, pair.ent) + } + + kd.Logger.Debugf("KD Readdir returning ents: %v", nineEnts) + return nineEnts, nil } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index d385b929a1a..986a11c6f51 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -64,7 +64,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pins, err := pd.core.Pin().Ls(handleContext, coreoptions.Pin.Type.Recursive()) if err != nil { cancel() - return pd.Qid, ipfsBlockSize, err + return pd.Qid, 0, err } // 9P representation diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 08714da451b..a9b777bb07e 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -302,21 +302,23 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { return qids, lastFile, nil } -func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]p9.Dirent, error) { +func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]entPair, error) { keys, err := core.Key().List(ctx) if err != nil { return nil, err } - ents := make([]p9.Dirent, 0, len(keys)) + ents := make([]entPair, 0, len(keys)) var offset = 1 - for _, key := range keys { + for i, key := range keys { dirEnt := &p9.Dirent{ Name: key.Name(), Offset: uint64(offset), } + fmt.Printf("[%d] %q:%q\n", i, key.Name(), key.Path()) + if err = coreStat(ctx, dirEnt, core, key.Path()); err != nil { //FIXME: bug in either the CoreAPI, http client, or somewhere else //if err == coreiface.ErrResolveFailed { @@ -326,7 +328,9 @@ func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]p9.Dirent, error) { } return nil, err } - ents = append(ents, *dirEnt) + + pair := entPair{ent: *dirEnt, key: key} + ents = append(ents, pair) offset++ } return ents, nil From 5ad516c570bce22af3c28e26844d9bb2f7fe6eea Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sun, 15 Sep 2019 21:36:21 -0400 Subject: [PATCH 050/102] log linting --- plugin/plugins/filesystem/nodes/ipfs.go | 22 +++++++++++----------- plugin/plugins/filesystem/nodes/ipns.go | 24 +++++++++++------------- plugin/plugins/filesystem/nodes/keyfs.go | 18 ++++++++---------- plugin/plugins/filesystem/nodes/pinfs.go | 14 +++++++------- plugin/plugins/filesystem/nodes/root.go | 20 ++++++++++---------- plugin/plugins/filesystem/nodes/utils.go | 6 +----- 6 files changed, 48 insertions(+), 56 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 17d31a4c37f..f9eb09eccf0 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -28,14 +28,14 @@ func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { } func (id *IPFS) Attach() (p9.File, error) { - id.Logger.Debugf("ID Attach") + id.Logger.Debugf("Attach") //TODO: check core connection here return id, nil } func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - id.Logger.Debugf("ID GetAttr") - id.Logger.Debugf("ID GetAttr path: %v", id.Path) + id.Logger.Debugf("GetAttr") + id.Logger.Debugf("GetAttr path: %v", id.Path) if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI return id.Qid, id.metaMask, id.meta, nil @@ -53,11 +53,11 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { } func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { - id.Logger.Debugf("ID Walk names %v", names) - id.Logger.Debugf("ID Walk myself: %s:%v", id.Path, id.Qid) + id.Logger.Debugf("Walk names %v", names) + id.Logger.Debugf("Walk myself: %s:%v", id.Path, id.Qid) if shouldClone(names) { - id.Logger.Debugf("ID Walk cloned") + id.Logger.Debugf("Walk cloned") return []p9.QID{id.Qid}, id, nil } @@ -89,12 +89,12 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { qids = append(qids, walkedNode.Qid) } - id.Logger.Debugf("ID Walk ret %v, %v", qids, walkedNode) + id.Logger.Debugf("Walk ret %v, %v", qids, walkedNode) return qids, walkedNode, err } func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - id.Logger.Debugf("ID Open %q", id.Path) + id.Logger.Debugf("Open %q", id.Path) // set up handle amenities var handleContext context.Context @@ -134,7 +134,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - id.Logger.Debugf("ID Readdir %d %d", offset, count) + id.Logger.Debugf("Readdir %d %d", offset, count) if id.directory == nil { return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) @@ -190,12 +190,12 @@ out: } } - id.Logger.Debugf("ID Readdir returning [%d]%v\n", len(ents), ents) + id.Logger.Debugf("Readdir returning [%d]%v\n", len(ents), ents) return ents, nil } func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { - id.Logger.Debugf("ID ReadAt") + id.Logger.Debugf("ReadAt") if id.file == nil { return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go index b7ded00f2df..4a0336cc0c3 100644 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -32,14 +32,14 @@ func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, key coreiface.Key } func (id *IPNS) Attach() (p9.File, error) { - id.Logger.Debugf("ID Attach") + id.Logger.Debugf("Attach") //TODO: check core connection here return id, nil } func (id *IPNS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - id.Logger.Debugf("ID GetAttr") - id.Logger.Debugf("ID GetAttr path: %v", id.Path) + id.Logger.Debugf("GetAttr") + id.Logger.Debugf("GetAttr path: %v", id.Path) if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI return id.Qid, id.metaMask, id.meta, nil @@ -57,11 +57,11 @@ func (id *IPNS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { } func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { - id.Logger.Debugf("ID Walk names %v", names) - id.Logger.Debugf("ID Walk myself: %s:%v", id.Path, id.Qid) + id.Logger.Debugf("Walk names %v", names) + id.Logger.Debugf("Walk myself: %s:%v", id.Path, id.Qid) if shouldClone(names) { - id.Logger.Debugf("ID Walk cloned") + id.Logger.Debugf("Walk cloned") return []p9.QID{id.Qid}, id, nil } @@ -72,8 +72,6 @@ func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { walkedNode := &IPNS{} // operate on a copy *walkedNode = *id - id.Logger.Errorf("self:\n%#v\nkey path:%q", id, id.key.Path()) - // first child requests will take care to translate the key's string name into a valid global path // this path will act as a subroot for further child requests if id.Path.Namespace() == "" && id.key != nil { @@ -123,12 +121,12 @@ func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { qids = append(qids, walkedNode.Qid) } - id.Logger.Debugf("ID Walk ret %v, %v", qids, walkedNode) + id.Logger.Debugf("Walk ret %v, %v", qids, walkedNode) return qids, walkedNode, err } func (id *IPNS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - id.Logger.Debugf("ID Open") + id.Logger.Debugf("Open") // set up handle amenities var handleContext context.Context @@ -166,7 +164,7 @@ func (id *IPNS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } func (id *IPNS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - id.Logger.Debugf("ID Readdir %d %d", offset, count) + id.Logger.Debugf("Readdir %d %d", offset, count) if id.directory == nil { return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) @@ -222,12 +220,12 @@ out: } } - id.Logger.Debugf("ID Readdir returning [%d]%v\n", len(ents), ents) + id.Logger.Debugf("Readdir returning [%d]%v\n", len(ents), ents) return ents, nil } func (id *IPNS) ReadAt(p []byte, offset uint64) (int, error) { - id.Logger.Debugf("ID ReadAt") + id.Logger.Debugf("ReadAt") if id.file == nil { return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index f3dff52ba49..0621dc49854 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -28,7 +28,7 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI) *KeyFS { } func (kd *KeyFS) Attach() (p9.File, error) { - kd.Logger.Debugf("KD Attach") + kd.Logger.Debugf("Attach") var subSystem walkRef = IPFSAttacher(kd.Ctx, kd.core) attacher, ok := subSystem.(p9.Attacher) @@ -45,17 +45,17 @@ func (kd *KeyFS) Attach() (p9.File, error) { } func (kd *KeyFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - kd.Logger.Debugf("KD GetAttr") + kd.Logger.Debugf("GetAttr") return kd.Qid, kd.metaMask, kd.meta, nil } func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { - kd.Logger.Debugf("KD Walk names %v", names) - kd.Logger.Debugf("KD Walk myself: %v", kd.Qid) + kd.Logger.Debugf("Walk names %v", names) + kd.Logger.Debugf("Walk myself: %v", kd.Qid) if shouldClone(names) { - kd.Logger.Debugf("KD Walk cloned") + kd.Logger.Debugf("Walk cloned") return []p9.QID{kd.Qid}, kd, nil } @@ -85,7 +85,7 @@ func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { } func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - kd.Logger.Debugf("KD Open") + kd.Logger.Debugf("Open") handleContext, cancel := context.WithCancel(kd.Ctx) kd.cancel = cancel @@ -96,13 +96,11 @@ func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return kd.Qid, 0, err } - kd.Logger.Errorf("key ents:%#v", kd.keyEnts) - return kd.Qid, ipfsBlockSize, nil } func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - kd.Logger.Debugf("KD Readdir") + kd.Logger.Debugf("Readdir") if kd.keyEnts == nil { return nil, fmt.Errorf("directory %q is not open for reading", kd.Path.String()) @@ -123,6 +121,6 @@ func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { nineEnts = append(nineEnts, pair.ent) } - kd.Logger.Debugf("KD Readdir returning ents: %v", nineEnts) + kd.Logger.Debugf("Readdir returning ents: %v", nineEnts) return nineEnts, nil } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 986a11c6f51..2b8863997b4 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -25,7 +25,7 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI) *PinFS { } func (pd *PinFS) Attach() (p9.File, error) { - pd.Logger.Debugf("PD Attach") + pd.Logger.Debugf("Attach") var subSystem walkRef = IPFSAttacher(pd.Ctx, pd.core) attacher, ok := subSystem.(p9.Attacher) @@ -42,20 +42,20 @@ func (pd *PinFS) Attach() (p9.File, error) { } func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - pd.Logger.Debugf("PD GetAttr") + pd.Logger.Debugf("GetAttr") return pd.Qid, pd.metaMask, pd.meta, nil } func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { - pd.Logger.Debugf("PD Walk names %v", names) - pd.Logger.Debugf("PD Walk myself: %v", pd.Qid) + pd.Logger.Debugf("Walk names %v", names) + pd.Logger.Debugf("Walk myself: %v", pd.Qid) return walker(pd, names) } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - pd.Logger.Debugf("PD Open") + pd.Logger.Debugf("Open") handleContext, cancel := context.WithCancel(pd.Ctx) pd.cancel = cancel @@ -88,7 +88,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - pd.Logger.Debugf("PD Readdir") + pd.Logger.Debugf("Readdir") if pd.ents == nil { return nil, fmt.Errorf("directory %q is not open for reading", pd.Path.String()) @@ -104,6 +104,6 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { subSlice = subSlice[:count] } - pd.Logger.Debugf("PD Readdir returning ents: %v", subSlice) + pd.Logger.Debugf("Readdir returning ents: %v", subSlice) return subSlice, nil } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index f63fc5c0ab9..cbee90f82ed 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -80,25 +80,25 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { } func (ri *RootIndex) Attach() (p9.File, error) { - ri.Logger.Debugf("RI Attach") + ri.Logger.Debugf("Attach") ri.parent = ri return ri, nil } func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - ri.Logger.Debugf("RI GetAttr") - ri.Logger.Debugf("RI mask: %v", req) + ri.Logger.Debugf("GetAttr") + ri.Logger.Debugf("mask: %v", req) return ri.Qid, ri.metaMask, ri.meta, nil } func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { - ri.Logger.Debugf("RI Walk names %v", names) - ri.Logger.Debugf("RI Walk myself: %v", ri.Qid) + ri.Logger.Debugf("Walk names %v", names) + ri.Logger.Debugf("Walk myself: %v", ri.Qid) if shouldClone(names) { - ri.Logger.Debugf("RI Walk cloned") + ri.Logger.Debugf("Walk cloned") return []p9.QID{ri.Qid}, ri, nil } @@ -131,12 +131,12 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { } func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - ri.Logger.Debugf("RI Open") + ri.Logger.Debugf("Open") return ri.Qid, 0, nil } func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - ri.Logger.Debugf("RI Readdir {%d}", count) + ri.Logger.Debugf("Readdir {%d}", count) shouldExit, err := boundCheck(offset, len(ri.subsystems)) if shouldExit { @@ -145,10 +145,10 @@ func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { offsetIndex := ri.subsystems[offset:] if len(offsetIndex) > int(count) { - ri.Logger.Debugf("RI Readdir returning [%d]%v\n", count, offsetIndex[:count]) + ri.Logger.Debugf("Readdir returning [%d]%v\n", count, offsetIndex[:count]) return offsetIndex[:count], nil } - ri.Logger.Debugf("RI Readdir returning [%d]%v\n", len(offsetIndex), offsetIndex) + ri.Logger.Debugf("Readdir returning [%d]%v\n", len(offsetIndex), offsetIndex) return offsetIndex, nil } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index a9b777bb07e..9ca96bfc945 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -271,7 +271,6 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { err error ) - var i int for len(names) != 0 { if names[0] == ".." { // climb to parent / leftwards requests p := ref.Parent() @@ -296,7 +295,6 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { qids = append(qids, subQids...) lastFile = curFile names = names[1:] - i++ } return qids, lastFile, nil @@ -311,14 +309,12 @@ func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]entPair, error) { ents := make([]entPair, 0, len(keys)) var offset = 1 - for i, key := range keys { + for _, key := range keys { dirEnt := &p9.Dirent{ Name: key.Name(), Offset: uint64(offset), } - fmt.Printf("[%d] %q:%q\n", i, key.Name(), key.Path()) - if err = coreStat(ctx, dirEnt, core, key.Path()); err != nil { //FIXME: bug in either the CoreAPI, http client, or somewhere else //if err == coreiface.ErrResolveFailed { From 02f69b81b240f08e546cd6289cd9b87ee11eab09 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 17 Sep 2019 16:17:48 -0400 Subject: [PATCH 051/102] walk logic - this is better but still broken --- plugin/plugins/filesystem/nodes/ipfs.go | 2 +- plugin/plugins/filesystem/nodes/root.go | 2 +- plugin/plugins/filesystem/nodes/utils.go | 54 ++++++++++++++---------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index f9eb09eccf0..c501ba16e39 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -63,7 +63,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { qids := make([]p9.QID, 0, len(names)) - walkedNode := &IPFS{} // operate on a copy + walkedNode := &IPFS{} // [walk(5)] id represents fid, walkedNode represents newfid *walkedNode = *id var err error diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index cbee90f82ed..917069f6567 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -66,7 +66,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { p9.Dirent }{ {"ipfs", rootDirTemplate}, - {"ipns", rootDirTemplate}, + //{"ipns", rootDirTemplate}, } { pathUnion.Dirent.Offset = uint64(i + 1) pathUnion.Dirent.Name = pathUnion.string diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 9ca96bfc945..7be37569241 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -3,7 +3,6 @@ package fsnodes import ( "context" "crypto/rand" - "errors" "fmt" "hash/fnv" "io" @@ -257,47 +256,58 @@ func boundCheck(offset uint64, length int) (bool, error) { } } -// walker acts as a dispatcher for intermediate filesystems -// sending path component requests to their appropriate system +// walker acts as a dispatcher for intermediate file systems +// sending individual path component requests to their appropriate target system +// regardless of (file system request) origin func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { + l := logging.Logger("walker") + l.Errorf("req: %v\n%v\n", ref, names) + // clone requests go right back to the caller if shouldClone(names) { return []p9.QID{ref.QID()}, ref, nil } var ( - qids, subQids []p9.QID - lastFile, curFile p9.File - err error + nextRef walkRef + qids, subQids []p9.QID + curFile p9.File + err error ) + var i int for len(names) != 0 { - if names[0] == ".." { // climb to parent / leftwards requests - p := ref.Parent() - if p == nil { - return []p9.QID{ref.QID()}, ref, errors.New("parent not assigned") - } + l.Errorf("[%d] loop: %v\n%v\n", i, ref, names) - subQids, curFile, err = p.Walk(names) + //prepare to step into next component + if names[0] == ".." { // climb to parent / leftwards requests + nextRef = ref.Parent() } else { // handle remainder / rightward requests - c := ref.Child() - if c == nil { - return []p9.QID{ref.QID()}, ref, errors.New("child not assigned") - } - - subQids, curFile, err = c.Walk(names) + nextRef = ref.Child() + } + if nextRef == nil { + return []p9.QID{ref.QID()}, nil, fmt.Errorf("system for target %q is not assigned", names[0]) } - if err != nil { - return qids, lastFile, err + // attempt the step + if subQids, curFile, err = nextRef.Walk(names); err != nil { + return qids, nil, err } + // we walked forward, prepare for next step qids = append(qids, subQids...) - lastFile = curFile names = names[1:] + ref = nextRef + + if len(names) != 0 { + curFile.Close() // we're not referencing this anymore + } // leave the last reference alive, for the caller to close } + i++ + + l.Errorf("walker ret: %v\n%v\n", qids, curFile) - return qids, lastFile, nil + return qids, curFile, nil } func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]entPair, error) { From eaff0c93b4bf15113cdf02a519c584ae77aae89e Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 18 Sep 2019 09:43:57 -0400 Subject: [PATCH 052/102] misc cleanup and context changes --- plugin/plugins/filesystem/filesystem.go | 5 +- plugin/plugins/filesystem/nodes/base.go | 15 +- plugin/plugins/filesystem/nodes/doc_test.go | 8 +- plugin/plugins/filesystem/nodes/ipfs.go | 72 +++--- plugin/plugins/filesystem/nodes/ipns.go | 252 -------------------- plugin/plugins/filesystem/nodes/keyfs.go | 2 +- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- plugin/plugins/filesystem/nodes/root.go | 12 +- 8 files changed, 65 insertions(+), 303 deletions(-) delete mode 100644 plugin/plugins/filesystem/nodes/ipns.go diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index bc4c1a4da76..27a2676280d 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -100,11 +100,12 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { fs.ctx, fs.cancel = context.WithCancel(context.Background()) fs.errorChan = make(chan error, 1) - var err error - if fs.listener, err = manet.Listen(fs.addr); err != nil { + listener, err := manet.Listen(fs.addr) + if err != nil { logger.Errorf("9P listen error: %s\n", err) return err } + fs.listener = listener // construct and run the 9P resource server s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core)) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 59f09cffcd2..2dbb8a9b9f3 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -32,7 +32,7 @@ type Base struct { meta p9.Attr metaMask p9.AttrMask - // The base context should be derived from a long running context (like Background) + // The base context should be valid for the lifetime of the file system // and used to derive call specific contexts from Ctx context.Context Logger logging.EventLogger @@ -52,12 +52,18 @@ type IPFSBase struct { // use it with the CoreAPI for something // and cancel it in another operation (like Close) // that pointer should be stored here between calls - cancel context.CancelFunc + operationsCancel context.CancelFunc } func (ib *IPFSBase) Close() error { - if ib.cancel != nil { - ib.cancel() + if ib.operationsCancel != nil { + ib.operationsCancel() + } + if ib.parent != nil { + ib.parent.Close() + } + if ib.child != nil { + ib.child.Close() } return nil } @@ -71,6 +77,7 @@ type directoryStream struct { entryChan <-chan coreiface.DirEntry cursor uint64 eos bool // have seen end of stream? + err error } type walkRef interface { diff --git a/plugin/plugins/filesystem/nodes/doc_test.go b/plugin/plugins/filesystem/nodes/doc_test.go index de9e03da879..7744c83cca6 100644 --- a/plugin/plugins/filesystem/nodes/doc_test.go +++ b/plugin/plugins/filesystem/nodes/doc_test.go @@ -1,14 +1,9 @@ package fsnodes -import ( - "strings" - - "github.com/djdv/p9/p9" -) - //TODO: compiler legal examples so Go tools don't get mad; needs client wrappers to look nice // `Output:` comments too +/* func ExampleRootIndex() { root, err := fsnodes.RootAttacher(ctx, coreAPI).Attach() _, file, err := root.Walk(strings.Split("ipfs/Qm.../subdir/file", "/")) @@ -31,3 +26,4 @@ func ExamplePinFS() { _, _, err := dir.Open(p9.ReadOnly) entries, err := dirClone.Readdir(offset, entryReturnCount) } +*/ diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index c501ba16e39..2c2932de7f0 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "time" "github.com/djdv/p9/p9" files "github.com/ipfs/go-ipfs-files" @@ -61,36 +62,42 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { return []p9.QID{id.Qid}, id, nil } - qids := make([]p9.QID, 0, len(names)) + var ( + qids = make([]p9.QID, 0, len(names)) + newFid = &IPFS{IPFSBase: newIPFSBase(id.Ctx, id.Path, 0, id.core, id.Logger)} + err error + ) - walkedNode := &IPFS{} // [walk(5)] id represents fid, walkedNode represents newfid - *walkedNode = *id - - var err error for _, name := range names { - //TODO: timerctx; don't want to hang forever - if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, corepath.Join(walkedNode.Path, name)); err != nil { + callCtx, cancel := context.WithTimeout(id.Ctx, 30*time.Second) + defer cancel() + + if newFid.Path, err = id.core.ResolvePath(callCtx, corepath.Join(newFid.Path, name)); err != nil { + cancel() + //TODO: switch off error, return appropriate errno (likely ENOENT) return qids, nil, err } - ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) + ipldNode, err := id.core.Dag().Get(callCtx, newFid.Path.Cid()) if err != nil { + cancel() return qids, nil, err } - //TODO: this is too opague; we want core path => qid, dirent isn't necessary + //TODO: this is too opaque; we want core path => qid, Dirent isn't /really/ necessary dirEnt := &p9.Dirent{} if err = ipldStat(dirEnt, ipldNode); err != nil { + cancel() return qids, nil, err } - walkedNode.Qid = dirEnt.QID + newFid.Qid = dirEnt.QID // - qids = append(qids, walkedNode.Qid) + qids = append(qids, newFid.Qid) } - id.Logger.Debugf("Walk ret %v, %v", qids, walkedNode) - return qids, walkedNode, err + id.Logger.Debugf("Walk ret %v, %v", qids, newFid) + return qids, newFid, err } func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { @@ -98,15 +105,13 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // set up handle amenities var handleContext context.Context - handleContext, cancel := context.WithCancel(id.Ctx) - id.cancel = cancel + handleContext, id.operationsCancel = context.WithCancel(id.Ctx) // handle directories if id.meta.Mode.IsDir() { c, err := id.core.Unixfs().Ls(handleContext, id.Path) if err != nil { - id.Logger.Errorf("hit\n%q\n%#v\n", id.Path, err) - cancel() + id.operationsCancel() return id.Qid, 0, err } @@ -119,16 +124,16 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // handle files apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) if err != nil { - id.Logger.Errorf("hit\n%q\n%#v\n", id.Path, err) - cancel() + id.operationsCancel() return id.Qid, 0, err } - var ok bool - if id.file, ok = apiNode.(files.File); !ok { - cancel() + fileNode, ok := apiNode.(files.File) + if !ok { + id.operationsCancel() return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) } + id.file = fileNode return id.Qid, ipfsBlockSize, nil } @@ -140,42 +145,39 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) } - if offset < 0 { - return nil, fmt.Errorf("offset %d can't be negative", offset) + if id.directory.err != nil { // previous request must have failed + return nil, id.directory.err } if id.directory.eos { if offset == id.directory.cursor-1 { - return nil, nil // valid end of stream request + return nil, nil // this is the only exception to offset being behind the cursor } } - if id.directory.eos && offset == id.directory.cursor-1 { - return nil, nil // valid end of stream request - } - if offset < id.directory.cursor { return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) } ents := make([]p9.Dirent, 0) -out: + for len(ents) < int(count) { select { case entry, open := <-id.directory.entryChan: if !open { + id.operationsCancel() id.directory.eos = true - break out + return ents, nil } if entry.Err != nil { - id.directory = nil + id.directory.err = entry.Err return nil, entry.Err } if offset <= id.directory.cursor { nineEnt, err := coreEntTo9Ent(entry) if err != nil { - id.directory = nil + id.directory.err = err return nil, err } nineEnt.Offset = id.directory.cursor @@ -185,8 +187,8 @@ out: id.directory.cursor++ case <-id.Ctx.Done(): - id.directory = nil - return ents, id.Ctx.Err() + id.directory.err = id.Ctx.Err() + return ents, id.directory.err } } diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go deleted file mode 100644 index 4a0336cc0c3..00000000000 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ /dev/null @@ -1,252 +0,0 @@ -package fsnodes - -import ( - "context" - "fmt" - "io" - - "github.com/djdv/p9/p9" - files "github.com/ipfs/go-ipfs-files" - logging "github.com/ipfs/go-log" - coreiface "github.com/ipfs/interface-go-ipfs-core" - corepath "github.com/ipfs/interface-go-ipfs-core/path" -) - -// IPNS exposes the IPNS API over a p9.File interface -// Walk does not expect a namespace, only its path argument -// e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipns", "Qm...", "subdir")` -type IPNS struct { - IPFSBase - ipfsHandle - key coreiface.Key -} - -func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, key coreiface.Key) *IPNS { - id := &IPNS{ - key: key, - IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, - core, logging.Logger("IPNS")), - } - id.meta, id.metaMask = defaultRootAttr() - return id -} - -func (id *IPNS) Attach() (p9.File, error) { - id.Logger.Debugf("Attach") - //TODO: check core connection here - return id, nil -} - -func (id *IPNS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - id.Logger.Debugf("GetAttr") - id.Logger.Debugf("GetAttr path: %v", id.Path) - - if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI - return id.Qid, id.metaMask, id.meta, nil - } - - if err := coreGetAttr(id.Ctx, &id.meta, req, id.core, id.Path); err != nil { - return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err - } - id.Qid.Type = id.meta.Mode.QIDType() - - metaClone := id.meta - metaClone.Filter(req) - - return id.Qid, req, metaClone, nil -} - -func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { - id.Logger.Debugf("Walk names %v", names) - id.Logger.Debugf("Walk myself: %s:%v", id.Path, id.Qid) - - if shouldClone(names) { - id.Logger.Debugf("Walk cloned") - return []p9.QID{id.Qid}, id, nil - } - - qids := make([]p9.QID, 0, len(names)) - - //TODO: [spec check] make sure we're not messing things up by doing this instead of mutating - // ^ Does internal library expect fid to mutate on success or does newfid clobber some external state anyway - walkedNode := &IPNS{} // operate on a copy - *walkedNode = *id - - // first child requests will take care to translate the key's string name into a valid global path - // this path will act as a subroot for further child requests - if id.Path.Namespace() == "" && id.key != nil { - //TODO [refactor]: repetition - var err error - if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, id.key.Path()); err != nil { - return qids, nil, err - } - - ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) - if err != nil { - return qids, nil, err - } - - //TODO: this is too opague; we want core path => qid, dirent isn't necessary - dirEnt := &p9.Dirent{} - if err = ipldStat(dirEnt, ipldNode); err != nil { - return qids, nil, err - } - - walkedNode.Qid = dirEnt.QID - // - qids = append(qids, walkedNode.Qid) - names = names[1:] - } - - var err error - for _, name := range names { - //TODO: timerctx; don't want to hang forever - if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, corepath.Join(walkedNode.Path, name)); err != nil { - return qids, nil, err - } - - ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) - if err != nil { - return qids, nil, err - } - - //TODO: this is too opague; we want core path => qid, dirent isn't necessary - dirEnt := &p9.Dirent{} - if err = ipldStat(dirEnt, ipldNode); err != nil { - return qids, nil, err - } - - walkedNode.Qid = dirEnt.QID - // - qids = append(qids, walkedNode.Qid) - } - - id.Logger.Debugf("Walk ret %v, %v", qids, walkedNode) - return qids, walkedNode, err -} - -func (id *IPNS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - id.Logger.Debugf("Open") - - // set up handle amenities - var handleContext context.Context - handleContext, cancel := context.WithCancel(id.Ctx) - id.cancel = cancel - - // handle directories - if id.meta.Mode.IsDir() { - c, err := id.core.Unixfs().Ls(handleContext, id.Path) - if err != nil { - cancel() - return id.Qid, 0, err - } - - id.directory = &directoryStream{ - entryChan: c, - } - return id.Qid, 0, nil - } - - // handle files - apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) - if err != nil { - cancel() - return id.Qid, 0, err - } - - var ok bool - if id.file, ok = apiNode.(files.File); !ok { - cancel() - return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) - } - - return id.Qid, ipfsBlockSize, nil -} - -func (id *IPNS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - id.Logger.Debugf("Readdir %d %d", offset, count) - - if id.directory == nil { - return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) - } - - if offset < 0 { - return nil, fmt.Errorf("offset %d can't be negative", offset) - } - - if id.directory.eos { - if offset == id.directory.cursor-1 { - return nil, nil // valid end of stream request - } - } - - if id.directory.eos && offset == id.directory.cursor-1 { - return nil, nil // valid end of stream request - } - - if offset < id.directory.cursor { - return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) - } - - ents := make([]p9.Dirent, 0) -out: - for len(ents) < int(count) { - select { - case entry, open := <-id.directory.entryChan: - if !open { - id.directory.eos = true - break out - } - if entry.Err != nil { - id.directory = nil - return nil, entry.Err - } - - if offset <= id.directory.cursor { - nineEnt, err := coreEntTo9Ent(entry) - if err != nil { - id.directory = nil - return nil, err - } - nineEnt.Offset = id.directory.cursor - ents = append(ents, nineEnt) - } - - id.directory.cursor++ - - case <-id.Ctx.Done(): - id.directory = nil - return ents, id.Ctx.Err() - } - } - - id.Logger.Debugf("Readdir returning [%d]%v\n", len(ents), ents) - return ents, nil -} - -func (id *IPNS) ReadAt(p []byte, offset uint64) (int, error) { - id.Logger.Debugf("ReadAt") - - if id.file == nil { - return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) - } - - if fileBound, err := id.file.Size(); err == nil { - if int64(offset) >= fileBound { - //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. - return 0, io.EOF - } - } - - if offset != 0 { - if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { - return -1, fmt.Errorf("Read - seek error: %s", err) - } - } - - readBytes, err := id.file.Read(p) - if err != nil && err != io.EOF { - return -1, err - } - return readBytes, err -} diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 0621dc49854..5020554598d 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -88,7 +88,7 @@ func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { kd.Logger.Debugf("Open") handleContext, cancel := context.WithCancel(kd.Ctx) - kd.cancel = cancel + kd.operationsCancel = cancel var err error if kd.keyEnts, err = getKeys(handleContext, kd.core); err != nil { diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 2b8863997b4..c2b2082dc16 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -58,7 +58,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") handleContext, cancel := context.WithCancel(pd.Ctx) - pd.cancel = cancel + pd.operationsCancel = cancel // IPFS core representation pins, err := pd.core.Pin().Ls(handleContext, coreoptions.Pin.Type.Recursive()) diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 917069f6567..277d3854d39 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -3,6 +3,7 @@ package fsnodes import ( "context" "fmt" + "syscall" "github.com/djdv/p9/p9" cid "github.com/ipfs/go-cid" @@ -113,7 +114,8 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { case "ipns": subSystem = KeyFSAttacher(ri.Ctx, ri.core) default: - return nil, nil, fmt.Errorf("%q is not provided by us", names[0]) + ri.Logger.Errorf("%q is not provided by us", names[0]) + return nil, nil, syscall.ENOENT //TODO: migrate to platform independant value } attacher, ok := subSystem.(p9.Attacher) @@ -121,7 +123,13 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { return nil, nil, fmt.Errorf("%q is not a valid file system", names[0]) } - if _, err = attacher.Attach(); err != nil { + // poke the filesystem to make sure it's alive, but don't do anything with it + tempReference, err := attacher.Attach() + if err != nil { + return nil, nil, err + } + //[p9-lib] "Close must be called even when Open has not been called." + if err = tempReference.Close(); err != nil { return nil, nil, err } From 3ab60290fd404e4f9d09628c973fe6a19c4134e1 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 18 Sep 2019 09:55:24 -0400 Subject: [PATCH 053/102] context consistency --- plugin/plugins/filesystem/filesystem_test.go | 2 +- plugin/plugins/filesystem/nodes/keyfs.go | 6 +++--- plugin/plugins/filesystem/nodes/pinfs.go | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 54a995e0dac..0e5ec2a03ab 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -61,7 +61,7 @@ func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { } //TODO: currently magic. As subsystems are implemented, rework this part of the test + lib to contain some list - if len(ents) != 2 || ents[0].Name != "ipfs" || ents[1].Name != "ipns" { + if len(ents) != 1 || ents[0].Name != "ipfs" { t.Fatalf("Failed, root has bad entries:: %v\n", ents) } diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 5020554598d..185993cf61b 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -87,12 +87,12 @@ func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { kd.Logger.Debugf("Open") - handleContext, cancel := context.WithCancel(kd.Ctx) - kd.operationsCancel = cancel + var handleContext context.Context + handleContext, kd.operationsCancel = context.WithCancel(kd.Ctx) var err error if kd.keyEnts, err = getKeys(handleContext, kd.core); err != nil { - cancel() + kd.operationsCancel() return kd.Qid, 0, err } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index c2b2082dc16..83988a83885 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -57,13 +57,13 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") - handleContext, cancel := context.WithCancel(pd.Ctx) - pd.operationsCancel = cancel + var handleContext context.Context + handleContext, pd.operationsCancel = context.WithCancel(pd.Ctx) // IPFS core representation pins, err := pd.core.Pin().Ls(handleContext, coreoptions.Pin.Type.Recursive()) if err != nil { - cancel() + pd.operationsCancel() return pd.Qid, 0, err } @@ -78,7 +78,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } if err = coreStat(handleContext, dirEnt, pd.core, pin.Path()); err != nil { - cancel() + pd.operationsCancel() return pd.Qid, 0, err } pd.ents = append(pd.ents, *dirEnt) From b6dee76e36d907f753e129d122bd8929b77da3d9 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 18 Sep 2019 17:10:33 -0400 Subject: [PATCH 054/102] less broken stat --- plugin/plugins/filesystem/nodes/ipfs.go | 48 ++--- plugin/plugins/filesystem/nodes/ipns.go | 251 +++++++++++++++++++++++ plugin/plugins/filesystem/nodes/pinfs.go | 24 ++- plugin/plugins/filesystem/nodes/utils.go | 102 ++++----- 4 files changed, 331 insertions(+), 94 deletions(-) create mode 100644 plugin/plugins/filesystem/nodes/ipns.go diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 2c2932de7f0..103a8a1ecfb 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -8,6 +8,7 @@ import ( "github.com/djdv/p9/p9" files "github.com/ipfs/go-ipfs-files" + ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" @@ -30,27 +31,14 @@ func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { func (id *IPFS) Attach() (p9.File, error) { id.Logger.Debugf("Attach") - //TODO: check core connection here return id, nil } func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - id.Logger.Debugf("GetAttr") id.Logger.Debugf("GetAttr path: %v", id.Path) - if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI - return id.Qid, id.metaMask, id.meta, nil - } - - if err := coreGetAttr(id.Ctx, &id.meta, req, id.core, id.Path); err != nil { - return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err - } - id.Qid.Type = id.meta.Mode.QIDType() - - metaClone := id.meta - metaClone.Filter(req) - - return id.Qid, req, metaClone, nil + // For IPFS, we set this up front in Walk + return id.Qid, id.metaMask, id.meta, nil } func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { @@ -63,9 +51,15 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { } var ( + // returned qids = make([]p9.QID, 0, len(names)) newFid = &IPFS{IPFSBase: newIPFSBase(id.Ctx, id.Path, 0, id.core, id.Logger)} - err error + // temporary + attr = &p9.Attr{} + requestType = p9.AttrMask{Mode: true} + err error + // last-set value is used + ipldNode ipld.Node ) for _, name := range names { @@ -78,24 +72,32 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { return qids, nil, err } - ipldNode, err := id.core.Dag().Get(callCtx, newFid.Path.Cid()) + ipldNode, err = id.core.Dag().Get(callCtx, newFid.Path.Cid()) if err != nil { cancel() return qids, nil, err } - //TODO: this is too opaque; we want core path => qid, Dirent isn't /really/ necessary - dirEnt := &p9.Dirent{} - if err = ipldStat(dirEnt, ipldNode); err != nil { + err, _ := ipldStat(callCtx, attr, ipldNode, requestType) + if err != nil { cancel() return qids, nil, err } - newFid.Qid = dirEnt.QID - // + newFid.Qid.Type = attr.Mode.QIDType() + newFid.Qid.Path = cidToQPath(ipldNode.Cid()) qids = append(qids, newFid.Qid) } + // stat up front for our returned file + err, filled := ipldStat(id.Ctx, &newFid.meta, ipldNode, p9.AttrMaskAll) + if err != nil { + return qids, nil, err + } + newFid.metaMask = filled + newFid.meta.Mode |= IRXA + newFid.meta.RDev, id.metaMask.RDev = dIPFS, true + id.Logger.Debugf("Walk ret %v, %v", qids, newFid) return qids, newFid, err } @@ -108,7 +110,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { handleContext, id.operationsCancel = context.WithCancel(id.Ctx) // handle directories - if id.meta.Mode.IsDir() { + if id.meta.Mode.IsDir() { //FIXME: meta is not being set anywhere c, err := id.core.Unixfs().Ls(handleContext, id.Path) if err != nil { id.operationsCancel() diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go new file mode 100644 index 00000000000..3252e7d34bf --- /dev/null +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -0,0 +1,251 @@ +package fsnodes + +import ( + "context" + + "github.com/djdv/p9/p9" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +// IPNS exposes the IPNS API over a p9.File interface +// Walk does not expect a namespace, only its path argument +// e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipns", "Qm...", "subdir")` +type IPNS struct { + IPFSBase + ipfsHandle + key coreiface.Key +} + +func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, key coreiface.Key) *IPNS { + id := &IPNS{ + key: key, + IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, + core, logging.Logger("IPNS")), + } + id.meta, id.metaMask = defaultRootAttr() + return id +} + +func (id *IPNS) Attach() (p9.File, error) { + id.Logger.Debugf("Attach") + //TODO: check core connection here + return id, nil +} + +/* + +func (id *IPNS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + id.Logger.Debugf("GetAttr") + id.Logger.Debugf("GetAttr path: %v", id.Path) + + if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI + return id.Qid, id.metaMask, id.meta, nil + } + + if err := coreGetAttr(id.Ctx, &id.meta, req, id.core, id.Path); err != nil { + return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err + } + id.Qid.Type = id.meta.Mode.QIDType() + + metaClone := id.meta + metaClone.Filter(req) + + return id.Qid, req, metaClone, nil +} + +func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { + id.Logger.Debugf("Walk names %v", names) + id.Logger.Debugf("Walk myself: %s:%v", id.Path, id.Qid) + + if shouldClone(names) { + id.Logger.Debugf("Walk cloned") + return []p9.QID{id.Qid}, id, nil + } + + qids := make([]p9.QID, 0, len(names)) + + //TODO: [spec check] make sure we're not messing things up by doing this instead of mutating + // ^ Does internal library expect fid to mutate on success or does newfid clobber some external state anyway + walkedNode := &IPNS{} // operate on a copy + *walkedNode = *id + + // first child requests will take care to translate the key's string name into a valid global path + // this path will act as a subroot for further child requests + if id.Path.Namespace() == "" && id.key != nil { + //TODO [refactor]: repetition + var err error + if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, id.key.Path()); err != nil { + return qids, nil, err + } + + ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) + if err != nil { + return qids, nil, err + } + + //TODO: this is too opague; we want core path => qid, dirent isn't necessary + dirEnt := &p9.Dirent{} + if err = ipldStat(dirEnt, ipldNode); err != nil { + return qids, nil, err + } + + walkedNode.Qid = dirEnt.QID + // + qids = append(qids, walkedNode.Qid) + names = names[1:] + } + + var err error + for _, name := range names { + //TODO: timerctx; don't want to hang forever + if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, corepath.Join(walkedNode.Path, name)); err != nil { + return qids, nil, err + } + + ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) + if err != nil { + return qids, nil, err + } + + //TODO: this is too opague; we want core path => qid, dirent isn't necessary + dirEnt := &p9.Dirent{} + if err = ipldStat(dirEnt, ipldNode); err != nil { + return qids, nil, err + } + + walkedNode.Qid = dirEnt.QID + // + qids = append(qids, walkedNode.Qid) + } + + id.Logger.Debugf("Walk ret %v, %v", qids, walkedNode) + return qids, walkedNode, err +} + +func (id *IPNS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + id.Logger.Debugf("Open") + + // set up handle amenities + var handleContext context.Context + handleContext, cancel := context.WithCancel(id.Ctx) + id.operationsCancel = cancel + + // handle directories + if id.meta.Mode.IsDir() { + c, err := id.core.Unixfs().Ls(handleContext, id.Path) + if err != nil { + cancel() + return id.Qid, 0, err + } + + id.directory = &directoryStream{ + entryChan: c, + } + return id.Qid, 0, nil + } + + // handle files + apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) + if err != nil { + cancel() + return id.Qid, 0, err + } + + var ok bool + if id.file, ok = apiNode.(files.File); !ok { + cancel() + return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) + } + + return id.Qid, ipfsBlockSize, nil +} + +func (id *IPNS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + id.Logger.Debugf("Readdir %d %d", offset, count) + + if id.directory == nil { + return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) + } + + if offset < 0 { + return nil, fmt.Errorf("offset %d can't be negative", offset) + } + + if id.directory.eos { + if offset == id.directory.cursor-1 { + return nil, nil // valid end of stream request + } + } + + if id.directory.eos && offset == id.directory.cursor-1 { + return nil, nil // valid end of stream request + } + + if offset < id.directory.cursor { + return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) + } + + ents := make([]p9.Dirent, 0) +out: + for len(ents) < int(count) { + select { + case entry, open := <-id.directory.entryChan: + if !open { + id.directory.eos = true + break out + } + if entry.Err != nil { + id.directory = nil + return nil, entry.Err + } + + if offset <= id.directory.cursor { + nineEnt, err := coreEntTo9Ent(entry) + if err != nil { + id.directory = nil + return nil, err + } + nineEnt.Offset = id.directory.cursor + ents = append(ents, nineEnt) + } + + id.directory.cursor++ + + case <-id.Ctx.Done(): + id.directory = nil + return ents, id.Ctx.Err() + } + } + + id.Logger.Debugf("Readdir returning [%d]%v\n", len(ents), ents) + return ents, nil +} + +func (id *IPNS) ReadAt(p []byte, offset uint64) (int, error) { + id.Logger.Debugf("ReadAt") + + if id.file == nil { + return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) + } + + if fileBound, err := id.file.Size(); err == nil { + if int64(offset) >= fileBound { + //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. + return 0, io.EOF + } + } + + if offset != 0 { + if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { + return -1, fmt.Errorf("Read - seek error: %s", err) + } + } + + readBytes, err := id.file.Read(p) + if err != nil && err != io.EOF { + return -1, err + } + return readBytes, err +} +*/ diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 83988a83885..b7ae5741bd1 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -70,18 +70,30 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // 9P representation pd.ents = make([]p9.Dirent, 0, len(pins)) + // temporary conversion storage + attr := &p9.Attr{} + requestType := p9.AttrMask{Mode: true} + // actual conversion for i, pin := range pins { - dirEnt := &p9.Dirent{ - Name: gopath.Base(pin.Path().String()), - Offset: uint64(i + 1), + ipldNode, err := pd.core.ResolveNode(handleContext, pin.Path()) + if err != nil { + pd.operationsCancel() + return pd.Qid, 0, err } - - if err = coreStat(handleContext, dirEnt, pd.core, pin.Path()); err != nil { + if err, _ = ipldStat(handleContext, attr, ipldNode, requestType); err != nil { pd.operationsCancel() return pd.Qid, 0, err } - pd.ents = append(pd.ents, *dirEnt) + + pd.ents = append(pd.ents, p9.Dirent{ + Name: gopath.Base(pin.Path().String()), + Offset: uint64(i + 1), + QID: p9.QID{ + Type: attr.Mode.QIDType(), + Path: cidToQPath(ipldNode.Cid()), + }, + }) } return pd.Qid, ipfsBlockSize, nil diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 7be37569241..d8e2ba4bf6d 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -6,6 +6,7 @@ import ( "fmt" "hash/fnv" "io" + gopath "path" "time" "github.com/djdv/p9/p9" @@ -52,76 +53,34 @@ func shouldClone(names []string) bool { } } -//TODO: rename this and/or extend -// it only does some of the stat and not what people probably expect -func coreStat(ctx context.Context, dirEnt *p9.Dirent, core coreiface.CoreAPI, path corepath.Path) error { - if ipldNode, err := core.ResolveNode(ctx, path); err != nil { - return err - } else { - return ipldStat(dirEnt, ipldNode) - } -} - -func coreGetAttr(ctx context.Context, attr *p9.Attr, attrMask p9.AttrMask, core coreiface.CoreAPI, path corepath.Path) (err error) { - ipldNode, err := core.ResolveNode(ctx, path) - if err != nil { - return err - } - ufsNode, err := unixfs.ExtractFSNode(ipldNode) +func ipldStat(ctx context.Context, attr *p9.Attr, node ipld.Node, mask p9.AttrMask) (error, p9.AttrMask) { + var filledAttrs p9.AttrMask + ufsNode, err := unixfs.ExtractFSNode(node) if err != nil { - return err + return err, filledAttrs } - if attrMask.Mode { - attr.Mode = IRXA //TODO: this should probably be the callers responsability; just document that permissions should be set afterwards or something + if mask.Mode { tBits, err := unixfsTypeTo9Type(ufsNode.Type()) if err != nil { - return err + return err, filledAttrs } attr.Mode |= tBits + filledAttrs.Mode = true } - if attrMask.Blocks { + if mask.Blocks { //TODO: when/if UFS supports this metadata field, use it instead - attr.BlockSize = ipfsBlockSize - } - - if attrMask.Size { - attr.Size, attrMask.Size = ufsNode.FileSize(), true + attr.BlockSize, filledAttrs.Blocks = ipfsBlockSize, true } - if attrMask.RDev { - switch path.Namespace() { - case "ipfs": - attr.RDev, attrMask.RDev = dIPFS, true - //case "ipns": - //attr.RDev, attrMask.RDev = dIPNS, true - //etc. - } + if mask.Size { + attr.Size, filledAttrs.Size = ufsNode.FileSize(), true } - //TODO [eventually]: switch off here for handling of time metadata in new UFS format standard + //TODO [eventually]: handle time metadata in new UFS format standard - return nil -} - -func ipldStat(dirEnt *p9.Dirent, node ipld.Node) error { - ufsNode, err := unixfs.ExtractFSNode(node) - - if err != nil { - return err - } - - nineType, err := unixfsTypeTo9Type(ufsNode.Type()) - if err != nil { - return err - } - - dirEnt.Type = nineType.QIDType() - dirEnt.QID.Type = nineType.QIDType() - dirEnt.QID.Path = cidToQPath(node.Cid()) - - return nil + return nil, filledAttrs } func cidToQPath(cid cid.Cid) uint64 { @@ -318,25 +277,38 @@ func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]entPair, error) { ents := make([]entPair, 0, len(keys)) - var offset = 1 - for _, key := range keys { - dirEnt := &p9.Dirent{ - Name: key.Name(), - Offset: uint64(offset), - } + // temporary conversion storage + attr := &p9.Attr{} + requestType := p9.AttrMask{Mode: true} - if err = coreStat(ctx, dirEnt, core, key.Path()); err != nil { + var offset uint64 = 1 + for _, key := range keys { + // + ipldNode, err := core.ResolveNode(ctx, key.Path()) + if err != nil { //FIXME: bug in either the CoreAPI, http client, or somewhere else //if err == coreiface.ErrResolveFailed { //HACK: if err.Error() == coreiface.ErrResolveFailed.Error() { - continue // skip unresolvable + continue // skip unresolvable keys (typical when a key exists but hasn't been published to } return nil, err } + if err, _ = ipldStat(ctx, attr, ipldNode, requestType); err != nil { + return nil, err + } - pair := entPair{ent: *dirEnt, key: key} - ents = append(ents, pair) + ents = append(ents, entPair{ + ent: p9.Dirent{ + Name: gopath.Base(key.Path().String()), + Offset: offset, + QID: p9.QID{ + Type: attr.Mode.QIDType(), + Path: cidToQPath(ipldNode.Cid()), + }, + }, + key: key, + }) offset++ } return ents, nil From 770bbceb22042433b062bb93115dbe718365269c Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 19 Sep 2019 09:27:40 -0400 Subject: [PATCH 055/102] small lint --- plugin/plugins/filesystem/nodes/utils.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index d8e2ba4bf6d..5fd25be612b 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -203,8 +203,6 @@ func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, c // returns true if the caller should return immediately with our values func boundCheck(offset uint64, length int) (bool, error) { switch { - case offset < 0: - return true, fmt.Errorf("offset %d can't be negative", offset) case offset == uint64(length): return true, nil // EOS case offset > uint64(length): @@ -234,10 +232,7 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { err error ) - var i int for len(names) != 0 { - l.Errorf("[%d] loop: %v\n%v\n", i, ref, names) - //prepare to step into next component if names[0] == ".." { // climb to parent / leftwards requests nextRef = ref.Parent() @@ -262,7 +257,6 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { curFile.Close() // we're not referencing this anymore } // leave the last reference alive, for the caller to close } - i++ l.Errorf("walker ret: %v\n%v\n", qids, curFile) From 4717f7367e40b7ca76be82684d7f04010c8c1b7b Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 19 Sep 2019 09:27:54 -0400 Subject: [PATCH 056/102] Revert "change config handling (needs tests)" This reverts commit b6d150654c2609f90df8b1c232cb06396fe688c7. --- plugin/plugins/filesystem/filesystem.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 27a2676280d..6a69ce1dafe 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -3,7 +3,6 @@ package filesystem import ( "context" "encoding/json" - "fmt" "os" "path/filepath" "runtime" @@ -60,17 +59,15 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } cfg := &Config{} - // config not being set is okay and will load defalts - // config being set with malformed data is not okay and will instruct the daemon to halt-and-catch-fire - rawConf, ok := (env.Config).(json.RawMessage) - if ok { - if err := json.Unmarshal(rawConf, cfg); err != nil { + if env.Config != nil { + byteRep, err := json.Marshal(env.Config) + if err != nil { return err } - } else { - if env.Config != nil { - return fmt.Errorf("plugin config does not appear to be correctly formatted: %#v", env.Config) + if err = json.Unmarshal(byteRep, cfg); err != nil { + return err } + } else { cfg = defaultConfig(env.Repo) } From 21b2d2cf1b13db5076405173d02a803f4a8c241f Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 21 Sep 2019 19:21:11 -0400 Subject: [PATCH 057/102] rework config, remove windows manet hacks, remove ipns --- go.mod | 1 + plugin/plugins/filesystem/filesystem.go | 39 ++-- plugin/plugins/filesystem/nodes/base.go | 43 ++-- plugin/plugins/filesystem/nodes/ipfs.go | 11 +- plugin/plugins/filesystem/nodes/ipns.go | 251 ----------------------- plugin/plugins/filesystem/nodes/keyfs.go | 126 ------------ plugin/plugins/filesystem/nodes/root.go | 6 +- plugin/plugins/filesystem/nodes/utils.go | 46 ----- plugin/plugins/filesystem/utils.go | 44 ++-- 9 files changed, 83 insertions(+), 484 deletions(-) delete mode 100644 plugin/plugins/filesystem/nodes/ipns.go delete mode 100644 plugin/plugins/filesystem/nodes/keyfs.go diff --git a/go.mod b/go.mod index fab1be06945..affef23cb50 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,7 @@ require ( github.com/libp2p/go-maddr-filter v0.0.5 github.com/mattn/go-runewidth v0.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 + github.com/mitchellh/mapstructure v1.1.2 github.com/mr-tron/base58 v1.1.2 github.com/multiformats/go-multiaddr v0.0.4 github.com/multiformats/go-multiaddr-dns v0.0.3 diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 6a69ce1dafe..beb9737966e 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -2,16 +2,15 @@ package filesystem import ( "context" - "encoding/json" "os" "path/filepath" - "runtime" "github.com/djdv/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" + "github.com/mitchellh/mapstructure" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) @@ -50,6 +49,8 @@ func (*FileSystemPlugin) Version() string { func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { logger.Info("Initializing 9P resource server...") + + // stabilise repo path if !filepath.IsAbs(env.Repo) { absRepo, err := filepath.Abs(env.Repo) if err != nil { @@ -59,27 +60,30 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } cfg := &Config{} + // load config from file or initialize it if env.Config != nil { - byteRep, err := json.Marshal(env.Config) - if err != nil { - return err - } - if err = json.Unmarshal(byteRep, cfg); err != nil { + if err := mapstructure.Decode(env.Config, cfg); err != nil { return err } } else { - cfg = defaultConfig(env.Repo) + cfg = defaultConfig() } - var err error - if envAddr := os.ExpandEnv(EnvAddr); envAddr == "" { - fs.addr, err = multiaddr.NewMultiaddr(cfg.Service[defaultService]) + var addrString string + // allow environment variable to override config values + if envAddr := os.ExpandEnv(EnvAddr); envAddr != "" { + addrString = EnvAddr } else { - fs.addr, err = multiaddr.NewMultiaddr(envAddr) + addrString = cfg.Service[defaultService] } + + // expand string templates and initialize listening addr + addrString = os.Expand(addrString, configVarMapper(env.Repo)) + ma, err := multiaddr.NewMultiaddr(addrString) if err != nil { return err } + fs.addr = ma logger.Info("9P resource server okay for launch") return nil @@ -88,15 +92,16 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { logger.Info("Starting 9P resource server...") - //TODO [manet]: unix sockets are not removed on process death (on Windows) - // so for now we just try to remove it before listening on it - if runtime.GOOS == "windows" { - removeUnixSockets(fs.addr) + // make sure sockets are not in use already (if we're using them) + err := removeUnixSockets(fs.addr) + if err != nil { + return err } fs.ctx, fs.cancel = context.WithCancel(context.Background()) fs.errorChan = make(chan error, 1) + // launch the listener listener, err := manet.Listen(fs.addr) if err != nil { logger.Errorf("9P listen error: %s\n", err) @@ -104,7 +109,7 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { } fs.listener = listener - // construct and run the 9P resource server + // construct and launch the 9P resource server s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core)) go func() { fs.errorChan <- s.Serve(manet.NetListener(fs.listener)) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 2dbb8a9b9f3..c2c9bbc3652 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -12,11 +12,8 @@ import ( ) const ( //device - attempts to comply with standard multicodec table - dMemory = 0x2f + dMemory = 0x2f // generic "path" dIPFS = 0xe4 - dIPNS = 0xe5 - //TODO: decide what MFS/Files API should be; IPLD? - //dFiles = 0xe2 ) var _ p9.File = (*Base)(nil) @@ -53,24 +50,40 @@ type IPFSBase struct { // and cancel it in another operation (like Close) // that pointer should be stored here between calls operationsCancel context.CancelFunc + + // operation handle storage + file files.File + directory *directoryStream +} + +func (b Base) Close() error { + if b.child != nil { + return b.child.Close() + } + + return nil } func (ib *IPFSBase) Close() error { - if ib.operationsCancel != nil { - ib.operationsCancel() + var lastErr error + + if err := ib.Base.Close(); err != nil { + ib.Logger.Errorf("base close: %s", err) + lastErr = err } - if ib.parent != nil { - ib.parent.Close() + + if ib.file != nil { + if err := ib.file.Close(); err != nil { + ib.Logger.Errorf("files.File close: %s", err) + lastErr = err + } } - if ib.child != nil { - ib.child.Close() + + if ib.operationsCancel != nil { + ib.operationsCancel() } - return nil -} -type ipfsHandle struct { - file files.File - directory *directoryStream + return lastErr } type directoryStream struct { diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 103a8a1ecfb..82992d66d15 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -19,7 +19,6 @@ import ( // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { IPFSBase - ipfsHandle } func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { @@ -66,19 +65,23 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { callCtx, cancel := context.WithTimeout(id.Ctx, 30*time.Second) defer cancel() - if newFid.Path, err = id.core.ResolvePath(callCtx, corepath.Join(newFid.Path, name)); err != nil { + corePath, err := id.core.ResolvePath(callCtx, corepath.Join(newFid.Path, name)) + if err != nil { cancel() - //TODO: switch off error, return appropriate errno (likely ENOENT) + //TODO: switch off error, return appropriate errno (ENOENT is going to be common here) + // ref: https://github.com/hugelgupf/p9/pull/12#discussion_r324991695 return qids, nil, err } + newFid.Path = corePath + ipldNode, err = id.core.Dag().Get(callCtx, newFid.Path.Cid()) if err != nil { cancel() return qids, nil, err } - err, _ := ipldStat(callCtx, attr, ipldNode, requestType) + err, _ = ipldStat(callCtx, attr, ipldNode, requestType) if err != nil { cancel() return qids, nil, err diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go deleted file mode 100644 index 3252e7d34bf..00000000000 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ /dev/null @@ -1,251 +0,0 @@ -package fsnodes - -import ( - "context" - - "github.com/djdv/p9/p9" - logging "github.com/ipfs/go-log" - coreiface "github.com/ipfs/interface-go-ipfs-core" -) - -// IPNS exposes the IPNS API over a p9.File interface -// Walk does not expect a namespace, only its path argument -// e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipns", "Qm...", "subdir")` -type IPNS struct { - IPFSBase - ipfsHandle - key coreiface.Key -} - -func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, key coreiface.Key) *IPNS { - id := &IPNS{ - key: key, - IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, - core, logging.Logger("IPNS")), - } - id.meta, id.metaMask = defaultRootAttr() - return id -} - -func (id *IPNS) Attach() (p9.File, error) { - id.Logger.Debugf("Attach") - //TODO: check core connection here - return id, nil -} - -/* - -func (id *IPNS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - id.Logger.Debugf("GetAttr") - id.Logger.Debugf("GetAttr path: %v", id.Path) - - if id.Path.Namespace() == nRoot { // metadata should have been initialized by attacher, don't consult CoreAPI - return id.Qid, id.metaMask, id.meta, nil - } - - if err := coreGetAttr(id.Ctx, &id.meta, req, id.core, id.Path); err != nil { - return p9.QID{}, p9.AttrMask{}, p9.Attr{}, err - } - id.Qid.Type = id.meta.Mode.QIDType() - - metaClone := id.meta - metaClone.Filter(req) - - return id.Qid, req, metaClone, nil -} - -func (id *IPNS) Walk(names []string) ([]p9.QID, p9.File, error) { - id.Logger.Debugf("Walk names %v", names) - id.Logger.Debugf("Walk myself: %s:%v", id.Path, id.Qid) - - if shouldClone(names) { - id.Logger.Debugf("Walk cloned") - return []p9.QID{id.Qid}, id, nil - } - - qids := make([]p9.QID, 0, len(names)) - - //TODO: [spec check] make sure we're not messing things up by doing this instead of mutating - // ^ Does internal library expect fid to mutate on success or does newfid clobber some external state anyway - walkedNode := &IPNS{} // operate on a copy - *walkedNode = *id - - // first child requests will take care to translate the key's string name into a valid global path - // this path will act as a subroot for further child requests - if id.Path.Namespace() == "" && id.key != nil { - //TODO [refactor]: repetition - var err error - if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, id.key.Path()); err != nil { - return qids, nil, err - } - - ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) - if err != nil { - return qids, nil, err - } - - //TODO: this is too opague; we want core path => qid, dirent isn't necessary - dirEnt := &p9.Dirent{} - if err = ipldStat(dirEnt, ipldNode); err != nil { - return qids, nil, err - } - - walkedNode.Qid = dirEnt.QID - // - qids = append(qids, walkedNode.Qid) - names = names[1:] - } - - var err error - for _, name := range names { - //TODO: timerctx; don't want to hang forever - if walkedNode.Path, err = id.core.ResolvePath(id.Ctx, corepath.Join(walkedNode.Path, name)); err != nil { - return qids, nil, err - } - - ipldNode, err := id.core.Dag().Get(id.Ctx, walkedNode.Path.Cid()) - if err != nil { - return qids, nil, err - } - - //TODO: this is too opague; we want core path => qid, dirent isn't necessary - dirEnt := &p9.Dirent{} - if err = ipldStat(dirEnt, ipldNode); err != nil { - return qids, nil, err - } - - walkedNode.Qid = dirEnt.QID - // - qids = append(qids, walkedNode.Qid) - } - - id.Logger.Debugf("Walk ret %v, %v", qids, walkedNode) - return qids, walkedNode, err -} - -func (id *IPNS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - id.Logger.Debugf("Open") - - // set up handle amenities - var handleContext context.Context - handleContext, cancel := context.WithCancel(id.Ctx) - id.operationsCancel = cancel - - // handle directories - if id.meta.Mode.IsDir() { - c, err := id.core.Unixfs().Ls(handleContext, id.Path) - if err != nil { - cancel() - return id.Qid, 0, err - } - - id.directory = &directoryStream{ - entryChan: c, - } - return id.Qid, 0, nil - } - - // handle files - apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) - if err != nil { - cancel() - return id.Qid, 0, err - } - - var ok bool - if id.file, ok = apiNode.(files.File); !ok { - cancel() - return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) - } - - return id.Qid, ipfsBlockSize, nil -} - -func (id *IPNS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - id.Logger.Debugf("Readdir %d %d", offset, count) - - if id.directory == nil { - return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) - } - - if offset < 0 { - return nil, fmt.Errorf("offset %d can't be negative", offset) - } - - if id.directory.eos { - if offset == id.directory.cursor-1 { - return nil, nil // valid end of stream request - } - } - - if id.directory.eos && offset == id.directory.cursor-1 { - return nil, nil // valid end of stream request - } - - if offset < id.directory.cursor { - return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) - } - - ents := make([]p9.Dirent, 0) -out: - for len(ents) < int(count) { - select { - case entry, open := <-id.directory.entryChan: - if !open { - id.directory.eos = true - break out - } - if entry.Err != nil { - id.directory = nil - return nil, entry.Err - } - - if offset <= id.directory.cursor { - nineEnt, err := coreEntTo9Ent(entry) - if err != nil { - id.directory = nil - return nil, err - } - nineEnt.Offset = id.directory.cursor - ents = append(ents, nineEnt) - } - - id.directory.cursor++ - - case <-id.Ctx.Done(): - id.directory = nil - return ents, id.Ctx.Err() - } - } - - id.Logger.Debugf("Readdir returning [%d]%v\n", len(ents), ents) - return ents, nil -} - -func (id *IPNS) ReadAt(p []byte, offset uint64) (int, error) { - id.Logger.Debugf("ReadAt") - - if id.file == nil { - return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) - } - - if fileBound, err := id.file.Size(); err == nil { - if int64(offset) >= fileBound { - //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. - return 0, io.EOF - } - } - - if offset != 0 { - if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { - return -1, fmt.Errorf("Read - seek error: %s", err) - } - } - - readBytes, err := id.file.Read(p) - if err != nil && err != io.EOF { - return -1, err - } - return readBytes, err -} -*/ diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go deleted file mode 100644 index 185993cf61b..00000000000 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ /dev/null @@ -1,126 +0,0 @@ -package fsnodes - -import ( - "context" - "fmt" - - "github.com/djdv/p9/p9" - logging "github.com/ipfs/go-log" - coreiface "github.com/ipfs/interface-go-ipfs-core" -) - -type entPair struct { - ent p9.Dirent - key coreiface.Key -} - -type KeyFS struct { - IPFSBase - keyEnts []entPair -} - -func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI) *KeyFS { - kd := &KeyFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipns"), p9.TypeDir, - core, logging.Logger("KeyFS"))} - kd.meta, kd.metaMask = defaultRootAttr() - kd.meta.Mode |= 0220 - return kd -} - -func (kd *KeyFS) Attach() (p9.File, error) { - kd.Logger.Debugf("Attach") - - var subSystem walkRef = IPFSAttacher(kd.Ctx, kd.core) - attacher, ok := subSystem.(p9.Attacher) - if !ok { - return nil, fmt.Errorf("subsystem %T is not a valid file system", subSystem) - } - - if _, err := attacher.Attach(); err != nil { - return nil, err - } - kd.child = subSystem - - return kd, nil -} - -func (kd *KeyFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - kd.Logger.Debugf("GetAttr") - - return kd.Qid, kd.metaMask, kd.meta, nil -} - -func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { - kd.Logger.Debugf("Walk names %v", names) - kd.Logger.Debugf("Walk myself: %v", kd.Qid) - - if shouldClone(names) { - kd.Logger.Debugf("Walk cloned") - return []p9.QID{kd.Qid}, kd, nil - } - - keys, err := kd.core.Key().List(kd.Ctx) - if err != nil { - return nil, nil, err - } - - var coreKey coreiface.Key - for _, key := range keys { - if names[0] == key.Name() { - coreKey = key - break - } - } - - if coreKey != nil { // use IPNS FS to allow write support to the key - ipns, err := IPNSAttacher(kd.Ctx, kd.core, coreKey).Attach() - if err != nil { - return nil, nil, err - } - return ipns.Walk(names) - } - - // if we don't own the key, treat this as a typical IPFS core-request - return walker(kd, names) -} - -func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - kd.Logger.Debugf("Open") - - var handleContext context.Context - handleContext, kd.operationsCancel = context.WithCancel(kd.Ctx) - - var err error - if kd.keyEnts, err = getKeys(handleContext, kd.core); err != nil { - kd.operationsCancel() - return kd.Qid, 0, err - } - - return kd.Qid, ipfsBlockSize, nil -} - -func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - kd.Logger.Debugf("Readdir") - - if kd.keyEnts == nil { - return nil, fmt.Errorf("directory %q is not open for reading", kd.Path.String()) - } - - shouldExit, err := boundCheck(offset, len(kd.keyEnts)) - if shouldExit { - return nil, err - } - - subSlice := kd.keyEnts[offset:] - if len(subSlice) > int(count) { - subSlice = subSlice[:count] - } - - nineEnts := make([]p9.Dirent, 0, len(subSlice)) - for _, pair := range subSlice { - nineEnts = append(nineEnts, pair.ent) - } - - kd.Logger.Debugf("Readdir returning ents: %v", nineEnts) - return nineEnts, nil -} diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 277d3854d39..30241a86fae 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -67,7 +67,6 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { p9.Dirent }{ {"ipfs", rootDirTemplate}, - //{"ipns", rootDirTemplate}, } { pathUnion.Dirent.Offset = uint64(i + 1) pathUnion.Dirent.Name = pathUnion.string @@ -111,8 +110,9 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { switch names[0] { case "ipfs": subSystem = PinFSAttacher(ri.Ctx, ri.core) - case "ipns": - subSystem = KeyFSAttacher(ri.Ctx, ri.core) + /* case "ipns": + subSystem = KeyFSAttacher(ri.Ctx, ri.core) + */ default: ri.Logger.Errorf("%q is not provided by us", names[0]) return nil, nil, syscall.ENOENT //TODO: migrate to platform independant value diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 5fd25be612b..e5f7d53d8b5 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -6,7 +6,6 @@ import ( "fmt" "hash/fnv" "io" - gopath "path" "time" "github.com/djdv/p9/p9" @@ -262,48 +261,3 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { return qids, curFile, nil } - -func getKeys(ctx context.Context, core coreiface.CoreAPI) ([]entPair, error) { - keys, err := core.Key().List(ctx) - if err != nil { - return nil, err - } - - ents := make([]entPair, 0, len(keys)) - - // temporary conversion storage - attr := &p9.Attr{} - requestType := p9.AttrMask{Mode: true} - - var offset uint64 = 1 - for _, key := range keys { - // - ipldNode, err := core.ResolveNode(ctx, key.Path()) - if err != nil { - //FIXME: bug in either the CoreAPI, http client, or somewhere else - //if err == coreiface.ErrResolveFailed { - //HACK: - if err.Error() == coreiface.ErrResolveFailed.Error() { - continue // skip unresolvable keys (typical when a key exists but hasn't been published to - } - return nil, err - } - if err, _ = ipldStat(ctx, attr, ipldNode, requestType); err != nil { - return nil, err - } - - ents = append(ents, entPair{ - ent: p9.Dirent{ - Name: gopath.Base(key.Path().String()), - Offset: offset, - QID: p9.QID{ - Type: attr.Mode.QIDType(), - Path: cidToQPath(ipldNode.Cid()), - }, - }, - key: key, - }) - offset++ - } - return ents, nil -} diff --git a/plugin/plugins/filesystem/utils.go b/plugin/plugins/filesystem/utils.go index fe5a9302d25..beed7c1fe37 100644 --- a/plugin/plugins/filesystem/utils.go +++ b/plugin/plugins/filesystem/utils.go @@ -1,10 +1,9 @@ package filesystem import ( + "fmt" "os" - gopath "path" "path/filepath" - "runtime" "strings" "github.com/multiformats/go-multiaddr" @@ -17,44 +16,45 @@ const ( //TODO [config]: move elsewhere; related: https://github.com/ipfs/go-ipfs/issues/6526 EnvAddr = "$IPFS_FS_ADDR" // multiaddr string - sockName = "filesystem.9P.sock" - defaultService = "9P" // (currently 9P2000.L) + defaultService = "9p" // (currently 9P2000.L) + sockName = "filesystem." + defaultService + ".sock" + + tmplHome = "IPFS_HOME" ) type Config struct { // NOTE: unstable/experimental // addresses for file system servers and clients - //e.g. "9P":"/ip4/localhost/tcp/564", "fuse":"/mountpoint", "🐇":"/rabbit-hutch/glenda", ... + //e.g. "9p":"/ip4/localhost/tcp/564", "fuse":"/mountpoint", "🐇":"/rabbit-hutch/glenda", ... Service map[string]string } -func defaultConfig(storagePath string) *Config { - serviceMap := make(map[string]string) - - sockTarget := gopath.Join(storagePath, sockName) - if runtime.GOOS == "windows" { - sockTarget = windowsToUnixFriendly(sockTarget) +func defaultConfig() *Config { + return &Config{ + map[string]string{ + defaultService: fmt.Sprintf("/unix/${%s}/%s", tmplHome, sockName), + }, } - - serviceMap[defaultService] = gopath.Join("/unix", sockTarget) - return &Config{serviceMap} } -func windowsToUnixFriendly(target string) string { - //TODO [manet]: doesn't like drive letters - //XXX: so for now we decap drive-spec-like paths and use the current working drive letter, relatively - if len(target) > 2 && target[1] == ':' { - target = target[2:] +func configVarMapper(repoPath string) func(string) string { + return func(s string) string { + switch s { + case tmplHome: + return repoPath + default: + return os.Getenv(s) + } } - return filepath.ToSlash(target) } // removeUnixSockets attempts to remove all unix domain paths from a multiaddr -// Does not stop on error, returns last encountered error +// does not stop on error, returns last encountered error, except "not exist" errors func removeUnixSockets(ma multiaddr.Multiaddr) error { var retErr error multiaddr.ForEach(ma, func(comp multiaddr.Component) bool { if comp.Protocol().Code == multiaddr.P_UNIX { - if err := os.Remove(strings.TrimPrefix(comp.String(), "/unix")); err != nil { + localPath := filepath.FromSlash(strings.TrimPrefix(comp.String(), "/unix")) + if err := os.Remove(localPath); err != nil && !os.IsNotExist(err) { retErr = err } } From 62560e419f6de378bee90ef5a2318c9f04e65efc Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sat, 21 Sep 2019 20:18:55 -0400 Subject: [PATCH 058/102] fix read offset and return values --- plugin/plugins/filesystem/nodes/ipfs.go | 21 +++++++++------------ plugin/plugins/filesystem/nodes/utils.go | 5 ----- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 82992d66d15..195dfa0cc0c 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -202,28 +202,25 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { } func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { - id.Logger.Debugf("ReadAt") + id.Logger.Debugf("ReadAt {%d/%d}%q", offset, id.meta.Size, id.Path.String()) if id.file == nil { - return -1, fmt.Errorf("file %q is not open for reading", id.Path.String()) + return 0, fmt.Errorf("file %q is not open for reading", id.Path.String()) } - if fileBound, err := id.file.Size(); err == nil { - if int64(offset) >= fileBound { - //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. - return 0, io.EOF - } + if offset >= id.meta.Size { + //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. + return 0, io.EOF } - if offset != 0 { - if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { - return -1, fmt.Errorf("Read - seek error: %s", err) - } + if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { + return 0, fmt.Errorf("Read - seek error: %s", err) } readBytes, err := id.file.Read(p) if err != nil && err != io.EOF { - return -1, err + return 0, err } + return readBytes, err } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index e5f7d53d8b5..8d89025cda6 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -216,9 +216,6 @@ func boundCheck(offset uint64, length int) (bool, error) { // sending individual path component requests to their appropriate target system // regardless of (file system request) origin func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { - l := logging.Logger("walker") - l.Errorf("req: %v\n%v\n", ref, names) - // clone requests go right back to the caller if shouldClone(names) { return []p9.QID{ref.QID()}, ref, nil @@ -257,7 +254,5 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { } // leave the last reference alive, for the caller to close } - l.Errorf("walker ret: %v\n%v\n", qids, curFile) - return qids, curFile, nil } From 11c5add1bb072bd42a4189314dda212bd48c9399 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Sun, 22 Sep 2019 13:18:04 -0400 Subject: [PATCH 059/102] change mount docs, context WIP --- docs/experimental-features.md | 7 +-- plugin/plugins/filesystem/nodes/base.go | 58 ++++++++++++++++++++---- plugin/plugins/filesystem/nodes/ipfs.go | 52 +++++++++++++++------ plugin/plugins/filesystem/nodes/pinfs.go | 29 ++++++++---- plugin/plugins/filesystem/nodes/root.go | 34 +++++++------- plugin/plugins/filesystem/nodes/utils.go | 4 +- 6 files changed, 129 insertions(+), 55 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 913aa1e9c0e..0004a8d08fb 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -716,9 +716,10 @@ Experimental, not included by default. This daemon plugin wraps the IPFS node and exposes file system services over a multiaddr listener. Currently we use the 9P2000.L protocol, and offer read-only support for `/ipfs` requests. With support for other (writable) systems coming next. -You may connect to this service using the `v9fs` client used in the Linux kernel. -By default we listen locally on the machine, using a Unix domain socket. -`mount -t 9p -o trans=unix $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` +You may connect to this service using the [`v9fs`](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/filesystems/9p.txt) client used in the Linux kernel. +Of the `v9fs` access modes, we currently only support `any` mode. +And by default we listen for requests on a Unix domain socket. +`mount -t 9p -o trans=unix,access=any $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` To configure the listening address and more, see the [package documentation](https://godoc.org/github.com/ipfs/go-ipfs/plugin/plugins/filesystem) See the [v9fs documentation](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/filesystems/9p.txt) for instructions on using transports, such as TCP. diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index c2c9bbc3652..9b54cc529bf 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -2,6 +2,8 @@ package fsnodes import ( "context" + "errors" + "fmt" "github.com/djdv/p9/p9" "github.com/djdv/p9/unimplfs" @@ -30,12 +32,17 @@ type Base struct { metaMask p9.AttrMask // The base context should be valid for the lifetime of the file system - // and used to derive call specific contexts from - Ctx context.Context - Logger logging.EventLogger + // and should be derived from the parent context on Attach() + // A cancel from either should cascade down, and invalidate all derived operations + // A call to fs.Close should leave no operations lingering + parentCtx context.Context + filesystemCtx context.Context + filesystemCancel context.CancelFunc + Logger logging.EventLogger parent walkRef // parent should be set and used to handle ".." requests child walkRef // child is an optional field, used to hand off child requests to another file system + root bool // am I a filesystem root? (as opposed to a file) } // IPFSBase is much like Base but extends it to hold IPFS specific metadata @@ -56,15 +63,46 @@ type IPFSBase struct { directory *directoryStream } -func (b Base) Close() error { +func (b *Base) Attach() (p9.File, error) { + if b.parentCtx == nil { + return nil, errors.New("Parent context was not set, no way to derive") + } + + select { + case <-b.parentCtx.Done(): + return nil, fmt.Errorf("Parent is done: %s", b.parentCtx.Err()) + default: + break + } + + if b.filesystemCtx != nil { + return nil, errors.New("Already attached") + } + b.filesystemCtx, b.filesystemCancel = context.WithCancel(b.parentCtx) + + b.root = true + + return b, nil +} + +func (b *Base) Close() error { + /* FIXME: close call ordering is unclear, currently this happens earlier than expected + maybe related to core bug? + + if b.root { + b.filesystemCancel() + } + if b.child != nil { return b.child.Close() } + */ return nil } func (ib *IPFSBase) Close() error { + ib.Logger.Debugf("Closing:%q", ib.Path.String()) var lastErr error if err := ib.Base.Close(); err != nil { @@ -79,9 +117,13 @@ func (ib *IPFSBase) Close() error { } } + /* FIXME: close call ordering is unclear, currently this happens earlier than expected + maybe related to core bug? + if ib.operationsCancel != nil { ib.operationsCancel() } + */ return lastErr } @@ -100,18 +142,18 @@ type walkRef interface { Child() walkRef } -func (b Base) Self() walkRef { +func (b *Base) Self() walkRef { return b } -func (b Base) Parent() walkRef { +func (b *Base) Parent() walkRef { return b.parent } -func (b Base) Child() walkRef { +func (b *Base) Child() walkRef { return b.child } -func (b Base) QID() p9.QID { +func (b *Base) QID() p9.QID { return b.Qid } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 195dfa0cc0c..255310ecdae 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -21,15 +21,25 @@ type IPFS struct { IPFSBase } -func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI) *IPFS { +func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) *IPFS { id := &IPFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, core, logging.Logger("IPFS"))} id.meta, id.metaMask = defaultRootAttr() + if parent != nil { + id.parent = parent + } else { + id.parent = id + } return id } func (id *IPFS) Attach() (p9.File, error) { id.Logger.Debugf("Attach") + _, err := id.Base.Attach() + if err != nil { + return nil, err + } + return id, nil } @@ -52,7 +62,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { var ( // returned qids = make([]p9.QID, 0, len(names)) - newFid = &IPFS{IPFSBase: newIPFSBase(id.Ctx, id.Path, 0, id.core, id.Logger)} + newFid = &IPFS{IPFSBase: newIPFSBase(id.parentCtx, id.Path, 0, id.core, id.Logger)} // temporary attr = &p9.Attr{} requestType = p9.AttrMask{Mode: true} @@ -60,12 +70,14 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { // last-set value is used ipldNode ipld.Node ) + //TODO: better construction/context init + newFid.filesystemCtx, newFid.filesystemCancel = id.filesystemCtx, id.filesystemCancel for _, name := range names { - callCtx, cancel := context.WithTimeout(id.Ctx, 30*time.Second) + callCtx, cancel := context.WithTimeout(newFid.filesystemCtx, 30*time.Second) defer cancel() - corePath, err := id.core.ResolvePath(callCtx, corepath.Join(newFid.Path, name)) + corePath, err := newFid.core.ResolvePath(callCtx, corepath.Join(newFid.Path, name)) if err != nil { cancel() //TODO: switch off error, return appropriate errno (ENOENT is going to be common here) @@ -75,7 +87,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { newFid.Path = corePath - ipldNode, err = id.core.Dag().Get(callCtx, newFid.Path.Cid()) + ipldNode, err = newFid.core.Dag().Get(callCtx, newFid.Path.Cid()) if err != nil { cancel() return qids, nil, err @@ -93,13 +105,13 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { } // stat up front for our returned file - err, filled := ipldStat(id.Ctx, &newFid.meta, ipldNode, p9.AttrMaskAll) + err, filled := ipldStat(newFid.filesystemCtx, &newFid.meta, ipldNode, p9.AttrMaskAll) if err != nil { return qids, nil, err } newFid.metaMask = filled newFid.meta.Mode |= IRXA - newFid.meta.RDev, id.metaMask.RDev = dIPFS, true + newFid.meta.RDev, newFid.metaMask.RDev = dIPFS, true id.Logger.Debugf("Walk ret %v, %v", qids, newFid) return qids, newFid, err @@ -110,7 +122,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // set up handle amenities var handleContext context.Context - handleContext, id.operationsCancel = context.WithCancel(id.Ctx) + handleContext, id.operationsCancel = context.WithCancel(id.filesystemCtx) // handle directories if id.meta.Mode.IsDir() { //FIXME: meta is not being set anywhere @@ -170,7 +182,7 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { select { case entry, open := <-id.directory.entryChan: if !open { - id.operationsCancel() + //id.operationsCancel() id.directory.eos = true return ents, nil } @@ -191,8 +203,8 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { id.directory.cursor++ - case <-id.Ctx.Done(): - id.directory.err = id.Ctx.Err() + case <-id.filesystemCtx.Done(): + id.directory.err = id.filesystemCtx.Err() return ents, id.directory.err } } @@ -202,10 +214,16 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { } func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { - id.Logger.Debugf("ReadAt {%d/%d}%q", offset, id.meta.Size, id.Path.String()) + const ( + readAtFmt = "ReadAt {%d/%d}%q" + readAtFmtErr = readAtFmt + ": %s" + ) + id.Logger.Debugf(readAtFmt, offset, id.meta.Size, id.Path.String()) if id.file == nil { - return 0, fmt.Errorf("file %q is not open for reading", id.Path.String()) + err := fmt.Errorf("file %q is not open for reading", id.Path.String()) + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.Path.String(), err) + return 0, err } if offset >= id.meta.Size { @@ -213,12 +231,18 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { return 0, io.EOF } + //FIXME: for some reason, the internal CoreAPI context is being canceled + // and it breaks everything if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { - return 0, fmt.Errorf("Read - seek error: %s", err) + id.operationsCancel() + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.Path.String(), err) + return 0, err } readBytes, err := id.file.Read(p) if err != nil && err != io.EOF { + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.Path.String(), err) + id.operationsCancel() return 0, err } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index b7ae5741bd1..e3ab5073f8f 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -16,27 +16,36 @@ type PinFS struct { ents []p9.Dirent } -func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI) *PinFS { +func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) *PinFS { pd := &PinFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, core, logging.Logger("PinFS"))} pd.meta, pd.metaMask = defaultRootAttr() - + if parent != nil { + pd.parent = parent + } else { + pd.parent = pd + } return pd } func (pd *PinFS) Attach() (p9.File, error) { pd.Logger.Debugf("Attach") + _, err := pd.Base.Attach() + if err != nil { + return nil, err + } - var subSystem walkRef = IPFSAttacher(pd.Ctx, pd.core) - attacher, ok := subSystem.(p9.Attacher) - if !ok { - return nil, fmt.Errorf("subsystem %T is not a valid file system", subSystem) + subSystem, err := IPFSAttacher(pd.filesystemCtx, pd.core, pd).Attach() + if err != nil { + return nil, fmt.Errorf("could not attach to subsystem: %s", err) } - if _, err := attacher.Attach(); err != nil { - return nil, err + walkRef, ok := subSystem.(walkRef) + if !ok { + return nil, fmt.Errorf("%q does not provide traverals methods", "ipfs") } - pd.child = subSystem + + pd.child = walkRef return pd, nil } @@ -58,7 +67,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") var handleContext context.Context - handleContext, pd.operationsCancel = context.WithCancel(pd.Ctx) + handleContext, pd.operationsCancel = context.WithCancel(pd.filesystemCtx) // IPFS core representation pins, err := pd.core.Pin().Ls(handleContext, coreoptions.Pin.Type.Recursive()) diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 30241a86fae..10ae16dc288 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -48,8 +48,8 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { core: core, subsystems: make([]p9.Dirent, 0, 1), //NOTE: capacity should be tied to root child count Base: Base{ - Logger: logging.Logger("RootFS"), - Ctx: ctx, + Logger: logging.Logger("RootFS"), + parentCtx: ctx, Qid: p9.QID{ Type: p9.TypeDir, Path: cidToQPath(rootPath("/").Cid()), @@ -81,6 +81,10 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { func (ri *RootIndex) Attach() (p9.File, error) { ri.Logger.Debugf("Attach") + _, err := ri.Base.Attach() + if err != nil { + return nil, err + } ri.parent = ri return ri, nil @@ -103,13 +107,17 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { } var ( - subSystem walkRef + //subSystem walkRef + subSystem p9.File err error ) switch names[0] { case "ipfs": - subSystem = PinFSAttacher(ri.Ctx, ri.core) + subSystem, err = PinFSAttacher(ri.filesystemCtx, ri.core, ri).Attach() + if err != nil { + return nil, nil, fmt.Errorf("could not attach to subsystem: %s", err) + } /* case "ipns": subSystem = KeyFSAttacher(ri.Ctx, ri.core) */ @@ -118,24 +126,14 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { return nil, nil, syscall.ENOENT //TODO: migrate to platform independant value } - attacher, ok := subSystem.(p9.Attacher) + walkRef, ok := subSystem.(walkRef) if !ok { - return nil, nil, fmt.Errorf("%q is not a valid file system", names[0]) - } - - // poke the filesystem to make sure it's alive, but don't do anything with it - tempReference, err := attacher.Attach() - if err != nil { - return nil, nil, err - } - //[p9-lib] "Close must be called even when Open has not been called." - if err = tempReference.Close(); err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("%q does not provide traverals methods", names[0]) } - ri.child = subSystem + ri.child = walkRef - return walker(subSystem, names[1:]) + return walker(ri.child, names[1:]) } func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 8d89025cda6..9ee7d1b32a8 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -188,8 +188,8 @@ func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, c Path: path, core: core, Base: Base{ - Logger: logger, - Ctx: ctx, + parentCtx: ctx, + Logger: logger, Qid: p9.QID{ Type: kind, Path: cidToQPath(path.Cid()), From dd3a8949ec173f800a6b3fb67e94b70f9e8b0b1a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 24 Sep 2019 15:34:19 -0400 Subject: [PATCH 060/102] walk refactor --- plugin/plugins/filesystem/filesystem.go | 2 +- plugin/plugins/filesystem/nodes/base.go | 70 ++++----- plugin/plugins/filesystem/nodes/ipfs.go | 39 +++-- plugin/plugins/filesystem/nodes/pinfs.go | 32 ++++- plugin/plugins/filesystem/nodes/root.go | 172 +++++++++++++++-------- plugin/plugins/filesystem/nodes/utils.go | 64 ++++++--- 6 files changed, 240 insertions(+), 139 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index beb9737966e..19520af8cc5 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -110,7 +110,7 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { fs.listener = listener // construct and launch the 9P resource server - s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core)) + s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core, nil)) go func() { fs.errorChan <- s.Serve(manet.NetListener(fs.listener)) }() diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 9b54cc529bf..a994f88f4d0 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -31,18 +31,21 @@ type Base struct { meta p9.Attr metaMask p9.AttrMask - // The base context should be valid for the lifetime of the file system - // and should be derived from the parent context on Attach() - // A cancel from either should cascade down, and invalidate all derived operations + // The parent context should be set prior to calling attach (usually in a consturctor) + // The Base filesystemCtx is derived from the parent context during Base.Attach + // and should be valid for the lifetime of the file system + // The fs context should be used to derive operation specific contexts from during calls + // A cancel should cascade down, and invalidate all derived contexts // A call to fs.Close should leave no operations lingering parentCtx context.Context filesystemCtx context.Context filesystemCancel context.CancelFunc Logger logging.EventLogger - parent walkRef // parent should be set and used to handle ".." requests - child walkRef // child is an optional field, used to hand off child requests to another file system - root bool // am I a filesystem root? (as opposed to a file) + parent p9.File // parent must be set, it is used to handle ".." requests; nodes without a parent must point back to themselves + child p9.File // child is an optional field, used to hand off walk requests to another file system + root bool // should be set to true on Attach if parent == self, this triggers the filesystemCancel on Close + open bool // should be set to true on Open and checked during Walk; do not walk open references walk(5) } // IPFSBase is much like Base but extends it to hold IPFS specific metadata @@ -52,8 +55,8 @@ type IPFSBase struct { Path corepath.Resolved core coreiface.CoreAPI - // you will typically want to derive a context from the base context within one operation (like Open) - // use it with the CoreAPI for something + // you will typically want to derive a context from the Base context within one operation (like Open) + // use it with the CoreAPI for something (like Get) // and cancel it in another operation (like Close) // that pointer should be stored here between calls operationsCancel context.CancelFunc @@ -63,6 +66,8 @@ type IPFSBase struct { directory *directoryStream } +// Base Attach should be called by all supersets during their Attach +// to initialize the file system context func (b *Base) Attach() (p9.File, error) { if b.parentCtx == nil { return nil, errors.New("Parent context was not set, no way to derive") @@ -80,51 +85,47 @@ func (b *Base) Attach() (p9.File, error) { } b.filesystemCtx, b.filesystemCancel = context.WithCancel(b.parentCtx) - b.root = true - return b, nil } +// Base Close should be called in all superset Close methods in order to +// close child subsystems and cancel the file system context func (b *Base) Close() error { - /* FIXME: close call ordering is unclear, currently this happens earlier than expected - maybe related to core bug? + b.Logger.Debugf("closing: {%v}", b.Qid.Path) if b.root { b.filesystemCancel() } + var err error if b.child != nil { - return b.child.Close() + if err = b.child.Close(); err != nil { + b.Logger.Error(err) + } } - */ - return nil + return err } func (ib *IPFSBase) Close() error { - ib.Logger.Debugf("Closing:%q", ib.Path.String()) + ib.Logger.Debugf("closing:{%v}%q", ib.Qid, ib.Path.String()) var lastErr error + if ib.operationsCancel != nil { + ib.operationsCancel() + } if err := ib.Base.Close(); err != nil { - ib.Logger.Errorf("base close: %s", err) + ib.Logger.Error(err) lastErr = err } if ib.file != nil { if err := ib.file.Close(); err != nil { - ib.Logger.Errorf("files.File close: %s", err) + ib.Logger.Error(err) lastErr = err } } - /* FIXME: close call ordering is unclear, currently this happens earlier than expected - maybe related to core bug? - - if ib.operationsCancel != nil { - ib.operationsCancel() - } - */ - return lastErr } @@ -137,23 +138,14 @@ type directoryStream struct { type walkRef interface { p9.File - QID() p9.QID - Parent() walkRef - Child() walkRef + Parent() p9.File + Child() p9.File } -func (b *Base) Self() walkRef { - return b -} - -func (b *Base) Parent() walkRef { +func (b *Base) Parent() p9.File { return b.parent } -func (b *Base) Child() walkRef { +func (b *Base) Child() p9.File { return b.child } - -func (b *Base) QID() p9.QID { - return b.Qid -} diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 255310ecdae..b221dd1df45 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -14,6 +14,9 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) +var _ p9.File = (*IPFS)(nil) +var _ walkRef = (*IPFS)(nil) + // IPFS exposes the IPFS API over a p9.File interface // Walk does not expect a namespace, only its path argument // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` @@ -21,7 +24,7 @@ type IPFS struct { IPFSBase } -func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) *IPFS { +func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) p9.Attacher { id := &IPFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, core, logging.Logger("IPFS"))} id.meta, id.metaMask = defaultRootAttr() @@ -40,6 +43,10 @@ func (id *IPFS) Attach() (p9.File, error) { return nil, err } + if id.parent == id { + id.root = true + } + return id, nil } @@ -54,28 +61,31 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { id.Logger.Debugf("Walk names %v", names) id.Logger.Debugf("Walk myself: %s:%v", id.Path, id.Qid) - if shouldClone(names) { + if id.open { + return nil, nil, errWalkOpened + } + + newFid := new(IPFS) + *newFid = *id + newFid.root = false + + if shouldClone(names, id.root) { id.Logger.Debugf("Walk cloned") - return []p9.QID{id.Qid}, id, nil + return []p9.QID{newFid.Qid}, newFid, nil } var ( // returned - qids = make([]p9.QID, 0, len(names)) - newFid = &IPFS{IPFSBase: newIPFSBase(id.parentCtx, id.Path, 0, id.core, id.Logger)} + qids = make([]p9.QID, 0, len(names)) // temporary - attr = &p9.Attr{} requestType = p9.AttrMask{Mode: true} err error // last-set value is used ipldNode ipld.Node ) - //TODO: better construction/context init - newFid.filesystemCtx, newFid.filesystemCancel = id.filesystemCtx, id.filesystemCancel for _, name := range names { callCtx, cancel := context.WithTimeout(newFid.filesystemCtx, 30*time.Second) - defer cancel() corePath, err := newFid.core.ResolvePath(callCtx, corepath.Join(newFid.Path, name)) if err != nil { @@ -93,6 +103,7 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { return qids, nil, err } + attr := &p9.Attr{} err, _ = ipldStat(callCtx, attr, ipldNode, requestType) if err != nil { cancel() @@ -102,13 +113,19 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { newFid.Qid.Type = attr.Mode.QIDType() newFid.Qid.Path = cidToQPath(ipldNode.Cid()) qids = append(qids, newFid.Qid) + + cancel() } // stat up front for our returned file - err, filled := ipldStat(newFid.filesystemCtx, &newFid.meta, ipldNode, p9.AttrMaskAll) + callCtx, cancel := context.WithTimeout(newFid.filesystemCtx, 30*time.Second) + defer cancel() + meta := &p9.Attr{} + err, filled := ipldStat(callCtx, meta, ipldNode, p9.AttrMaskAll) if err != nil { return qids, nil, err } + newFid.meta = *meta newFid.metaMask = filled newFid.meta.Mode |= IRXA newFid.meta.RDev, newFid.metaMask.RDev = dIPFS, true @@ -218,7 +235,7 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { readAtFmt = "ReadAt {%d/%d}%q" readAtFmtErr = readAtFmt + ": %s" ) - id.Logger.Debugf(readAtFmt, offset, id.meta.Size, id.Path.String()) + //id.Logger.Debugf(readAtFmt, offset, id.meta.Size, id.Path.String()) if id.file == nil { err := fmt.Errorf("file %q is not open for reading", id.Path.String()) diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index e3ab5073f8f..1e855124105 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -11,13 +11,16 @@ import ( coreoptions "github.com/ipfs/interface-go-ipfs-core/options" ) +var _ p9.File = (*PinFS)(nil) +var _ walkRef = (*PinFS)(nil) + type PinFS struct { IPFSBase ents []p9.Dirent } -func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) *PinFS { - pd := &PinFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, +func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) p9.Attacher { + pd := &PinFS{IPFSBase: newIPFSBase(ctx, rootPath("/PinFS"), p9.TypeDir, core, logging.Logger("PinFS"))} pd.meta, pd.metaMask = defaultRootAttr() if parent != nil { @@ -35,14 +38,14 @@ func (pd *PinFS) Attach() (p9.File, error) { return nil, err } - subSystem, err := IPFSAttacher(pd.filesystemCtx, pd.core, pd).Attach() + subsystem, err := IPFSAttacher(pd.filesystemCtx, pd.core, pd).Attach() if err != nil { - return nil, fmt.Errorf("could not attach to subsystem: %s", err) + return nil, fmt.Errorf(errFmtWalkSubsystem, err) } - walkRef, ok := subSystem.(walkRef) + walkRef, ok := subsystem.(walkRef) if !ok { - return nil, fmt.Errorf("%q does not provide traverals methods", "ipfs") + return nil, fmt.Errorf(errFmtExternalWalk, "ipfs") } pd.child = walkRef @@ -60,7 +63,22 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { pd.Logger.Debugf("Walk names %v", names) pd.Logger.Debugf("Walk myself: %v", pd.Qid) - return walker(pd, names) + qids := []p9.QID{pd.Qid} + + if pd.open { + return qids, nil, errWalkOpened + } + + newFid := new(PinFS) + *newFid = *pd + newFid.root = false + + if shouldClone(names, pd.root) { + pd.Logger.Debugf("Walk cloned") + return qids, newFid, nil + } + + return stepper(pd, names) } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 10ae16dc288..3ca14960ac0 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -2,7 +2,7 @@ package fsnodes import ( "context" - "fmt" + "sort" "syscall" "github.com/djdv/p9/p9" @@ -12,16 +12,18 @@ import ( "github.com/multiformats/go-multihash" ) -const nRoot = "" // root namespace is intentionally left blank var _ p9.File = (*RootIndex)(nil) +var _ walkRef = (*RootIndex)(nil) + +const nRoot = "" // root namespace is intentionally left blank type rootPath string func (rp rootPath) String() string { return string(rp) } func (rootPath) Namespace() string { return nRoot } -func (rp rootPath) Mutable() bool { return true } -func (rp rootPath) IsValid() error { return nil } -func (rootPath) Root() cid.Cid { return cid.Cid{} } //TODO: reference the root CID when overlay is finished +func (rootPath) Mutable() bool { return true } +func (rootPath) IsValid() error { return nil } +func (rootPath) Root() cid.Cid { return cid.Cid{} } func (rootPath) Remainder() string { return "" } func (rp rootPath) Cid() cid.Cid { prefix := cid.V1Builder{Codec: cid.DagCBOR, MhType: multihash.BLAKE2B_MIN} @@ -32,21 +34,31 @@ func (rp rootPath) Cid() cid.Cid { return c } +type systemTuple struct { + file p9.File + dirent p9.Dirent +} + +type systemSlice []systemTuple + +func (ss systemSlice) Len() int { return len(ss) } +func (ss systemSlice) Swap(i, j int) { ss[i], ss[j] = ss[j], ss[i] } +func (ss systemSlice) Less(i, j int) bool { return ss[i].dirent.Offset < ss[j].dirent.Offset } + +//TODO: rename, while this is likely to be the root, it doesn't have to be; maybe "IPFSOverlay" // RootIndex is a virtual directory file system, that maps a set of file system implementations to a hierarchy -// Currently: "/":RootIndex, "/ipfs":PinFS, "/ipfs/*:IPFS +// Currently: "/ipfs":PinFS, "/ipfs/*:IPFS type RootIndex struct { Base - - subsystems []p9.Dirent - - core coreiface.CoreAPI + //subsystems []p9.Dirent + subsystems map[string]systemTuple + core coreiface.CoreAPI } // RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it -func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { +func RootAttacher(ctx context.Context, core coreiface.CoreAPI, parent p9.File) p9.Attacher { ri := &RootIndex{ - core: core, - subsystems: make([]p9.Dirent, 0, 1), //NOTE: capacity should be tied to root child count + core: core, Base: Base{ Logger: logging.Logger("RootFS"), parentCtx: ctx, @@ -58,22 +70,48 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI) *RootIndex { } ri.meta, ri.metaMask = defaultRootAttr() - rootDirTemplate := p9.Dirent{ + // + rootDirent := p9.Dirent{ Type: p9.TypeDir, - QID: p9.QID{Type: p9.TypeDir}} + QID: p9.QID{Type: p9.TypeDir}, + } - for i, pathUnion := range [...]struct { + type subattacher func(context.Context, coreiface.CoreAPI, walkRef) p9.Attacher + type attachTuple struct { string - p9.Dirent - }{ - {"ipfs", rootDirTemplate}, - } { - pathUnion.Dirent.Offset = uint64(i + 1) - pathUnion.Dirent.Name = pathUnion.string + subattacher + //p9.Dirent + } + subsystems := [...]attachTuple{ + {"ipfs", PinFSAttacher}, + //{"ipns", KeyFsAttacher}, + //{"files", needs CoreAPI changes}, + } + + ri.subsystems = make(map[string]systemTuple, len(subsystems)) + + for i, subsystem := range subsystems { + fs, err := subsystem.subattacher(ctx, core, ri).Attach() + if err != nil { + panic(err) // hard implementation error + } + + //for i, pathUnion := range [...]struct { + rootDirent.Offset = uint64(i + 1) + rootDirent.Name = subsystem.string + + rootDirent.QID.Path = cidToQPath(rootPath("/" + subsystem.string).Cid()) + + ri.subsystems[subsystem.string] = systemTuple{ + file: fs, + dirent: rootDirent, + } + } - pathNode := rootPath("/" + pathUnion.string) - pathUnion.Dirent.QID.Path = cidToQPath(pathNode.Cid()) - ri.subsystems = append(ri.subsystems, pathUnion.Dirent) + if parent != nil { + ri.parent = parent + } else { + ri.parent = ri } return ri @@ -86,13 +124,16 @@ func (ri *RootIndex) Attach() (p9.File, error) { return nil, err } - ri.parent = ri + if ri.parent == ri { + ri.root = true + } + return ri, nil } func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { ri.Logger.Debugf("GetAttr") - ri.Logger.Debugf("mask: %v", req) + //ri.Logger.Debugf("mask: %v", req) return ri.Qid, ri.metaMask, ri.meta, nil } @@ -101,60 +142,75 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { ri.Logger.Debugf("Walk names %v", names) ri.Logger.Debugf("Walk myself: %v", ri.Qid) - if shouldClone(names) { - ri.Logger.Debugf("Walk cloned") - return []p9.QID{ri.Qid}, ri, nil + qids := []p9.QID{ri.Qid} + + if ri.open { + return qids, nil, errWalkOpened } - var ( - //subSystem walkRef - subSystem p9.File - err error - ) + newFid := new(RootIndex) + *newFid = *ri + newFid.root = false - switch names[0] { - case "ipfs": - subSystem, err = PinFSAttacher(ri.filesystemCtx, ri.core, ri).Attach() - if err != nil { - return nil, nil, fmt.Errorf("could not attach to subsystem: %s", err) - } - /* case "ipns": - subSystem = KeyFSAttacher(ri.Ctx, ri.core) - */ - default: - ri.Logger.Errorf("%q is not provided by us", names[0]) - return nil, nil, syscall.ENOENT //TODO: migrate to platform independant value + if shouldClone(names, ri.root) { + ri.Logger.Debugf("Walk cloned") + return qids, newFid, nil } - walkRef, ok := subSystem.(walkRef) + subSys, ok := ri.subsystems[names[0]] if !ok { - return nil, nil, fmt.Errorf("%q does not provide traverals methods", names[0]) + ri.Logger.Errorf("%q is not provided by us", names[0]) + return nil, nil, syscall.ENOENT //TODO: migrate to platform independent value } - ri.child = walkRef + newFid.child = subSys.file - return walker(ri.child, names[1:]) + return stepper(newFid, names[1:]) } func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { ri.Logger.Debugf("Open") + ri.open = true return ri.Qid, 0, nil } func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { ri.Logger.Debugf("Readdir {%d}", count) - shouldExit, err := boundCheck(offset, len(ri.subsystems)) + subs := len(ri.subsystems) + + shouldExit, err := boundCheck(offset, subs) if shouldExit { return nil, err } - offsetIndex := ri.subsystems[offset:] - if len(offsetIndex) > int(count) { - ri.Logger.Debugf("Readdir returning [%d]%v\n", count, offsetIndex[:count]) - return offsetIndex[:count], nil + // TODO: [perf] iterating over the map and slot populating an array is probably the least wasted effort + // map => ordered set + orderedSet := make(systemSlice, 0, subs) + for _, pair := range ri.subsystems { + orderedSet = append(orderedSet, pair) + } + sort.Sort(orderedSet) + + offsetIndex := subs - int(offset) + + // n-tuple => singleton + ents := make([]p9.Dirent, 0, offsetIndex) + for _, pair := range orderedSet[offset:] { + ents = append(ents, pair.dirent) } - ri.Logger.Debugf("Readdir returning [%d]%v\n", len(offsetIndex), offsetIndex) - return offsetIndex, nil + // set => trimmed-set + if offsetIndex > int(count) { + return ents[:count], nil + } + + return ents, nil +} + +func (ri *RootIndex) Close() error { + ri.Logger.Debugf("closing: {%v} root", ri.Qid) + err := ri.Base.Close() + ri.open = false + return err } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 9ee7d1b32a8..2c6ee8c80d9 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -3,6 +3,7 @@ package fsnodes import ( "context" "crypto/rand" + "errors" "fmt" "hash/fnv" "io" @@ -25,8 +26,13 @@ const ( // context: https://github.com/ipfs/go-ipfs/pull/6612/files#r322989041 ipfsBlockSize = 256 << 10 saltSize = 32 + + errFmtWalkSubsystem = "could not attach to subsystem: %q" + errFmtExternalWalk = "%q does not provide external fs walk methods" ) +var errWalkOpened = errors.New("this fid is open") + // NOTE [2019.09.12]: QID's have a high collision probability // as a result we add a salt to hashes to attempt to mitigate this // for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 @@ -40,13 +46,13 @@ func init() { } } -func shouldClone(names []string) bool { +func shouldClone(names []string, isRoot bool) bool { switch len(names) { case 0: // empty path return true case 1: // self? pc := names[0] - return pc == ".." || pc == "." || pc == "" + return pc == "." || pc == "" || (isRoot && pc == "..") default: return false } @@ -212,46 +218,58 @@ func boundCheck(offset uint64, length int) (bool, error) { } } -// walker acts as a dispatcher for intermediate file systems -// sending individual path component requests to their appropriate target system -// regardless of (file system request) origin -func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { - // clone requests go right back to the caller - if shouldClone(names) { - return []p9.QID{ref.QID()}, ref, nil - } - +//TODO: English sanity check; does this make sense without reading the code? +//TODO: we need to throw a lot of path tests at this; the recursive nature of walk->stepper->walk->stepper may be troublesome as well +// stepper acts as a dispatcher for intermediate file systems +// sending path component requests to a separate file system based on the name args +// clone requests (no names) will return a clone of the attached child +func stepper(ref walkRef, names []string) ([]p9.QID, p9.File, error) { var ( - nextRef walkRef + nextRef p9.File qids, subQids []p9.QID curFile p9.File err error + ok bool ) + if len(names) == 0 { + //return qids, nil, errors.New("no subnames were provided") + c := ref.Child() + if c == nil { + return qids, nil, errors.New("clone requested but child file system attached") + } + + return c.Walk(nil) + } + for len(names) != 0 { - //prepare to step into next component - if names[0] == ".." { // climb to parent / leftwards requests + //prepare to step into next component(s) + if names[0] == ".." { // climb to parent / leftward requests nextRef = ref.Parent() - } else { // handle remainder / rightward requests + } else { // descend into remainder / rightward requests nextRef = ref.Child() } if nextRef == nil { - return []p9.QID{ref.QID()}, nil, fmt.Errorf("system for target %q is not assigned", names[0]) + return qids, nil, fmt.Errorf("%q is not attached to another file system", names[0]) } - // attempt the step - if subQids, curFile, err = nextRef.Walk(names); err != nil { + ref, ok = nextRef.(walkRef) //TODO: see if we can implement this without runtime assertion + if !ok { + return qids, nil, fmt.Errorf(errFmtExternalWalk, names[0]) + } + + // attempt the step(s) + if subQids, curFile, err = ref.Walk(names); err != nil { return qids, nil, err } - // we walked forward, prepare for next step + // we walked forward, prepare for next step(s) qids = append(qids, subQids...) - names = names[1:] - ref = nextRef + names = names[len(subQids):] - if len(names) != 0 { + if len(names) != 0 { // leave the last reference alive, for the caller to close curFile.Close() // we're not referencing this anymore - } // leave the last reference alive, for the caller to close + } } return qids, curFile, nil From 1636e1b203a3b7b2bc9f0ad8932030d318fe7453 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 24 Sep 2019 15:39:12 -0400 Subject: [PATCH 061/102] unbreak tests --- plugin/plugins/filesystem/filesystem_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 0e5ec2a03ab..31e94d5eae2 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -42,7 +42,7 @@ func TestAll(t *testing.T) { } func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - nineRoot, err := fsnodes.RootAttacher(ctx, core).Attach() + nineRoot, err := fsnodes.RootAttacher(ctx, core, nil).Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource: %s\n", err) } @@ -69,7 +69,7 @@ func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { } func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - pinRoot, err := fsnodes.PinFSAttacher(ctx, core).Attach() + pinRoot, err := fsnodes.PinFSAttacher(ctx, core, nil).Attach() if err != nil { t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) } @@ -137,7 +137,7 @@ func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { t.Fatalf("Failed to attach to local resource %q: %s\n", env, err) } - ipfsRoot, err := fsnodes.IPFSAttacher(ctx, core).Attach() + ipfsRoot, err := fsnodes.IPFSAttacher(ctx, core, nil).Attach() if err != nil { t.Fatalf("Failed to attach to IPFS resource: %s\n", err) } From 63f6d2786db9d71551d6b71cb90ec7132f3280a8 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 25 Sep 2019 09:50:25 -0400 Subject: [PATCH 062/102] possible fix for re-attachment --- plugin/plugins/filesystem/nodes/base.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index a994f88f4d0..e8ec597c867 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -81,7 +81,12 @@ func (b *Base) Attach() (p9.File, error) { } if b.filesystemCtx != nil { - return nil, errors.New("Already attached") + select { + case <-b.filesystemCtx.Done(): + break + default: + return nil, errors.New("Attach was called already and file system is in use") + } } b.filesystemCtx, b.filesystemCancel = context.WithCancel(b.parentCtx) From 97f36a99bcba8dec51f374afd703999c58ecbd00 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 25 Sep 2019 10:37:15 -0400 Subject: [PATCH 063/102] socket path template nit --- plugin/plugins/filesystem/filesystem.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 19520af8cc5..7a71b2780ad 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "github.com/djdv/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" @@ -50,7 +51,7 @@ func (*FileSystemPlugin) Version() string { func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { logger.Info("Initializing 9P resource server...") - // stabilise repo path + // stabilise repo path; our template depends on this if !filepath.IsAbs(env.Repo) { absRepo, err := filepath.Abs(env.Repo) if err != nil { @@ -78,7 +79,15 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { } // expand string templates and initialize listening addr - addrString = os.Expand(addrString, configVarMapper(env.Repo)) + templateRepoPath := env.Repo + if strings.HasPrefix(addrString, "/unix") { + // prevent template from expanding to double slashed paths like `/unix//home/...` + // but allow it to expand to `/unix/C:\Users...` + templateRepoPath = strings.TrimPrefix(templateRepoPath, "/") + } + + addrString = os.Expand(addrString, configVarMapper(templateRepoPath)) + ma, err := multiaddr.NewMultiaddr(addrString) if err != nil { return err From 3c56f5f7428bc15dedb73fb8bcfcc7e6850e7043 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 26 Sep 2019 09:36:24 -0400 Subject: [PATCH 064/102] change attacher-constructors and walk clone-semantics --- plugin/plugins/filesystem/filesystem.go | 2 +- plugin/plugins/filesystem/filesystem_test.go | 6 ++-- plugin/plugins/filesystem/nodes/base.go | 10 +----- plugin/plugins/filesystem/nodes/ipfs.go | 20 +++++------ .../filesystem/nodes/options/options.go | 36 +++++++++++++++++++ plugin/plugins/filesystem/nodes/pinfs.go | 19 ++++++---- plugin/plugins/filesystem/nodes/root.go | 36 +++++++++---------- 7 files changed, 82 insertions(+), 47 deletions(-) create mode 100644 plugin/plugins/filesystem/nodes/options/options.go diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 7a71b2780ad..7c2febf578b 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -119,7 +119,7 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { fs.listener = listener // construct and launch the 9P resource server - s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core, nil)) + s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core)) go func() { fs.errorChan <- s.Serve(manet.NetListener(fs.listener)) }() diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 31e94d5eae2..0e5ec2a03ab 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -42,7 +42,7 @@ func TestAll(t *testing.T) { } func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - nineRoot, err := fsnodes.RootAttacher(ctx, core, nil).Attach() + nineRoot, err := fsnodes.RootAttacher(ctx, core).Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource: %s\n", err) } @@ -69,7 +69,7 @@ func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { } func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - pinRoot, err := fsnodes.PinFSAttacher(ctx, core, nil).Attach() + pinRoot, err := fsnodes.PinFSAttacher(ctx, core).Attach() if err != nil { t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) } @@ -137,7 +137,7 @@ func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { t.Fatalf("Failed to attach to local resource %q: %s\n", env, err) } - ipfsRoot, err := fsnodes.IPFSAttacher(ctx, core, nil).Attach() + ipfsRoot, err := fsnodes.IPFSAttacher(ctx, core).Attach() if err != nil { t.Fatalf("Failed to attach to IPFS resource: %s\n", err) } diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index e8ec597c867..4d70ddb0469 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -75,19 +75,11 @@ func (b *Base) Attach() (p9.File, error) { select { case <-b.parentCtx.Done(): - return nil, fmt.Errorf("Parent is done: %s", b.parentCtx.Err()) + return nil, fmt.Errorf("Parent context is done: %s", b.parentCtx.Err()) default: break } - if b.filesystemCtx != nil { - select { - case <-b.filesystemCtx.Done(): - break - default: - return nil, errors.New("Attach was called already and file system is in use") - } - } b.filesystemCtx, b.filesystemCancel = context.WithCancel(b.parentCtx) return b, nil diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index b221dd1df45..c348d7f01f9 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -8,6 +8,7 @@ import ( "github.com/djdv/p9/p9" files "github.com/ipfs/go-ipfs-files" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" @@ -24,14 +25,17 @@ type IPFS struct { IPFSBase } -func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) p9.Attacher { +func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.Option) p9.Attacher { id := &IPFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, core, logging.Logger("IPFS"))} id.meta, id.metaMask = defaultRootAttr() - if parent != nil { - id.parent = parent + + options := nodeopts.AttachOps(ops...) + if options.Parent != nil { + id.parent = options.Parent } else { id.parent = id + id.root = true } return id } @@ -43,10 +47,6 @@ func (id *IPFS) Attach() (p9.File, error) { return nil, err } - if id.parent == id { - id.root = true - } - return id, nil } @@ -67,13 +67,14 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { newFid := new(IPFS) *newFid = *id - newFid.root = false - if shouldClone(names, id.root) { + if shouldClone(names, newFid.root) { id.Logger.Debugf("Walk cloned") return []p9.QID{newFid.Qid}, newFid, nil } + newFid.root = false + var ( // returned qids = make([]p9.QID, 0, len(names)) @@ -130,7 +131,6 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { newFid.meta.Mode |= IRXA newFid.meta.RDev, newFid.metaMask.RDev = dIPFS, true - id.Logger.Debugf("Walk ret %v, %v", qids, newFid) return qids, newFid, err } diff --git a/plugin/plugins/filesystem/nodes/options/options.go b/plugin/plugins/filesystem/nodes/options/options.go new file mode 100644 index 00000000000..e040aba1a6d --- /dev/null +++ b/plugin/plugins/filesystem/nodes/options/options.go @@ -0,0 +1,36 @@ +package nodeopts + +import ( + "github.com/djdv/p9/p9" + "github.com/jbenet/goprocess" +) + +type Options struct { + Parent p9.File + Process goprocess.Process +} + +type Option func(*Options) + +// if NOT provided, we assume the file system is to be treated as a root, assigning itself as a parent +func Parent(p p9.File) Option { + return func(ops *Options) { + ops.Parent = p + } +} + +//TODO: this isn't true yet +// if provided, file systems implemented here will utilize this to create a cascading Close() tree +func Process(p goprocess.Process) Option { + return func(ops *Options) { + ops.Process = p + } +} + +func AttachOps(options ...Option) *Options { + ops := &Options{} // only nil defaults at this time + for _, op := range options { + op(ops) + } + return ops +} diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 1e855124105..eec2afbdc56 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -6,6 +6,7 @@ import ( gopath "path" "github.com/djdv/p9/p9" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" coreoptions "github.com/ipfs/interface-go-ipfs-core/options" @@ -19,14 +20,17 @@ type PinFS struct { ents []p9.Dirent } -func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, parent walkRef) p9.Attacher { +func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.Option) p9.Attacher { pd := &PinFS{IPFSBase: newIPFSBase(ctx, rootPath("/PinFS"), p9.TypeDir, core, logging.Logger("PinFS"))} pd.meta, pd.metaMask = defaultRootAttr() - if parent != nil { - pd.parent = parent + + options := nodeopts.AttachOps(ops...) + if options.Parent != nil { + pd.parent = options.Parent } else { pd.parent = pd + pd.root = true } return pd } @@ -38,7 +42,9 @@ func (pd *PinFS) Attach() (p9.File, error) { return nil, err } - subsystem, err := IPFSAttacher(pd.filesystemCtx, pd.core, pd).Attach() + opts := []nodeopts.Option{nodeopts.Parent(pd)} + + subsystem, err := IPFSAttacher(pd.filesystemCtx, pd.core, opts...).Attach() if err != nil { return nil, fmt.Errorf(errFmtWalkSubsystem, err) } @@ -71,14 +77,15 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { newFid := new(PinFS) *newFid = *pd - newFid.root = false if shouldClone(names, pd.root) { pd.Logger.Debugf("Walk cloned") return qids, newFid, nil } - return stepper(pd, names) + newFid.root = false + + return stepper(newFid, names) } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 3ca14960ac0..3371c68fc39 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -7,6 +7,7 @@ import ( "github.com/djdv/p9/p9" cid "github.com/ipfs/go-cid" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" "github.com/multiformats/go-multihash" @@ -56,7 +57,7 @@ type RootIndex struct { } // RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it -func RootAttacher(ctx context.Context, core coreiface.CoreAPI, parent p9.File) p9.Attacher { +func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.Option) p9.Attacher { ri := &RootIndex{ core: core, Base: Base{ @@ -70,33 +71,29 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, parent p9.File) p } ri.meta, ri.metaMask = defaultRootAttr() - // rootDirent := p9.Dirent{ Type: p9.TypeDir, QID: p9.QID{Type: p9.TypeDir}, } - type subattacher func(context.Context, coreiface.CoreAPI, walkRef) p9.Attacher + type subattacher func(context.Context, coreiface.CoreAPI, ...nodeopts.Option) p9.Attacher type attachTuple struct { string subattacher - //p9.Dirent } subsystems := [...]attachTuple{ {"ipfs", PinFSAttacher}, - //{"ipns", KeyFsAttacher}, - //{"files", needs CoreAPI changes}, } ri.subsystems = make(map[string]systemTuple, len(subsystems)) + opts := []nodeopts.Option{nodeopts.Parent(ri)} for i, subsystem := range subsystems { - fs, err := subsystem.subattacher(ctx, core, ri).Attach() + fs, err := subsystem.subattacher(ctx, core, opts...).Attach() if err != nil { panic(err) // hard implementation error } - //for i, pathUnion := range [...]struct { rootDirent.Offset = uint64(i + 1) rootDirent.Name = subsystem.string @@ -108,10 +105,12 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, parent p9.File) p } } - if parent != nil { - ri.parent = parent + options := nodeopts.AttachOps(ops...) + if options.Parent != nil { + ri.parent = options.Parent } else { ri.parent = ri + ri.root = true } return ri @@ -119,16 +118,16 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, parent p9.File) p func (ri *RootIndex) Attach() (p9.File, error) { ri.Logger.Debugf("Attach") - _, err := ri.Base.Attach() + + newFid := new(RootIndex) + *newFid = *ri + + _, err := newFid.Base.Attach() if err != nil { return nil, err } - if ri.parent == ri { - ri.root = true - } - - return ri, nil + return newFid, nil } func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { @@ -150,13 +149,14 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { newFid := new(RootIndex) *newFid = *ri - newFid.root = false - if shouldClone(names, ri.root) { + if shouldClone(names, newFid.root) { ri.Logger.Debugf("Walk cloned") return qids, newFid, nil } + newFid.root = false + subSys, ok := ri.subsystems[names[0]] if !ok { ri.Logger.Errorf("%q is not provided by us", names[0]) From 4c1ad5d6e5f774a72faf150a8f29da7e15460a28 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Thu, 26 Sep 2019 09:59:53 -0400 Subject: [PATCH 065/102] add basic root tests --- plugin/plugins/filesystem/filesystem_test.go | 75 ++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 0e5ec2a03ab..fd57c209a4d 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -36,11 +36,86 @@ func TestAll(t *testing.T) { t.Fatalf("Failed to construct IPFS node: %s\n", err) } + t.Run("RootFS Attach", func(t *testing.T) { testRootAttacher(ctx, t, core) }) + t.Run("RootFS Clone", func(t *testing.T) { testRootClones(ctx, t, core) }) t.Run("RootFS", func(t *testing.T) { testRootFS(ctx, t, core) }) t.Run("PinFS", func(t *testing.T) { testPinFS(ctx, t, core) }) t.Run("IPFS", func(t *testing.T) { testIPFS(ctx, t, core) }) } +func testRootAttacher(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + rootAttacher := fsnodes.RootAttacher(ctx, core) + + // 2 individual instances + nineRoot, err := rootAttacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + } + + if err = nineRoot.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + nineRootTheRevenge, err := rootAttacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) + } + + if err = nineRootTheRevenge.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // 2 instances at the same time + nineRoot, err = rootAttacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + } + + nineRootTheRevenge, err = rootAttacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) + } + + if err = nineRootTheRevenge.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + if err = nineRoot.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } +} + +func testRootClones(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + nineRoot, err := fsnodes.RootAttacher(ctx, core).Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + } + + _, nineRef, err := nineRoot.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + // this shouldn't affect the parent it's derived from + if err = nineRef.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + _, anotherNineRef, err := nineRoot.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + if err = anotherNineRef.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + if err = nineRoot.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + +} + func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { nineRoot, err := fsnodes.RootAttacher(ctx, core).Attach() if err != nil { From e3db71a55202ea430b6076bcbb48aab19a1b5f6a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 27 Sep 2019 10:10:41 -0400 Subject: [PATCH 066/102] remove runtime check, already static checked --- plugin/plugins/filesystem/nodes/pinfs.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index eec2afbdc56..8156c546275 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -49,12 +49,7 @@ func (pd *PinFS) Attach() (p9.File, error) { return nil, fmt.Errorf(errFmtWalkSubsystem, err) } - walkRef, ok := subsystem.(walkRef) - if !ok { - return nil, fmt.Errorf(errFmtExternalWalk, "ipfs") - } - - pd.child = walkRef + pd.child = subsystem return pd, nil } From b2a43b8da3ddfcc0451bc2db12216c6642a8ac04 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 27 Sep 2019 10:20:18 -0400 Subject: [PATCH 067/102] fix attach for other roots too --- plugin/plugins/filesystem/nodes/ipfs.go | 8 ++++++-- plugin/plugins/filesystem/nodes/pinfs.go | 14 +++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index c348d7f01f9..c88985974e0 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -42,12 +42,16 @@ func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.O func (id *IPFS) Attach() (p9.File, error) { id.Logger.Debugf("Attach") - _, err := id.Base.Attach() + + newFid := new(IPFS) + *newFid = *id + + _, err := newFid.Base.Attach() if err != nil { return nil, err } - return id, nil + return newFid, nil } func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 8156c546275..11b83ee4213 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -37,21 +37,25 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. func (pd *PinFS) Attach() (p9.File, error) { pd.Logger.Debugf("Attach") - _, err := pd.Base.Attach() + + newFid := new(PinFS) + *newFid = *pd + + _, err := newFid.Base.Attach() if err != nil { return nil, err } - opts := []nodeopts.Option{nodeopts.Parent(pd)} + opts := []nodeopts.Option{nodeopts.Parent(newFid)} - subsystem, err := IPFSAttacher(pd.filesystemCtx, pd.core, opts...).Attach() + subsystem, err := IPFSAttacher(newFid.filesystemCtx, newFid.core, opts...).Attach() if err != nil { return nil, fmt.Errorf(errFmtWalkSubsystem, err) } - pd.child = subsystem + newFid.child = subsystem - return pd, nil + return newFid, nil } func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { From 9cc06aee3de575a7701c9b6881d7e03e4c6fc320 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 2 Oct 2019 23:12:47 -0400 Subject: [PATCH 068/102] traversal/construction refactor --- plugin/plugins/filesystem/filesystem.go | 4 +- plugin/plugins/filesystem/filesystem_test.go | 37 +- plugin/plugins/filesystem/nodes/base.go | 200 ++++---- plugin/plugins/filesystem/nodes/ipfs.go | 275 ++++++----- plugin/plugins/filesystem/nodes/mfs.go | 438 ++++++++++++++++++ .../filesystem/nodes/options/options.go | 57 ++- plugin/plugins/filesystem/nodes/pinfs.go | 128 +++-- plugin/plugins/filesystem/nodes/root.go | 181 ++++---- plugin/plugins/filesystem/nodes/utils.go | 230 +++++---- 9 files changed, 1077 insertions(+), 473 deletions(-) create mode 100644 plugin/plugins/filesystem/nodes/mfs.go diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 7c2febf578b..083e4e22088 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -9,6 +9,7 @@ import ( "github.com/djdv/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" "github.com/mitchellh/mapstructure" @@ -119,7 +120,8 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { fs.listener = listener // construct and launch the 9P resource server - s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core)) + opts := []nodeopts.AttachOption{nodeopts.Logger(logging.Logger("9root"))} + s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core, opts...)) go func() { fs.errorChan <- s.Serve(manet.NetListener(fs.listener)) }() diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index fd57c209a4d..bedab1edbeb 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -24,10 +24,13 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) -var attrMaskIPFSTest = p9.AttrMask{ - Mode: true, - Size: true, -} +var ( + attrMaskIPFSTest = p9.AttrMask{ + Mode: true, + Size: true, + } + rootSubsystems = []string{"ipfs"} +) func TestAll(t *testing.T) { ctx := context.TODO() @@ -135,12 +138,15 @@ func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { t.Fatalf("Failed to read root: %s\n", err) } - //TODO: currently magic. As subsystems are implemented, rework this part of the test + lib to contain some list - if len(ents) != 1 || ents[0].Name != "ipfs" { - t.Fatalf("Failed, root has bad entries:: %v\n", ents) + if len(ents) != len(rootSubsystems) { + t.Fatalf("Failed, root has bad entries:\nHave:%v\nWant:%v\n", ents, rootSubsystems) } - - //TODO: type checking + for i, name := range rootSubsystems { + if ents[i].Name != name { + t.Fatalf("Failed, root has bad entries:\nHave:%v\nWant:%v\n", ents, rootSubsystems) + } + } + //TODO: deeper compare than just the names / name order } func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { @@ -225,7 +231,7 @@ func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { t.Fatalf("Failed to clone IPFS environment handle: %s\n", err) } - testCompareTreeModes(t, localEnv, ipfsEnv) + testCompareTreeAttrs(t, localEnv, ipfsEnv) // test readdir bounds //TODO: compare against a table, not just lengths @@ -242,7 +248,7 @@ func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { } } -func testCompareTreeModes(t *testing.T, f1, f2 p9.File) { +func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { var expand func(p9.File) (map[string]p9.Attr, error) expand = func(nineRef p9.File) (map[string]p9.Attr, error) { ents, err := p9Readdir(nineRef) @@ -320,6 +326,15 @@ func testCompareTreeModes(t *testing.T, f1, f2 p9.File) { ) return false } + + bSize := baseAttr.Size + tSize := target[path].Size + if bSize != tSize { + t.Fatalf("size for %q doesn't match\nbase:%d\ntarget:%d\n", + path, + bSize, + tSize) + } } return true } diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 4d70ddb0469..98968cb7564 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -2,12 +2,12 @@ package fsnodes import ( "context" - "errors" - "fmt" + gopath "path" + "time" "github.com/djdv/p9/p9" "github.com/djdv/p9/unimplfs" - files "github.com/ipfs/go-ipfs-files" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" @@ -20,129 +20,155 @@ const ( //device - attempts to comply with standard multicodec table var _ p9.File = (*Base)(nil) -// Base is a foundational file system node that provides common file metadata as well as stubs for unimplemented methods +type p9Path = uint64 + +// Base provides a foundation to build file system nodes which contain file meta data +// as well as stubs for unimplemented file system methods type Base struct { // Provide stubs for unimplemented methods unimplfs.NoopFile p9.DefaultWalkGetAttr + Trail []string // FS "breadcrumb" trail from node's root + // Storage for file's metadata Qid p9.QID - meta p9.Attr - metaMask p9.AttrMask - - // The parent context should be set prior to calling attach (usually in a consturctor) - // The Base filesystemCtx is derived from the parent context during Base.Attach - // and should be valid for the lifetime of the file system - // The fs context should be used to derive operation specific contexts from during calls - // A cancel should cascade down, and invalidate all derived contexts - // A call to fs.Close should leave no operations lingering - parentCtx context.Context - filesystemCtx context.Context - filesystemCancel context.CancelFunc - Logger logging.EventLogger + meta *p9.Attr + metaMask *p9.AttrMask + //open bool // should be set to true on Open and checked during Walk; cloning while open is allowed, walking open references is not; walk(5) + Logger logging.EventLogger +} + +func (b *Base) QID() p9.QID { return b.Qid } + +func (b *Base) NinePath() p9Path { return b.Qid.Path } + +func newBase(ops ...nodeopts.AttachOption) Base { + options := nodeopts.AttachOps(ops...) + + return Base{ + Logger: options.Logger, + meta: new(p9.Attr), + metaMask: new(p9.AttrMask), + } +} - parent p9.File // parent must be set, it is used to handle ".." requests; nodes without a parent must point back to themselves - child p9.File // child is an optional field, used to hand off walk requests to another file system - root bool // should be set to true on Attach if parent == self, this triggers the filesystemCancel on Close - open bool // should be set to true on Open and checked during Walk; do not walk open references walk(5) +func (b *Base) Derive() Base { + newFid := newBase(nodeopts.Logger(b.Logger)) + + newFid.Qid = b.Qid + newFid.meta, newFid.metaMask = b.meta, b.metaMask + newFid.Trail = make([]string, len(b.Trail)) + copy(newFid.Trail, b.Trail) + + return newFid } // IPFSBase is much like Base but extends it to hold IPFS specific metadata type IPFSBase struct { Base + OverlayFileMeta - Path corepath.Resolved - core coreiface.CoreAPI + // The parent context should be set prior to `Attach` + //parentCtx context.Context - // you will typically want to derive a context from the Base context within one operation (like Open) + // The file system contex should be derived from the parentCtx during `Attach` + // and is expected to be valid for the lifetime of the file system + filesystemCtx context.Context + // cancel should be called when `Close` is called on the root instance returned from `Attach` + filesystemCancel context.CancelFunc + + // Typically you'll want to derive a context from the fs ctx within one operation (like Open) // use it with the CoreAPI for something (like Get) // and cancel it in another operation (like Close) - // that pointer should be stored here between calls - operationsCancel context.CancelFunc - - // operation handle storage - file files.File - directory *directoryStream + // those pointers should be stored here between operation calls + operationsContext context.Context + operationsCancel context.CancelFunc + + // Format the namespace as if it were a rooted directory, sans trailing slash + // e.g. `/ipfs` + // the base relative path is appended to the namespace for core requests upon calling `.CorePath()` + coreNamespace string + core coreiface.CoreAPI } -// Base Attach should be called by all supersets during their Attach -// to initialize the file system context -func (b *Base) Attach() (p9.File, error) { - if b.parentCtx == nil { - return nil, errors.New("Parent context was not set, no way to derive") - } - - select { - case <-b.parentCtx.Done(): - return nil, fmt.Errorf("Parent context is done: %s", b.parentCtx.Err()) - default: - break - } +func (b *Base) StringPath() string { + return gopath.Join(b.Trail...) +} - b.filesystemCtx, b.filesystemCancel = context.WithCancel(b.parentCtx) +func (ib *IPFSBase) StringPath() string { + return gopath.Join(append([]string{ib.coreNamespace}, ib.Base.StringPath())...) +} - return b, nil +func (ib *IPFSBase) CorePath(names ...string) corepath.Path { + return corepath.Join(rootPath(ib.coreNamespace), append(ib.Trail, names...)...) } -// Base Close should be called in all superset Close methods in order to -// close child subsystems and cancel the file system context -func (b *Base) Close() error { - b.Logger.Debugf("closing: {%v}", b.Qid.Path) +//func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.FileMode, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { +func newIPFSBase(ctx context.Context, coreNamespace string, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { + options := nodeopts.AttachOps(ops...) - if b.root { - b.filesystemCancel() + base := IPFSBase{ + Base: newBase(ops...), + coreNamespace: coreNamespace, + core: core, } + base.filesystemCtx, base.filesystemCancel = context.WithCancel(ctx) - var err error - if b.child != nil { - if err = b.child.Close(); err != nil { - b.Logger.Error(err) + if options.Parent != nil { // parent is optional + parentRef, ok := options.Parent.(walkRef) // interface is not + if !ok { + panic("parent node lacks overlay traversal methods") } + base.OverlayFileMeta.parent = parentRef } - - return err + return base } -func (ib *IPFSBase) Close() error { - ib.Logger.Debugf("closing:{%v}%q", ib.Qid, ib.Path.String()) - var lastErr error - if ib.operationsCancel != nil { - ib.operationsCancel() - } - - if err := ib.Base.Close(); err != nil { - ib.Logger.Error(err) - lastErr = err +func (ib *IPFSBase) Derive() IPFSBase { + newFid := IPFSBase{ + Base: ib.Base.Derive(), + OverlayFileMeta: ib.OverlayFileMeta, + coreNamespace: ib.coreNamespace, + core: ib.core, } + newFid.filesystemCtx, newFid.filesystemCancel = context.WithCancel(ib.filesystemCtx) - if ib.file != nil { - if err := ib.file.Close(); err != nil { - ib.Logger.Error(err) - lastErr = err - } - } + return newFid +} - return lastErr +func (b *IPFSBase) Flush() error { + b.Logger.Debugf("flushing: {%d}%q", b.Qid.Path, b.StringPath()) + return nil } -type directoryStream struct { - entryChan <-chan coreiface.DirEntry - cursor uint64 - eos bool // have seen end of stream? - err error +func (b *Base) Close() error { + b.Logger.Debugf("closing: {%d}%q", b.Qid.Path, b.StringPath()) + return nil } -type walkRef interface { - p9.File - Parent() p9.File - Child() p9.File +func (ib *IPFSBase) Close() error { + ib.Logger.Debugf("closing: {%d}%q", ib.Qid.Path, ib.StringPath()) + + var err error + if ib.filesystemCancel != nil { + if ib.proxy != nil { + if err = ib.proxy.Close(); err != nil { + ib.Logger.Error(err) + } + } + ib.filesystemCancel() + } + + return err } -func (b *Base) Parent() p9.File { - return b.parent +func (b *Base) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + b.Logger.Debugf("GetAttr") + + return b.Qid, *b.metaMask, *b.meta, nil } -func (b *Base) Child() p9.File { - return b.child +func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { + return context.WithTimeout(b.filesystemCtx, 30*time.Second) } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index c88985974e0..333158f3e07 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -4,13 +4,10 @@ import ( "context" "fmt" "io" - "time" "github.com/djdv/p9/p9" files "github.com/ipfs/go-ipfs-files" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" - ipld "github.com/ipfs/go-ipld-format" - logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -23,21 +20,21 @@ var _ walkRef = (*IPFS)(nil) // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { IPFSBase + IPFSFileMeta } -func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.Option) p9.Attacher { - id := &IPFS{IPFSBase: newIPFSBase(ctx, rootPath("/ipfs"), p9.TypeDir, - core, logging.Logger("IPFS"))} - id.meta, id.metaMask = defaultRootAttr() - - options := nodeopts.AttachOps(ops...) - if options.Parent != nil { - id.parent = options.Parent - } else { - id.parent = id - id.root = true - } - return id +type IPFSFileMeta struct { + // operation handle storage + file files.File + directory *directoryStream +} + +func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { + return &IPFS{IPFSBase: newIPFSBase(ctx, "/ipfs", core, ops...)} +} + +func (id *IPFS) Derive() walkRef { + return &IPFS{IPFSBase: id.IPFSBase.Derive()} } func (id *IPFS) Attach() (p9.File, error) { @@ -46,110 +43,71 @@ func (id *IPFS) Attach() (p9.File, error) { newFid := new(IPFS) *newFid = *id - _, err := newFid.Base.Attach() - if err != nil { - return nil, err - } - return newFid, nil } -func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - id.Logger.Debugf("GetAttr path: %v", id.Path) - - // For IPFS, we set this up front in Walk - return id.Qid, id.metaMask, id.meta, nil -} - -func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { - id.Logger.Debugf("Walk names %v", names) - id.Logger.Debugf("Walk myself: %s:%v", id.Path, id.Qid) - - if id.open { - return nil, nil, errWalkOpened +func coreAttr(ctx context.Context, attr *p9.Attr, path corepath.Resolved, core coreiface.CoreAPI, req p9.AttrMask) (p9.AttrMask, error) { + ipldNode, err := core.Dag().Get(ctx, path.Cid()) + if err != nil { + return p9.AttrMask{}, err } - newFid := new(IPFS) - *newFid = *id - - if shouldClone(names, newFid.root) { - id.Logger.Debugf("Walk cloned") - return []p9.QID{newFid.Qid}, newFid, nil - } + return ipldStat(ctx, attr, ipldNode, req) +} - newFid.root = false +func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + id.Logger.Debugf("GetAttr") + // TODO: we need to re use storage on id + // merge existing members with request members fetched via request + // rather than generating a new copy each time var ( - // returned - qids = make([]p9.QID, 0, len(names)) - // temporary - requestType = p9.AttrMask{Mode: true} - err error - // last-set value is used - ipldNode ipld.Node + attr p9.Attr + filled p9.AttrMask + callCtx, cancel = id.callCtx() ) + defer cancel() - for _, name := range names { - callCtx, cancel := context.WithTimeout(newFid.filesystemCtx, 30*time.Second) - - corePath, err := newFid.core.ResolvePath(callCtx, corepath.Join(newFid.Path, name)) - if err != nil { - cancel() - //TODO: switch off error, return appropriate errno (ENOENT is going to be common here) - // ref: https://github.com/hugelgupf/p9/pull/12#discussion_r324991695 - return qids, nil, err - } - - newFid.Path = corePath + corePath, err := id.core.ResolvePath(callCtx, id.CorePath()) + if err != nil { + return id.Qid, filled, attr, err + } - ipldNode, err = newFid.core.Dag().Get(callCtx, newFid.Path.Cid()) - if err != nil { - cancel() - return qids, nil, err - } + filled, err = coreAttr(callCtx, &attr, corePath, id.core, req) + if err != nil { + id.Logger.Error(err) + return id.Qid, filled, attr, err + } - attr := &p9.Attr{} - err, _ = ipldStat(callCtx, attr, ipldNode, requestType) - if err != nil { - cancel() - return qids, nil, err - } + if req.RDev { + attr.RDev, filled.RDev = dIPFS, true + } - newFid.Qid.Type = attr.Mode.QIDType() - newFid.Qid.Path = cidToQPath(ipldNode.Cid()) - qids = append(qids, newFid.Qid) + attr.Mode |= IRXA - cancel() - } + return id.Qid, filled, attr, err +} - // stat up front for our returned file - callCtx, cancel := context.WithTimeout(newFid.filesystemCtx, 30*time.Second) - defer cancel() - meta := &p9.Attr{} - err, filled := ipldStat(callCtx, meta, ipldNode, p9.AttrMaskAll) - if err != nil { - return qids, nil, err - } - newFid.meta = *meta - newFid.metaMask = filled - newFid.meta.Mode |= IRXA - newFid.meta.RDev, newFid.metaMask.RDev = dIPFS, true +func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { + id.Logger.Debugf("Walk names: %v", names) + id.Logger.Debugf("Walk myself: %q:{%d}", id.StringPath(), id.NinePath()) - return qids, newFid, err + return walker(id, names) } func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - id.Logger.Debugf("Open %q", id.Path) + id.Logger.Debugf("Open: %s", id.StringPath()) // set up handle amenities - var handleContext context.Context - handleContext, id.operationsCancel = context.WithCancel(id.filesystemCtx) + //var handleContext context.Context + //handleContext, id.operationsCancel = context.WithCancel(id.filesystemCtx) // handle directories - if id.meta.Mode.IsDir() { //FIXME: meta is not being set anywhere - c, err := id.core.Unixfs().Ls(handleContext, id.Path) + if id.Qid.Type == p9.TypeDir { + // c, err := id.core.Unixfs().Ls(handleContext, id.CorePath()) + c, err := id.core.Unixfs().Ls(id.filesystemCtx, id.CorePath()) if err != nil { - id.operationsCancel() + //id.operationsCancel() return id.Qid, 0, err } @@ -160,27 +118,33 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } // handle files - apiNode, err := id.core.Unixfs().Get(handleContext, id.Path) + apiNode, err := id.core.Unixfs().Get(id.filesystemCtx, id.CorePath()) if err != nil { - id.operationsCancel() + //id.operationsCancel() return id.Qid, 0, err } fileNode, ok := apiNode.(files.File) if !ok { - id.operationsCancel() - return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.Path.String(), apiNode) + //id.operationsCancel() + return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.StringPath(), apiNode) } id.file = fileNode + s, err := id.file.Size() + if err != nil { + //id.operationsCancel() + return id.Qid, 0, err + } + id.meta.Size, id.metaMask.Size = uint64(s), true return id.Qid, ipfsBlockSize, nil } func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - id.Logger.Debugf("Readdir %d %d", offset, count) + id.Logger.Debugf("Readdir %q %d %d", id.StringPath(), offset, count) if id.directory == nil { - return nil, fmt.Errorf("directory %q is not open for reading", id.Path.String()) + return nil, fmt.Errorf("directory %q is not open for reading", id.StringPath()) } if id.directory.err != nil { // previous request must have failed @@ -226,6 +190,7 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { case <-id.filesystemCtx.Done(): id.directory.err = id.filesystemCtx.Err() + id.Logger.Error(id.directory.err) return ents, id.directory.err } } @@ -239,11 +204,11 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { readAtFmt = "ReadAt {%d/%d}%q" readAtFmtErr = readAtFmt + ": %s" ) - //id.Logger.Debugf(readAtFmt, offset, id.meta.Size, id.Path.String()) + //id.Logger.Debugf(readAtFmt, offset, id.meta.Size, id.StringPath()) if id.file == nil { - err := fmt.Errorf("file %q is not open for reading", id.Path.String()) - id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.Path.String(), err) + err := fmt.Errorf("file is not open for reading") + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.StringPath(), err) return 0, err } @@ -252,20 +217,108 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { return 0, io.EOF } - //FIXME: for some reason, the internal CoreAPI context is being canceled - // and it breaks everything if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { - id.operationsCancel() - id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.Path.String(), err) + //id.operationsCancel() + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.StringPath(), err) return 0, err } readBytes, err := id.file.Read(p) if err != nil && err != io.EOF { - id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.Path.String(), err) - id.operationsCancel() + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.StringPath(), err) + //id.operationsCancel() return 0, err } return readBytes, err } + +func (id *IPFS) Close() error { + lastErr := id.IPFSBase.Close() + if lastErr != nil { + id.Logger.Error(lastErr) + } + + if id.file != nil { + if err := id.file.Close(); err != nil { + id.Logger.Error(err) + lastErr = err + } + id.file = nil + } + id.directory = nil + + return lastErr +} + +func coreToQid(ctx context.Context, path corepath.Path, core coreiface.CoreAPI) (p9.QID, error) { + var qid p9.QID + // translate from abstract path to CoreAPI resolved path + resolvedPath, err := core.ResolvePath(ctx, path) + if err != nil { + return qid, err + } + + // inspected to derive 9P QID + attr := new(p9.Attr) + _, err = coreAttr(ctx, attr, resolvedPath, core, p9.AttrMask{Mode: true}) + if err != nil { + return qid, err + } + + qid.Type = attr.Mode.QIDType() + qid.Path = cidToQPath(resolvedPath.Cid()) + return qid, nil +} + +// IPFS appends "name" to its current path, and returns itself +func (id *IPFS) Step(name string) (walkRef, error) { + callCtx, cancel := id.callCtx() + defer cancel() + + qid, err := coreToQid(callCtx, id.CorePath(name), id.core) + if err != nil { + return nil, err + } + + // we stepped successfully, so set up a newFid to return + newFid := id.Derive().(*IPFS) + newFid.Trail = append(newFid.Trail, name) + newFid.Qid = qid + + return newFid, nil +} + +func (id *IPFS) Backtrack() (walkRef, error) { + // if we're the root + if len(id.Trail) == 0 { + // return our parent, or ourselves if we don't have one + if id.parent != nil { + return id.parent, nil + } + return id, nil + } + + // otherwise step back + tLen := len(id.Trail) + breadCrumb := make([]string, tLen) + copy(breadCrumb, id.Trail) + + id.Trail = id.Trail[:tLen-1] + + // reset QID + callCtx, cancel := id.callCtx() + defer cancel() + + qid, err := coreToQid(callCtx, id.CorePath(), id.core) + if err != nil { + // recover path on failure + id.Trail = breadCrumb + return nil, err + } + + // set on success; we stepped back + id.Qid = qid + + return id, nil +} diff --git a/plugin/plugins/filesystem/nodes/mfs.go b/plugin/plugins/filesystem/nodes/mfs.go new file mode 100644 index 00000000000..31368b1e5da --- /dev/null +++ b/plugin/plugins/filesystem/nodes/mfs.go @@ -0,0 +1,438 @@ +package fsnodes + +import ( + "context" + "fmt" + "io" + gopath "path" + "time" + + "github.com/djdv/p9/p9" + cid "github.com/ipfs/go-cid" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + dag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-mfs" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +var _ p9.File = (*MFS)(nil) +var _ walkRef = (*MFS)(nil) + +// TODO: break this up into 2 file systems? +// MFS + MFS Overlay? +// TODO: docs +type MFS struct { + IPFSBase + MFSFileMeta + + //ref uint //TODO: rename, root refcount + //key coreiface.Key // optional value, if set, publish to IPNS key on MFS change + //roots map[string]*mfs.Root //share the same mfs root across calls + mroot *mfs.Root +} + +type MFSFileMeta struct { + //TODO: re-use base path for this + //relativePath string + //rootName string + file mfs.FileDescriptor + directory *mfs.Directory +} + +func MFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { + options := nodeopts.AttachOps(ops...) + + mroot, err := cidToMFSRoot(ctx, options.MFSRoot, core, options.MFSPublish) + if err != nil { + panic(err) + } + + md := &MFS{ + IPFSBase: newIPFSBase(ctx, "/ipld", core, ops...), + mroot: mroot, + } + md.meta.Mode, md.metaMask.Mode = p9.ModeDirectory|IRXA|0220, true + + return md +} + +func (md *MFS) Derive() walkRef { + newFid := &MFS{ + IPFSBase: md.IPFSBase.Derive(), + //MFSFileMeta: md.MFSFileMeta, + mroot: md.mroot, + } + + return newFid +} + +func (md *MFS) Attach() (p9.File, error) { + md.Logger.Debugf("Attach") + + newFid := new(MFS) + *newFid = *md + + return newFid, nil +} + +func (md *MFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + md.Logger.Debugf("GetAttr path: %s", md.StringPath()) + md.Logger.Debugf("%p", md) + + callCtx, cancel := md.callCtx() + defer cancel() + + filled, err := mfsAttr(callCtx, md.meta, md.mroot, p9.AttrMaskAll, md.Trail...) + if err != nil { + return md.Qid, filled, *md.meta, err + } + + md.meta.Mode |= IRXA | 0220 + if req.RDev { + md.meta.RDev, filled.RDev = dMemory, true + } + + return md.Qid, filled, *md.meta, nil +} + +func (md *MFS) Walk(names []string) ([]p9.QID, p9.File, error) { + md.Logger.Debugf("Walk names: %v", names) + md.Logger.Debugf("Walk myself: %q:{%d}", md.StringPath(), md.NinePath()) + + md.Logger.Debugf("%p", md) + return walker(md, names) +} + +func (md *MFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + md.Logger.Debugf("Open %q {Mode:%v OSFlags:%v, String:%s}", md.StringPath(), mode.Mode(), mode.OSFlags(), mode.String()) + md.Logger.Debugf("%p", md) + + if md.mroot == nil { + return md.Qid, 0, fmt.Errorf("TODO: message; root not set") + } + + //TODO: current: lookup -> mfsnoode -> md.file | md.directory = mfsNode.open(fsctx) + + mNode, err := mfs.Lookup(md.mroot, gopath.Join(md.Trail...)) + if err != nil { + return md.Qid, 0, err + } + + // handle directories + if md.meta.Mode.IsDir() { + dir, ok := mNode.(*mfs.Directory) + if !ok { + return md.Qid, 0, fmt.Errorf("type mismatch %q is %T not a directory", md.StringPath(), mNode) + } + md.directory = dir + } else { + mFile, ok := mNode.(*mfs.File) + if !ok { + return md.Qid, 0, fmt.Errorf("type mismatch %q is %T not a file", md.StringPath(), mNode) + } + + openFile, err := mFile.Open(mfs.Flags{Read: true, Write: true}) + if err != nil { + return md.Qid, 0, err + } + s, err := openFile.Size() + if err != nil { + return md.Qid, 0, err + } + + md.file = openFile + md.meta.Size, md.metaMask.Size = uint64(s), true + } + + return md.Qid, ipfsBlockSize, nil +} + +func (md *MFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + md.Logger.Debugf("Readdir %d %d", offset, count) + + if md.directory == nil { + return nil, fmt.Errorf("directory %q is not open for reading", md.StringPath()) + } + + //TODO: resetable context; for { ...; ctx.reset() } + callCtx, cancel := context.WithCancel(md.filesystemCtx) + defer cancel() + + ents := make([]p9.Dirent, 0) + + var index uint64 + var done bool + err := md.directory.ForEachEntry(callCtx, func(nl mfs.NodeListing) error { + if done { + cancel() + return nil + } + + if index < offset { + index++ //skip + return nil + } + + ent, err := mfsEntTo9Ent(nl) + if err != nil { + return err + } + + ent.Offset = index + 1 + + ents = append(ents, ent) + if len(ents) == int(count) { + done = true + } + + index++ + return nil + }) + + return ents, err +} + +func (md *MFS) ReadAt(p []byte, offset uint64) (int, error) { + const ( + readAtFmt = "ReadAt {%d/%d}%q" + readAtFmtErr = readAtFmt + ": %s" + ) + + if md.file == nil { + err := fmt.Errorf("file is not open for reading") + md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) + return 0, err + } + + if offset >= md.meta.Size { + //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. + return 0, io.EOF + } + + if _, err := md.file.Seek(int64(offset), io.SeekStart); err != nil { + md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) + return 0, err + } + + //TODO: remove, debug + + nbytes, err := md.file.Read(p) + if err != nil { + md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) + } + + return nbytes, err + // + + //return md.file.Read(p) +} + +func (md *MFS) SetAttr(valid p9.SetAttrMask, attr p9.SetAttr) error { + md.Logger.Debugf("SetAttr %v %v", valid, attr) + md.Logger.Debugf("%p", md) + + if valid.Size && attr.Size < md.meta.Size { + if md.file == nil { + err := fmt.Errorf("file %q is not open, cannot change size", md.StringPath()) + md.Logger.Error(err) + return err + } + + if err := md.file.Truncate(int64(attr.Size)); err != nil { + return err + } + } + + md.meta.Apply(valid, attr) + return nil +} + +func (md *MFS) WriteAt(p []byte, offset uint64) (int, error) { + const ( + readAtFmt = "WriteAt {%d/%d}%q" + readAtFmtErr = readAtFmt + ": %s" + ) + if md.file == nil { + err := fmt.Errorf("file is not open for writing") + md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) + return 0, err + } + + //TODO: remove, debug + + nbytes, err := md.file.WriteAt(p, int64(offset)) + if err != nil { + md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) + } + + return nbytes, err + // + + //return md.file.WriteAt(p, int64(offset)) +} + +func (md *MFS) Close() error { + md.Logger.Debugf("closing: %q:{%d}", md.StringPath(), md.Qid.Path) + md.Logger.Debugf("%p", md) + + var lastErr error + if err := md.Base.Close(); err != nil { + md.Logger.Error(err) + lastErr = err + } + + if md.file != nil { + if err := md.file.Close(); err != nil { + md.Logger.Error(err) + lastErr = err + } + } + + return lastErr +} + +/* +{ + Base: { + coreNamespace: `/ipld`, + Trail: []string{"folder", "file.txt"} + } + mroot: fromCid(`QmVuDpaFj55JnUH7UYxTAydx6ayrs2cB3Gb7cdPr61wLv5`) +} +=> +`/ipld/QmVuDpaFj55JnUH7UYxTAydx6ayrs2cB3Gb7cdPr61wLv5/folder/file.txt` +*/ +func (md *MFS) StringPath() string { + rootNode, err := md.mroot.GetDirectory().GetNode() + if err != nil { + panic(err) + } + return gopath.Join(append([]string{md.coreNamespace, rootNode.Cid().String()}, md.Trail...)...) +} + +func (md *MFS) Backtrack() (walkRef, error) { + // if we're the root + if len(md.Trail) == 0 { + // return our parent, or ourselves if we don't have one + if md.parent != nil { + return md.parent, nil + } + return md, nil + } + + // otherwise step back + tLen := len(md.Trail) + breadCrumb := make([]string, tLen) + copy(breadCrumb, md.Trail) + + md.Trail = md.Trail[:tLen-1] + + // reset QID + callCtx, cancel := md.callCtx() + defer cancel() + + qid, err := coreToQid(callCtx, md.CorePath(), md.core) + if err != nil { + // recover path on failure + md.Trail = breadCrumb + return nil, err + } + + // set on success; we stepped back + md.Qid = qid + + return md, nil +} + +func mfsAttr(ctx context.Context, attr *p9.Attr, mroot *mfs.Root, req p9.AttrMask, names ...string) (p9.AttrMask, error) { + mfsNode, err := mfs.Lookup(mroot, gopath.Join(names...)) + if err != nil { + return p9.AttrMask{}, err + } + + ipldNode, err := mfsNode.GetNode() + if err != nil { + return p9.AttrMask{}, err + } + + return ipldStat(ctx, attr, ipldNode, req) +} + +func mfsToQid(ctx context.Context, mroot *mfs.Root, names ...string) (p9.QID, error) { + mNode, err := mfs.Lookup(mroot, gopath.Join(names...)) + if err != nil { + return p9.QID{}, err + } + + t, err := mfsTypeToNineType(mNode.Type()) + if err != nil { + return p9.QID{}, err + } + + ipldNode, err := mNode.GetNode() + if err != nil { + return p9.QID{}, err + } + + return p9.QID{ + Type: t, + Path: cidToQPath(ipldNode.Cid()), + }, nil +} + +func (md *MFS) Step(name string) (walkRef, error) { + callCtx, cancel := md.callCtx() + defer cancel() + + breadCrumb := append(md.Trail, name) + qid, err := mfsToQid(callCtx, md.mroot, breadCrumb...) + if err != nil { + return nil, err + } + + // set on success; we stepped + md.Trail = breadCrumb + md.Qid = qid + return md, nil +} + +/* +func (md *MFS) RootPath(keyName string, components ...string) (corepath.Path, error) { + if keyName == "" { + return nil, fmt.Errorf("no path key was provided") + } + + rootCid, err := cid.Decode(keyName) + if err != nil { + return nil, err + } + + return corepath.Join(corepath.IpldPath(rootCid), components...), nil +} + +func (md *MFS) ResolvedPath(names ...string) (corepath.Path, error) { + callCtx, cancel := md.callCtx() + defer cancel() + + return md.core.ResolvePath(callCtx, md.KeyPath(names[0], names[1:]...)) + + corePath = corepath.IpldPath(md.Tail[0]) + return md.core.ResolvePath(callCtx, corepath.Join(corePath, append(md.Tail[1:], names)...)) +} +*/ + +func cidToMFSRoot(ctx context.Context, rootCid *cid.Cid, core coreiface.CoreAPI, publish mfs.PubFunc) (*mfs.Root, error) { + callCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + ipldNode, err := core.Dag().Get(callCtx, *rootCid) + if err != nil { + return nil, err + } + + pbNode, ok := ipldNode.(*dag.ProtoNode) + if !ok { + return nil, fmt.Errorf("%q has incompatible type %T", rootCid.String(), ipldNode) + } + + return mfs.NewRoot(ctx, core.Dag(), pbNode, publish) +} diff --git a/plugin/plugins/filesystem/nodes/options/options.go b/plugin/plugins/filesystem/nodes/options/options.go index e040aba1a6d..f11be5048cf 100644 --- a/plugin/plugins/filesystem/nodes/options/options.go +++ b/plugin/plugins/filesystem/nodes/options/options.go @@ -3,34 +3,65 @@ package nodeopts import ( "github.com/djdv/p9/p9" "github.com/jbenet/goprocess" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" + "github.com/ipfs/go-mfs" ) -type Options struct { - Parent p9.File - Process goprocess.Process +//TODO: we're doing runtime hacks to check if this is a walkref +// we don't use a walkref because import cycle +// amend this +type AttachOptions struct { + //Parent fsnodes.walkRef // node directly behind self + Parent p9.File // node directly behind self + Logger logging.EventLogger // what subsystem you are + Process goprocess.Process // TODO: I documented this somewhere else + MFSRoot *cid.Cid // required when attaching to MFS + MFSPublish mfs.PubFunc } -type Option func(*Options) +type AttachOption func(*AttachOptions) + +func AttachOps(options ...AttachOption) *AttachOptions { + ops := &AttachOptions{ + Logger: logging.Logger("FS"), + } + for _, op := range options { + op(ops) + } + return ops +} // if NOT provided, we assume the file system is to be treated as a root, assigning itself as a parent -func Parent(p p9.File) Option { - return func(ops *Options) { +func Parent(p p9.File) AttachOption { + return func(ops *AttachOptions) { ops.Parent = p } } +func Logger(l logging.EventLogger) AttachOption { + return func(ops *AttachOptions) { + ops.Logger = l + } +} + //TODO: this isn't true yet // if provided, file systems implemented here will utilize this to create a cascading Close() tree -func Process(p goprocess.Process) Option { - return func(ops *Options) { +func Process(p goprocess.Process) AttachOption { + return func(ops *AttachOptions) { ops.Process = p } } -func AttachOps(options ...Option) *Options { - ops := &Options{} // only nil defaults at this time - for _, op := range options { - op(ops) +func MFSRoot(rcid *cid.Cid) AttachOption { + return func(ops *AttachOptions) { + ops.MFSRoot = rcid + } +} + +func MFSPublish(p mfs.PubFunc) AttachOption { + return func(ops *AttachOptions) { + ops.MFSPublish = p } - return ops } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 11b83ee4213..104cabdc6a2 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -20,83 +20,60 @@ type PinFS struct { ents []p9.Dirent } -func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.Option) p9.Attacher { - pd := &PinFS{IPFSBase: newIPFSBase(ctx, rootPath("/PinFS"), p9.TypeDir, - core, logging.Logger("PinFS"))} - pd.meta, pd.metaMask = defaultRootAttr() - - options := nodeopts.AttachOps(ops...) - if options.Parent != nil { - pd.parent = options.Parent - } else { - pd.parent = pd - pd.root = true +func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { + pd := &PinFS{IPFSBase: newIPFSBase(ctx, "/pinfs", core, ops...)} + pd.Qid.Type = p9.TypeDir + pd.meta.Mode, pd.metaMask.Mode = p9.ModeDirectory|IRXA, true + + // set up our subsystem, used to relay walk names to IPFS + subOpts := []nodeopts.AttachOption{ + nodeopts.Parent(pd), + nodeopts.Logger(logging.Logger("IPFS")), } - return pd -} - -func (pd *PinFS) Attach() (p9.File, error) { - pd.Logger.Debugf("Attach") - - newFid := new(PinFS) - *newFid = *pd - _, err := newFid.Base.Attach() + subsystem, err := IPFSAttacher(ctx, core, subOpts...).Attach() if err != nil { - return nil, err + panic(err) } - opts := []nodeopts.Option{nodeopts.Parent(newFid)} - - subsystem, err := IPFSAttacher(newFid.filesystemCtx, newFid.core, opts...).Attach() - if err != nil { - return nil, fmt.Errorf(errFmtWalkSubsystem, err) - } + pd.proxy = subsystem.(walkRef) - newFid.child = subsystem + return pd +} - return newFid, nil +func (pd *PinFS) Derive() walkRef { + newFid := &PinFS{ + IPFSBase: pd.IPFSBase.Derive(), + } + return newFid } -func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - pd.Logger.Debugf("GetAttr") +func (pd *PinFS) Attach() (p9.File, error) { + pd.Logger.Debugf("Attach") + return pd, nil +} - return pd.Qid, pd.metaMask, pd.meta, nil +// PinFS proxies steps to the IPFS root that was set during construction +func (pd *PinFS) Step(name string) (walkRef, error) { + // derive copy of IPFS root + p := pd.proxy.Derive() + // proxy the request for "name" + return p.Step(name) } func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { pd.Logger.Debugf("Walk names %v", names) pd.Logger.Debugf("Walk myself: %v", pd.Qid) - qids := []p9.QID{pd.Qid} - - if pd.open { - return qids, nil, errWalkOpened - } - - newFid := new(PinFS) - *newFid = *pd - - if shouldClone(names, pd.root) { - pd.Logger.Debugf("Walk cloned") - return qids, newFid, nil - } - - newFid.root = false - - return stepper(newFid, names) + return walker(pd, names) } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") - var handleContext context.Context - handleContext, pd.operationsCancel = context.WithCancel(pd.filesystemCtx) - // IPFS core representation - pins, err := pd.core.Pin().Ls(handleContext, coreoptions.Pin.Type.Recursive()) + pins, err := pd.core.Pin().Ls(pd.filesystemCtx, coreoptions.Pin.Type.Recursive()) if err != nil { - pd.operationsCancel() return pd.Qid, 0, err } @@ -109,13 +86,14 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // actual conversion for i, pin := range pins { - ipldNode, err := pd.core.ResolveNode(handleContext, pin.Path()) + callCtx, cancel := pd.callCtx() + ipldNode, err := pd.core.ResolveNode(callCtx, pin.Path()) if err != nil { - pd.operationsCancel() + cancel() return pd.Qid, 0, err } - if err, _ = ipldStat(handleContext, attr, ipldNode, requestType); err != nil { - pd.operationsCancel() + if _, err = ipldStat(callCtx, attr, ipldNode, requestType); err != nil { + cancel() return pd.Qid, 0, err } @@ -127,6 +105,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { Path: cidToQPath(ipldNode.Cid()), }, }) + cancel() } return pd.Qid, ipfsBlockSize, nil @@ -136,19 +115,32 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { pd.Logger.Debugf("Readdir") if pd.ents == nil { - return nil, fmt.Errorf("directory %q is not open for reading", pd.Path.String()) + return nil, fmt.Errorf("directory %q is not open for reading", pd.StringPath()) } - shouldExit, err := boundCheck(offset, len(pd.ents)) - if shouldExit { - return nil, err - } + return flatReaddir(pd.ents, offset, count) +} - subSlice := pd.ents[offset:] - if len(subSlice) > int(count) { - subSlice = subSlice[:count] +func (pd *PinFS) Backtrack() (walkRef, error) { + // return the parent if we are the root + if len(pd.Trail) == 0 { + return pd.parent, nil } - pd.Logger.Debugf("Readdir returning ents: %v", subSlice) - return subSlice, nil + // otherwise step back + pd.Trail = pd.Trail[1:] + + // reset meta + return pd, nil +} + +func (pd *PinFS) Flush() error { + pd.Logger.Errorf("flushing:%q:%d", pd.StringPath(), pd.NinePath()) + return nil +} + +func (pd *PinFS) Close() error { + + pd.ents = nil + return nil } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 3371c68fc39..9601ea50ee9 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -2,7 +2,6 @@ package fsnodes import ( "context" - "sort" "syscall" "github.com/djdv/p9/p9" @@ -36,7 +35,7 @@ func (rp rootPath) Cid() cid.Cid { } type systemTuple struct { - file p9.File + file walkRef dirent p9.Dirent } @@ -50,127 +49,118 @@ func (ss systemSlice) Less(i, j int) bool { return ss[i].dirent.Offset < ss[j].d // RootIndex is a virtual directory file system, that maps a set of file system implementations to a hierarchy // Currently: "/ipfs":PinFS, "/ipfs/*:IPFS type RootIndex struct { - Base - //subsystems []p9.Dirent + IPFSBase subsystems map[string]systemTuple - core coreiface.CoreAPI } -// RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it -func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.Option) p9.Attacher { - ri := &RootIndex{ - core: core, - Base: Base{ - Logger: logging.Logger("RootFS"), - parentCtx: ctx, - Qid: p9.QID{ - Type: p9.TypeDir, - Path: cidToQPath(rootPath("/").Cid()), - }, - }, - } - ri.meta, ri.metaMask = defaultRootAttr() - - rootDirent := p9.Dirent{ - Type: p9.TypeDir, - QID: p9.QID{Type: p9.TypeDir}, - } +// OverlayFileMeta holds data relevant to file system nodes themselves +type OverlayFileMeta struct { + // parent may be used to send ".." requests to another file system + // during `Backtrack` + parent walkRef + // proxy may be used to send requests to another file system + // during `Step` + proxy walkRef +} - type subattacher func(context.Context, coreiface.CoreAPI, ...nodeopts.Option) p9.Attacher +// RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it +func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { + // construct root node + ri := &RootIndex{IPFSBase: newIPFSBase(ctx, "/", core, ops...)} + ri.Qid.Type = p9.TypeDir + ri.meta.Mode, ri.metaMask.Mode = p9.ModeDirectory|IRXA, true + + // attach to subsystems + // used for proxying walk requests to other filesystems + type subattacher func(context.Context, coreiface.CoreAPI, ...nodeopts.AttachOption) p9.Attacher type attachTuple struct { string subattacher + logging.EventLogger } + + // "mount/bind" table: + // "path"=>filesystem subsystems := [...]attachTuple{ - {"ipfs", PinFSAttacher}, + {"ipfs", PinFSAttacher, logging.Logger("PinFS")}, } + // prealloc what we can ri.subsystems = make(map[string]systemTuple, len(subsystems)) - opts := []nodeopts.Option{nodeopts.Parent(ri)} + opts := []nodeopts.AttachOption{nodeopts.Parent(ri)} + rootDirent := p9.Dirent{ //template + Type: p9.TypeDir, + QID: p9.QID{Type: p9.TypeDir}, + } + // couple the strings to their implementations for i, subsystem := range subsystems { - fs, err := subsystem.subattacher(ctx, core, opts...).Attach() + logOpt := nodeopts.Logger(subsystem.EventLogger) + // the file system implementation + fs, err := subsystem.subattacher(ctx, core, append(opts, logOpt)...).Attach() if err != nil { panic(err) // hard implementation error } + // create a directory entry for it rootDirent.Offset = uint64(i + 1) rootDirent.Name = subsystem.string rootDirent.QID.Path = cidToQPath(rootPath("/" + subsystem.string).Cid()) + // add it as a unionlike thing ri.subsystems[subsystem.string] = systemTuple{ - file: fs, + file: fs.(walkRef), dirent: rootDirent, } } - options := nodeopts.AttachOps(ops...) - if options.Parent != nil { - ri.parent = options.Parent - } else { - ri.parent = ri - ri.root = true - } - return ri } -func (ri *RootIndex) Attach() (p9.File, error) { - ri.Logger.Debugf("Attach") - - newFid := new(RootIndex) - *newFid = *ri - - _, err := newFid.Base.Attach() - if err != nil { - return nil, err +func (ri *RootIndex) Derive() walkRef { + newFid := &RootIndex{ + IPFSBase: ri.IPFSBase.Derive(), + subsystems: ri.subsystems, } - return newFid, nil + return newFid } -func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - ri.Logger.Debugf("GetAttr") - //ri.Logger.Debugf("mask: %v", req) - - return ri.Qid, ri.metaMask, ri.meta, nil +func (ri *RootIndex) Attach() (p9.File, error) { + ri.Logger.Debugf("Attach") + return ri, nil } func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { ri.Logger.Debugf("Walk names %v", names) - ri.Logger.Debugf("Walk myself: %v", ri.Qid) - - qids := []p9.QID{ri.Qid} - - if ri.open { - return qids, nil, errWalkOpened - } - - newFid := new(RootIndex) - *newFid = *ri + ri.Logger.Debugf("Walk myself: %v", ri.Qid.Path) - if shouldClone(names, newFid.root) { - ri.Logger.Debugf("Walk cloned") - return qids, newFid, nil - } - - newFid.root = false + return walker(ri, names) +} - subSys, ok := ri.subsystems[names[0]] +// The RootIndex checks if it has attached to "name" +// derives a node from it, and returns it +func (ri *RootIndex) Step(name string) (walkRef, error) { + // consume fs/access name + subSys, ok := ri.subsystems[name] if !ok { - ri.Logger.Errorf("%q is not provided by us", names[0]) - return nil, nil, syscall.ENOENT //TODO: migrate to platform independent value + ri.Logger.Errorf("%q is not provided by us", name) + return nil, syscall.ENOENT //TODO: migrate to platform independent value } - newFid.child = subSys.file + // create ready to use clone of target + target := subSys.file.Derive() - return stepper(newFid, names[1:]) + // return the proxied fs/resource + return target, nil } func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { ri.Logger.Debugf("Open") - ri.open = true + + // we're always storing entries on the instance + return ri.Qid, 0, nil } @@ -184,33 +174,34 @@ func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, err } - // TODO: [perf] iterating over the map and slot populating an array is probably the least wasted effort - // map => ordered set - orderedSet := make(systemSlice, 0, subs) - for _, pair := range ri.subsystems { - orderedSet = append(orderedSet, pair) - } - sort.Sort(orderedSet) - - offsetIndex := subs - int(offset) + relativeEnd := subs - int(offset) - // n-tuple => singleton - ents := make([]p9.Dirent, 0, offsetIndex) - for _, pair := range orderedSet[offset:] { - ents = append(ents, pair.dirent) + // use the lesser for allocating the slice + var ents []p9.Dirent + if count < uint32(relativeEnd) { + ents = make([]p9.Dirent, count) + } else { + ents = make([]p9.Dirent, relativeEnd) } - // set => trimmed-set - if offsetIndex > int(count) { - return ents[:count], nil + // use ents from map within request bounds to populate slice + for _, pair := range ri.subsystems { + if count == 0 { + break + } + if pair.dirent.Offset >= offset && pair.dirent.Offset <= uint64(relativeEnd) { + ents[pair.dirent.Offset-1] = pair.dirent + count-- + } } return ents, nil } -func (ri *RootIndex) Close() error { - ri.Logger.Debugf("closing: {%v} root", ri.Qid) - err := ri.Base.Close() - ri.open = false - return err +func (ri *RootIndex) Backtrack() (walkRef, error) { + // return our parent, or ourselves if we don't have one + if ri.parent != nil { + return ri.parent, nil + } + return ri, nil } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 2c6ee8c80d9..732ccfad5ef 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -13,10 +13,11 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" + "github.com/ipfs/go-mfs" "github.com/ipfs/go-unixfs" unixpb "github.com/ipfs/go-unixfs/pb" coreiface "github.com/ipfs/interface-go-ipfs-core" - corepath "github.com/ipfs/interface-go-ipfs-core/path" + coreoptions "github.com/ipfs/interface-go-ipfs-core/options" ) const ( @@ -29,15 +30,89 @@ const ( errFmtWalkSubsystem = "could not attach to subsystem: %q" errFmtExternalWalk = "%q does not provide external fs walk methods" + //errFmtKeyNoLongerExists = "key %q was not found in the key store" ) var errWalkOpened = errors.New("this fid is open") +var errKeyNotInStore = errors.New("requested key was not found in the key store") // NOTE [2019.09.12]: QID's have a high collision probability // as a result we add a salt to hashes to attempt to mitigate this // for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 var salt []byte +type directoryStream struct { + entryChan <-chan coreiface.DirEntry + cursor uint64 + eos bool // have seen end of stream? + err error +} + +type walkRef interface { + p9.File + // returns the QID of the reference + QID() p9.QID // implemented on Base + + // returns a derived instance that is ready to step + Derive() walkRef // implemented per filesystem + + // Step should directly modify the node's path, to track its current path + "name" + // and assign the passed in node as its parent + // returns the node at resulting "name" path + Step(name string) (walkRef, error) + + // Backtrack returns a reference to the node's parent + Backtrack() (parentRef walkRef, err error) +} + +func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { + dbgL := logging.Logger("walker") //TODO: remove this or use the reference log, or something + dbgL.Debugf("ref: (%p){%T}", ref, ref) + + qids, cloneRequest := allocQids(names, ref.QID()) + // walk(5) + // It is legal for nwname to be zero, in which case newfid will represent the same file as fid + // and the walk will usually succeed + if cloneRequest { + curRef := ref.Derive() + dbgL.Debugf("cloning: %p -> %p", ref, curRef) + return qids, curRef, nil + } + + curRef := ref + var err error + for _, name := range names { + switch name { + default: + // step forward + curRef, err = curRef.Step(name) + dbgL.Debugf("stepped to: %q", name) + + case ".": + // stay + // qid = qids[len(qids)-1] + // continue + dbgL.Debugf("dot, staying put") + + case "..": + // step back + curRef, err = curRef.Backtrack() + dbgL.Debugf("backtracking") + } + + if err != nil { + dbgL.Error(err) + return qids, nil, err + } + + dbgL.Debugf("currentRef: (%p){%T}", curRef, curRef) + qids = append(qids, curRef.QID()) + } + + dbgL.Debugf("returned ref: (%p){%T}", curRef, curRef) + return qids, curRef, nil +} + func init() { salt = make([]byte, saltSize) _, err := io.ReadFull(rand.Reader, salt) @@ -46,31 +121,31 @@ func init() { } } -func shouldClone(names []string, isRoot bool) bool { +func shouldClone(names []string) bool { switch len(names) { case 0: // empty path return true case 1: // self? pc := names[0] - return pc == "." || pc == "" || (isRoot && pc == "..") + return pc == "." || pc == "" default: return false } } -func ipldStat(ctx context.Context, attr *p9.Attr, node ipld.Node, mask p9.AttrMask) (error, p9.AttrMask) { +func ipldStat(ctx context.Context, attr *p9.Attr, node ipld.Node, mask p9.AttrMask) (p9.AttrMask, error) { var filledAttrs p9.AttrMask ufsNode, err := unixfs.ExtractFSNode(node) if err != nil { - return err, filledAttrs + return filledAttrs, err } if mask.Mode { tBits, err := unixfsTypeTo9Type(ufsNode.Type()) if err != nil { - return err, filledAttrs + return filledAttrs, err } - attr.Mode |= tBits + attr.Mode = tBits filledAttrs.Mode = true } @@ -85,7 +160,7 @@ func ipldStat(ctx context.Context, attr *p9.Attr, node ipld.Node, mask p9.AttrMa //TODO [eventually]: handle time metadata in new UFS format standard - return nil, filledAttrs + return filledAttrs, nil } func cidToQPath(cid cid.Cid) uint64 { @@ -145,6 +220,40 @@ func coreEntTo9Ent(coreEnt coreiface.DirEntry) (p9.Dirent, error) { }, nil } +func mfsTypeToNineType(nt mfs.NodeType) (entType p9.QIDType, err error) { + switch nt { + //mfsEnt.Type; mfs.NodeType(t) { + case mfs.TFile: + entType = p9.TypeRegular + case mfs.TDir: + entType = p9.TypeDir + default: + err = fmt.Errorf("unexpected node type %v", nt) + } + return +} + +func mfsEntTo9Ent(mfsEnt mfs.NodeListing) (p9.Dirent, error) { + pathCid, err := cid.Decode(mfsEnt.Hash) + if err != nil { + return p9.Dirent{}, err + } + + t, err := mfsTypeToNineType(mfs.NodeType(mfsEnt.Type)) + if err != nil { + return p9.Dirent{}, err + } + + return p9.Dirent{ + Name: mfsEnt.Name, + Type: t, + QID: p9.QID{ + Type: t, + Path: cidToQPath(pathCid), + }, + }, nil +} + const ( // pedantic POSIX stuff S_IROTH p9.FileMode = p9.Read S_IWOTH = p9.Write @@ -166,16 +275,6 @@ const ( // pedantic POSIX stuff IRXA = IRWXA &^ (S_IWUSR | S_IWGRP | S_IWOTH) // 0555 ) -func defaultRootAttr() (attr p9.Attr, attrMask p9.AttrMask) { - attr.Mode = p9.ModeDirectory | IRXA - attr.RDev = dMemory - attrMask.Mode = true - attrMask.RDev = true - attrMask.Size = true - //timeStamp(&attr, attrMask) - return attr, attrMask -} - func timeStamp(attr *p9.Attr, mask p9.AttrMask) { now := time.Now() if mask.ATime { @@ -189,19 +288,33 @@ func timeStamp(attr *p9.Attr, mask p9.AttrMask) { } } -func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.QIDType, core coreiface.CoreAPI, logger logging.EventLogger) IPFSBase { - return IPFSBase{ - Path: path, - core: core, - Base: Base{ - parentCtx: ctx, - Logger: logger, - Qid: p9.QID{ - Type: kind, - Path: cidToQPath(path.Cid()), - }, - }, +func offlineAPI(core coreiface.CoreAPI) coreiface.CoreAPI { + oAPI, err := core.WithOptions(coreoptions.Api.Offline(true)) + if err != nil { + panic(err) } + return oAPI +} + +// returns a slice of qid's, and true if this is a clone request +func allocQids(names []string, self p9.QID) ([]p9.QID, bool) { + if shouldClone(names) { + return []p9.QID{self}, true + } + return make([]p9.QID, 0, len(names)), false +} + +func flatReaddir(ents []p9.Dirent, offset uint64, count uint32) ([]p9.Dirent, error) { + shouldExit, err := boundCheck(offset, len(ents)) + if shouldExit { + return nil, err + } + + subSlice := ents[offset:] + if len(subSlice) > int(count) { + subSlice = subSlice[:count] + } + return subSlice, nil } // boundCheck assures operation arguments are valid @@ -217,60 +330,3 @@ func boundCheck(offset uint64, length int) (bool, error) { return false, nil } } - -//TODO: English sanity check; does this make sense without reading the code? -//TODO: we need to throw a lot of path tests at this; the recursive nature of walk->stepper->walk->stepper may be troublesome as well -// stepper acts as a dispatcher for intermediate file systems -// sending path component requests to a separate file system based on the name args -// clone requests (no names) will return a clone of the attached child -func stepper(ref walkRef, names []string) ([]p9.QID, p9.File, error) { - var ( - nextRef p9.File - qids, subQids []p9.QID - curFile p9.File - err error - ok bool - ) - - if len(names) == 0 { - //return qids, nil, errors.New("no subnames were provided") - c := ref.Child() - if c == nil { - return qids, nil, errors.New("clone requested but child file system attached") - } - - return c.Walk(nil) - } - - for len(names) != 0 { - //prepare to step into next component(s) - if names[0] == ".." { // climb to parent / leftward requests - nextRef = ref.Parent() - } else { // descend into remainder / rightward requests - nextRef = ref.Child() - } - if nextRef == nil { - return qids, nil, fmt.Errorf("%q is not attached to another file system", names[0]) - } - - ref, ok = nextRef.(walkRef) //TODO: see if we can implement this without runtime assertion - if !ok { - return qids, nil, fmt.Errorf(errFmtExternalWalk, names[0]) - } - - // attempt the step(s) - if subQids, curFile, err = ref.Walk(names); err != nil { - return qids, nil, err - } - - // we walked forward, prepare for next step(s) - qids = append(qids, subQids...) - names = names[len(subQids):] - - if len(names) != 0 { // leave the last reference alive, for the caller to close - curFile.Close() // we're not referencing this anymore - } - } - - return qids, curFile, nil -} From e79e371aec21ff5daf5e8db87b9376aaff9684f5 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 9 Oct 2019 12:42:20 -0400 Subject: [PATCH 069/102] ipns refactor WIP --- plugin/plugins/filesystem/filesystem_test.go | 2 +- plugin/plugins/filesystem/nodes/base.go | 22 +++--- plugin/plugins/filesystem/nodes/ipns.go | 21 +++++ plugin/plugins/filesystem/nodes/keyfs.go | 80 ++++++++++++++++++++ plugin/plugins/filesystem/nodes/pinfs.go | 1 - plugin/plugins/filesystem/nodes/root.go | 1 + 6 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 plugin/plugins/filesystem/nodes/ipns.go create mode 100644 plugin/plugins/filesystem/nodes/keyfs.go diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index bedab1edbeb..2eed29a853a 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -29,7 +29,7 @@ var ( Mode: true, Size: true, } - rootSubsystems = []string{"ipfs"} + rootSubsystems = []string{"ipfs", "ipns"} ) func TestAll(t *testing.T) { diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 98968cb7564..3d7a0bf1d15 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -35,8 +35,7 @@ type Base struct { Qid p9.QID meta *p9.Attr metaMask *p9.AttrMask - //open bool // should be set to true on Open and checked during Walk; cloning while open is allowed, walking open references is not; walk(5) - Logger logging.EventLogger + Logger logging.EventLogger } func (b *Base) QID() p9.QID { return b.Qid } @@ -69,21 +68,26 @@ type IPFSBase struct { Base OverlayFileMeta - // The parent context should be set prior to `Attach` - //parentCtx context.Context + /* For file systems, + this context should be set prior to `Attach` - // The file system contex should be derived from the parentCtx during `Attach` - // and is expected to be valid for the lifetime of the file system + For files, + this context should be overwritten with a context derived from the existing fs context + during `Walk` + + The context is expected to be valid for the lifetime of the file system / file respectively + to be used during operations, such as `Walk`, `Open`, `Read` etc. + */ filesystemCtx context.Context - // cancel should be called when `Close` is called on the root instance returned from `Attach` + // cancel should be called upon `Close` + // closing a file system returned from `Attach` + // or a derived file previously returned from `Walk` filesystemCancel context.CancelFunc // Typically you'll want to derive a context from the fs ctx within one operation (like Open) // use it with the CoreAPI for something (like Get) // and cancel it in another operation (like Close) // those pointers should be stored here between operation calls - operationsContext context.Context - operationsCancel context.CancelFunc // Format the namespace as if it were a rooted directory, sans trailing slash // e.g. `/ipfs` diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go new file mode 100644 index 00000000000..be020ae22a8 --- /dev/null +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -0,0 +1,21 @@ +package fsnodes + +import ( + "context" + + "github.com/djdv/p9/p9" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +var _ p9.File = (*IPNS)(nil) +var _ walkRef = (*IPNS)(nil) + +// IPNS exposes the IPNS API over a p9.File interface +// Walk does not expect a namespace, only its path argument +// e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipns", "Qm...", "subdir")` +type IPNS = IPFS + +func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { + return &IPNS{IPFSBase: newIPFSBase(ctx, "/ipns", core, ops...)} +} diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go new file mode 100644 index 00000000000..3d212bbc09d --- /dev/null +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -0,0 +1,80 @@ +package fsnodes + +import ( + "context" + "fmt" + + "github.com/djdv/p9/p9" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + logging "github.com/ipfs/go-log" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +var _ p9.File = (*KeyFS)(nil) +var _ walkRef = (*KeyFS)(nil) + +type KeyFS struct { + IPFSBase +} + +func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { + kd := &KeyFS{IPFSBase: newIPFSBase(ctx, "/keyfs", core, ops...)} + kd.Qid.Type = p9.TypeDir + kd.meta.Mode, kd.metaMask.Mode = p9.ModeDirectory|IRXA|0220, true + + // non-keyed requests fall through to IPNS + opts := []nodeopts.AttachOption{ + nodeopts.Parent(kd), + nodeopts.Logger(logging.Logger("IPNS")), + } + + subsystem, err := IPNSAttacher(ctx, core, opts...).Attach() + if err != nil { + panic(fmt.Errorf(errFmtWalkSubsystem, err)) + } + + kd.proxy = subsystem.(walkRef) + + return kd +} + +func (kd *KeyFS) Derive() walkRef { + newFid := &KeyFS{IPFSBase: kd.IPFSBase.Derive()} + return newFid +} + +func (kd *KeyFS) Attach() (p9.File, error) { + kd.Logger.Debugf("Attach") + return kd, nil +} + +func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { + kd.Logger.Debugf("Walk names %v", names) + kd.Logger.Debugf("Walk myself: %v", kd.Qid) + + return walker(kd, names) +} + +func (kd *KeyFS) Step(keyName string) (walkRef, error) { + // derive copy of IPFS root + p := kd.proxy.Derive() + // proxy the request for "keyName" + return p.Step(keyName) +} + +func (kd *KeyFS) Backtrack() (walkRef, error) { + // return our parent, or ourselves if we don't have one + if kd.parent != nil { + return kd.parent, nil + } + return kd, nil +} + +// temporary stubs to allow forwarding requests on empty directory +func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + kd.Logger.Debugf("Open") + return kd.Qid, 0, nil +} +func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + return nil, nil +} diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 104cabdc6a2..2ada099e864 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -140,7 +140,6 @@ func (pd *PinFS) Flush() error { } func (pd *PinFS) Close() error { - pd.ents = nil return nil } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 9601ea50ee9..6a1f9109084 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -83,6 +83,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A // "path"=>filesystem subsystems := [...]attachTuple{ {"ipfs", PinFSAttacher, logging.Logger("PinFS")}, + {"ipns", KeyFSAttacher, logging.Logger("KeyFS")}, } // prealloc what we can From d1a1170570d5fda01d222b706257d895304f03a0 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 9 Oct 2019 13:40:45 -0400 Subject: [PATCH 070/102] change/fix node derive logic --- plugin/plugins/filesystem/nodes/base.go | 7 ++++++- plugin/plugins/filesystem/nodes/keyfs.go | 10 ++-------- plugin/plugins/filesystem/nodes/pinfs.go | 6 ++---- plugin/plugins/filesystem/nodes/root.go | 15 ++------------ plugin/plugins/filesystem/nodes/utils.go | 25 +++++++++--------------- 5 files changed, 21 insertions(+), 42 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 3d7a0bf1d15..487ef7bcef8 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -168,7 +168,7 @@ func (ib *IPFSBase) Close() error { } func (b *Base) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - b.Logger.Debugf("GetAttr") + b.Logger.Debugf("GetAttr {%d}:%q", b.Qid.Path, b.StringPath()) return b.Qid, *b.metaMask, *b.meta, nil } @@ -176,3 +176,8 @@ func (b *Base) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { return context.WithTimeout(b.filesystemCtx, 30*time.Second) } + +func (b *IPFSBase) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { + b.Logger.Debugf("Open") + return b.Qid, 0, nil +} diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 3d212bbc09d..6f7bdcfe1f7 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -56,10 +56,8 @@ func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { } func (kd *KeyFS) Step(keyName string) (walkRef, error) { - // derive copy of IPFS root - p := kd.proxy.Derive() // proxy the request for "keyName" - return p.Step(keyName) + return kd.proxy.Step(keyName) } func (kd *KeyFS) Backtrack() (walkRef, error) { @@ -70,11 +68,7 @@ func (kd *KeyFS) Backtrack() (walkRef, error) { return kd, nil } -// temporary stubs to allow forwarding requests on empty directory -func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - kd.Logger.Debugf("Open") - return kd.Qid, 0, nil -} +// temporary stub to allow forwarding requests on empty directory func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, nil } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 2ada099e864..04804195031 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -55,10 +55,8 @@ func (pd *PinFS) Attach() (p9.File, error) { // PinFS proxies steps to the IPFS root that was set during construction func (pd *PinFS) Step(name string) (walkRef, error) { - // derive copy of IPFS root - p := pd.proxy.Derive() - // proxy the request for "name" - return p.Step(name) + // proxy the request for "name" to IPFS root (set on us during construction) + return pd.proxy.Step(name) } func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 6a1f9109084..137cd22e3c6 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -150,19 +150,8 @@ func (ri *RootIndex) Step(name string) (walkRef, error) { return nil, syscall.ENOENT //TODO: migrate to platform independent value } - // create ready to use clone of target - target := subSys.file.Derive() - - // return the proxied fs/resource - return target, nil -} - -func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - ri.Logger.Debugf("Open") - - // we're always storing entries on the instance - - return ri.Qid, 0, nil + // return a ready to use derivative of it + return subSys.file.Derive(), nil } func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 732ccfad5ef..37e3dbd0738 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -56,11 +56,10 @@ type walkRef interface { // returns a derived instance that is ready to step Derive() walkRef // implemented per filesystem - // Step should directly modify the node's path, to track its current path + "name" - // and assign the passed in node as its parent - // returns the node at resulting "name" path + // Step should try to step to "name" and return a ready to use derivative of the result Step(name string) (walkRef, error) + //TODO: implement on base class // Backtrack returns a reference to the node's parent Backtrack() (parentRef walkRef, err error) } @@ -69,17 +68,19 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { dbgL := logging.Logger("walker") //TODO: remove this or use the reference log, or something dbgL.Debugf("ref: (%p){%T}", ref, ref) - qids, cloneRequest := allocQids(names, ref.QID()) + curRef := ref.Derive() + // walk(5) // It is legal for nwname to be zero, in which case newfid will represent the same file as fid // and the walk will usually succeed - if cloneRequest { - curRef := ref.Derive() + if shouldClone(names) { dbgL.Debugf("cloning: %p -> %p", ref, curRef) - return qids, curRef, nil + return []p9.QID{curRef.QID()}, curRef, nil } - curRef := ref + qids := make([]p9.QID, 0, len(names)) + //qids = append(qids, curRef.QID()) + var err error for _, name := range names { switch name { @@ -296,14 +297,6 @@ func offlineAPI(core coreiface.CoreAPI) coreiface.CoreAPI { return oAPI } -// returns a slice of qid's, and true if this is a clone request -func allocQids(names []string, self p9.QID) ([]p9.QID, bool) { - if shouldClone(names) { - return []p9.QID{self}, true - } - return make([]p9.QID, 0, len(names)), false -} - func flatReaddir(ents []p9.Dirent, offset uint64, count uint32) ([]p9.Dirent, error) { shouldExit, err := boundCheck(offset, len(ents)) if shouldExit { From be83005ad7cac6754bb0efaf5f97c0841124c2b0 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 9 Oct 2019 14:40:59 -0400 Subject: [PATCH 071/102] update experimental doc --- docs/experimental-features.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 0004a8d08fb..2a8941e57da 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -717,9 +717,8 @@ This daemon plugin wraps the IPFS node and exposes file system services over a m Currently we use the 9P2000.L protocol, and offer read-only support for `/ipfs` requests. With support for other (writable) systems coming next. You may connect to this service using the [`v9fs`](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/filesystems/9p.txt) client used in the Linux kernel. -Of the `v9fs` access modes, we currently only support `any` mode. -And by default we listen for requests on a Unix domain socket. -`mount -t 9p -o trans=unix,access=any $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` +By default we listen for requests on a Unix domain socket. +`mount -t 9p -o trans=unix $IPFS_PATH/filesystem.9p.sock ~/ipfs-mount` To configure the listening address and more, see the [package documentation](https://godoc.org/github.com/ipfs/go-ipfs/plugin/plugins/filesystem) See the [v9fs documentation](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/Documentation/filesystems/9p.txt) for instructions on using transports, such as TCP. From fbac8981fbfc2a75b1fb99d8d7bd003e6d950752 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 9 Oct 2019 14:42:18 -0400 Subject: [PATCH 072/102] linting --- plugin/plugins/filesystem/nodes/keyfs.go | 17 +++++++++-------- plugin/plugins/filesystem/nodes/utils.go | 13 ++++--------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 6f7bdcfe1f7..9708136a9af 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -2,7 +2,6 @@ package fsnodes import ( "context" - "fmt" "github.com/djdv/p9/p9" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" @@ -30,7 +29,7 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. subsystem, err := IPNSAttacher(ctx, core, opts...).Attach() if err != nil { - panic(fmt.Errorf(errFmtWalkSubsystem, err)) + panic(err) } kd.proxy = subsystem.(walkRef) @@ -39,7 +38,9 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. } func (kd *KeyFS) Derive() walkRef { - newFid := &KeyFS{IPFSBase: kd.IPFSBase.Derive()} + newFid := &KeyFS{ + IPFSBase: kd.IPFSBase.Derive(), + } return newFid } @@ -48,6 +49,11 @@ func (kd *KeyFS) Attach() (p9.File, error) { return kd, nil } +func (kd *KeyFS) Step(keyName string) (walkRef, error) { + // proxy the request for "keyName" to IPFS root (set on us during construction) + return kd.proxy.Step(keyName) +} + func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { kd.Logger.Debugf("Walk names %v", names) kd.Logger.Debugf("Walk myself: %v", kd.Qid) @@ -55,11 +61,6 @@ func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { return walker(kd, names) } -func (kd *KeyFS) Step(keyName string) (walkRef, error) { - // proxy the request for "keyName" - return kd.proxy.Step(keyName) -} - func (kd *KeyFS) Backtrack() (walkRef, error) { // return our parent, or ourselves if we don't have one if kd.parent != nil { diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 37e3dbd0738..4a66929b2bf 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -27,10 +27,6 @@ const ( // context: https://github.com/ipfs/go-ipfs/pull/6612/files#r322989041 ipfsBlockSize = 256 << 10 saltSize = 32 - - errFmtWalkSubsystem = "could not attach to subsystem: %q" - errFmtExternalWalk = "%q does not provide external fs walk methods" - //errFmtKeyNoLongerExists = "key %q was not found in the key store" ) var errWalkOpened = errors.New("this fid is open") @@ -56,7 +52,7 @@ type walkRef interface { // returns a derived instance that is ready to step Derive() walkRef // implemented per filesystem - // Step should try to step to "name" and return a ready to use derivative of the result + // Step should try to step to "name", resulting in a ready to use derivative Step(name string) (walkRef, error) //TODO: implement on base class @@ -79,26 +75,25 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { } qids := make([]p9.QID, 0, len(names)) - //qids = append(qids, curRef.QID()) var err error for _, name := range names { switch name { default: // step forward + dbgL.Debugf("stepping to: %q", name) curRef, err = curRef.Step(name) - dbgL.Debugf("stepped to: %q", name) case ".": // stay // qid = qids[len(qids)-1] // continue - dbgL.Debugf("dot, staying put") + dbgL.Debugf(`staying put: "."`) case "..": // step back + dbgL.Debugf(`backtracking to: ".."`) curRef, err = curRef.Backtrack() - dbgL.Debugf("backtracking") } if err != nil { From 62cf47ecc5e62712cc818122777f23ac16bfa74c Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 9 Oct 2019 16:36:17 -0400 Subject: [PATCH 073/102] test: don't compare directory size --- plugin/plugins/filesystem/filesystem_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 2eed29a853a..92fe5cd9c46 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -327,13 +327,16 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { return false } - bSize := baseAttr.Size - tSize := target[path].Size - if bSize != tSize { - t.Fatalf("size for %q doesn't match\nbase:%d\ntarget:%d\n", - path, - bSize, - tSize) + if bMode.FileType() != p9.ModeDirectory { + bSize := baseAttr.Size + tSize := target[path].Size + + if bSize != tSize { + t.Fatalf("size for %q doesn't match\nbase:%d\ntarget:%d\n", + path, + bSize, + tSize) + } } } return true From e8ccc6b4201704e58e665ac38e13a08dda5ea63c Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 9 Oct 2019 21:02:49 -0400 Subject: [PATCH 074/102] list ipns in the root --- plugin/plugins/filesystem/nodes/doc.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/plugins/filesystem/nodes/doc.go b/plugin/plugins/filesystem/nodes/doc.go index a1c2fd67e83..edf14c78239 100644 --- a/plugin/plugins/filesystem/nodes/doc.go +++ b/plugin/plugins/filesystem/nodes/doc.go @@ -11,5 +11,6 @@ The current mapping looks like this: - / - points back to itself, returning the root index - /ipfs - PinFS - /ipfs/* - IPFS + - /ipns/* - IPNS */ package fsnodes From 6d92612f2d40b12dab0376ad994837f17264749f Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 14 Oct 2019 15:35:59 -0400 Subject: [PATCH 075/102] context refactor wip --- plugin/plugins/filesystem/filesystem_test.go | 264 +++++------ plugin/plugins/filesystem/ipfs_test.go | 58 +++ plugin/plugins/filesystem/nodes/base.go | 177 ++++++-- plugin/plugins/filesystem/nodes/ipfs.go | 145 +++--- plugin/plugins/filesystem/nodes/ipns.go | 5 +- plugin/plugins/filesystem/nodes/keyfs.go | 27 +- plugin/plugins/filesystem/nodes/mfs.go | 438 ------------------- plugin/plugins/filesystem/nodes/pinfs.go | 45 +- plugin/plugins/filesystem/nodes/root.go | 42 +- plugin/plugins/filesystem/nodes/utils.go | 197 +++++++-- plugin/plugins/filesystem/pinfs_test.go | 68 +++ plugin/plugins/filesystem/root_test.go | 70 +++ 12 files changed, 721 insertions(+), 815 deletions(-) create mode 100644 plugin/plugins/filesystem/ipfs_test.go delete mode 100644 plugin/plugins/filesystem/nodes/mfs.go create mode 100644 plugin/plugins/filesystem/pinfs_test.go create mode 100644 plugin/plugins/filesystem/root_test.go diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 92fe5cd9c46..463911424c1 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -3,54 +3,60 @@ package filesystem import ( "context" "fmt" + "io" "io/ioutil" "math/rand" "os" gopath "path" "path/filepath" - "sort" "testing" "time" - "github.com/djdv/p9/localfs" "github.com/djdv/p9/p9" files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/coreapi" - fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" coreiface "github.com/ipfs/interface-go-ipfs-core" coreoptions "github.com/ipfs/interface-go-ipfs-core/options" corepath "github.com/ipfs/interface-go-ipfs-core/path" ) -var ( - attrMaskIPFSTest = p9.AttrMask{ - Mode: true, - Size: true, - } - rootSubsystems = []string{"ipfs", "ipns"} -) +var attrMaskIPFSTest = p9.AttrMask{ + Mode: true, + Size: true, +} func TestAll(t *testing.T) { ctx := context.TODO() - core, err := initCore(ctx) + core, err := InitCore(ctx) if err != nil { t.Fatalf("Failed to construct IPFS node: %s\n", err) } - t.Run("RootFS Attach", func(t *testing.T) { testRootAttacher(ctx, t, core) }) - t.Run("RootFS Clone", func(t *testing.T) { testRootClones(ctx, t, core) }) t.Run("RootFS", func(t *testing.T) { testRootFS(ctx, t, core) }) t.Run("PinFS", func(t *testing.T) { testPinFS(ctx, t, core) }) t.Run("IPFS", func(t *testing.T) { testIPFS(ctx, t, core) }) } -func testRootAttacher(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - rootAttacher := fsnodes.RootAttacher(ctx, core) +type baseAttacher func(context.Context, coreiface.CoreAPI, ...nodeopts.AttachOption) p9.Attacher + +func baseLine(ctx context.Context, t *testing.T, core coreiface.CoreAPI, attachFn baseAttacher) { + attacher := attachFn(ctx, core) + + t.Run("attacher", func(t *testing.T) { testAttacher(ctx, t, attacher) }) + root, err := attacher.Attach() + if err != nil { + t.Fatalf("Attach test passed but attach failed: %s\n", err) + } + t.Run("walk", func(t *testing.T) { testClones(ctx, t, root) }) + +} - // 2 individual instances - nineRoot, err := rootAttacher.Attach() +func testAttacher(ctx context.Context, t *testing.T, attacher p9.Attacher) { + // 2 individual instances, one after another + nineRoot, err := attacher.Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource: %s\n", err) } @@ -59,7 +65,7 @@ func testRootAttacher(ctx context.Context, t *testing.T, core coreiface.CoreAPI) t.Fatalf("Close errored: %s\n", err) } - nineRootTheRevenge, err := rootAttacher.Attach() + nineRootTheRevenge, err := attacher.Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) } @@ -69,12 +75,12 @@ func testRootAttacher(ctx context.Context, t *testing.T, core coreiface.CoreAPI) } // 2 instances at the same time - nineRoot, err = rootAttacher.Attach() + nineRoot, err = attacher.Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource: %s\n", err) } - nineRootTheRevenge, err = rootAttacher.Attach() + nineRootTheRevenge, err = attacher.Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) } @@ -86,165 +92,92 @@ func testRootAttacher(ctx context.Context, t *testing.T, core coreiface.CoreAPI) if err = nineRoot.Close(); err != nil { t.Fatalf("Close errored: %s\n", err) } -} -func testRootClones(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - nineRoot, err := fsnodes.RootAttacher(ctx, core).Attach() + // final instance + nineRoot, err = attacher.Attach() if err != nil { t.Fatalf("Failed to attach to 9P root resource: %s\n", err) } - _, nineRef, err := nineRoot.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) - } - - // this shouldn't affect the parent it's derived from - if err = nineRef.Close(); err != nil { + if err = nineRoot.Close(); err != nil { t.Fatalf("Close errored: %s\n", err) } +} - _, anotherNineRef, err := nineRoot.Walk(nil) +func testClones(ctx context.Context, t *testing.T, nineRef p9.File) { + // clone the node we were passed; 1st generation + _, newRef, err := nineRef.Walk(nil) if err != nil { t.Fatalf("Failed to clone root: %s\n", err) } - if err = anotherNineRef.Close(); err != nil { + // this `Close` shouldn't affect the parent it's derived from + // only descendants + if err = newRef.Close(); err != nil { t.Fatalf("Close errored: %s\n", err) } - if err = nineRoot.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - -} - -func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - nineRoot, err := fsnodes.RootAttacher(ctx, core).Attach() - if err != nil { - t.Fatalf("Failed to attach to 9P root resource: %s\n", err) - } - - _, nineRef, err := nineRoot.Walk(nil) + // remake the clone from the original; 1st generation again + _, gen1, err := nineRef.Walk(nil) if err != nil { - t.Fatalf("Failed to walk root: %s\n", err) - } - if _, _, err = nineRef.Open(p9.ReadOnly); err != nil { - t.Fatalf("Failed to open root: %s\n", err) + t.Fatalf("Failed to clone root: %s\n", err) } - ents, err := nineRef.Readdir(0, ^uint32(0)) + // clone a 2nd generation from the 1st + _, gen2, err := gen1.Walk(nil) if err != nil { - t.Fatalf("Failed to read root: %s\n", err) - } - - if len(ents) != len(rootSubsystems) { - t.Fatalf("Failed, root has bad entries:\nHave:%v\nWant:%v\n", ents, rootSubsystems) - } - for i, name := range rootSubsystems { - if ents[i].Name != name { - t.Fatalf("Failed, root has bad entries:\nHave:%v\nWant:%v\n", ents, rootSubsystems) - } + t.Fatalf("Failed to clone root: %s\n", err) } - //TODO: deeper compare than just the names / name order -} -func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - pinRoot, err := fsnodes.PinFSAttacher(ctx, core).Attach() + // 3rd from the 2nd + _, gen3, err := gen2.Walk(nil) if err != nil { - t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) + t.Fatalf("Failed to clone root: %s\n", err) } - same := func(base, target []string) bool { - if len(base) != len(target) { - return false - } - sort.Strings(base) - sort.Strings(target) - - for i := len(base) - 1; i >= 0; i-- { - if target[i] != base[i] { - return false - } - } - return true + // close the 2nd reference + if err = gen2.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) } - shallowCompare := func() { - basePins, err := pinNames(ctx, core) - if err != nil { - t.Fatalf("Failed to list IPFS pins: %s\n", err) - } - p9Pins, err := p9PinNames(pinRoot) - if err != nil { - t.Fatalf("Failed to list 9P pins: %s\n", err) - } - - if !same(basePins, p9Pins) { - t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) - } + // try to clone from the 2nd reference + // this should fail since we closed it + _, undead, err := gen2.Walk(nil) + if err == nil { + t.Fatalf("Clone (%p)%q succeeded when parent (%p)%q was closed\n", undead, undead, gen2, gen2) } - //test default (likely empty) test repo pins - shallowCompare() - - // test modifying pinset +1; initEnv pins its IPFS environment - env, _, err := initEnv(ctx, core) + // 4th from the 3rd + // should still succeed regardless of 2's state + _, gen4, err := gen3.Walk(nil) if err != nil { - t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + t.Fatalf("Failed to clone root: %s\n", err) } - defer os.RemoveAll(env) - shallowCompare() - // test modifying pinset +1 again; generate garbage and pin it - if err := generateGarbage(env); err != nil { - t.Fatalf("Failed to generate test data: %s\n", err) - } - if _, err = pinAddDir(ctx, core, env); err != nil { - t.Fatalf("Failed to add directory to IPFS: %s\n", err) + // close the 3rd reference + if err = gen3.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) } - shallowCompare() -} -func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { - env, iEnv, err := initEnv(ctx, core) - if err != nil { - t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + // close the 4th reference + if err = gen4.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) } - defer os.RemoveAll(env) - localEnv, err := localfs.Attacher(env).Attach() + // clone a 2nd generation from the 1st again + _, gen2, err = gen1.Walk(nil) if err != nil { - t.Fatalf("Failed to attach to local resource %q: %s\n", env, err) + t.Fatalf("Failed to clone root: %s\n", err) } - ipfsRoot, err := fsnodes.IPFSAttacher(ctx, core).Attach() - if err != nil { - t.Fatalf("Failed to attach to IPFS resource: %s\n", err) - } - _, ipfsEnv, err := ipfsRoot.Walk([]string{gopath.Base(iEnv.String())}) - if err != nil { - t.Fatalf("Failed to walk to IPFS test environment: %s\n", err) - } - _, envClone, err := ipfsEnv.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone IPFS environment handle: %s\n", err) + // close the 1st + if err = gen1.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) } - testCompareTreeAttrs(t, localEnv, ipfsEnv) - - // test readdir bounds - //TODO: compare against a table, not just lengths - _, _, err = envClone.Open(p9.ReadOnly) - if err != nil { - t.Fatalf("Failed to open IPFS test directory: %s\n", err) - } - ents, err := envClone.Readdir(2, 2) // start at ent 2, return max 2 - if err != nil { - t.Fatalf("Failed to read IPFS test directory: %s\n", err) - } - if l := len(ents); l == 0 || l > 2 { - t.Fatalf("IPFS test directory contents don't match read request: %v\n", ents) + // close the 2nd + if err = gen2.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) } } @@ -268,7 +201,6 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { return nil, err } res[ent.Name] = attr - //p9.AttrMaskAll if ent.Type == p9.TypeDir { subRes, err := expand(child) @@ -279,6 +211,9 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { res[gopath.Join(ent.Name, name)] = attr } } + if err = child.Close(); err != nil { + return nil, err + } } return res, nil } @@ -346,7 +281,7 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { } } -func initCore(ctx context.Context) (coreiface.CoreAPI, error) { +func InitCore(ctx context.Context) (coreiface.CoreAPI, error) { node, err := core.NewNode(ctx, &core.BuildCfg{ Online: false, Permanent: false, @@ -406,6 +341,39 @@ func initEnv(ctx context.Context, core coreiface.CoreAPI) (string, corepath.Reso return testDir, iPath, err } +func p9Readdir(dir p9.File) ([]p9.Dirent, error) { + _, dirClone, err := dir.Walk(nil) + if err != nil { + return nil, err + } + + _, _, err = dirClone.Open(p9.ReadOnly) + if err != nil { + return nil, err + } + defer dirClone.Close() + + var ( + offset uint64 + ents []p9.Dirent + ) + for { + curEnts, err := dirClone.Readdir(offset, ^uint32(0)) + ents = append(ents, curEnts...) + if err != nil { + break + } + + offset += uint64(len(curEnts)) + } + + if err == io.EOF { + err = nil + } + + return ents, err +} + func pinAddDir(ctx context.Context, core coreiface.CoreAPI, path string) (corepath.Resolved, error) { fi, err := os.Stat(path) if err != nil { @@ -471,20 +439,6 @@ func p9PinNames(root p9.File) ([]string, error) { return names, nil } -func p9Readdir(dir p9.File) ([]p9.Dirent, error) { - _, dirClone, err := dir.Walk(nil) - if err != nil { - return nil, err - } - - _, _, err = dirClone.Open(p9.ReadOnly) - if err != nil { - return nil, err - } - defer dirClone.Close() - return dirClone.Readdir(0, ^uint32(0)) -} - //TODO: // NOTE: compares a subset of attributes, matching those of IPFS func testIPFSCompare(t *testing.T, f1, f2 p9.File) { diff --git a/plugin/plugins/filesystem/ipfs_test.go b/plugin/plugins/filesystem/ipfs_test.go new file mode 100644 index 00000000000..1779d595027 --- /dev/null +++ b/plugin/plugins/filesystem/ipfs_test.go @@ -0,0 +1,58 @@ +package filesystem + +import ( + "context" + "os" + gopath "path" + "testing" + + "github.com/djdv/p9/localfs" + "github.com/djdv/p9/p9" + fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + t.Run("Baseline", func(t *testing.T) { baseLine(ctx, t, core, fsnodes.IPFSAttacher) }) + + rootRef, err := fsnodes.IPFSAttacher(ctx, core).Attach() + if err != nil { + t.Fatalf("Baseline test passed but attach failed: %s\n", err) + } + + env, iEnv, err := initEnv(ctx, core) + if err != nil { + t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + } + defer os.RemoveAll(env) + + localEnv, err := localfs.Attacher(env).Attach() + if err != nil { + t.Fatalf("Failed to attach to local resource %q: %s\n", env, err) + } + + _, ipfsEnv, err := rootRef.Walk([]string{gopath.Base(iEnv.String())}) + if err != nil { + t.Fatalf("Failed to walk to IPFS test environment: %s\n", err) + } + _, envClone, err := ipfsEnv.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone IPFS environment handle: %s\n", err) + } + + testCompareTreeAttrs(t, localEnv, ipfsEnv) + + // test readdir bounds + //TODO: compare against a table, not just lengths + _, _, err = envClone.Open(p9.ReadOnly) + if err != nil { + t.Fatalf("Failed to open IPFS test directory: %s\n", err) + } + ents, err := envClone.Readdir(2, 2) // start at ent 2, return max 2 + if err != nil { + t.Fatalf("Failed to read IPFS test directory: %s\n", err) + } + if l := len(ents); l == 0 || l > 2 { + t.Fatalf("IPFS test directory contents don't match read request: %v\n", ents) + } +} diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 487ef7bcef8..cd746dfc8d1 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -2,6 +2,7 @@ package fsnodes import ( "context" + "errors" gopath "path" "time" @@ -32,58 +33,76 @@ type Base struct { Trail []string // FS "breadcrumb" trail from node's root // Storage for file's metadata - Qid p9.QID + Qid *p9.QID meta *p9.Attr metaMask *p9.AttrMask Logger logging.EventLogger -} -func (b *Base) QID() p9.QID { return b.Qid } + closed bool // set to true upon close; this reference should not be used again for anything + modified bool // set to true when the `Trail` has been modified (usually by `Step`) + // reset to false when Qid has been populated with the current path in `Trail` (usually by `QID`) -func (b *Base) NinePath() p9Path { return b.Qid.Path } +} func newBase(ops ...nodeopts.AttachOption) Base { options := nodeopts.AttachOps(ops...) return Base{ Logger: options.Logger, + Qid: new(p9.QID), meta: new(p9.Attr), metaMask: new(p9.AttrMask), } } -func (b *Base) Derive() Base { - newFid := newBase(nodeopts.Logger(b.Logger)) +func (b *Base) clone() Base { + return *b +} - newFid.Qid = b.Qid - newFid.meta, newFid.metaMask = b.meta, b.metaMask - newFid.Trail = make([]string, len(b.Trail)) +func (b *Base) Fork() Base { + newFid := b.clone() + newFid.Trail = make([]string, len(b.Trail)+1) // NOTE: +1 is preallocated space for `Step`; not required copy(newFid.Trail, b.Trail) return newFid } +func (b *Base) String() string { + return gopath.Join(b.Trail...) +} + +func (b *Base) NinePath() p9Path { return b.Qid.Path } + +func (b *Base) QID() (p9.QID, error) { return *b.Qid, nil } + // IPFSBase is much like Base but extends it to hold IPFS specific metadata type IPFSBase struct { Base OverlayFileMeta - /* For file systems, - this context should be set prior to `Attach` - - For files, - this context should be overwritten with a context derived from the existing fs context - during `Walk` + /* The filesystem context should be set prior to `Attach` + During `fs.Attach`, a new reference to `fs` should be created + with its fs-context + fs-cancel populated with one derived from the existing fs context - The context is expected to be valid for the lifetime of the file system / file respectively - to be used during operations, such as `Walk`, `Open`, `Read` etc. + This context is expected to be valid for the lifetime of the file system + and canceled on `Close` by references returned from `Attach` only. */ - filesystemCtx context.Context - // cancel should be called upon `Close` - // closing a file system returned from `Attach` - // or a derived file previously returned from `Walk` + filesystemCtx context.Context filesystemCancel context.CancelFunc + /* During `file.Walk` a new reference to `file` should be created + with its op-context +op-cancel populated with one derived from the existing fs context + This context is expected to be valid as long as the file is being referenced by a particular FID + it should be canceled during `Close` + + for the lifetime of the file system + to be used during operations, such as `Open`, `Read` etc. + and canceled on `Close` by references returned from `Attach` only. + */ + + operationsCtx context.Context + operationsCancel context.CancelFunc + // Typically you'll want to derive a context from the fs ctx within one operation (like Open) // use it with the CoreAPI for something (like Get) // and cancel it in another operation (like Close) @@ -96,29 +115,16 @@ type IPFSBase struct { core coreiface.CoreAPI } -func (b *Base) StringPath() string { - return gopath.Join(b.Trail...) -} - -func (ib *IPFSBase) StringPath() string { - return gopath.Join(append([]string{ib.coreNamespace}, ib.Base.StringPath())...) -} - -func (ib *IPFSBase) CorePath(names ...string) corepath.Path { - return corepath.Join(rootPath(ib.coreNamespace), append(ib.Trail, names...)...) -} - //func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.FileMode, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { func newIPFSBase(ctx context.Context, coreNamespace string, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { - options := nodeopts.AttachOps(ops...) - base := IPFSBase{ Base: newBase(ops...), coreNamespace: coreNamespace, core: core, + filesystemCtx: ctx, } - base.filesystemCtx, base.filesystemCancel = context.WithCancel(ctx) + options := nodeopts.AttachOps(ops...) if options.Parent != nil { // parent is optional parentRef, ok := options.Parent.(walkRef) // interface is not if !ok { @@ -129,30 +135,70 @@ func newIPFSBase(ctx context.Context, coreNamespace string, core coreiface.CoreA return base } -func (ib *IPFSBase) Derive() IPFSBase { +func (ib *IPFSBase) clone() IPFSBase { newFid := IPFSBase{ - Base: ib.Base.Derive(), + Base: ib.Base.clone(), OverlayFileMeta: ib.OverlayFileMeta, coreNamespace: ib.coreNamespace, core: ib.core, + filesystemCtx: ib.filesystemCtx, } - newFid.filesystemCtx, newFid.filesystemCancel = context.WithCancel(ib.filesystemCtx) - return newFid } +func (ib *IPFSBase) Fork() (IPFSBase, error) { + newFid := ib.clone() + err := newFid.newOperations() + + return newFid, err +} + +func (ib *IPFSBase) newFilesystem() error { + if err := ib.checkFSCtx(); err != nil { + return err + } + ib.filesystemCtx, ib.filesystemCancel = context.WithCancel(ib.filesystemCtx) + return nil +} + +func (ib *IPFSBase) newOperations() error { + if err := ib.checkFSCtx(); err != nil { + return err + } + ib.operationsCtx, ib.operationsCancel = context.WithCancel(ib.filesystemCtx) + return nil +} + +func (ib *IPFSBase) checkFSCtx() error { + select { + case <-ib.filesystemCtx.Done(): + return ib.filesystemCtx.Err() + default: + return nil + } +} + +func (ib *IPFSBase) String() string { + return gopath.Join(append([]string{ib.coreNamespace}, ib.Base.String())...) +} + +func (ib *IPFSBase) CorePath(names ...string) corepath.Path { + return corepath.Join(rootPath(ib.coreNamespace), append(ib.Trail, names...)...) +} + func (b *IPFSBase) Flush() error { - b.Logger.Debugf("flushing: {%d}%q", b.Qid.Path, b.StringPath()) + b.Logger.Debugf("flush requested: {%d}%q", b.Qid.Path, b.String()) return nil } func (b *Base) Close() error { - b.Logger.Debugf("closing: {%d}%q", b.Qid.Path, b.StringPath()) + b.Logger.Debugf("closing: {%d}%q", b.Qid.Path, b.String()) + b.closed = true return nil } func (ib *IPFSBase) Close() error { - ib.Logger.Debugf("closing: {%d}%q", ib.Qid.Path, ib.StringPath()) + ib.Logger.Debugf("closing: {%d}%q", ib.Qid.Path, ib.String()) var err error if ib.filesystemCancel != nil { @@ -164,13 +210,15 @@ func (ib *IPFSBase) Close() error { ib.filesystemCancel() } + ib.closed = true + return err } func (b *Base) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - b.Logger.Debugf("GetAttr {%d}:%q", b.Qid.Path, b.StringPath()) + b.Logger.Debugf("GetAttr {%d}:%q", b.Qid.Path, b.String()) - return b.Qid, *b.metaMask, *b.meta, nil + return *b.Qid, *b.metaMask, *b.meta, nil } func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { @@ -179,5 +227,42 @@ func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { func (b *IPFSBase) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { b.Logger.Debugf("Open") - return b.Qid, 0, nil + return *b.Qid, 0, nil +} + +func (ib *IPFSBase) step(self walkRef, name string) (walkRef, error) { + if ib.Qid.Type != p9.TypeDir { + return nil, ENOTDIR + } + + if ib.closed == true { + return nil, errors.New("TODO: ref was previously closed err") + } + + ib.Trail = append(ib.Trail, name) + ib.modified = true + return self, nil +} + +func (ib *IPFSBase) backtrack(self walkRef) (walkRef, error) { + // if we're the root + if len(ib.Trail) == 0 { + // return our parent, or ourselves if we don't have one + if ib.parent != nil { + return ib.parent, nil + } + return self, nil + } + + // otherwise step back + ib.Trail = ib.Trail[:len(ib.Trail)-1] + + return self, nil +} + +func (b *Base) CheckWalk() error { + if b.closed { + return errors.New("TODO: already closed msg") + } + return nil } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 333158f3e07..0cea66c4e2b 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -30,11 +30,18 @@ type IPFSFileMeta struct { } func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { - return &IPFS{IPFSBase: newIPFSBase(ctx, "/ipfs", core, ops...)} + id := &IPFS{IPFSBase: newIPFSBase(ctx, "/ipfs", core, ops...)} + id.Qid.Type = p9.TypeDir + id.meta.Mode, id.metaMask.Mode = p9.ModeDirectory|IRXA, true + return id } -func (id *IPFS) Derive() walkRef { - return &IPFS{IPFSBase: id.IPFSBase.Derive()} +func (id *IPFS) Fork() (walkRef, error) { + base, err := id.IPFSBase.Fork() + if err != nil { + return nil, err + } + return &IPFS{IPFSBase: base}, nil } func (id *IPFS) Attach() (p9.File, error) { @@ -65,86 +72,89 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { attr p9.Attr filled p9.AttrMask callCtx, cancel = id.callCtx() + qid = *id.Qid ) defer cancel() corePath, err := id.core.ResolvePath(callCtx, id.CorePath()) if err != nil { - return id.Qid, filled, attr, err + return qid, filled, attr, err } filled, err = coreAttr(callCtx, &attr, corePath, id.core, req) if err != nil { id.Logger.Error(err) - return id.Qid, filled, attr, err + return qid, filled, attr, err } if req.RDev { attr.RDev, filled.RDev = dIPFS, true } - attr.Mode |= IRXA + if req.Mode { + attr.Mode |= IRXA + qid.Type = attr.Mode.QIDType() + id.Qid.Type = attr.Mode.QIDType() + } - return id.Qid, filled, attr, err + return qid, filled, attr, err } func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { id.Logger.Debugf("Walk names: %v", names) - id.Logger.Debugf("Walk myself: %q:{%d}", id.StringPath(), id.NinePath()) + id.Logger.Debugf("Walk myself: %q:{%d}", id.String(), id.NinePath()) return walker(id, names) } func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - id.Logger.Debugf("Open: %s", id.StringPath()) + id.Logger.Debugf("Open: %s", id.String()) - // set up handle amenities - //var handleContext context.Context - //handleContext, id.operationsCancel = context.WithCancel(id.filesystemCtx) + qid := *id.Qid // handle directories - if id.Qid.Type == p9.TypeDir { + if qid.Type == p9.TypeDir { // c, err := id.core.Unixfs().Ls(handleContext, id.CorePath()) c, err := id.core.Unixfs().Ls(id.filesystemCtx, id.CorePath()) if err != nil { //id.operationsCancel() - return id.Qid, 0, err + return qid, 0, err } id.directory = &directoryStream{ entryChan: c, } - return id.Qid, 0, nil + return qid, 0, nil } // handle files apiNode, err := id.core.Unixfs().Get(id.filesystemCtx, id.CorePath()) if err != nil { //id.operationsCancel() - return id.Qid, 0, err + return qid, 0, err } fileNode, ok := apiNode.(files.File) if !ok { //id.operationsCancel() - return id.Qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.StringPath(), apiNode) + return qid, 0, fmt.Errorf("%q does not appear to be a file: %T", id.String(), apiNode) } id.file = fileNode s, err := id.file.Size() if err != nil { //id.operationsCancel() - return id.Qid, 0, err + return qid, 0, err } id.meta.Size, id.metaMask.Size = uint64(s), true - return id.Qid, ipfsBlockSize, nil + return qid, ipfsBlockSize, nil } func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - id.Logger.Debugf("Readdir %q %d %d", id.StringPath(), offset, count) + id.Logger.Debugf("Readdir %q %d %d", id.String(), offset, count) if id.directory == nil { - return nil, fmt.Errorf("directory %q is not open for reading", id.StringPath()) + return nil, fmt.Errorf("directory %q is not open for reading", id.String()) } if id.directory.err != nil { // previous request must have failed @@ -152,8 +162,8 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { } if id.directory.eos { - if offset == id.directory.cursor-1 { - return nil, nil // this is the only exception to offset being behind the cursor + if offset == id.directory.cursor { + return nil, io.EOF // this is the only exception to offset being behind the cursor } } @@ -169,7 +179,7 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { if !open { //id.operationsCancel() id.directory.eos = true - return ents, nil + return ents, io.EOF } if entry.Err != nil { id.directory.err = entry.Err @@ -182,7 +192,7 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { id.directory.err = err return nil, err } - nineEnt.Offset = id.directory.cursor + nineEnt.Offset = id.directory.cursor + 1 ents = append(ents, nineEnt) } @@ -204,11 +214,11 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { readAtFmt = "ReadAt {%d/%d}%q" readAtFmtErr = readAtFmt + ": %s" ) - //id.Logger.Debugf(readAtFmt, offset, id.meta.Size, id.StringPath()) + //id.Logger.Debugf(readAtFmt, offset, id.meta.Size, id.String()) if id.file == nil { err := fmt.Errorf("file is not open for reading") - id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.StringPath(), err) + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.String(), err) return 0, err } @@ -219,13 +229,13 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { if _, err := id.file.Seek(int64(offset), io.SeekStart); err != nil { //id.operationsCancel() - id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.StringPath(), err) + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.String(), err) return 0, err } readBytes, err := id.file.Read(p) if err != nil && err != io.EOF { - id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.StringPath(), err) + id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.String(), err) //id.operationsCancel() return 0, err } @@ -251,74 +261,27 @@ func (id *IPFS) Close() error { return lastErr } -func coreToQid(ctx context.Context, path corepath.Path, core coreiface.CoreAPI) (p9.QID, error) { - var qid p9.QID - // translate from abstract path to CoreAPI resolved path - resolvedPath, err := core.ResolvePath(ctx, path) - if err != nil { - return qid, err - } - - // inspected to derive 9P QID - attr := new(p9.Attr) - _, err = coreAttr(ctx, attr, resolvedPath, core, p9.AttrMask{Mode: true}) - if err != nil { - return qid, err - } - - qid.Type = attr.Mode.QIDType() - qid.Path = cidToQPath(resolvedPath.Cid()) - return qid, nil -} - // IPFS appends "name" to its current path, and returns itself func (id *IPFS) Step(name string) (walkRef, error) { - callCtx, cancel := id.callCtx() - defer cancel() - - qid, err := coreToQid(callCtx, id.CorePath(name), id.core) - if err != nil { - return nil, err - } - - // we stepped successfully, so set up a newFid to return - newFid := id.Derive().(*IPFS) - newFid.Trail = append(newFid.Trail, name) - newFid.Qid = qid - - return newFid, nil + return id.step(id, name) } -func (id *IPFS) Backtrack() (walkRef, error) { - // if we're the root - if len(id.Trail) == 0 { - // return our parent, or ourselves if we don't have one - if id.parent != nil { - return id.parent, nil - } - return id, nil - } - - // otherwise step back - tLen := len(id.Trail) - breadCrumb := make([]string, tLen) - copy(breadCrumb, id.Trail) - - id.Trail = id.Trail[:tLen-1] +func (id *IPFS) QID() (p9.QID, error) { + if id.modified { + callCtx, cancel := id.callCtx() + defer cancel() - // reset QID - callCtx, cancel := id.callCtx() - defer cancel() - - qid, err := coreToQid(callCtx, id.CorePath(), id.core) - if err != nil { - // recover path on failure - id.Trail = breadCrumb - return nil, err + qid, err := coreToQid(callCtx, id.CorePath(), id.core) + if err != nil { + return qid, err + } + id.modified = false + id.Qid = &qid } - // set on success; we stepped back - id.Qid = qid + return *id.Qid, nil +} - return id, nil +func (id *IPFS) Backtrack() (walkRef, error) { + return id.IPFSBase.backtrack(id) } diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go index be020ae22a8..f771c8fc4d2 100644 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -17,5 +17,8 @@ var _ walkRef = (*IPNS)(nil) type IPNS = IPFS func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { - return &IPNS{IPFSBase: newIPFSBase(ctx, "/ipns", core, ops...)} + id := &IPNS{IPFSBase: newIPFSBase(ctx, "/ipns", core, ops...)} + id.Qid.Type = p9.TypeDir + id.meta.Mode, id.metaMask.Mode = p9.ModeDirectory|IRXA, true + return id } diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 9708136a9af..dbde706b624 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -37,21 +37,30 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return kd } -func (kd *KeyFS) Derive() walkRef { - newFid := &KeyFS{ - IPFSBase: kd.IPFSBase.Derive(), - } - return newFid +func (kd *KeyFS) Fork() (walkRef, error) { + newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change + // set new operations context + err := newFid.newOperations() + return newFid, err } func (kd *KeyFS) Attach() (p9.File, error) { kd.Logger.Debugf("Attach") - return kd, nil + + newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change + // set new fs context + err := newFid.newFilesystem() + return newFid, err } -func (kd *KeyFS) Step(keyName string) (walkRef, error) { - // proxy the request for "keyName" to IPFS root (set on us during construction) - return kd.proxy.Step(keyName) +// KeyFS forks the IPFS root that was set during construction +// and calls step on it rather than itself +func (kd *KeyFS) Step(name string) (walkRef, error) { + newFid, err := kd.proxy.Fork() + if err != nil { + return nil, err + } + return newFid.Step(name) } func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { diff --git a/plugin/plugins/filesystem/nodes/mfs.go b/plugin/plugins/filesystem/nodes/mfs.go deleted file mode 100644 index 31368b1e5da..00000000000 --- a/plugin/plugins/filesystem/nodes/mfs.go +++ /dev/null @@ -1,438 +0,0 @@ -package fsnodes - -import ( - "context" - "fmt" - "io" - gopath "path" - "time" - - "github.com/djdv/p9/p9" - cid "github.com/ipfs/go-cid" - nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" - dag "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-mfs" - coreiface "github.com/ipfs/interface-go-ipfs-core" -) - -var _ p9.File = (*MFS)(nil) -var _ walkRef = (*MFS)(nil) - -// TODO: break this up into 2 file systems? -// MFS + MFS Overlay? -// TODO: docs -type MFS struct { - IPFSBase - MFSFileMeta - - //ref uint //TODO: rename, root refcount - //key coreiface.Key // optional value, if set, publish to IPNS key on MFS change - //roots map[string]*mfs.Root //share the same mfs root across calls - mroot *mfs.Root -} - -type MFSFileMeta struct { - //TODO: re-use base path for this - //relativePath string - //rootName string - file mfs.FileDescriptor - directory *mfs.Directory -} - -func MFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { - options := nodeopts.AttachOps(ops...) - - mroot, err := cidToMFSRoot(ctx, options.MFSRoot, core, options.MFSPublish) - if err != nil { - panic(err) - } - - md := &MFS{ - IPFSBase: newIPFSBase(ctx, "/ipld", core, ops...), - mroot: mroot, - } - md.meta.Mode, md.metaMask.Mode = p9.ModeDirectory|IRXA|0220, true - - return md -} - -func (md *MFS) Derive() walkRef { - newFid := &MFS{ - IPFSBase: md.IPFSBase.Derive(), - //MFSFileMeta: md.MFSFileMeta, - mroot: md.mroot, - } - - return newFid -} - -func (md *MFS) Attach() (p9.File, error) { - md.Logger.Debugf("Attach") - - newFid := new(MFS) - *newFid = *md - - return newFid, nil -} - -func (md *MFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - md.Logger.Debugf("GetAttr path: %s", md.StringPath()) - md.Logger.Debugf("%p", md) - - callCtx, cancel := md.callCtx() - defer cancel() - - filled, err := mfsAttr(callCtx, md.meta, md.mroot, p9.AttrMaskAll, md.Trail...) - if err != nil { - return md.Qid, filled, *md.meta, err - } - - md.meta.Mode |= IRXA | 0220 - if req.RDev { - md.meta.RDev, filled.RDev = dMemory, true - } - - return md.Qid, filled, *md.meta, nil -} - -func (md *MFS) Walk(names []string) ([]p9.QID, p9.File, error) { - md.Logger.Debugf("Walk names: %v", names) - md.Logger.Debugf("Walk myself: %q:{%d}", md.StringPath(), md.NinePath()) - - md.Logger.Debugf("%p", md) - return walker(md, names) -} - -func (md *MFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - md.Logger.Debugf("Open %q {Mode:%v OSFlags:%v, String:%s}", md.StringPath(), mode.Mode(), mode.OSFlags(), mode.String()) - md.Logger.Debugf("%p", md) - - if md.mroot == nil { - return md.Qid, 0, fmt.Errorf("TODO: message; root not set") - } - - //TODO: current: lookup -> mfsnoode -> md.file | md.directory = mfsNode.open(fsctx) - - mNode, err := mfs.Lookup(md.mroot, gopath.Join(md.Trail...)) - if err != nil { - return md.Qid, 0, err - } - - // handle directories - if md.meta.Mode.IsDir() { - dir, ok := mNode.(*mfs.Directory) - if !ok { - return md.Qid, 0, fmt.Errorf("type mismatch %q is %T not a directory", md.StringPath(), mNode) - } - md.directory = dir - } else { - mFile, ok := mNode.(*mfs.File) - if !ok { - return md.Qid, 0, fmt.Errorf("type mismatch %q is %T not a file", md.StringPath(), mNode) - } - - openFile, err := mFile.Open(mfs.Flags{Read: true, Write: true}) - if err != nil { - return md.Qid, 0, err - } - s, err := openFile.Size() - if err != nil { - return md.Qid, 0, err - } - - md.file = openFile - md.meta.Size, md.metaMask.Size = uint64(s), true - } - - return md.Qid, ipfsBlockSize, nil -} - -func (md *MFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - md.Logger.Debugf("Readdir %d %d", offset, count) - - if md.directory == nil { - return nil, fmt.Errorf("directory %q is not open for reading", md.StringPath()) - } - - //TODO: resetable context; for { ...; ctx.reset() } - callCtx, cancel := context.WithCancel(md.filesystemCtx) - defer cancel() - - ents := make([]p9.Dirent, 0) - - var index uint64 - var done bool - err := md.directory.ForEachEntry(callCtx, func(nl mfs.NodeListing) error { - if done { - cancel() - return nil - } - - if index < offset { - index++ //skip - return nil - } - - ent, err := mfsEntTo9Ent(nl) - if err != nil { - return err - } - - ent.Offset = index + 1 - - ents = append(ents, ent) - if len(ents) == int(count) { - done = true - } - - index++ - return nil - }) - - return ents, err -} - -func (md *MFS) ReadAt(p []byte, offset uint64) (int, error) { - const ( - readAtFmt = "ReadAt {%d/%d}%q" - readAtFmtErr = readAtFmt + ": %s" - ) - - if md.file == nil { - err := fmt.Errorf("file is not open for reading") - md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) - return 0, err - } - - if offset >= md.meta.Size { - //NOTE [styx]: If the offset field is greater than or equal to the number of bytes in the file, a count of zero will be returned. - return 0, io.EOF - } - - if _, err := md.file.Seek(int64(offset), io.SeekStart); err != nil { - md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) - return 0, err - } - - //TODO: remove, debug - - nbytes, err := md.file.Read(p) - if err != nil { - md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) - } - - return nbytes, err - // - - //return md.file.Read(p) -} - -func (md *MFS) SetAttr(valid p9.SetAttrMask, attr p9.SetAttr) error { - md.Logger.Debugf("SetAttr %v %v", valid, attr) - md.Logger.Debugf("%p", md) - - if valid.Size && attr.Size < md.meta.Size { - if md.file == nil { - err := fmt.Errorf("file %q is not open, cannot change size", md.StringPath()) - md.Logger.Error(err) - return err - } - - if err := md.file.Truncate(int64(attr.Size)); err != nil { - return err - } - } - - md.meta.Apply(valid, attr) - return nil -} - -func (md *MFS) WriteAt(p []byte, offset uint64) (int, error) { - const ( - readAtFmt = "WriteAt {%d/%d}%q" - readAtFmtErr = readAtFmt + ": %s" - ) - if md.file == nil { - err := fmt.Errorf("file is not open for writing") - md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) - return 0, err - } - - //TODO: remove, debug - - nbytes, err := md.file.WriteAt(p, int64(offset)) - if err != nil { - md.Logger.Errorf(readAtFmtErr, offset, md.meta.Size, md.StringPath(), err) - } - - return nbytes, err - // - - //return md.file.WriteAt(p, int64(offset)) -} - -func (md *MFS) Close() error { - md.Logger.Debugf("closing: %q:{%d}", md.StringPath(), md.Qid.Path) - md.Logger.Debugf("%p", md) - - var lastErr error - if err := md.Base.Close(); err != nil { - md.Logger.Error(err) - lastErr = err - } - - if md.file != nil { - if err := md.file.Close(); err != nil { - md.Logger.Error(err) - lastErr = err - } - } - - return lastErr -} - -/* -{ - Base: { - coreNamespace: `/ipld`, - Trail: []string{"folder", "file.txt"} - } - mroot: fromCid(`QmVuDpaFj55JnUH7UYxTAydx6ayrs2cB3Gb7cdPr61wLv5`) -} -=> -`/ipld/QmVuDpaFj55JnUH7UYxTAydx6ayrs2cB3Gb7cdPr61wLv5/folder/file.txt` -*/ -func (md *MFS) StringPath() string { - rootNode, err := md.mroot.GetDirectory().GetNode() - if err != nil { - panic(err) - } - return gopath.Join(append([]string{md.coreNamespace, rootNode.Cid().String()}, md.Trail...)...) -} - -func (md *MFS) Backtrack() (walkRef, error) { - // if we're the root - if len(md.Trail) == 0 { - // return our parent, or ourselves if we don't have one - if md.parent != nil { - return md.parent, nil - } - return md, nil - } - - // otherwise step back - tLen := len(md.Trail) - breadCrumb := make([]string, tLen) - copy(breadCrumb, md.Trail) - - md.Trail = md.Trail[:tLen-1] - - // reset QID - callCtx, cancel := md.callCtx() - defer cancel() - - qid, err := coreToQid(callCtx, md.CorePath(), md.core) - if err != nil { - // recover path on failure - md.Trail = breadCrumb - return nil, err - } - - // set on success; we stepped back - md.Qid = qid - - return md, nil -} - -func mfsAttr(ctx context.Context, attr *p9.Attr, mroot *mfs.Root, req p9.AttrMask, names ...string) (p9.AttrMask, error) { - mfsNode, err := mfs.Lookup(mroot, gopath.Join(names...)) - if err != nil { - return p9.AttrMask{}, err - } - - ipldNode, err := mfsNode.GetNode() - if err != nil { - return p9.AttrMask{}, err - } - - return ipldStat(ctx, attr, ipldNode, req) -} - -func mfsToQid(ctx context.Context, mroot *mfs.Root, names ...string) (p9.QID, error) { - mNode, err := mfs.Lookup(mroot, gopath.Join(names...)) - if err != nil { - return p9.QID{}, err - } - - t, err := mfsTypeToNineType(mNode.Type()) - if err != nil { - return p9.QID{}, err - } - - ipldNode, err := mNode.GetNode() - if err != nil { - return p9.QID{}, err - } - - return p9.QID{ - Type: t, - Path: cidToQPath(ipldNode.Cid()), - }, nil -} - -func (md *MFS) Step(name string) (walkRef, error) { - callCtx, cancel := md.callCtx() - defer cancel() - - breadCrumb := append(md.Trail, name) - qid, err := mfsToQid(callCtx, md.mroot, breadCrumb...) - if err != nil { - return nil, err - } - - // set on success; we stepped - md.Trail = breadCrumb - md.Qid = qid - return md, nil -} - -/* -func (md *MFS) RootPath(keyName string, components ...string) (corepath.Path, error) { - if keyName == "" { - return nil, fmt.Errorf("no path key was provided") - } - - rootCid, err := cid.Decode(keyName) - if err != nil { - return nil, err - } - - return corepath.Join(corepath.IpldPath(rootCid), components...), nil -} - -func (md *MFS) ResolvedPath(names ...string) (corepath.Path, error) { - callCtx, cancel := md.callCtx() - defer cancel() - - return md.core.ResolvePath(callCtx, md.KeyPath(names[0], names[1:]...)) - - corePath = corepath.IpldPath(md.Tail[0]) - return md.core.ResolvePath(callCtx, corepath.Join(corePath, append(md.Tail[1:], names)...)) -} -*/ - -func cidToMFSRoot(ctx context.Context, rootCid *cid.Cid, core coreiface.CoreAPI, publish mfs.PubFunc) (*mfs.Root, error) { - callCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - ipldNode, err := core.Dag().Get(callCtx, *rootCid) - if err != nil { - return nil, err - } - - pbNode, ok := ipldNode.(*dag.ProtoNode) - if !ok { - return nil, fmt.Errorf("%q has incompatible type %T", rootCid.String(), ipldNode) - } - - return mfs.NewRoot(ctx, core.Dag(), pbNode, publish) -} diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 04804195031..445bb0948c9 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -41,22 +41,30 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return pd } -func (pd *PinFS) Derive() walkRef { - newFid := &PinFS{ - IPFSBase: pd.IPFSBase.Derive(), - } - return newFid +func (pd *PinFS) Fork() (walkRef, error) { + newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change + // set new operations context + err := newFid.newOperations() + return newFid, err } func (pd *PinFS) Attach() (p9.File, error) { pd.Logger.Debugf("Attach") - return pd, nil + + newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change + // set new fs context + err := newFid.newFilesystem() + return newFid, err } -// PinFS proxies steps to the IPFS root that was set during construction +// PinFS forks the IPFS root that was set during construction +// and calls step on it rather than itself func (pd *PinFS) Step(name string) (walkRef, error) { - // proxy the request for "name" to IPFS root (set on us during construction) - return pd.proxy.Step(name) + newFid, err := pd.proxy.Fork() + if err != nil { + return nil, err + } + return newFid.Step(name) } func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { @@ -69,10 +77,12 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") + qid := *pd.Qid + // IPFS core representation pins, err := pd.core.Pin().Ls(pd.filesystemCtx, coreoptions.Pin.Type.Recursive()) if err != nil { - return pd.Qid, 0, err + return qid, 0, err } // 9P representation @@ -88,11 +98,11 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { ipldNode, err := pd.core.ResolveNode(callCtx, pin.Path()) if err != nil { cancel() - return pd.Qid, 0, err + return qid, 0, err } if _, err = ipldStat(callCtx, attr, ipldNode, requestType); err != nil { cancel() - return pd.Qid, 0, err + return qid, 0, err } pd.ents = append(pd.ents, p9.Dirent{ @@ -106,14 +116,14 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { cancel() } - return pd.Qid, ipfsBlockSize, nil + return qid, ipfsBlockSize, nil } func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { pd.Logger.Debugf("Readdir") if pd.ents == nil { - return nil, fmt.Errorf("directory %q is not open for reading", pd.StringPath()) + return nil, fmt.Errorf("directory %q is not open for reading", pd.String()) } return flatReaddir(pd.ents, offset, count) @@ -128,15 +138,10 @@ func (pd *PinFS) Backtrack() (walkRef, error) { // otherwise step back pd.Trail = pd.Trail[1:] - // reset meta + // TODO: reset meta return pd, nil } -func (pd *PinFS) Flush() error { - pd.Logger.Errorf("flushing:%q:%d", pd.StringPath(), pd.NinePath()) - return nil -} - func (pd *PinFS) Close() error { pd.ents = nil return nil diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 137cd22e3c6..2c71b0922bb 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -2,7 +2,6 @@ package fsnodes import ( "context" - "syscall" "github.com/djdv/p9/p9" cid "github.com/ipfs/go-cid" @@ -39,12 +38,6 @@ type systemTuple struct { dirent p9.Dirent } -type systemSlice []systemTuple - -func (ss systemSlice) Len() int { return len(ss) } -func (ss systemSlice) Swap(i, j int) { ss[i], ss[j] = ss[j], ss[i] } -func (ss systemSlice) Less(i, j int) bool { return ss[i].dirent.Offset < ss[j].dirent.Offset } - //TODO: rename, while this is likely to be the root, it doesn't have to be; maybe "IPFSOverlay" // RootIndex is a virtual directory file system, that maps a set of file system implementations to a hierarchy // Currently: "/ipfs":PinFS, "/ipfs/*:IPFS @@ -77,24 +70,27 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A string subattacher logging.EventLogger + // *logging.EventLogger } - // "mount/bind" table: - // "path"=>filesystem + // 9P Access names mapped to IPFS attacher functions subsystems := [...]attachTuple{ {"ipfs", PinFSAttacher, logging.Logger("PinFS")}, {"ipns", KeyFSAttacher, logging.Logger("KeyFS")}, } - // prealloc what we can + // allocate root entry pairs + // assign inherent options, + // and instantiate a template root entry ri.subsystems = make(map[string]systemTuple, len(subsystems)) opts := []nodeopts.AttachOption{nodeopts.Parent(ri)} - rootDirent := p9.Dirent{ //template + rootDirent := p9.Dirent{ Type: p9.TypeDir, QID: p9.QID{Type: p9.TypeDir}, } // couple the strings to their implementations + // "aname"=>{filesystem,entry} for i, subsystem := range subsystems { logOpt := nodeopts.Logger(subsystem.EventLogger) // the file system implementation @@ -109,7 +105,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A rootDirent.QID.Path = cidToQPath(rootPath("/" + subsystem.string).Cid()) - // add it as a unionlike thing + // add the fs+entry to the list of subsystems ri.subsystems[subsystem.string] = systemTuple{ file: fs.(walkRef), dirent: rootDirent, @@ -119,18 +115,28 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A return ri } -func (ri *RootIndex) Derive() walkRef { +func (ri *RootIndex) Fork() (walkRef, error) { newFid := &RootIndex{ - IPFSBase: ri.IPFSBase.Derive(), + IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change subsystems: ri.subsystems, } - return newFid + // set new operations context + err := newFid.newOperations() + return newFid, err } func (ri *RootIndex) Attach() (p9.File, error) { ri.Logger.Debugf("Attach") - return ri, nil + + newFid := &RootIndex{ + IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change + subsystems: ri.subsystems, + } + + // set new fs context + err := newFid.newFilesystem() + return newFid, err } func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { @@ -147,11 +153,11 @@ func (ri *RootIndex) Step(name string) (walkRef, error) { subSys, ok := ri.subsystems[name] if !ok { ri.Logger.Errorf("%q is not provided by us", name) - return nil, syscall.ENOENT //TODO: migrate to platform independent value + return nil, ENOENT } // return a ready to use derivative of it - return subSys.file.Derive(), nil + return subSys.file.Fork() } func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 4a66929b2bf..b71bd3ba0c9 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -7,6 +7,7 @@ import ( "fmt" "hash/fnv" "io" + "syscall" "time" "github.com/djdv/p9/p9" @@ -18,6 +19,7 @@ import ( unixpb "github.com/ipfs/go-unixfs/pb" coreiface "github.com/ipfs/interface-go-ipfs-core" coreoptions "github.com/ipfs/interface-go-ipfs-core/options" + corepath "github.com/ipfs/interface-go-ipfs-core/path" ) const ( @@ -27,15 +29,42 @@ const ( // context: https://github.com/ipfs/go-ipfs/pull/6612/files#r322989041 ipfsBlockSize = 256 << 10 saltSize = 32 + + // TODO: move these + // Linux errno values for non-Linux systems; 9p2000.L compliance + ENOTDIR = syscall.Errno(0x14) + ENOENT = syscall.ENOENT //TODO: migrate to platform independent value + + // pedantic POSIX stuff + S_IROTH p9.FileMode = p9.Read + S_IWOTH = p9.Write + S_IXOTH = p9.Exec + + S_IRGRP = S_IROTH << 3 + S_IWGRP = S_IWOTH << 3 + S_IXGRP = S_IXOTH << 3 + + S_IRUSR = S_IRGRP << 3 + S_IWUSR = S_IWGRP << 3 + S_IXUSR = S_IXGRP << 3 + + S_IRWXO = S_IROTH | S_IWOTH | S_IXOTH + S_IRWXG = S_IRGRP | S_IWGRP | S_IXGRP + S_IRWXU = S_IRUSR | S_IWUSR | S_IXUSR + + IRWXA = S_IRWXU | S_IRWXG | S_IRWXO // 0777 + IRXA = IRWXA &^ (S_IWUSR | S_IWGRP | S_IWOTH) // 0555 ) -var errWalkOpened = errors.New("this fid is open") -var errKeyNotInStore = errors.New("requested key was not found in the key store") +var ( + errWalkOpened = errors.New("this fid is open") + errKeyNotInStore = errors.New("requested key was not found in the key store") -// NOTE [2019.09.12]: QID's have a high collision probability -// as a result we add a salt to hashes to attempt to mitigate this -// for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 -var salt []byte + // NOTE [2019.09.12]: QID's have a high collision probability + // as a result we add a salt to hashes to attempt to mitigate this + // for more context see: https://github.com/ipfs/go-ipfs/pull/6612#discussion_r321038649 + salt []byte +) type directoryStream struct { entryChan <-chan coreiface.DirEntry @@ -46,21 +75,48 @@ type directoryStream struct { type walkRef interface { p9.File - // returns the QID of the reference - QID() p9.QID // implemented on Base - - // returns a derived instance that is ready to step - Derive() walkRef // implemented per filesystem - // Step should try to step to "name", resulting in a ready to use derivative + /* CheckWalk should make sure that the current reference adheard to the restrictions + of 'walk(5)' + In particular the reference must not be open for I/O, or otherwise already closed + */ + CheckWalk() error + + /* Fork allocates a reference dervied from itself + The returned reference should be at the same path as the existing walkRef + the new reference is to act like the starting point `newfid` during `Walk` + e.g. + `newFid` originally references the same data as the origin `walkRef` + but is closed seperatley from the origin + operations such as `Walk` will modify `newFid` without affecting the origin `walkRef` + operations such as `Open` should prevent all references to the same path from opening + etc. in compliance with 'walk(5)' + */ + Fork() (walkRef, error) + + /* QID should check that the node's path is walkable + by constructing the QID for its path + */ + QID() (p9.QID, error) + + /* Step should return a reference that is tracking the result of + the node's current path + "name" + implementaion of this is fs specific + it is valid to return a new reference or the same reference modified + within or outside of your own fs boundaries + as long as `QID` is ready to be called on the resulting node + */ Step(name string) (walkRef, error) - //TODO: implement on base class - // Backtrack returns a reference to the node's parent + /* Backtrack must handle `..` request + returning a reference to the node behind the current node (or itself in the case of the root) + the same comment about implementation of `Step` applies here + */ Backtrack() (parentRef walkRef, err error) } -func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { +/* +func walkerV1(ref walkRef, names []string) ([]p9.QID, p9.File, error) { dbgL := logging.Logger("walker") //TODO: remove this or use the reference log, or something dbgL.Debugf("ref: (%p){%T}", ref, ref) @@ -108,6 +164,74 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { dbgL.Debugf("returned ref: (%p){%T}", curRef, curRef) return qids, curRef, nil } +*/ + +func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { + dbgL := logging.Logger("walker") //TODO: remove this or use the reference log, or something + dbgL.Debugf("ref: (%p){%T}", ref, ref) + + err := ref.CheckWalk() + if err != nil { + return nil, nil, err + } + + curRef, err := ref.Fork() + if err != nil { + return nil, nil, err + } + + // walk(5) + // It is legal for nwname to be zero, in which case newfid will represent the same file as fid + // and the walk will usually succeed + if shouldClone(names) { + dbgL.Debugf("cloning: %p -> %p", ref, curRef) + qid, err := ref.QID() + if err != nil { + return nil, nil, err + } + return []p9.QID{qid}, curRef, nil + } + + qids := make([]p9.QID, 0, len(names)) + + for _, name := range names { + switch name { + default: + // get ready to step forward; maybe across FS bounds + dbgL.Debugf("stepping to: %q", name) + curRef, err = curRef.Step(name) + + case ".": + // don't prepare to move at all + dbgL.Debugf(`staying put: "."`) + + case "..": + // get ready to step backwards; maybe across FS bounds + dbgL.Debugf(`backtracking to: ".."`) + curRef, err = curRef.Backtrack() + } + + if err != nil { + dbgL.Error(err) + return qids, nil, err + } + + // commit to the step + qid, err := curRef.QID() + if err != nil { + dbgL.Error(err) // we fell down + return qids, nil, err + } + + // set on success, we stepped forward + qids = append(qids, qid) + + dbgL.Debugf("currentRef: (%p){%T}", curRef, curRef) + } + + dbgL.Debugf("returned ref: (%p){%T}", curRef, curRef) + return qids, curRef, nil +} func init() { salt = make([]byte, saltSize) @@ -250,27 +374,6 @@ func mfsEntTo9Ent(mfsEnt mfs.NodeListing) (p9.Dirent, error) { }, nil } -const ( // pedantic POSIX stuff - S_IROTH p9.FileMode = p9.Read - S_IWOTH = p9.Write - S_IXOTH = p9.Exec - - S_IRGRP = S_IROTH << 3 - S_IWGRP = S_IWOTH << 3 - S_IXGRP = S_IXOTH << 3 - - S_IRUSR = S_IRGRP << 3 - S_IWUSR = S_IWGRP << 3 - S_IXUSR = S_IXGRP << 3 - - S_IRWXO = S_IROTH | S_IWOTH | S_IXOTH - S_IRWXG = S_IRGRP | S_IWGRP | S_IXGRP - S_IRWXU = S_IRUSR | S_IWUSR | S_IXUSR - - IRWXA = S_IRWXU | S_IRWXG | S_IRWXO // 0777 - IRXA = IRWXA &^ (S_IWUSR | S_IWGRP | S_IWOTH) // 0555 -) - func timeStamp(attr *p9.Attr, mask p9.AttrMask) { now := time.Now() if mask.ATime { @@ -310,7 +413,7 @@ func flatReaddir(ents []p9.Dirent, offset uint64, count uint32) ([]p9.Dirent, er func boundCheck(offset uint64, length int) (bool, error) { switch { case offset == uint64(length): - return true, nil // EOS + return true, io.EOF // EOS case offset > uint64(length): return true, fmt.Errorf("offset %d extends beyond directory bound %d", offset, length) default: @@ -318,3 +421,23 @@ func boundCheck(offset uint64, length int) (bool, error) { return false, nil } } + +func coreToQid(ctx context.Context, path corepath.Path, core coreiface.CoreAPI) (p9.QID, error) { + var qid p9.QID + // translate from abstract path to CoreAPI resolved path + resolvedPath, err := core.ResolvePath(ctx, path) + if err != nil { + return qid, err + } + + // inspected to derive 9P QID + attr := new(p9.Attr) + _, err = coreAttr(ctx, attr, resolvedPath, core, p9.AttrMask{Mode: true}) + if err != nil { + return qid, err + } + + qid.Type = attr.Mode.QIDType() + qid.Path = cidToQPath(resolvedPath.Cid()) + return qid, nil +} diff --git a/plugin/plugins/filesystem/pinfs_test.go b/plugin/plugins/filesystem/pinfs_test.go new file mode 100644 index 00000000000..9dbeff2090b --- /dev/null +++ b/plugin/plugins/filesystem/pinfs_test.go @@ -0,0 +1,68 @@ +package filesystem + +import ( + "context" + "os" + "sort" + "testing" + + fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + pinRoot, err := fsnodes.PinFSAttacher(ctx, core).Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) + } + + same := func(base, target []string) bool { + if len(base) != len(target) { + return false + } + sort.Strings(base) + sort.Strings(target) + + for i := len(base) - 1; i >= 0; i-- { + if target[i] != base[i] { + return false + } + } + return true + } + + shallowCompare := func() { + basePins, err := pinNames(ctx, core) + if err != nil { + t.Fatalf("Failed to list IPFS pins: %s\n", err) + } + p9Pins, err := p9PinNames(pinRoot) + if err != nil { + t.Fatalf("Failed to list 9P pins: %s\n", err) + } + + if !same(basePins, p9Pins) { + t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + } + } + + //test default (likely empty) test repo pins + shallowCompare() + + // test modifying pinset +1; initEnv pins its IPFS environment + env, _, err := initEnv(ctx, core) + if err != nil { + t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + } + defer os.RemoveAll(env) + shallowCompare() + + // test modifying pinset +1 again; generate garbage and pin it + if err := generateGarbage(env); err != nil { + t.Fatalf("Failed to generate test data: %s\n", err) + } + if _, err = pinAddDir(ctx, core, env); err != nil { + t.Fatalf("Failed to add directory to IPFS: %s\n", err) + } + shallowCompare() +} diff --git a/plugin/plugins/filesystem/root_test.go b/plugin/plugins/filesystem/root_test.go new file mode 100644 index 00000000000..90525e7bc00 --- /dev/null +++ b/plugin/plugins/filesystem/root_test.go @@ -0,0 +1,70 @@ +package filesystem + +import ( + "context" + "errors" + "fmt" + "io" + "testing" + + "github.com/djdv/p9/p9" + fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" + coreiface "github.com/ipfs/interface-go-ipfs-core" +) + +var rootSubsystems = []p9.Dirent{ + { + Name: "ipfs", + Offset: 1, + Type: p9.TypeDir, + QID: p9.QID{ + Type: p9.TypeDir, + }, + }, { + Name: "ipns", + Offset: 2, + Type: p9.TypeDir, + QID: p9.QID{ + Type: p9.TypeDir, + }, + }, +} + +func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + t.Run("Baseline", func(t *testing.T) { baseLine(ctx, t, core, fsnodes.RootAttacher) }) + + rootRef, err := fsnodes.RootAttacher(ctx, core).Attach() + if err != nil { + t.Fatalf("Baseline test passed but attach failed: %s\n", err) + } + _, root, err := rootRef.Walk(nil) + if err != nil { + t.Fatalf("Baseline test passed but walk failed: %s\n", err) + } + + t.Run("Root directory entries", func(t *testing.T) { testRootDir(ctx, t, root) }) +} + +func testRootDir(ctx context.Context, t *testing.T, root p9.File) { + root.Open(p9.ReadOnly) + + ents, err := root.Readdir(0, uint32(len(rootSubsystems))) + if err != nil { + t.Fatal(err) + } + + if _, err = root.Readdir(uint64(len(ents)), ^uint32(0)); err != io.EOF { + t.Fatal(errors.New("entry count mismatch")) + } + + for i, ent := range ents { + // TODO: for now we trust the QID from the server + // we should generate these paths separately during init + rootSubsystems[i].QID.Path = ent.QID.Path + + if ent != rootSubsystems[i] { + t.Fatal(fmt.Errorf("ent %v != expected %v", ent, rootSubsystems[i])) + } + } + +} From 294c3ea94d031b0428de4b6e3eba865ae7677279 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 14 Oct 2019 16:07:34 -0400 Subject: [PATCH 076/102] actually cancel the context + lint --- plugin/plugins/filesystem/nodes/base.go | 23 +++++--- plugin/plugins/filesystem/nodes/ipfs.go | 11 ++-- plugin/plugins/filesystem/nodes/utils.go | 71 ++---------------------- 3 files changed, 26 insertions(+), 79 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index cd746dfc8d1..2d943b00565 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -40,7 +40,7 @@ type Base struct { closed bool // set to true upon close; this reference should not be used again for anything modified bool // set to true when the `Trail` has been modified (usually by `Step`) - // reset to false when Qid has been populated with the current path in `Trail` (usually by `QID`) + // reset to false when `Qid` has been populated with the current path in `Trail` (usually by `QID`) } @@ -91,7 +91,7 @@ type IPFSBase struct { filesystemCancel context.CancelFunc /* During `file.Walk` a new reference to `file` should be created - with its op-context +op-cancel populated with one derived from the existing fs context + with its op-context + op-cancel populated with one derived from the existing fs context This context is expected to be valid as long as the file is being referenced by a particular FID it should be canceled during `Close` @@ -198,25 +198,32 @@ func (b *Base) Close() error { } func (ib *IPFSBase) Close() error { - ib.Logger.Debugf("closing: {%d}%q", ib.Qid.Path, ib.String()) + lastErr := ib.Base.Close() + if lastErr != nil { + ib.Logger.Error(lastErr) + } - var err error if ib.filesystemCancel != nil { + /* TODO: only do this on the last close to the root if ib.proxy != nil { - if err = ib.proxy.Close(); err != nil { + if err := ib.proxy.Close(); err != nil { ib.Logger.Error(err) } + lastErr = err } + */ ib.filesystemCancel() } - ib.closed = true + if ib.operationsCancel != nil { + ib.operationsCancel() + } - return err + return lastErr } func (b *Base) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - b.Logger.Debugf("GetAttr {%d}:%q", b.Qid.Path, b.String()) + //b.Logger.Debugf("GetAttr {%d}:%q", b.Qid.Path, b.String()) return *b.Qid, *b.metaMask, *b.meta, nil } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 0cea66c4e2b..f8a72e88df3 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -244,11 +244,9 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { } func (id *IPFS) Close() error { - lastErr := id.IPFSBase.Close() - if lastErr != nil { - id.Logger.Error(lastErr) - } + var lastErr error + //TODO: timeout and cancel the context if Close takes too long if id.file != nil { if err := id.file.Close(); err != nil { id.Logger.Error(err) @@ -258,6 +256,11 @@ func (id *IPFS) Close() error { } id.directory = nil + lastErr = id.IPFSBase.Close() + if lastErr != nil { + id.Logger.Error(lastErr) + } + return lastErr } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index b71bd3ba0c9..9b36dc9dc5d 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -13,7 +13,6 @@ import ( "github.com/djdv/p9/p9" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - logging "github.com/ipfs/go-log" "github.com/ipfs/go-mfs" "github.com/ipfs/go-unixfs" unixpb "github.com/ipfs/go-unixfs/pb" @@ -76,18 +75,18 @@ type directoryStream struct { type walkRef interface { p9.File - /* CheckWalk should make sure that the current reference adheard to the restrictions + /* CheckWalk should make sure that the current reference adheres to the restrictions of 'walk(5)' In particular the reference must not be open for I/O, or otherwise already closed */ CheckWalk() error - /* Fork allocates a reference dervied from itself + /* Fork allocates a reference derived from itself The returned reference should be at the same path as the existing walkRef the new reference is to act like the starting point `newfid` during `Walk` e.g. `newFid` originally references the same data as the origin `walkRef` - but is closed seperatley from the origin + but is closed separately from the origin operations such as `Walk` will modify `newFid` without affecting the origin `walkRef` operations such as `Open` should prevent all references to the same path from opening etc. in compliance with 'walk(5)' @@ -101,7 +100,7 @@ type walkRef interface { /* Step should return a reference that is tracking the result of the node's current path + "name" - implementaion of this is fs specific + implementation of this is fs specific it is valid to return a new reference or the same reference modified within or outside of your own fs boundaries as long as `QID` is ready to be called on the resulting node @@ -115,61 +114,7 @@ type walkRef interface { Backtrack() (parentRef walkRef, err error) } -/* -func walkerV1(ref walkRef, names []string) ([]p9.QID, p9.File, error) { - dbgL := logging.Logger("walker") //TODO: remove this or use the reference log, or something - dbgL.Debugf("ref: (%p){%T}", ref, ref) - - curRef := ref.Derive() - - // walk(5) - // It is legal for nwname to be zero, in which case newfid will represent the same file as fid - // and the walk will usually succeed - if shouldClone(names) { - dbgL.Debugf("cloning: %p -> %p", ref, curRef) - return []p9.QID{curRef.QID()}, curRef, nil - } - - qids := make([]p9.QID, 0, len(names)) - - var err error - for _, name := range names { - switch name { - default: - // step forward - dbgL.Debugf("stepping to: %q", name) - curRef, err = curRef.Step(name) - - case ".": - // stay - // qid = qids[len(qids)-1] - // continue - dbgL.Debugf(`staying put: "."`) - - case "..": - // step back - dbgL.Debugf(`backtracking to: ".."`) - curRef, err = curRef.Backtrack() - } - - if err != nil { - dbgL.Error(err) - return qids, nil, err - } - - dbgL.Debugf("currentRef: (%p){%T}", curRef, curRef) - qids = append(qids, curRef.QID()) - } - - dbgL.Debugf("returned ref: (%p){%T}", curRef, curRef) - return qids, curRef, nil -} -*/ - func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { - dbgL := logging.Logger("walker") //TODO: remove this or use the reference log, or something - dbgL.Debugf("ref: (%p){%T}", ref, ref) - err := ref.CheckWalk() if err != nil { return nil, nil, err @@ -184,7 +129,6 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { // It is legal for nwname to be zero, in which case newfid will represent the same file as fid // and the walk will usually succeed if shouldClone(names) { - dbgL.Debugf("cloning: %p -> %p", ref, curRef) qid, err := ref.QID() if err != nil { return nil, nil, err @@ -198,38 +142,31 @@ func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { switch name { default: // get ready to step forward; maybe across FS bounds - dbgL.Debugf("stepping to: %q", name) curRef, err = curRef.Step(name) case ".": // don't prepare to move at all - dbgL.Debugf(`staying put: "."`) case "..": // get ready to step backwards; maybe across FS bounds - dbgL.Debugf(`backtracking to: ".."`) curRef, err = curRef.Backtrack() } if err != nil { - dbgL.Error(err) return qids, nil, err } // commit to the step qid, err := curRef.QID() if err != nil { - dbgL.Error(err) // we fell down return qids, nil, err } // set on success, we stepped forward qids = append(qids, qid) - dbgL.Debugf("currentRef: (%p){%T}", curRef, curRef) } - dbgL.Debugf("returned ref: (%p){%T}", curRef, curRef) return qids, curRef, nil } From 8ce5e518c35422f1e48c023f362e87e78e4d2f2f Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 14 Oct 2019 16:18:24 -0400 Subject: [PATCH 077/102] actually use the context too --- plugin/plugins/filesystem/nodes/ipfs.go | 4 ++-- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index f8a72e88df3..6a5168be8fb 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -115,7 +115,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // handle directories if qid.Type == p9.TypeDir { // c, err := id.core.Unixfs().Ls(handleContext, id.CorePath()) - c, err := id.core.Unixfs().Ls(id.filesystemCtx, id.CorePath()) + c, err := id.core.Unixfs().Ls(id.operationsCtx, id.CorePath()) if err != nil { //id.operationsCancel() return qid, 0, err @@ -128,7 +128,7 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { } // handle files - apiNode, err := id.core.Unixfs().Get(id.filesystemCtx, id.CorePath()) + apiNode, err := id.core.Unixfs().Get(id.operationsCtx, id.CorePath()) if err != nil { //id.operationsCancel() return qid, 0, err diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 445bb0948c9..20062535e51 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -80,7 +80,7 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { qid := *pd.Qid // IPFS core representation - pins, err := pd.core.Pin().Ls(pd.filesystemCtx, coreoptions.Pin.Type.Recursive()) + pins, err := pd.core.Pin().Ls(pd.operationsCtx, coreoptions.Pin.Type.Recursive()) if err != nil { return qid, 0, err } From 83284858b44da12378ef9794f7f4654afa5e76f9 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 14 Oct 2019 21:22:12 -0400 Subject: [PATCH 078/102] don't readdir forever during Linux tests --- plugin/plugins/filesystem/filesystem_test.go | 12 ++++-------- plugin/plugins/filesystem/nodes/ipfs.go | 9 +++------ plugin/plugins/filesystem/nodes/utils.go | 2 +- plugin/plugins/filesystem/root_test.go | 3 +-- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 463911424c1..3f6b7f251a9 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -3,7 +3,6 @@ package filesystem import ( "context" "fmt" - "io" "io/ioutil" "math/rand" "os" @@ -359,16 +358,13 @@ func p9Readdir(dir p9.File) ([]p9.Dirent, error) { ) for { curEnts, err := dirClone.Readdir(offset, ^uint32(0)) - ents = append(ents, curEnts...) - if err != nil { + lEnts := len(curEnts) + if err != nil || lEnts == 0 { break } - offset += uint64(len(curEnts)) - } - - if err == io.EOF { - err = nil + ents = append(ents, curEnts...) + offset += uint64(lEnts) } return ents, err diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 6a5168be8fb..9e0b05ca29a 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -114,7 +114,6 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // handle directories if qid.Type == p9.TypeDir { - // c, err := id.core.Unixfs().Ls(handleContext, id.CorePath()) c, err := id.core.Unixfs().Ls(id.operationsCtx, id.CorePath()) if err != nil { //id.operationsCancel() @@ -161,10 +160,8 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, id.directory.err } - if id.directory.eos { - if offset == id.directory.cursor { - return nil, io.EOF // this is the only exception to offset being behind the cursor - } + if id.directory.eos && offset == id.directory.cursor { + return nil, nil // EOS } if offset < id.directory.cursor { @@ -179,7 +176,7 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { if !open { //id.operationsCancel() id.directory.eos = true - return ents, io.EOF + return ents, nil } if entry.Err != nil { id.directory.err = entry.Err diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 9b36dc9dc5d..be45f2e715d 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -350,7 +350,7 @@ func flatReaddir(ents []p9.Dirent, offset uint64, count uint32) ([]p9.Dirent, er func boundCheck(offset uint64, length int) (bool, error) { switch { case offset == uint64(length): - return true, io.EOF // EOS + return true, nil // EOS case offset > uint64(length): return true, fmt.Errorf("offset %d extends beyond directory bound %d", offset, length) default: diff --git a/plugin/plugins/filesystem/root_test.go b/plugin/plugins/filesystem/root_test.go index 90525e7bc00..58d9089892a 100644 --- a/plugin/plugins/filesystem/root_test.go +++ b/plugin/plugins/filesystem/root_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "testing" "github.com/djdv/p9/p9" @@ -53,7 +52,7 @@ func testRootDir(ctx context.Context, t *testing.T, root p9.File) { t.Fatal(err) } - if _, err = root.Readdir(uint64(len(ents)), ^uint32(0)); err != io.EOF { + if _, err = root.Readdir(uint64(len(ents)), ^uint32(0)); err != nil { t.Fatal(errors.New("entry count mismatch")) } From db0bcbad36a724cf7cc7fe83bf8ce346a6090d41 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 14 Oct 2019 21:25:42 -0400 Subject: [PATCH 079/102] don't lose err value --- plugin/plugins/filesystem/filesystem_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 3f6b7f251a9..5a546821912 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -357,7 +357,8 @@ func p9Readdir(dir p9.File) ([]p9.Dirent, error) { ents []p9.Dirent ) for { - curEnts, err := dirClone.Readdir(offset, ^uint32(0)) + var curEnts []p9.Dirent + curEnts, err = dirClone.Readdir(offset, ^uint32(0)) lEnts := len(curEnts) if err != nil || lEnts == 0 { break From 7bb94a4da28eb8a6a4b322a35128264d07891711 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 15 Oct 2019 14:03:15 -0400 Subject: [PATCH 080/102] readdir fixes --- plugin/plugins/filesystem/nodes/ipfs.go | 28 +++++++++++------------- plugin/plugins/filesystem/nodes/utils.go | 1 - 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 9e0b05ca29a..86b9dc6b2fc 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -160,10 +160,6 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, id.directory.err } - if id.directory.eos && offset == id.directory.cursor { - return nil, nil // EOS - } - if offset < id.directory.cursor { return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) } @@ -175,7 +171,6 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { case entry, open := <-id.directory.entryChan: if !open { //id.operationsCancel() - id.directory.eos = true return ents, nil } if entry.Err != nil { @@ -183,18 +178,21 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, entry.Err } - if offset <= id.directory.cursor { - nineEnt, err := coreEntTo9Ent(entry) - if err != nil { - id.directory.err = err - return nil, err - } - nineEnt.Offset = id.directory.cursor + 1 - ents = append(ents, nineEnt) - } - + // we consumed an entry id.directory.cursor++ + // skip processing the entry if its below the request offset + if offset > id.directory.cursor { + continue + } + nineEnt, err := coreEntTo9Ent(entry) + if err != nil { + id.directory.err = err + return nil, err + } + nineEnt.Offset = id.directory.cursor + ents = append(ents, nineEnt) + case <-id.filesystemCtx.Done(): id.directory.err = id.filesystemCtx.Err() id.Logger.Error(id.directory.err) diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index be45f2e715d..fa9c73968bf 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -68,7 +68,6 @@ var ( type directoryStream struct { entryChan <-chan coreiface.DirEntry cursor uint64 - eos bool // have seen end of stream? err error } From e029c7de79ef23c2f2c8af621e6d6669f42ba37b Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 15 Oct 2019 16:21:03 -0400 Subject: [PATCH 081/102] gomod deps --- go.mod | 4 +++- go.sum | 4 ++-- plugin/plugins/filesystem/filesystem.go | 2 +- plugin/plugins/filesystem/filesystem_test.go | 2 +- plugin/plugins/filesystem/ipfs_test.go | 4 ++-- plugin/plugins/filesystem/nodes/base.go | 4 ++-- plugin/plugins/filesystem/nodes/ipfs.go | 2 +- plugin/plugins/filesystem/nodes/ipns.go | 2 +- plugin/plugins/filesystem/nodes/keyfs.go | 2 +- plugin/plugins/filesystem/nodes/options/options.go | 2 +- plugin/plugins/filesystem/nodes/pinfs.go | 2 +- plugin/plugins/filesystem/nodes/root.go | 2 +- plugin/plugins/filesystem/nodes/utils.go | 2 +- plugin/plugins/filesystem/root_test.go | 2 +- 14 files changed, 19 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index affef23cb50..18d32ceb3f2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d - github.com/djdv/p9 v0.0.0-20190907154417-3fdf0632585f github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fatih/color v1.7.0 // indirect @@ -16,6 +15,7 @@ require ( github.com/golangci/golangci-lint v1.17.1 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/golang-lru v0.5.3 + github.com/hugelgupf/p9 v0.0.0-20190923080529-a5e7bac5e555 github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.0 @@ -121,3 +121,5 @@ require ( replace github.com/golangci/golangci-lint => github.com/mhutchinson/golangci-lint v1.17.2-0.20190819125825-d18f2136e32b go 1.12 + +replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af diff --git a/go.sum b/go.sum index 431224ae038..9cb788e9bac 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/djdv/p9 v0.0.0-20190907154417-3fdf0632585f h1:CUjmKarvWU1sBD/hUbKDes68c9ApZcmHnYaxRL9wvgc= -github.com/djdv/p9 v0.0.0-20190907154417-3fdf0632585f/go.mod h1:58+5oyIFRxi9jt7uQrdqpmUryJDTsxY8N8GxhnNocLQ= +github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af h1:Fz2zlKtMYbMpX1BPo4begzCE855g+UGcj+5hmn4Bf04= +github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af/go.mod h1:igFKfChBTNZnGzM9Arxcki7QfgT01VJneIZrNY8OqKI= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 083e4e22088..f3c8f92fa8b 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" plugin "github.com/ipfs/go-ipfs/plugin" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 5a546821912..9d3da2bd194 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipfs/core" diff --git a/plugin/plugins/filesystem/ipfs_test.go b/plugin/plugins/filesystem/ipfs_test.go index 1779d595027..795d27fd775 100644 --- a/plugin/plugins/filesystem/ipfs_test.go +++ b/plugin/plugins/filesystem/ipfs_test.go @@ -6,8 +6,8 @@ import ( gopath "path" "testing" - "github.com/djdv/p9/localfs" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/localfs" + "github.com/hugelgupf/p9/p9" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" coreiface "github.com/ipfs/interface-go-ipfs-core" ) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 2d943b00565..3cf0eea1ff9 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -6,8 +6,8 @@ import ( gopath "path" "time" - "github.com/djdv/p9/p9" - "github.com/djdv/p9/unimplfs" + "github.com/hugelgupf/p9/p9" + "github.com/hugelgupf/p9/unimplfs" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 86b9dc6b2fc..6928f373891 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" files "github.com/ipfs/go-ipfs-files" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" coreiface "github.com/ipfs/interface-go-ipfs-core" diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go index f771c8fc4d2..2f5513553ac 100644 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -3,7 +3,7 @@ package fsnodes import ( "context" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" coreiface "github.com/ipfs/interface-go-ipfs-core" ) diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index dbde706b624..a457450767d 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -3,7 +3,7 @@ package fsnodes import ( "context" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" diff --git a/plugin/plugins/filesystem/nodes/options/options.go b/plugin/plugins/filesystem/nodes/options/options.go index f11be5048cf..2f0c6e96ba9 100644 --- a/plugin/plugins/filesystem/nodes/options/options.go +++ b/plugin/plugins/filesystem/nodes/options/options.go @@ -1,7 +1,7 @@ package nodeopts import ( - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" "github.com/jbenet/goprocess" "github.com/ipfs/go-cid" diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 20062535e51..c69a1f87d6b 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -5,7 +5,7 @@ import ( "fmt" gopath "path" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 2c71b0922bb..03d85c5077a 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -3,7 +3,7 @@ package fsnodes import ( "context" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" cid "github.com/ipfs/go-cid" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" logging "github.com/ipfs/go-log" diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index fa9c73968bf..f8edd719f52 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -10,7 +10,7 @@ import ( "syscall" "time" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-mfs" diff --git a/plugin/plugins/filesystem/root_test.go b/plugin/plugins/filesystem/root_test.go index 58d9089892a..ad5a5f6c63f 100644 --- a/plugin/plugins/filesystem/root_test.go +++ b/plugin/plugins/filesystem/root_test.go @@ -6,7 +6,7 @@ import ( "fmt" "testing" - "github.com/djdv/p9/p9" + "github.com/hugelgupf/p9/p9" fsnodes "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes" coreiface "github.com/ipfs/interface-go-ipfs-core" ) From 8cee086b55721be3675a3f01f7e12e71a4830133 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 21 Oct 2019 12:12:01 -0400 Subject: [PATCH 082/102] changes and tests around plugin close --- go.mod | 3 +- go.sum | 2 + plugin/plugins/filesystem/filesystem.go | 58 ++- .../filesystem/filesystem_common_test.go | 435 +++++++++++++++++ plugin/plugins/filesystem/filesystem_test.go | 449 ++---------------- 5 files changed, 532 insertions(+), 415 deletions(-) create mode 100644 plugin/plugins/filesystem/filesystem_common_test.go diff --git a/go.mod b/go.mod index 18d32ceb3f2..3db357e3978 100644 --- a/go.mod +++ b/go.mod @@ -122,4 +122,5 @@ replace github.com/golangci/golangci-lint => github.com/mhutchinson/golangci-lin go 1.12 -replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af +//TODO: remove this replace when upstream merges changes +replace github.com/hugelgupf/p9 => github.com/djdv/p9 v0.0.0-20191018144345-781c6a62f733 diff --git a/go.sum b/go.sum index 9cb788e9bac..6bb4b982840 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af h1:Fz2zlKtMYbMpX1BPo4begzCE855g+UGcj+5hmn4Bf04= github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af/go.mod h1:igFKfChBTNZnGzM9Arxcki7QfgT01VJneIZrNY8OqKI= +github.com/djdv/p9 v0.0.0-20191018144345-781c6a62f733 h1:2jtqdlho/DLOfSxzPs1SM6fpENZwZii2o7lFtvHVhmc= +github.com/djdv/p9 v0.0.0-20191018144345-781c6a62f733/go.mod h1:igFKfChBTNZnGzM9Arxcki7QfgT01VJneIZrNY8OqKI= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 h1:QV0ZrfBLpFc2KDk+a4LJefDczXnonRwrYrQJY/9L4dA= diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index f3c8f92fa8b..c82a4ab021d 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -2,6 +2,9 @@ package filesystem import ( "context" + "errors" + "fmt" + "net" "os" "path/filepath" "strings" @@ -38,7 +41,9 @@ type FileSystemPlugin struct { addr multiaddr.Multiaddr listener manet.Listener - errorChan chan error + closed chan struct{} + closing bool //TODO: p9 lib should probably have a `server.Close` that closes the listener and swallows the final `Accept` error instead of us managing it in this pkg + serverErr error } func (*FileSystemPlugin) Name() string { @@ -51,6 +56,9 @@ func (*FileSystemPlugin) Version() string { func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { logger.Info("Initializing 9P resource server...") + if fs.addr != nil { + return fmt.Errorf("already initialized with %s", fs.addr.String()) + } // stabilise repo path; our template depends on this if !filepath.IsAbs(env.Repo) { @@ -101,6 +109,14 @@ func (fs *FileSystemPlugin) Init(env *plugin.Environment) error { func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { logger.Info("Starting 9P resource server...") + if fs.addr == nil { + return fmt.Errorf("Start called before plugin Init") + } + + // make sure we're not in use already + if fs.listener != nil { + return fmt.Errorf("already started and listening on %s", fs.listener.Addr()) + } // make sure sockets are not in use already (if we're using them) err := removeUnixSockets(fs.addr) @@ -108,9 +124,6 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { return err } - fs.ctx, fs.cancel = context.WithCancel(context.Background()) - fs.errorChan = make(chan error, 1) - // launch the listener listener, err := manet.Listen(fs.addr) if err != nil { @@ -120,10 +133,24 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { fs.listener = listener // construct and launch the 9P resource server + fs.ctx, fs.cancel = context.WithCancel(context.Background()) + fs.closed = make(chan struct{}) + opts := []nodeopts.AttachOption{nodeopts.Logger(logging.Logger("9root"))} - s := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core, opts...)) + server := p9.NewServer(fsnodes.RootAttacher(fs.ctx, core, opts...)) go func() { - fs.errorChan <- s.Serve(manet.NetListener(fs.listener)) + // run the server until the listener closes + // store error on the fs object then close our syncing channel (see use in `Close` below) + fs.serverErr = server.Serve(manet.NetListener(fs.listener)) + + if fs.closing { // we expect `Accept` to fail when `Close` is called + var opErr *net.OpError + if errors.As(fs.serverErr, &opErr) && opErr.Op == "accept" { + fs.serverErr = nil + } + } + + close(fs.closed) }() logger.Infof("9P service is listening on %s\n", fs.listener.Addr()) @@ -132,7 +159,20 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { func (fs *FileSystemPlugin) Close() error { logger.Info("9P server requested to close") - fs.listener.Close() - fs.cancel() - return <-fs.errorChan + if fs.addr == nil { // forbidden + return fmt.Errorf("Close called before plugin Init") + } + + // synchronization between plugin interface <-> fs server + if fs.closed != nil { // implies `Start` was called prior + fs.closing = true // lets the goroutine know that accept is now allowed to fail + fs.listener.Close() // stop accepting actually + fs.cancel() // stop any lingering fs operations + <-fs.closed // wait for the server thread to set the error value + fs.closing = false // reset our own condition + fs.listener = nil // reset `Start` conditions + fs.closed = nil + } + // otherwise we were never started to begin with; default/initial value will be returned + return fs.serverErr } diff --git a/plugin/plugins/filesystem/filesystem_common_test.go b/plugin/plugins/filesystem/filesystem_common_test.go new file mode 100644 index 00000000000..37aa412ffe7 --- /dev/null +++ b/plugin/plugins/filesystem/filesystem_common_test.go @@ -0,0 +1,435 @@ +package filesystem + +import ( + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + gopath "path" + "path/filepath" + "testing" + "time" + + "github.com/hugelgupf/p9/p9" + files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/core/coreapi" + nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + coreiface "github.com/ipfs/interface-go-ipfs-core" + coreoptions "github.com/ipfs/interface-go-ipfs-core/options" + corepath "github.com/ipfs/interface-go-ipfs-core/path" +) + +type baseAttacher func(context.Context, coreiface.CoreAPI, ...nodeopts.AttachOption) p9.Attacher + +func baseLine(ctx context.Context, t *testing.T, core coreiface.CoreAPI, attachFn baseAttacher) { + attacher := attachFn(ctx, core) + + t.Run("attacher", func(t *testing.T) { testAttacher(ctx, t, attacher) }) + root, err := attacher.Attach() + if err != nil { + t.Fatalf("Attach test passed but attach failed: %s\n", err) + } + t.Run("walk", func(t *testing.T) { testClones(ctx, t, root) }) + +} + +func testAttacher(ctx context.Context, t *testing.T, attacher p9.Attacher) { + // 2 individual instances, one after another + nineRoot, err := attacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + } + + if err = nineRoot.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + nineRootTheRevenge, err := attacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) + } + + if err = nineRootTheRevenge.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // 2 instances at the same time + nineRoot, err = attacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + } + + nineRootTheRevenge, err = attacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) + } + + if err = nineRootTheRevenge.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + if err = nineRoot.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // final instance + nineRoot, err = attacher.Attach() + if err != nil { + t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + } + + if err = nineRoot.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } +} + +func testClones(ctx context.Context, t *testing.T, nineRef p9.File) { + // clone the node we were passed; 1st generation + _, newRef, err := nineRef.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + // this `Close` shouldn't affect the parent it's derived from + // only descendants + if err = newRef.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // remake the clone from the original; 1st generation again + _, gen1, err := nineRef.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + // clone a 2nd generation from the 1st + _, gen2, err := gen1.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + // 3rd from the 2nd + _, gen3, err := gen2.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + // close the 2nd reference + if err = gen2.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // try to clone from the 2nd reference + // this should fail since we closed it + _, undead, err := gen2.Walk(nil) + if err == nil { + t.Fatalf("Clone (%p)%q succeeded when parent (%p)%q was closed\n", undead, undead, gen2, gen2) + } + + // 4th from the 3rd + // should still succeed regardless of 2's state + _, gen4, err := gen3.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + // close the 3rd reference + if err = gen3.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // close the 4th reference + if err = gen4.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // clone a 2nd generation from the 1st again + _, gen2, err = gen1.Walk(nil) + if err != nil { + t.Fatalf("Failed to clone root: %s\n", err) + } + + // close the 1st + if err = gen1.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } + + // close the 2nd + if err = gen2.Close(); err != nil { + t.Fatalf("Close errored: %s\n", err) + } +} + +func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { + var expand func(p9.File) (map[string]p9.Attr, error) + expand = func(nineRef p9.File) (map[string]p9.Attr, error) { + ents, err := p9Readdir(nineRef) + if err != nil { + return nil, err + } + + res := make(map[string]p9.Attr) + for _, ent := range ents { + _, child, err := nineRef.Walk([]string{ent.Name}) + if err != nil { + return nil, err + } + + _, _, attr, err := child.GetAttr(attrMaskIPFSTest) + if err != nil { + return nil, err + } + res[ent.Name] = attr + + if ent.Type == p9.TypeDir { + subRes, err := expand(child) + if err != nil { + return nil, err + } + for name, attr := range subRes { + res[gopath.Join(ent.Name, name)] = attr + } + } + if err = child.Close(); err != nil { + return nil, err + } + } + return res, nil + } + + f1Map, err := expand(f1) + if err != nil { + t.Fatal(err) + } + + f2Map, err := expand(f2) + if err != nil { + t.Fatal(err) + } + + same := func(permissionContains p9.FileMode, base, target map[string]p9.Attr) bool { + if len(base) != len(target) { + + var baseNames []string + var targetNames []string + for name := range base { + baseNames = append(baseNames, name) + } + for name := range target { + targetNames = append(targetNames, name) + } + + t.Fatalf("map lengths don't match:\nbase:%v\ntarget:%v\n", baseNames, targetNames) + return false + } + + for path, baseAttr := range base { + bMode := baseAttr.Mode + tMode := target[path].Mode + + if bMode.FileType() != tMode.FileType() { + t.Fatalf("type for %q don't match:\nbase:%v\ntarget:%v\n", path, bMode, tMode) + return false + } + + if ((bMode.Permissions() & permissionContains) & (tMode.Permissions() & permissionContains)) == 0 { + t.Fatalf("permissions for %q don't match\n(unfiltered)\nbase:%v\ntarget:%v\n(filtered)\nbase:%v\ntarget:%v\n", + path, + bMode.Permissions(), tMode.Permissions(), + bMode.Permissions()&permissionContains, tMode.Permissions()&permissionContains, + ) + return false + } + + if bMode.FileType() != p9.ModeDirectory { + bSize := baseAttr.Size + tSize := target[path].Size + + if bSize != tSize { + t.Fatalf("size for %q doesn't match\nbase:%d\ntarget:%d\n", + path, + bSize, + tSize) + } + } + } + return true + } + if !same(p9.Read, f1Map, f2Map) { + t.Fatalf("contents don't match \nf1:%v\nf2:%v\n", f1Map, f2Map) + } +} + +func InitCore(ctx context.Context) (coreiface.CoreAPI, error) { + node, err := core.NewNode(ctx, &core.BuildCfg{ + Online: false, + Permanent: false, + DisableEncryptedConnections: true, + }) + if err != nil { + return nil, err + } + + return coreapi.NewCoreAPI(node) +} + +const incantation = "May the bits passing through this device somehow help bring peace to this world" + +func initEnv(ctx context.Context, core coreiface.CoreAPI) (string, corepath.Resolved, error) { + testDir, err := ioutil.TempDir("", "ipfs-") + if err != nil { + return "", nil, err + } + if err := os.Chmod(testDir, 0775); err != nil { + return "", nil, err + } + + if err = ioutil.WriteFile(filepath.Join(testDir, "empty"), + []byte(nil), + 0644); err != nil { + return "", nil, err + } + + if err = ioutil.WriteFile(filepath.Join(testDir, "small"), + []byte(incantation), + 0644); err != nil { + return "", nil, err + } + + if err := generateGarbage(testDir); err != nil { + return "", nil, err + } + + testSubDir, err := ioutil.TempDir(testDir, "ipfs-") + if err != nil { + return "", nil, err + } + if err := os.Chmod(testSubDir, 0775); err != nil { + return "", nil, err + } + + if err := generateGarbage(testSubDir); err != nil { + return "", nil, err + } + + iPath, err := pinAddDir(ctx, core, testDir) + if err != nil { + return "", nil, err + } + + return testDir, iPath, err +} + +func p9Readdir(dir p9.File) ([]p9.Dirent, error) { + _, dirClone, err := dir.Walk(nil) + if err != nil { + return nil, err + } + + _, _, err = dirClone.Open(p9.ReadOnly) + if err != nil { + return nil, err + } + defer dirClone.Close() + + var ( + offset uint64 + ents []p9.Dirent + ) + for { + var curEnts []p9.Dirent + curEnts, err = dirClone.Readdir(offset, ^uint32(0)) + lEnts := len(curEnts) + if err != nil || lEnts == 0 { + break + } + + ents = append(ents, curEnts...) + offset += uint64(lEnts) + } + + return ents, err +} + +func pinAddDir(ctx context.Context, core coreiface.CoreAPI, path string) (corepath.Resolved, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + + node, err := files.NewSerialFile(path, false, fi) + if err != nil { + return nil, err + } + + iPath, err := core.Unixfs().Add(ctx, node.(files.Directory), coreoptions.Unixfs.Pin(true)) + if err != nil { + return nil, err + } + return iPath, nil +} + +func generateGarbage(tempDir string) error { + randDev := rand.New(rand.NewSource(time.Now().UnixNano())) + + for _, size := range []int{4, 8, 16, 32} { + buf := make([]byte, size<<(10*2)) + if _, err := randDev.Read(buf); err != nil { + return err + } + + name := fmt.Sprintf("%dMiB", size) + if err := ioutil.WriteFile(filepath.Join(tempDir, name), + buf, + 0644); err != nil { + return err + } + } + + return nil +} + +func pinNames(ctx context.Context, core coreiface.CoreAPI) ([]string, error) { + pins, err := core.Pin().Ls(ctx, coreoptions.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + names := make([]string, 0, len(pins)) + for _, pin := range pins { + names = append(names, gopath.Base(pin.Path().String())) + } + return names, nil +} + +func p9PinNames(root p9.File) ([]string, error) { + ents, err := p9Readdir(root) + if err != nil { + return nil, err + } + + names := make([]string, 0, len(ents)) + + for _, ent := range ents { + names = append(names, ent.Name) + } + + return names, nil +} + +//TODO: +// NOTE: compares a subset of attributes, matching those of IPFS +func testIPFSCompare(t *testing.T, f1, f2 p9.File) { + _, _, f1Attr, err := f1.GetAttr(attrMaskIPFSTest) + if err != nil { + t.Errorf("Attr(%v) = %v, want nil", f1, err) + } + _, _, f2Attr, err := f2.GetAttr(attrMaskIPFSTest) + if err != nil { + t.Errorf("Attr(%v) = %v, want nil", f2, err) + } + if f1Attr != f2Attr { + t.Errorf("Attributes of same files do not match: %v and %v", f1Attr, f2Attr) + } +} diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 9d3da2bd194..7b2555ee982 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -2,24 +2,11 @@ package filesystem import ( "context" - "fmt" - "io/ioutil" - "math/rand" - "os" - gopath "path" - "path/filepath" "testing" - "time" "github.com/hugelgupf/p9/p9" - files "github.com/ipfs/go-ipfs-files" - - "github.com/ipfs/go-ipfs/core" - "github.com/ipfs/go-ipfs/core/coreapi" - nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + plugin "github.com/ipfs/go-ipfs/plugin" coreiface "github.com/ipfs/interface-go-ipfs-core" - coreoptions "github.com/ipfs/interface-go-ipfs-core/options" - corepath "github.com/ipfs/interface-go-ipfs-core/path" ) var attrMaskIPFSTest = p9.AttrMask{ @@ -29,425 +16,77 @@ var attrMaskIPFSTest = p9.AttrMask{ func TestAll(t *testing.T) { ctx := context.TODO() + core, err := InitCore(ctx) if err != nil { - t.Fatalf("Failed to construct IPFS node: %s\n", err) + t.Fatalf("Failed to construct CoreAPI: %s\n", err) } t.Run("RootFS", func(t *testing.T) { testRootFS(ctx, t, core) }) t.Run("PinFS", func(t *testing.T) { testPinFS(ctx, t, core) }) t.Run("IPFS", func(t *testing.T) { testIPFS(ctx, t, core) }) -} - -type baseAttacher func(context.Context, coreiface.CoreAPI, ...nodeopts.AttachOption) p9.Attacher - -func baseLine(ctx context.Context, t *testing.T, core coreiface.CoreAPI, attachFn baseAttacher) { - attacher := attachFn(ctx, core) - - t.Run("attacher", func(t *testing.T) { testAttacher(ctx, t, attacher) }) - root, err := attacher.Attach() - if err != nil { - t.Fatalf("Attach test passed but attach failed: %s\n", err) - } - t.Run("walk", func(t *testing.T) { testClones(ctx, t, root) }) - -} - -func testAttacher(ctx context.Context, t *testing.T, attacher p9.Attacher) { - // 2 individual instances, one after another - nineRoot, err := attacher.Attach() - if err != nil { - t.Fatalf("Failed to attach to 9P root resource: %s\n", err) - } - - if err = nineRoot.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - nineRootTheRevenge, err := attacher.Attach() - if err != nil { - t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) - } - - if err = nineRootTheRevenge.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - // 2 instances at the same time - nineRoot, err = attacher.Attach() - if err != nil { - t.Fatalf("Failed to attach to 9P root resource: %s\n", err) - } - - nineRootTheRevenge, err = attacher.Attach() - if err != nil { - t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) - } - - if err = nineRootTheRevenge.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - if err = nineRoot.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - // final instance - nineRoot, err = attacher.Attach() - if err != nil { - t.Fatalf("Failed to attach to 9P root resource: %s\n", err) - } - - if err = nineRoot.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } -} - -func testClones(ctx context.Context, t *testing.T, nineRef p9.File) { - // clone the node we were passed; 1st generation - _, newRef, err := nineRef.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) - } - - // this `Close` shouldn't affect the parent it's derived from - // only descendants - if err = newRef.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - // remake the clone from the original; 1st generation again - _, gen1, err := nineRef.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) - } - - // clone a 2nd generation from the 1st - _, gen2, err := gen1.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) - } - - // 3rd from the 2nd - _, gen3, err := gen2.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) - } - - // close the 2nd reference - if err = gen2.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - // try to clone from the 2nd reference - // this should fail since we closed it - _, undead, err := gen2.Walk(nil) - if err == nil { - t.Fatalf("Clone (%p)%q succeeded when parent (%p)%q was closed\n", undead, undead, gen2, gen2) - } - - // 4th from the 3rd - // should still succeed regardless of 2's state - _, gen4, err := gen3.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) - } - - // close the 3rd reference - if err = gen3.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - // close the 4th reference - if err = gen4.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - // clone a 2nd generation from the 1st again - _, gen2, err = gen1.Walk(nil) - if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) - } - - // close the 1st - if err = gen1.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } - - // close the 2nd - if err = gen2.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) - } -} - -func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { - var expand func(p9.File) (map[string]p9.Attr, error) - expand = func(nineRef p9.File) (map[string]p9.Attr, error) { - ents, err := p9Readdir(nineRef) - if err != nil { - return nil, err - } - - res := make(map[string]p9.Attr) - for _, ent := range ents { - _, child, err := nineRef.Walk([]string{ent.Name}) - if err != nil { - return nil, err - } - - _, _, attr, err := child.GetAttr(attrMaskIPFSTest) - if err != nil { - return nil, err - } - res[ent.Name] = attr - if ent.Type == p9.TypeDir { - subRes, err := expand(child) - if err != nil { - return nil, err - } - for name, attr := range subRes { - res[gopath.Join(ent.Name, name)] = attr - } - } - if err = child.Close(); err != nil { - return nil, err - } - } - return res, nil - } - - f1Map, err := expand(f1) - if err != nil { - t.Fatal(err) - } - - f2Map, err := expand(f2) - if err != nil { - t.Fatal(err) - } - - same := func(permissionContains p9.FileMode, base, target map[string]p9.Attr) bool { - if len(base) != len(target) { - - var baseNames []string - var targetNames []string - for name := range base { - baseNames = append(baseNames, name) - } - for name := range target { - targetNames = append(targetNames, name) - } - - t.Fatalf("map lengths don't match:\nbase:%v\ntarget:%v\n", baseNames, targetNames) - return false - } - - for path, baseAttr := range base { - bMode := baseAttr.Mode - tMode := target[path].Mode - - if bMode.FileType() != tMode.FileType() { - t.Fatalf("type for %q don't match:\nbase:%v\ntarget:%v\n", path, bMode, tMode) - return false - } - - if ((bMode.Permissions() & permissionContains) & (tMode.Permissions() & permissionContains)) == 0 { - t.Fatalf("permissions for %q don't match\n(unfiltered)\nbase:%v\ntarget:%v\n(filtered)\nbase:%v\ntarget:%v\n", - path, - bMode.Permissions(), tMode.Permissions(), - bMode.Permissions()&permissionContains, tMode.Permissions()&permissionContains, - ) - return false - } - - if bMode.FileType() != p9.ModeDirectory { - bSize := baseAttr.Size - tSize := target[path].Size - - if bSize != tSize { - t.Fatalf("size for %q doesn't match\nbase:%d\ntarget:%d\n", - path, - bSize, - tSize) - } - } - } - return true - } - if !same(p9.Read, f1Map, f2Map) { - t.Fatalf("contents don't match \nf1:%v\nf2:%v\n", f1Map, f2Map) - } -} - -func InitCore(ctx context.Context) (coreiface.CoreAPI, error) { - node, err := core.NewNode(ctx, &core.BuildCfg{ - Online: false, - Permanent: false, - DisableEncryptedConnections: true, - }) - if err != nil { - return nil, err - } - - return coreapi.NewCoreAPI(node) -} - -const incantation = "May the bits passing through this device somehow help bring peace to this world" - -func initEnv(ctx context.Context, core coreiface.CoreAPI) (string, corepath.Resolved, error) { - testDir, err := ioutil.TempDir("", "ipfs-") - if err != nil { - return "", nil, err - } - if err := os.Chmod(testDir, 0775); err != nil { - return "", nil, err - } - - if err = ioutil.WriteFile(filepath.Join(testDir, "empty"), - []byte(nil), - 0644); err != nil { - return "", nil, err - } - - if err = ioutil.WriteFile(filepath.Join(testDir, "small"), - []byte(incantation), - 0644); err != nil { - return "", nil, err - } - - if err := generateGarbage(testDir); err != nil { - return "", nil, err - } - - testSubDir, err := ioutil.TempDir(testDir, "ipfs-") - if err != nil { - return "", nil, err - } - if err := os.Chmod(testSubDir, 0775); err != nil { - return "", nil, err - } - - if err := generateGarbage(testSubDir); err != nil { - return "", nil, err - } - - iPath, err := pinAddDir(ctx, core, testDir) - if err != nil { - return "", nil, err - } - - return testDir, iPath, err + pluginEnv := &plugin.Environment{Config: defaultConfig()} + t.Run("Plugin", func(t *testing.T) { testPlugin(t, pluginEnv, core) }) } -func p9Readdir(dir p9.File) ([]p9.Dirent, error) { - _, dirClone, err := dir.Walk(nil) - if err != nil { - return nil, err - } - - _, _, err = dirClone.Open(p9.ReadOnly) - if err != nil { - return nil, err - } - defer dirClone.Close() - +func testPlugin(t *testing.T, pluginEnv *plugin.Environment, core coreiface.CoreAPI) { + // NOTE: all restrictive comments are in relation to our plugin, not all plugins var ( - offset uint64 - ents []p9.Dirent + module FileSystemPlugin + err error ) - for { - var curEnts []p9.Dirent - curEnts, err = dirClone.Readdir(offset, ^uint32(0)) - lEnts := len(curEnts) - if err != nil || lEnts == 0 { - break - } - ents = append(ents, curEnts...) - offset += uint64(lEnts) + // close and start before init are NOT allowed + if err = module.Close(); err == nil { + t.Fatal("plugin was not initialized but Close succeeded") + // also should not hang } - - return ents, err -} - -func pinAddDir(ctx context.Context, core coreiface.CoreAPI, path string) (corepath.Resolved, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err + if err = module.Start(core); err == nil { + t.Fatal("plugin was not initialized but Start succeeded") + // also should not hang } - node, err := files.NewSerialFile(path, false, fi) - if err != nil { - return nil, err + // initialize the module + if err = module.Init(pluginEnv); err != nil { + t.Fatal("Plugin couldn't be initialized: ", err) } - iPath, err := core.Unixfs().Add(ctx, node.(files.Directory), coreoptions.Unixfs.Pin(true)) - if err != nil { - return nil, err + // double init is NOT allowed + if err = module.Init(pluginEnv); err == nil { + t.Fatal("init isn't intended to succeed twice") } - return iPath, nil -} - -func generateGarbage(tempDir string) error { - randDev := rand.New(rand.NewSource(time.Now().UnixNano())) - - for _, size := range []int{4, 8, 16, 32} { - buf := make([]byte, size<<(10*2)) - if _, err := randDev.Read(buf); err != nil { - return err - } - name := fmt.Sprintf("%dMiB", size) - if err := ioutil.WriteFile(filepath.Join(tempDir, name), - buf, - 0644); err != nil { - return err - } + // close before start is allowed + if err = module.Close(); err != nil { + t.Fatal("plugin isn't busy, but it can't close: ", err) + // also should not hang } - return nil -} - -func pinNames(ctx context.Context, core coreiface.CoreAPI) ([]string, error) { - pins, err := core.Pin().Ls(ctx, coreoptions.Pin.Type.Recursive()) - if err != nil { - return nil, err - } - names := make([]string, 0, len(pins)) - for _, pin := range pins { - names = append(names, gopath.Base(pin.Path().String())) + // double close is allowed + if err = module.Close(); err != nil { + t.Fatal("plugin couldn't close twice: ", err) } - return names, nil -} -func p9PinNames(root p9.File) ([]string, error) { - ents, err := p9Readdir(root) - if err != nil { - return nil, err + // start the module + if err = module.Start(core); err != nil { + t.Fatal("module could not start: ", err) } - names := make([]string, 0, len(ents)) - - for _, ent := range ents { - names = append(names, ent.Name) + // double start is NOT allowed + if err = module.Start(core); err == nil { + t.Fatal("module is intended to be exclusive but was allowed to start twice") } - return names, nil -} - -//TODO: -// NOTE: compares a subset of attributes, matching those of IPFS -func testIPFSCompare(t *testing.T, f1, f2 p9.File) { - _, _, f1Attr, err := f1.GetAttr(attrMaskIPFSTest) - if err != nil { - t.Errorf("Attr(%v) = %v, want nil", f1, err) - } - _, _, f2Attr, err := f2.GetAttr(attrMaskIPFSTest) - if err != nil { - t.Errorf("Attr(%v) = %v, want nil", f2, err) + // actual close + if err = module.Close(); err != nil { + t.Fatalf("plugin isn't busy, but it can't close: %#v", err) + t.Fatal("plugin isn't busy, but it can't close: ", err) } - if f1Attr != f2Attr { - t.Errorf("Attributes of same files do not match: %v and %v", f1Attr, f2Attr) + + // another redundant close + if err = module.Close(); err != nil { + t.Fatal("plugin isn't busy, but it can't close: ", err) + // also should not hang } } From 2a495c73b9b2a8cef7065d2f53fc821a6deb42a1 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 21 Oct 2019 12:16:53 -0400 Subject: [PATCH 083/102] lol unsaved buffers --- plugin/plugins/filesystem/filesystem.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index c82a4ab021d..57c14f3945a 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -143,7 +143,7 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { // store error on the fs object then close our syncing channel (see use in `Close` below) fs.serverErr = server.Serve(manet.NetListener(fs.listener)) - if fs.closing { // we expect `Accept` to fail when `Close` is called + if fs.closing { //[async] we expect `Accept` to fail while `Close` is in progress var opErr *net.OpError if errors.As(fs.serverErr, &opErr) && opErr.Op == "accept" { fs.serverErr = nil @@ -169,7 +169,7 @@ func (fs *FileSystemPlugin) Close() error { fs.listener.Close() // stop accepting actually fs.cancel() // stop any lingering fs operations <-fs.closed // wait for the server thread to set the error value - fs.closing = false // reset our own condition + fs.closing = false // reset our async condition fs.listener = nil // reset `Start` conditions fs.closed = nil } From 161d86a2ffe100ace7481c83b10fdb1e1ebec26e Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 21 Oct 2019 12:20:06 -0400 Subject: [PATCH 084/102] expose WalkRef and generalize Backtrack --- plugin/plugins/filesystem/nodes/base.go | 10 +++++----- plugin/plugins/filesystem/nodes/ipfs.go | 8 ++++---- plugin/plugins/filesystem/nodes/ipns.go | 2 +- plugin/plugins/filesystem/nodes/keyfs.go | 16 ++++++---------- plugin/plugins/filesystem/nodes/pinfs.go | 21 ++++++--------------- plugin/plugins/filesystem/nodes/root.go | 22 +++++++++------------- plugin/plugins/filesystem/nodes/utils.go | 10 +++++----- 7 files changed, 36 insertions(+), 53 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 3cf0eea1ff9..b2f3d8d781d 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -126,7 +126,7 @@ func newIPFSBase(ctx context.Context, coreNamespace string, core coreiface.CoreA options := nodeopts.AttachOps(ops...) if options.Parent != nil { // parent is optional - parentRef, ok := options.Parent.(walkRef) // interface is not + parentRef, ok := options.Parent.(WalkRef) // interface is not if !ok { panic("parent node lacks overlay traversal methods") } @@ -237,7 +237,7 @@ func (b *IPFSBase) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return *b.Qid, 0, nil } -func (ib *IPFSBase) step(self walkRef, name string) (walkRef, error) { +func (ib *IPFSBase) step(self WalkRef, name string) (WalkRef, error) { if ib.Qid.Type != p9.TypeDir { return nil, ENOTDIR } @@ -251,10 +251,10 @@ func (ib *IPFSBase) step(self walkRef, name string) (walkRef, error) { return self, nil } -func (ib *IPFSBase) backtrack(self walkRef) (walkRef, error) { - // if we're the root +// XXX: go receiver type trickery +func (ib *IPFSBase) backtrack(self WalkRef) (WalkRef, error) { + // if we're a root return our parent, or ourselves if we don't have one if len(ib.Trail) == 0 { - // return our parent, or ourselves if we don't have one if ib.parent != nil { return ib.parent, nil } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 6928f373891..7fbff6e9300 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -13,7 +13,7 @@ import ( ) var _ p9.File = (*IPFS)(nil) -var _ walkRef = (*IPFS)(nil) +var _ WalkRef = (*IPFS)(nil) // IPFS exposes the IPFS API over a p9.File interface // Walk does not expect a namespace, only its path argument @@ -36,7 +36,7 @@ func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A return id } -func (id *IPFS) Fork() (walkRef, error) { +func (id *IPFS) Fork() (WalkRef, error) { base, err := id.IPFSBase.Fork() if err != nil { return nil, err @@ -260,7 +260,7 @@ func (id *IPFS) Close() error { } // IPFS appends "name" to its current path, and returns itself -func (id *IPFS) Step(name string) (walkRef, error) { +func (id *IPFS) Step(name string) (WalkRef, error) { return id.step(id, name) } @@ -280,6 +280,6 @@ func (id *IPFS) QID() (p9.QID, error) { return *id.Qid, nil } -func (id *IPFS) Backtrack() (walkRef, error) { +func (id *IPFS) Backtrack() (WalkRef, error) { return id.IPFSBase.backtrack(id) } diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go index 2f5513553ac..697ebc2dd5f 100644 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -9,7 +9,7 @@ import ( ) var _ p9.File = (*IPNS)(nil) -var _ walkRef = (*IPNS)(nil) +var _ WalkRef = (*IPNS)(nil) // IPNS exposes the IPNS API over a p9.File interface // Walk does not expect a namespace, only its path argument diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index a457450767d..2b278f79e85 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -10,7 +10,7 @@ import ( ) var _ p9.File = (*KeyFS)(nil) -var _ walkRef = (*KeyFS)(nil) +var _ WalkRef = (*KeyFS)(nil) type KeyFS struct { IPFSBase @@ -32,12 +32,12 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. panic(err) } - kd.proxy = subsystem.(walkRef) + kd.proxy = subsystem.(WalkRef) return kd } -func (kd *KeyFS) Fork() (walkRef, error) { +func (kd *KeyFS) Fork() (WalkRef, error) { newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change // set new operations context err := newFid.newOperations() @@ -55,7 +55,7 @@ func (kd *KeyFS) Attach() (p9.File, error) { // KeyFS forks the IPFS root that was set during construction // and calls step on it rather than itself -func (kd *KeyFS) Step(name string) (walkRef, error) { +func (kd *KeyFS) Step(name string) (WalkRef, error) { newFid, err := kd.proxy.Fork() if err != nil { return nil, err @@ -70,12 +70,8 @@ func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { return walker(kd, names) } -func (kd *KeyFS) Backtrack() (walkRef, error) { - // return our parent, or ourselves if we don't have one - if kd.parent != nil { - return kd.parent, nil - } - return kd, nil +func (kd *KeyFS) Backtrack() (WalkRef, error) { + return kd.IPFSBase.backtrack(kd) } // temporary stub to allow forwarding requests on empty directory diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index c69a1f87d6b..3d68c77826b 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -13,7 +13,7 @@ import ( ) var _ p9.File = (*PinFS)(nil) -var _ walkRef = (*PinFS)(nil) +var _ WalkRef = (*PinFS)(nil) type PinFS struct { IPFSBase @@ -36,12 +36,12 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. panic(err) } - pd.proxy = subsystem.(walkRef) + pd.proxy = subsystem.(WalkRef) return pd } -func (pd *PinFS) Fork() (walkRef, error) { +func (pd *PinFS) Fork() (WalkRef, error) { newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change // set new operations context err := newFid.newOperations() @@ -59,7 +59,7 @@ func (pd *PinFS) Attach() (p9.File, error) { // PinFS forks the IPFS root that was set during construction // and calls step on it rather than itself -func (pd *PinFS) Step(name string) (walkRef, error) { +func (pd *PinFS) Step(name string) (WalkRef, error) { newFid, err := pd.proxy.Fork() if err != nil { return nil, err @@ -129,17 +129,8 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return flatReaddir(pd.ents, offset, count) } -func (pd *PinFS) Backtrack() (walkRef, error) { - // return the parent if we are the root - if len(pd.Trail) == 0 { - return pd.parent, nil - } - - // otherwise step back - pd.Trail = pd.Trail[1:] - - // TODO: reset meta - return pd, nil +func (pd *PinFS) Backtrack() (WalkRef, error) { + return pd.IPFSBase.backtrack(pd) } func (pd *PinFS) Close() error { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 03d85c5077a..d2aea18042a 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -12,7 +12,7 @@ import ( ) var _ p9.File = (*RootIndex)(nil) -var _ walkRef = (*RootIndex)(nil) +var _ WalkRef = (*RootIndex)(nil) const nRoot = "" // root namespace is intentionally left blank @@ -34,7 +34,7 @@ func (rp rootPath) Cid() cid.Cid { } type systemTuple struct { - file walkRef + file WalkRef dirent p9.Dirent } @@ -50,10 +50,10 @@ type RootIndex struct { type OverlayFileMeta struct { // parent may be used to send ".." requests to another file system // during `Backtrack` - parent walkRef + parent WalkRef // proxy may be used to send requests to another file system // during `Step` - proxy walkRef + proxy WalkRef } // RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it @@ -107,7 +107,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A // add the fs+entry to the list of subsystems ri.subsystems[subsystem.string] = systemTuple{ - file: fs.(walkRef), + file: fs.(WalkRef), dirent: rootDirent, } } @@ -115,7 +115,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A return ri } -func (ri *RootIndex) Fork() (walkRef, error) { +func (ri *RootIndex) Fork() (WalkRef, error) { newFid := &RootIndex{ IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change subsystems: ri.subsystems, @@ -148,7 +148,7 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { // The RootIndex checks if it has attached to "name" // derives a node from it, and returns it -func (ri *RootIndex) Step(name string) (walkRef, error) { +func (ri *RootIndex) Step(name string) (WalkRef, error) { // consume fs/access name subSys, ok := ri.subsystems[name] if !ok { @@ -194,10 +194,6 @@ func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return ents, nil } -func (ri *RootIndex) Backtrack() (walkRef, error) { - // return our parent, or ourselves if we don't have one - if ri.parent != nil { - return ri.parent, nil - } - return ri, nil +func (ri *RootIndex) Backtrack() (WalkRef, error) { + return ri.IPFSBase.backtrack(ri) } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index f8edd719f52..1c39171421d 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -71,7 +71,7 @@ type directoryStream struct { err error } -type walkRef interface { +type WalkRef interface { p9.File /* CheckWalk should make sure that the current reference adheres to the restrictions @@ -90,7 +90,7 @@ type walkRef interface { operations such as `Open` should prevent all references to the same path from opening etc. in compliance with 'walk(5)' */ - Fork() (walkRef, error) + Fork() (WalkRef, error) /* QID should check that the node's path is walkable by constructing the QID for its path @@ -104,16 +104,16 @@ type walkRef interface { within or outside of your own fs boundaries as long as `QID` is ready to be called on the resulting node */ - Step(name string) (walkRef, error) + Step(name string) (WalkRef, error) /* Backtrack must handle `..` request returning a reference to the node behind the current node (or itself in the case of the root) the same comment about implementation of `Step` applies here */ - Backtrack() (parentRef walkRef, err error) + Backtrack() (parentRef WalkRef, err error) } -func walker(ref walkRef, names []string) ([]p9.QID, p9.File, error) { +func walker(ref WalkRef, names []string) ([]p9.QID, p9.File, error) { err := ref.CheckWalk() if err != nil { return nil, nil, err From 9ef8e6ad9c4fc3f0213cea2836e9baf585c7987e Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 21 Oct 2019 15:51:30 -0400 Subject: [PATCH 085/102] extract WalkRef and cleanup related use --- plugin/plugins/filesystem/filesystem.go | 2 +- plugin/plugins/filesystem/nodes/base.go | 61 ++++----- plugin/plugins/filesystem/nodes/ipfs.go | 23 ++-- plugin/plugins/filesystem/nodes/ipns.go | 5 +- plugin/plugins/filesystem/nodes/keyfs.go | 17 +-- .../filesystem/nodes/options/options.go | 10 +- plugin/plugins/filesystem/nodes/pinfs.go | 19 +-- plugin/plugins/filesystem/nodes/root.go | 24 ++-- plugin/plugins/filesystem/nodes/utils.go | 110 --------------- plugin/plugins/filesystem/utils/utils.go | 129 ++++++++++++++++++ 10 files changed, 207 insertions(+), 193 deletions(-) create mode 100644 plugin/plugins/filesystem/utils/utils.go diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 57c14f3945a..804e88bff40 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -175,4 +175,4 @@ func (fs *FileSystemPlugin) Close() error { } // otherwise we were never started to begin with; default/initial value will be returned return fs.serverErr -} +} \ No newline at end of file diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index b2f3d8d781d..4be3bfbab73 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -9,6 +9,7 @@ import ( "github.com/hugelgupf/p9/p9" "github.com/hugelgupf/p9/unimplfs" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" @@ -33,14 +34,14 @@ type Base struct { Trail []string // FS "breadcrumb" trail from node's root // Storage for file's metadata - Qid *p9.QID + qid *p9.QID meta *p9.Attr metaMask *p9.AttrMask Logger logging.EventLogger closed bool // set to true upon close; this reference should not be used again for anything modified bool // set to true when the `Trail` has been modified (usually by `Step`) - // reset to false when `Qid` has been populated with the current path in `Trail` (usually by `QID`) + // reset to false when `qid` has been populated with the current path in `Trail` (usually by `QID`) } @@ -49,7 +50,7 @@ func newBase(ops ...nodeopts.AttachOption) Base { return Base{ Logger: options.Logger, - Qid: new(p9.QID), + qid: new(p9.QID), meta: new(p9.Attr), metaMask: new(p9.AttrMask), } @@ -61,8 +62,9 @@ func (b *Base) clone() Base { func (b *Base) Fork() Base { newFid := b.clone() - newFid.Trail = make([]string, len(b.Trail)+1) // NOTE: +1 is preallocated space for `Step`; not required - copy(newFid.Trail, b.Trail) + + tLen := len(b.Trail) + newFid.Trail = b.Trail[:tLen:tLen] // make a new path slice for the new reference return newFid } @@ -71,9 +73,9 @@ func (b *Base) String() string { return gopath.Join(b.Trail...) } -func (b *Base) NinePath() p9Path { return b.Qid.Path } +func (b *Base) NinePath() p9Path { return b.qid.Path } -func (b *Base) QID() (p9.QID, error) { return *b.Qid, nil } +func (b *Base) QID() (p9.QID, error) { return *b.qid, nil } // IPFSBase is much like Base but extends it to hold IPFS specific metadata type IPFSBase struct { @@ -117,33 +119,26 @@ type IPFSBase struct { //func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.FileMode, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { func newIPFSBase(ctx context.Context, coreNamespace string, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { - base := IPFSBase{ - Base: newBase(ops...), + options := nodeopts.AttachOps(ops...) + return IPFSBase{ + Base: newBase(ops...), + OverlayFileMeta: OverlayFileMeta{ + parent: options.Parent, + }, coreNamespace: coreNamespace, core: core, filesystemCtx: ctx, } - - options := nodeopts.AttachOps(ops...) - if options.Parent != nil { // parent is optional - parentRef, ok := options.Parent.(WalkRef) // interface is not - if !ok { - panic("parent node lacks overlay traversal methods") - } - base.OverlayFileMeta.parent = parentRef - } - return base } func (ib *IPFSBase) clone() IPFSBase { - newFid := IPFSBase{ + return IPFSBase{ Base: ib.Base.clone(), OverlayFileMeta: ib.OverlayFileMeta, coreNamespace: ib.coreNamespace, core: ib.core, filesystemCtx: ib.filesystemCtx, } - return newFid } func (ib *IPFSBase) Fork() (IPFSBase, error) { @@ -187,12 +182,12 @@ func (ib *IPFSBase) CorePath(names ...string) corepath.Path { } func (b *IPFSBase) Flush() error { - b.Logger.Debugf("flush requested: {%d}%q", b.Qid.Path, b.String()) + b.Logger.Debugf("flush requested: {%d}%q", b.qid.Path, b.String()) return nil } func (b *Base) Close() error { - b.Logger.Debugf("closing: {%d}%q", b.Qid.Path, b.String()) + b.Logger.Debugf("closing: {%d}%q", b.qid.Path, b.String()) b.closed = true return nil } @@ -223,9 +218,9 @@ func (ib *IPFSBase) Close() error { } func (b *Base) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - //b.Logger.Debugf("GetAttr {%d}:%q", b.Qid.Path, b.String()) + //b.Logger.Debugf("GetAttr {%d}:%q", b.qid.Path, b.String()) - return *b.Qid, *b.metaMask, *b.meta, nil + return *b.qid, *b.metaMask, *b.meta, nil } func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { @@ -234,25 +229,25 @@ func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { func (b *IPFSBase) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { b.Logger.Debugf("Open") - return *b.Qid, 0, nil + return *b.qid, 0, nil } -func (ib *IPFSBase) step(self WalkRef, name string) (WalkRef, error) { - if ib.Qid.Type != p9.TypeDir { +func (b *Base) step(self fsutils.WalkRef, name string) (fsutils.WalkRef, error) { + if b.qid.Type != p9.TypeDir { return nil, ENOTDIR } - if ib.closed == true { + if b.closed == true { return nil, errors.New("TODO: ref was previously closed err") } - ib.Trail = append(ib.Trail, name) - ib.modified = true + tLen := len(b.Trail) + b.Trail = append(b.Trail[:tLen:tLen], name) + b.modified = true return self, nil } -// XXX: go receiver type trickery -func (ib *IPFSBase) backtrack(self WalkRef) (WalkRef, error) { +func (ib *IPFSBase) backtrack(self fsutils.WalkRef) (fsutils.WalkRef, error) { // if we're a root return our parent, or ourselves if we don't have one if len(ib.Trail) == 0 { if ib.parent != nil { diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 7fbff6e9300..ad5b2c97879 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -8,12 +8,13 @@ import ( "github.com/hugelgupf/p9/p9" files "github.com/ipfs/go-ipfs-files" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" coreiface "github.com/ipfs/interface-go-ipfs-core" corepath "github.com/ipfs/interface-go-ipfs-core/path" ) var _ p9.File = (*IPFS)(nil) -var _ WalkRef = (*IPFS)(nil) +var _ fsutils.WalkRef = (*IPFS)(nil) // IPFS exposes the IPFS API over a p9.File interface // Walk does not expect a namespace, only its path argument @@ -31,12 +32,12 @@ type IPFSFileMeta struct { func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { id := &IPFS{IPFSBase: newIPFSBase(ctx, "/ipfs", core, ops...)} - id.Qid.Type = p9.TypeDir + id.qid.Type = p9.TypeDir id.meta.Mode, id.metaMask.Mode = p9.ModeDirectory|IRXA, true return id } -func (id *IPFS) Fork() (WalkRef, error) { +func (id *IPFS) Fork() (fsutils.WalkRef, error) { base, err := id.IPFSBase.Fork() if err != nil { return nil, err @@ -72,7 +73,7 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { attr p9.Attr filled p9.AttrMask callCtx, cancel = id.callCtx() - qid = *id.Qid + qid = *id.qid ) defer cancel() @@ -94,7 +95,7 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { if req.Mode { attr.Mode |= IRXA qid.Type = attr.Mode.QIDType() - id.Qid.Type = attr.Mode.QIDType() + id.qid.Type = attr.Mode.QIDType() } return qid, filled, attr, err @@ -104,13 +105,13 @@ func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { id.Logger.Debugf("Walk names: %v", names) id.Logger.Debugf("Walk myself: %q:{%d}", id.String(), id.NinePath()) - return walker(id, names) + return fsutils.Walker(id, names) } func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { id.Logger.Debugf("Open: %s", id.String()) - qid := *id.Qid + qid := *id.qid // handle directories if qid.Type == p9.TypeDir { @@ -260,7 +261,7 @@ func (id *IPFS) Close() error { } // IPFS appends "name" to its current path, and returns itself -func (id *IPFS) Step(name string) (WalkRef, error) { +func (id *IPFS) Step(name string) (fsutils.WalkRef, error) { return id.step(id, name) } @@ -274,12 +275,12 @@ func (id *IPFS) QID() (p9.QID, error) { return qid, err } id.modified = false - id.Qid = &qid + id.qid = &qid } - return *id.Qid, nil + return *id.qid, nil } -func (id *IPFS) Backtrack() (WalkRef, error) { +func (id *IPFS) Backtrack() (fsutils.WalkRef, error) { return id.IPFSBase.backtrack(id) } diff --git a/plugin/plugins/filesystem/nodes/ipns.go b/plugin/plugins/filesystem/nodes/ipns.go index 697ebc2dd5f..fe706a5b69a 100644 --- a/plugin/plugins/filesystem/nodes/ipns.go +++ b/plugin/plugins/filesystem/nodes/ipns.go @@ -5,11 +5,12 @@ import ( "github.com/hugelgupf/p9/p9" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" coreiface "github.com/ipfs/interface-go-ipfs-core" ) var _ p9.File = (*IPNS)(nil) -var _ WalkRef = (*IPNS)(nil) +var _ fsutils.WalkRef = (*IPNS)(nil) // IPNS exposes the IPNS API over a p9.File interface // Walk does not expect a namespace, only its path argument @@ -18,7 +19,7 @@ type IPNS = IPFS func IPNSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { id := &IPNS{IPFSBase: newIPFSBase(ctx, "/ipns", core, ops...)} - id.Qid.Type = p9.TypeDir + id.qid.Type = p9.TypeDir id.meta.Mode, id.metaMask.Mode = p9.ModeDirectory|IRXA, true return id } diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 2b278f79e85..bb18979374c 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -5,12 +5,13 @@ import ( "github.com/hugelgupf/p9/p9" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" ) var _ p9.File = (*KeyFS)(nil) -var _ WalkRef = (*KeyFS)(nil) +var _ fsutils.WalkRef = (*KeyFS)(nil) type KeyFS struct { IPFSBase @@ -18,7 +19,7 @@ type KeyFS struct { func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { kd := &KeyFS{IPFSBase: newIPFSBase(ctx, "/keyfs", core, ops...)} - kd.Qid.Type = p9.TypeDir + kd.qid.Type = p9.TypeDir kd.meta.Mode, kd.metaMask.Mode = p9.ModeDirectory|IRXA|0220, true // non-keyed requests fall through to IPNS @@ -32,12 +33,12 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. panic(err) } - kd.proxy = subsystem.(WalkRef) + kd.proxy = subsystem.(fsutils.WalkRef) return kd } -func (kd *KeyFS) Fork() (WalkRef, error) { +func (kd *KeyFS) Fork() (fsutils.WalkRef, error) { newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change // set new operations context err := newFid.newOperations() @@ -55,7 +56,7 @@ func (kd *KeyFS) Attach() (p9.File, error) { // KeyFS forks the IPFS root that was set during construction // and calls step on it rather than itself -func (kd *KeyFS) Step(name string) (WalkRef, error) { +func (kd *KeyFS) Step(name string) (fsutils.WalkRef, error) { newFid, err := kd.proxy.Fork() if err != nil { return nil, err @@ -65,12 +66,12 @@ func (kd *KeyFS) Step(name string) (WalkRef, error) { func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { kd.Logger.Debugf("Walk names %v", names) - kd.Logger.Debugf("Walk myself: %v", kd.Qid) + kd.Logger.Debugf("Walk myself: %v", kd.qid) - return walker(kd, names) + return fsutils.Walker(kd, names) } -func (kd *KeyFS) Backtrack() (WalkRef, error) { +func (kd *KeyFS) Backtrack() (fsutils.WalkRef, error) { return kd.IPFSBase.backtrack(kd) } diff --git a/plugin/plugins/filesystem/nodes/options/options.go b/plugin/plugins/filesystem/nodes/options/options.go index 2f0c6e96ba9..366e1b46c28 100644 --- a/plugin/plugins/filesystem/nodes/options/options.go +++ b/plugin/plugins/filesystem/nodes/options/options.go @@ -1,20 +1,16 @@ package nodeopts import ( - "github.com/hugelgupf/p9/p9" "github.com/jbenet/goprocess" "github.com/ipfs/go-cid" + fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" "github.com/ipfs/go-mfs" ) -//TODO: we're doing runtime hacks to check if this is a walkref -// we don't use a walkref because import cycle -// amend this type AttachOptions struct { - //Parent fsnodes.walkRef // node directly behind self - Parent p9.File // node directly behind self + Parent fsutils.WalkRef // node directly behind self Logger logging.EventLogger // what subsystem you are Process goprocess.Process // TODO: I documented this somewhere else MFSRoot *cid.Cid // required when attaching to MFS @@ -34,7 +30,7 @@ func AttachOps(options ...AttachOption) *AttachOptions { } // if NOT provided, we assume the file system is to be treated as a root, assigning itself as a parent -func Parent(p p9.File) AttachOption { +func Parent(p fsutils.WalkRef) AttachOption { return func(ops *AttachOptions) { ops.Parent = p } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 3d68c77826b..6d7d4fbbda6 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -7,13 +7,14 @@ import ( "github.com/hugelgupf/p9/p9" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" coreoptions "github.com/ipfs/interface-go-ipfs-core/options" ) var _ p9.File = (*PinFS)(nil) -var _ WalkRef = (*PinFS)(nil) +var _ fsutils.WalkRef = (*PinFS)(nil) type PinFS struct { IPFSBase @@ -22,7 +23,7 @@ type PinFS struct { func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { pd := &PinFS{IPFSBase: newIPFSBase(ctx, "/pinfs", core, ops...)} - pd.Qid.Type = p9.TypeDir + pd.qid.Type = p9.TypeDir pd.meta.Mode, pd.metaMask.Mode = p9.ModeDirectory|IRXA, true // set up our subsystem, used to relay walk names to IPFS @@ -36,12 +37,12 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. panic(err) } - pd.proxy = subsystem.(WalkRef) + pd.proxy = subsystem.(fsutils.WalkRef) return pd } -func (pd *PinFS) Fork() (WalkRef, error) { +func (pd *PinFS) Fork() (fsutils.WalkRef, error) { newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change // set new operations context err := newFid.newOperations() @@ -59,7 +60,7 @@ func (pd *PinFS) Attach() (p9.File, error) { // PinFS forks the IPFS root that was set during construction // and calls step on it rather than itself -func (pd *PinFS) Step(name string) (WalkRef, error) { +func (pd *PinFS) Step(name string) (fsutils.WalkRef, error) { newFid, err := pd.proxy.Fork() if err != nil { return nil, err @@ -69,15 +70,15 @@ func (pd *PinFS) Step(name string) (WalkRef, error) { func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { pd.Logger.Debugf("Walk names %v", names) - pd.Logger.Debugf("Walk myself: %v", pd.Qid) + pd.Logger.Debugf("Walk myself: %v", pd.qid) - return walker(pd, names) + return fsutils.Walker(pd, names) } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") - qid := *pd.Qid + qid := *pd.qid // IPFS core representation pins, err := pd.core.Pin().Ls(pd.operationsCtx, coreoptions.Pin.Type.Recursive()) @@ -129,7 +130,7 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return flatReaddir(pd.ents, offset, count) } -func (pd *PinFS) Backtrack() (WalkRef, error) { +func (pd *PinFS) Backtrack() (fsutils.WalkRef, error) { return pd.IPFSBase.backtrack(pd) } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index d2aea18042a..582d5613c0f 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -6,13 +6,14 @@ import ( "github.com/hugelgupf/p9/p9" cid "github.com/ipfs/go-cid" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" + fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" coreiface "github.com/ipfs/interface-go-ipfs-core" "github.com/multiformats/go-multihash" ) var _ p9.File = (*RootIndex)(nil) -var _ WalkRef = (*RootIndex)(nil) +var _ fsutils.WalkRef = (*RootIndex)(nil) const nRoot = "" // root namespace is intentionally left blank @@ -34,7 +35,7 @@ func (rp rootPath) Cid() cid.Cid { } type systemTuple struct { - file WalkRef + file fsutils.WalkRef dirent p9.Dirent } @@ -50,17 +51,17 @@ type RootIndex struct { type OverlayFileMeta struct { // parent may be used to send ".." requests to another file system // during `Backtrack` - parent WalkRef + parent fsutils.WalkRef // proxy may be used to send requests to another file system // during `Step` - proxy WalkRef + proxy fsutils.WalkRef } // RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { // construct root node ri := &RootIndex{IPFSBase: newIPFSBase(ctx, "/", core, ops...)} - ri.Qid.Type = p9.TypeDir + ri.qid.Type = p9.TypeDir ri.meta.Mode, ri.metaMask.Mode = p9.ModeDirectory|IRXA, true // attach to subsystems @@ -70,7 +71,6 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A string subattacher logging.EventLogger - // *logging.EventLogger } // 9P Access names mapped to IPFS attacher functions @@ -107,7 +107,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A // add the fs+entry to the list of subsystems ri.subsystems[subsystem.string] = systemTuple{ - file: fs.(WalkRef), + file: fs.(fsutils.WalkRef), dirent: rootDirent, } } @@ -115,7 +115,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A return ri } -func (ri *RootIndex) Fork() (WalkRef, error) { +func (ri *RootIndex) Fork() (fsutils.WalkRef, error) { newFid := &RootIndex{ IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change subsystems: ri.subsystems, @@ -141,14 +141,14 @@ func (ri *RootIndex) Attach() (p9.File, error) { func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { ri.Logger.Debugf("Walk names %v", names) - ri.Logger.Debugf("Walk myself: %v", ri.Qid.Path) + ri.Logger.Debugf("Walk myself: %v", ri.qid.Path) - return walker(ri, names) + return fsutils.Walker(ri, names) } // The RootIndex checks if it has attached to "name" // derives a node from it, and returns it -func (ri *RootIndex) Step(name string) (WalkRef, error) { +func (ri *RootIndex) Step(name string) (fsutils.WalkRef, error) { // consume fs/access name subSys, ok := ri.subsystems[name] if !ok { @@ -194,6 +194,6 @@ func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return ents, nil } -func (ri *RootIndex) Backtrack() (WalkRef, error) { +func (ri *RootIndex) Backtrack() (fsutils.WalkRef, error) { return ri.IPFSBase.backtrack(ri) } diff --git a/plugin/plugins/filesystem/nodes/utils.go b/plugin/plugins/filesystem/nodes/utils.go index 1c39171421d..a6fdb8a12d6 100644 --- a/plugin/plugins/filesystem/nodes/utils.go +++ b/plugin/plugins/filesystem/nodes/utils.go @@ -71,104 +71,6 @@ type directoryStream struct { err error } -type WalkRef interface { - p9.File - - /* CheckWalk should make sure that the current reference adheres to the restrictions - of 'walk(5)' - In particular the reference must not be open for I/O, or otherwise already closed - */ - CheckWalk() error - - /* Fork allocates a reference derived from itself - The returned reference should be at the same path as the existing walkRef - the new reference is to act like the starting point `newfid` during `Walk` - e.g. - `newFid` originally references the same data as the origin `walkRef` - but is closed separately from the origin - operations such as `Walk` will modify `newFid` without affecting the origin `walkRef` - operations such as `Open` should prevent all references to the same path from opening - etc. in compliance with 'walk(5)' - */ - Fork() (WalkRef, error) - - /* QID should check that the node's path is walkable - by constructing the QID for its path - */ - QID() (p9.QID, error) - - /* Step should return a reference that is tracking the result of - the node's current path + "name" - implementation of this is fs specific - it is valid to return a new reference or the same reference modified - within or outside of your own fs boundaries - as long as `QID` is ready to be called on the resulting node - */ - Step(name string) (WalkRef, error) - - /* Backtrack must handle `..` request - returning a reference to the node behind the current node (or itself in the case of the root) - the same comment about implementation of `Step` applies here - */ - Backtrack() (parentRef WalkRef, err error) -} - -func walker(ref WalkRef, names []string) ([]p9.QID, p9.File, error) { - err := ref.CheckWalk() - if err != nil { - return nil, nil, err - } - - curRef, err := ref.Fork() - if err != nil { - return nil, nil, err - } - - // walk(5) - // It is legal for nwname to be zero, in which case newfid will represent the same file as fid - // and the walk will usually succeed - if shouldClone(names) { - qid, err := ref.QID() - if err != nil { - return nil, nil, err - } - return []p9.QID{qid}, curRef, nil - } - - qids := make([]p9.QID, 0, len(names)) - - for _, name := range names { - switch name { - default: - // get ready to step forward; maybe across FS bounds - curRef, err = curRef.Step(name) - - case ".": - // don't prepare to move at all - - case "..": - // get ready to step backwards; maybe across FS bounds - curRef, err = curRef.Backtrack() - } - - if err != nil { - return qids, nil, err - } - - // commit to the step - qid, err := curRef.QID() - if err != nil { - return qids, nil, err - } - - // set on success, we stepped forward - qids = append(qids, qid) - - } - - return qids, curRef, nil -} - func init() { salt = make([]byte, saltSize) _, err := io.ReadFull(rand.Reader, salt) @@ -177,18 +79,6 @@ func init() { } } -func shouldClone(names []string) bool { - switch len(names) { - case 0: // empty path - return true - case 1: // self? - pc := names[0] - return pc == "." || pc == "" - default: - return false - } -} - func ipldStat(ctx context.Context, attr *p9.Attr, node ipld.Node, mask p9.AttrMask) (p9.AttrMask, error) { var filledAttrs p9.AttrMask ufsNode, err := unixfs.ExtractFSNode(node) diff --git a/plugin/plugins/filesystem/utils/utils.go b/plugin/plugins/filesystem/utils/utils.go new file mode 100644 index 00000000000..d64907f46b3 --- /dev/null +++ b/plugin/plugins/filesystem/utils/utils.go @@ -0,0 +1,129 @@ +package fsutils + +import "github.com/hugelgupf/p9/p9" + +// WalkRef is used to generalize 9P `Walk` operations within filesystem boundries +// and allows for traversal across those boundaries if intended by the implementation +// +// The reference root node implementation links filesystems together using a parent/child relation +// sending the appropriate node back to the caller by using the linakge between nodes +// combined with inspection of a nodes current path +// it tries its best to avoid copying by modifying a node directly where possible +// falling back to derived copies when crossing filesystem boundaries during "movement" +type WalkRef interface { + p9.File + + /* CheckWalk should make sure that the current reference adheres to the restrictions + of 'walk(5)' + In particular the reference must not be open for I/O, or otherwise already closed + */ + CheckWalk() error + + /* Fork allocates a new reference, derived from the existing reference + Acting as the `newfid` construct mentioned in the documentation for the protocol + A subset of the semantics will be noted here in a generalized way + Please see 'walk(5)' for more information on the standard + + The returned reference node must "stand" beside the existing `WalkRef` + Meaning the node must be "at"/contain the same location/path as the existing reference. + + The returned node must also adhear to 'walk(5)' `newfid` semantics + Meaning that... + `newfid` must be allowed to `Close` seperatley from the original reference + `newfid`'s path may be modified during `Walk` without affecting the original `WalkRef` + `Open` must flag all references within the same system, at the same path, as open + etc. in compliance with 'walk(5)' + */ + Fork() (WalkRef, error) + + /* QID checks that the node's current path contains an existing file + and returns the QID for it + */ + QID() (p9.QID, error) + + /* Step should return a reference that is tracking the result of + the node's current-path + "name" + + It is valid to return a newly constructed reference or modify and return the existing reference + as long as `QID` is ready to be called on the resulting node + and resources are reaped where sensible within the fs implementation + */ + Step(name string) (WalkRef, error) + + /* Backtrack is the handler for `..` requests + it is effectivley the inverse of `Step` + if called on the root node, the node should return itself + */ + Backtrack() (parentRef WalkRef, err error) +} + +// Walker implements the 9P `Walk` operation +func Walker(ref WalkRef, names []string) ([]p9.QID, p9.File, error) { + // operations check + err := ref.CheckWalk() + if err != nil { + return nil, nil, err + } + + // no matter the outcome, we start with a `newfid` + curRef, err := ref.Fork() + if err != nil { + return nil, nil, err + } + + if shouldClone(names) { + qid, err := ref.QID() // validate the node is "walkable" + if err != nil { + return nil, nil, err + } + return []p9.QID{qid}, curRef, nil + } + + qids := make([]p9.QID, 0, len(names)) + + for _, name := range names { + switch name { + default: + // get ready to step forward; maybe across FS bounds + curRef, err = curRef.Step(name) + + case ".": + // don't prepare to move at all + + case "..": + // get ready to step backwards; maybe across FS bounds + curRef, err = curRef.Backtrack() + } + + if err != nil { + return qids, nil, err + } + + // commit to the step + qid, err := curRef.QID() + if err != nil { + return qids, nil, err + } + + // set on success, we stepped forward + qids = append(qids, qid) + } + + return qids, curRef, nil +} + +/* walk(5): +It is legal for `nwname` to be zero, in which case `newfid` will represent the same `file` as `fid` +and the `walk` will usually succeed; this is equivalent to walking to dot. +*/ +func shouldClone(names []string) bool { + switch len(names) { + case 0: // truly empty path + return true + case 1: // self or empty but not nil + pc := names[0] + return pc == "." || pc == "" + default: + return false + } +} From 545d666ea74696673191aa59274ee5fa4fb83e95 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 21 Oct 2019 17:06:55 -0400 Subject: [PATCH 086/102] unexport NinePath --- plugin/plugins/filesystem/nodes/base.go | 2 +- plugin/plugins/filesystem/nodes/ipfs.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 4be3bfbab73..dfbc55f3892 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -73,7 +73,7 @@ func (b *Base) String() string { return gopath.Join(b.Trail...) } -func (b *Base) NinePath() p9Path { return b.qid.Path } +func (b *Base) ninePath() p9Path { return b.qid.Path } func (b *Base) QID() (p9.QID, error) { return *b.qid, nil } diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index ad5b2c97879..f069b5ddc14 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -103,7 +103,7 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { id.Logger.Debugf("Walk names: %v", names) - id.Logger.Debugf("Walk myself: %q:{%d}", id.String(), id.NinePath()) + id.Logger.Debugf("Walk myself: %q:{%d}", id.String(), id.ninePath()) return fsutils.Walker(id, names) } From 9024b957f3687f78d0c0d4b6b4b5833dfba38d4b Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 21 Oct 2019 18:35:54 -0400 Subject: [PATCH 087/102] inheritance changes related to the base clases --- go.mod | 1 + go.sum | 2 + plugin/plugins/filesystem/nodes/base.go | 135 ++++++++++------------- plugin/plugins/filesystem/nodes/ipfs.go | 88 ++++++++------- plugin/plugins/filesystem/nodes/keyfs.go | 44 +++++--- plugin/plugins/filesystem/nodes/pinfs.go | 68 +++++++----- plugin/plugins/filesystem/nodes/root.go | 79 +++++++------ plugin/plugins/filesystem/pinfs_test.go | 2 + 8 files changed, 221 insertions(+), 198 deletions(-) diff --git a/go.mod b/go.mod index 3db357e3978..50a4bcbf697 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d + github.com/djdv/p9 v0.0.0-20190730163255-e22b6295de2a github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fatih/color v1.7.0 // indirect diff --git a/go.sum b/go.sum index 6bb4b982840..80a6b463260 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/djdv/p9 v0.0.0-20190730163255-e22b6295de2a h1:crOdRmBZDY7MHBGlOKvrsVfEt72n6FgVWZITGOz7qDM= +github.com/djdv/p9 v0.0.0-20190730163255-e22b6295de2a/go.mod h1:TCADIMV+5QUCC44hQ6p6HMpgdj7LuxJLzIZR2Fhvns4= github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af h1:Fz2zlKtMYbMpX1BPo4begzCE855g+UGcj+5hmn4Bf04= github.com/djdv/p9 v0.0.0-20191015201628-1d10538c99af/go.mod h1:igFKfChBTNZnGzM9Arxcki7QfgT01VJneIZrNY8OqKI= github.com/djdv/p9 v0.0.0-20191018144345-781c6a62f733 h1:2jtqdlho/DLOfSxzPs1SM6fpENZwZii2o7lFtvHVhmc= diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index dfbc55f3892..766d0073fb3 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -7,7 +7,6 @@ import ( "time" "github.com/hugelgupf/p9/p9" - "github.com/hugelgupf/p9/unimplfs" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" @@ -20,17 +19,11 @@ const ( //device - attempts to comply with standard multicodec table dIPFS = 0xe4 ) -var _ p9.File = (*Base)(nil) - type p9Path = uint64 // Base provides a foundation to build file system nodes which contain file meta data -// as well as stubs for unimplemented file system methods +// as well as some base methods type Base struct { - // Provide stubs for unimplemented methods - unimplfs.NoopFile - p9.DefaultWalkGetAttr - Trail []string // FS "breadcrumb" trail from node's root // Storage for file's metadata @@ -56,27 +49,6 @@ func newBase(ops ...nodeopts.AttachOption) Base { } } -func (b *Base) clone() Base { - return *b -} - -func (b *Base) Fork() Base { - newFid := b.clone() - - tLen := len(b.Trail) - newFid.Trail = b.Trail[:tLen:tLen] // make a new path slice for the new reference - - return newFid -} - -func (b *Base) String() string { - return gopath.Join(b.Trail...) -} - -func (b *Base) ninePath() p9Path { return b.qid.Path } - -func (b *Base) QID() (p9.QID, error) { return *b.qid, nil } - // IPFSBase is much like Base but extends it to hold IPFS specific metadata type IPFSBase struct { Base @@ -131,69 +103,57 @@ func newIPFSBase(ctx context.Context, coreNamespace string, core coreiface.CoreA } } -func (ib *IPFSBase) clone() IPFSBase { - return IPFSBase{ - Base: ib.Base.clone(), - OverlayFileMeta: ib.OverlayFileMeta, - coreNamespace: ib.coreNamespace, - core: ib.core, - filesystemCtx: ib.filesystemCtx, - } -} +/* general helpers */ -func (ib *IPFSBase) Fork() (IPFSBase, error) { - newFid := ib.clone() - err := newFid.newOperations() +func (b *Base) ninePath() p9Path { return b.qid.Path } +func (b *Base) String() string { return gopath.Join(b.Trail...) } - return newFid, err +func (ib *IPFSBase) String() string { + return gopath.Join(append([]string{ib.coreNamespace}, ib.Base.String())...) } -func (ib *IPFSBase) newFilesystem() error { - if err := ib.checkFSCtx(); err != nil { +// see filesystemCtx section in IPFSBase comments +func (ib *IPFSBase) forkFilesystem() error { + if err := ib.filesystemCtx.Err(); err != nil { return err } ib.filesystemCtx, ib.filesystemCancel = context.WithCancel(ib.filesystemCtx) return nil } -func (ib *IPFSBase) newOperations() error { - if err := ib.checkFSCtx(); err != nil { +// see operationsCtx section in IPFSBase comments +func (ib *IPFSBase) forkOperations() error { + if err := ib.filesystemCtx.Err(); err != nil { return err } ib.operationsCtx, ib.operationsCancel = context.WithCancel(ib.filesystemCtx) return nil } -func (ib *IPFSBase) checkFSCtx() error { - select { - case <-ib.filesystemCtx.Done(): - return ib.filesystemCtx.Err() - default: - return nil - } -} - -func (ib *IPFSBase) String() string { - return gopath.Join(append([]string{ib.coreNamespace}, ib.Base.String())...) +func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { + return context.WithTimeout(b.filesystemCtx, 30*time.Second) } func (ib *IPFSBase) CorePath(names ...string) corepath.Path { return corepath.Join(rootPath(ib.coreNamespace), append(ib.Trail, names...)...) } -func (b *IPFSBase) Flush() error { - b.Logger.Debugf("flush requested: {%d}%q", b.qid.Path, b.String()) - return nil +/* base operation methods to build on */ + +func (b *Base) getAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + //b.Logger.Debugf("GetAttr {%d}:%q", b.qid.Path, b.String()) + + return *b.qid, *b.metaMask, *b.meta, nil } -func (b *Base) Close() error { +func (b *Base) close() error { b.Logger.Debugf("closing: {%d}%q", b.qid.Path, b.String()) b.closed = true return nil } -func (ib *IPFSBase) Close() error { - lastErr := ib.Base.Close() +func (ib *IPFSBase) close() error { + lastErr := ib.Base.close() if lastErr != nil { ib.Logger.Error(lastErr) } @@ -217,19 +177,45 @@ func (ib *IPFSBase) Close() error { return lastErr } -func (b *Base) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - //b.Logger.Debugf("GetAttr {%d}:%q", b.qid.Path, b.String()) +/* WalkRef relevant */ - return *b.qid, *b.metaMask, *b.meta, nil +func (b *Base) checkWalk() error { + if b.closed { + return errors.New("TODO: already closed msg") + } + return nil } -func (b *IPFSBase) callCtx() (context.Context, context.CancelFunc) { - return context.WithTimeout(b.filesystemCtx, 30*time.Second) +func (b *Base) qID() (p9.QID, error) { return *b.qid, nil } +func (b *Base) clone() Base { return *b } + +func (ib *IPFSBase) clone() IPFSBase { + return IPFSBase{ + Base: ib.Base.clone(), + OverlayFileMeta: ib.OverlayFileMeta, + coreNamespace: ib.coreNamespace, + core: ib.core, + filesystemCtx: ib.filesystemCtx, + } } -func (b *IPFSBase) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { - b.Logger.Debugf("Open") - return *b.qid, 0, nil +func (b *Base) fork() Base { + newFid := b.clone() + + tLen := len(b.Trail) + newFid.Trail = b.Trail[:tLen:tLen] // make a new path slice for the new reference + + return newFid +} + +func (ib *IPFSBase) fork() (IPFSBase, error) { + newFid := ib.clone() + err := newFid.forkOperations() + if err != nil { + return IPFSBase{}, err + } + + return newFid, nil } func (b *Base) step(self fsutils.WalkRef, name string) (fsutils.WalkRef, error) { @@ -261,10 +247,3 @@ func (ib *IPFSBase) backtrack(self fsutils.WalkRef) (fsutils.WalkRef, error) { return self, nil } - -func (b *Base) CheckWalk() error { - if b.closed { - return errors.New("TODO: already closed msg") - } - return nil -} diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index f069b5ddc14..81ea8eb473a 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -6,6 +6,7 @@ import ( "io" "github.com/hugelgupf/p9/p9" + "github.com/hugelgupf/p9/unimplfs" files "github.com/ipfs/go-ipfs-files" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" @@ -20,6 +21,9 @@ var _ fsutils.WalkRef = (*IPFS)(nil) // Walk does not expect a namespace, only its path argument // e.g. `ipfs.Walk([]string("Qm...", "subdir")` not `ipfs.Walk([]string("ipfs", "Qm...", "subdir")` type IPFS struct { + unimplfs.NoopFile + p9.DefaultWalkGetAttr + IPFSBase IPFSFileMeta } @@ -37,14 +41,6 @@ func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A return id } -func (id *IPFS) Fork() (fsutils.WalkRef, error) { - base, err := id.IPFSBase.Fork() - if err != nil { - return nil, err - } - return &IPFS{IPFSBase: base}, nil -} - func (id *IPFS) Attach() (p9.File, error) { id.Logger.Debugf("Attach") @@ -54,15 +50,6 @@ func (id *IPFS) Attach() (p9.File, error) { return newFid, nil } -func coreAttr(ctx context.Context, attr *p9.Attr, path corepath.Resolved, core coreiface.CoreAPI, req p9.AttrMask) (p9.AttrMask, error) { - ipldNode, err := core.Dag().Get(ctx, path.Cid()) - if err != nil { - return p9.AttrMask{}, err - } - - return ipldStat(ctx, attr, ipldNode, req) -} - func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { id.Logger.Debugf("GetAttr") @@ -101,13 +88,6 @@ func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { return qid, filled, attr, err } -func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { - id.Logger.Debugf("Walk names: %v", names) - id.Logger.Debugf("Walk myself: %q:{%d}", id.String(), id.ninePath()) - - return fsutils.Walker(id, names) -} - func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { id.Logger.Debugf("Open: %s", id.String()) @@ -150,6 +130,27 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return qid, ipfsBlockSize, nil } +func (id *IPFS) Close() error { + var lastErr error + + //TODO: timeout and cancel the context if Close takes too long + if id.file != nil { + if err := id.file.Close(); err != nil { + id.Logger.Error(err) + lastErr = err + } + id.file = nil + } + id.directory = nil + + lastErr = id.IPFSBase.close() + if lastErr != nil { + id.Logger.Error(lastErr) + } + + return lastErr +} + func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { id.Logger.Debugf("Readdir %q %d %d", id.String(), offset, count) @@ -239,25 +240,32 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { return readBytes, err } -func (id *IPFS) Close() error { - var lastErr error - - //TODO: timeout and cancel the context if Close takes too long - if id.file != nil { - if err := id.file.Close(); err != nil { - id.Logger.Error(err) - lastErr = err - } - id.file = nil +func coreAttr(ctx context.Context, attr *p9.Attr, path corepath.Resolved, core coreiface.CoreAPI, req p9.AttrMask) (p9.AttrMask, error) { + ipldNode, err := core.Dag().Get(ctx, path.Cid()) + if err != nil { + return p9.AttrMask{}, err } - id.directory = nil - lastErr = id.IPFSBase.Close() - if lastErr != nil { - id.Logger.Error(lastErr) - } + return ipldStat(ctx, attr, ipldNode, req) +} - return lastErr +func (id *IPFS) Walk(names []string) ([]p9.QID, p9.File, error) { + id.Logger.Debugf("Walk names: %v", names) + id.Logger.Debugf("Walk myself: %q:{%d}", id.String(), id.ninePath()) + + return fsutils.Walker(id, names) +} + +/* WalkRef relevant */ + +func (id *IPFS) CheckWalk() error { return id.Base.checkWalk() } + +func (id *IPFS) Fork() (fsutils.WalkRef, error) { + base, err := id.IPFSBase.fork() + if err != nil { + return nil, err + } + return &IPFS{IPFSBase: base}, nil } // IPFS appends "name" to its current path, and returns itself diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index bb18979374c..19ca6d350ed 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -4,6 +4,7 @@ import ( "context" "github.com/hugelgupf/p9/p9" + "github.com/hugelgupf/p9/unimplfs" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" @@ -14,6 +15,9 @@ var _ p9.File = (*KeyFS)(nil) var _ fsutils.WalkRef = (*KeyFS)(nil) type KeyFS struct { + unimplfs.NoopFile + p9.DefaultWalkGetAttr + IPFSBase } @@ -38,19 +42,30 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return kd } -func (kd *KeyFS) Fork() (fsutils.WalkRef, error) { +func (kd *KeyFS) Attach() (p9.File, error) { + kd.Logger.Debugf("Attach") + newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change - // set new operations context - err := newFid.newOperations() + // set new fs context + err := newFid.forkFilesystem() return newFid, err } -func (kd *KeyFS) Attach() (p9.File, error) { - kd.Logger.Debugf("Attach") +func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return *kd.qid, 0, nil } +func (kd *KeyFS) Close() error { return kd.IPFSBase.close() } + +// temporary stub to allow forwarding requests on empty directory +// will contain keys later +func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { + return nil, nil +} +/* WalkRef relevant */ + +func (kd *KeyFS) Fork() (fsutils.WalkRef, error) { newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change - // set new fs context - err := newFid.newFilesystem() + // set new operations context + err := newFid.forkOperations() return newFid, err } @@ -64,18 +79,15 @@ func (kd *KeyFS) Step(name string) (fsutils.WalkRef, error) { return newFid.Step(name) } +func (kd *KeyFS) CheckWalk() error { return kd.Base.checkWalk() } +func (kd *KeyFS) QID() (p9.QID, error) { return kd.Base.qID() } +func (kd *KeyFS) Backtrack() (fsutils.WalkRef, error) { return kd.IPFSBase.backtrack(kd) } + +/* base class boilerplate */ + func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { kd.Logger.Debugf("Walk names %v", names) kd.Logger.Debugf("Walk myself: %v", kd.qid) return fsutils.Walker(kd, names) } - -func (kd *KeyFS) Backtrack() (fsutils.WalkRef, error) { - return kd.IPFSBase.backtrack(kd) -} - -// temporary stub to allow forwarding requests on empty directory -func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { - return nil, nil -} diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index 6d7d4fbbda6..e17f9048710 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -6,6 +6,7 @@ import ( gopath "path" "github.com/hugelgupf/p9/p9" + "github.com/hugelgupf/p9/unimplfs" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" logging "github.com/ipfs/go-log" @@ -17,6 +18,9 @@ var _ p9.File = (*PinFS)(nil) var _ fsutils.WalkRef = (*PinFS)(nil) type PinFS struct { + unimplfs.NoopFile + p9.DefaultWalkGetAttr + IPFSBase ents []p9.Dirent } @@ -42,39 +46,15 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return pd } -func (pd *PinFS) Fork() (fsutils.WalkRef, error) { - newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change - // set new operations context - err := newFid.newOperations() - return newFid, err -} - func (pd *PinFS) Attach() (p9.File, error) { pd.Logger.Debugf("Attach") newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change // set new fs context - err := newFid.newFilesystem() + err := newFid.forkFilesystem() return newFid, err } -// PinFS forks the IPFS root that was set during construction -// and calls step on it rather than itself -func (pd *PinFS) Step(name string) (fsutils.WalkRef, error) { - newFid, err := pd.proxy.Fork() - if err != nil { - return nil, err - } - return newFid.Step(name) -} - -func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { - pd.Logger.Debugf("Walk names %v", names) - pd.Logger.Debugf("Walk myself: %v", pd.qid) - - return fsutils.Walker(pd, names) -} - func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") @@ -120,6 +100,11 @@ func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return qid, ipfsBlockSize, nil } +func (pd *PinFS) Close() error { + pd.ents = nil + return pd.IPFSBase.close() +} + func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { pd.Logger.Debugf("Readdir") @@ -130,11 +115,34 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return flatReaddir(pd.ents, offset, count) } -func (pd *PinFS) Backtrack() (fsutils.WalkRef, error) { - return pd.IPFSBase.backtrack(pd) +/* WalkRef relevant */ + +func (pd *PinFS) Fork() (fsutils.WalkRef, error) { + newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change + // set new operations context + err := newFid.forkOperations() + return newFid, err } -func (pd *PinFS) Close() error { - pd.ents = nil - return nil +// PinFS forks the IPFS root that was set during construction +// and calls step on it rather than itself +func (pd *PinFS) Step(name string) (fsutils.WalkRef, error) { + newFid, err := pd.proxy.Fork() + if err != nil { + return nil, err + } + return newFid.Step(name) +} + +func (pd *PinFS) CheckWalk() error { return pd.Base.checkWalk() } +func (pd *PinFS) QID() (p9.QID, error) { return pd.Base.qID() } +func (pd *PinFS) Backtrack() (fsutils.WalkRef, error) { return pd.IPFSBase.backtrack(pd) } + +/* base class boilerplate */ + +func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { + pd.Logger.Debugf("Walk names %v", names) + pd.Logger.Debugf("Walk myself: %v", pd.qid) + + return fsutils.Walker(pd, names) } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 582d5613c0f..740505b7cb5 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -4,6 +4,7 @@ import ( "context" "github.com/hugelgupf/p9/p9" + "github.com/hugelgupf/p9/unimplfs" cid "github.com/ipfs/go-cid" nodeopts "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/nodes/options" fsutils "github.com/ipfs/go-ipfs/plugin/plugins/filesystem/utils" @@ -43,6 +44,9 @@ type systemTuple struct { // RootIndex is a virtual directory file system, that maps a set of file system implementations to a hierarchy // Currently: "/ipfs":PinFS, "/ipfs/*:IPFS type RootIndex struct { + unimplfs.NoopFile + p9.DefaultWalkGetAttr + IPFSBase subsystems map[string]systemTuple } @@ -115,17 +119,6 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A return ri } -func (ri *RootIndex) Fork() (fsutils.WalkRef, error) { - newFid := &RootIndex{ - IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change - subsystems: ri.subsystems, - } - - // set new operations context - err := newFid.newOperations() - return newFid, err -} - func (ri *RootIndex) Attach() (p9.File, error) { ri.Logger.Debugf("Attach") @@ -135,30 +128,12 @@ func (ri *RootIndex) Attach() (p9.File, error) { } // set new fs context - err := newFid.newFilesystem() + err := newFid.forkFilesystem() return newFid, err } -func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { - ri.Logger.Debugf("Walk names %v", names) - ri.Logger.Debugf("Walk myself: %v", ri.qid.Path) - - return fsutils.Walker(ri, names) -} - -// The RootIndex checks if it has attached to "name" -// derives a node from it, and returns it -func (ri *RootIndex) Step(name string) (fsutils.WalkRef, error) { - // consume fs/access name - subSys, ok := ri.subsystems[name] - if !ok { - ri.Logger.Errorf("%q is not provided by us", name) - return nil, ENOENT - } - - // return a ready to use derivative of it - return subSys.file.Fork() -} +func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return *ri.qid, 0, nil } +func (ri *RootIndex) Close() error { return ri.IPFSBase.close() } func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { ri.Logger.Debugf("Readdir {%d}", count) @@ -194,6 +169,42 @@ func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return ents, nil } -func (ri *RootIndex) Backtrack() (fsutils.WalkRef, error) { - return ri.IPFSBase.backtrack(ri) +/* WalkRef relevant */ + +func (ri *RootIndex) Fork() (fsutils.WalkRef, error) { + newFid := &RootIndex{ + IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change + subsystems: ri.subsystems, + } + + // set new operations context + err := newFid.forkOperations() + return newFid, err +} + +// The RootIndex checks if it has attached to "name" +// derives a node from it, and returns it +func (ri *RootIndex) Step(name string) (fsutils.WalkRef, error) { + // consume fs/access name + subSys, ok := ri.subsystems[name] + if !ok { + ri.Logger.Errorf("%q is not provided by us", name) + return nil, ENOENT + } + + // return a ready to use derivative of it + return subSys.file.Fork() +} + +func (ri *RootIndex) CheckWalk() error { return ri.Base.checkWalk() } +func (ri *RootIndex) QID() (p9.QID, error) { return ri.Base.qID() } +func (ri *RootIndex) Backtrack() (fsutils.WalkRef, error) { return ri.IPFSBase.backtrack(ri) } + +/* base class boilerplate */ + +func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { + ri.Logger.Debugf("Walk names %v", names) + ri.Logger.Debugf("Walk myself: %v", ri.qid.Path) + + return fsutils.Walker(ri, names) } diff --git a/plugin/plugins/filesystem/pinfs_test.go b/plugin/plugins/filesystem/pinfs_test.go index 9dbeff2090b..72b82d6ca4f 100644 --- a/plugin/plugins/filesystem/pinfs_test.go +++ b/plugin/plugins/filesystem/pinfs_test.go @@ -11,6 +11,8 @@ import ( ) func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { + t.Run("Baseline", func(t *testing.T) { baseLine(ctx, t, core, fsnodes.PinFSAttacher) }) + pinRoot, err := fsnodes.PinFSAttacher(ctx, core).Attach() if err != nil { t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) From a0465f4924a8855db424f081b5c4e808aaae4580 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Mon, 21 Oct 2019 18:39:15 -0400 Subject: [PATCH 088/102] small lint and docs --- plugin/plugins/filesystem/nodes/base.go | 13 +++++-------- plugin/plugins/filesystem/nodes/ipfs.go | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 766d0073fb3..952fee4a9c2 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -140,12 +140,6 @@ func (ib *IPFSBase) CorePath(names ...string) corepath.Path { /* base operation methods to build on */ -func (b *Base) getAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - //b.Logger.Debugf("GetAttr {%d}:%q", b.qid.Path, b.String()) - - return *b.qid, *b.metaMask, *b.meta, nil -} - func (b *Base) close() error { b.Logger.Debugf("closing: {%d}%q", b.qid.Path, b.String()) b.closed = true @@ -199,6 +193,7 @@ func (ib *IPFSBase) clone() IPFSBase { } } +// see fsutils.WalkRef.Fork documentation func (b *Base) fork() Base { newFid := b.clone() @@ -218,13 +213,14 @@ func (ib *IPFSBase) fork() (IPFSBase, error) { return newFid, nil } +// see fsutils.WalkRef.Step documentation func (b *Base) step(self fsutils.WalkRef, name string) (fsutils.WalkRef, error) { if b.qid.Type != p9.TypeDir { return nil, ENOTDIR } - if b.closed == true { - return nil, errors.New("TODO: ref was previously closed err") + if b.closed { + return nil, errors.New("ref was previously closed") //TODO: use a 9P error value } tLen := len(b.Trail) @@ -233,6 +229,7 @@ func (b *Base) step(self fsutils.WalkRef, name string) (fsutils.WalkRef, error) return self, nil } +// see fsutils.WalkRef.Backtrack documentation func (ib *IPFSBase) backtrack(self fsutils.WalkRef) (fsutils.WalkRef, error) { // if we're a root return our parent, or ourselves if we don't have one if len(ib.Trail) == 0 { diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 81ea8eb473a..0b390d611b9 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -143,9 +143,9 @@ func (id *IPFS) Close() error { } id.directory = nil - lastErr = id.IPFSBase.close() - if lastErr != nil { + if err := id.IPFSBase.close(); err != nil { id.Logger.Error(lastErr) + lastErr = id.IPFSBase.close() } return lastErr From 2aead06090d5b826cf9a4b5b9589a0f87ff185ed Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 22 Oct 2019 06:42:53 -0400 Subject: [PATCH 089/102] change attach clone method --- plugin/plugins/filesystem/nodes/ipfs.go | 7 ++++--- plugin/plugins/filesystem/nodes/keyfs.go | 6 ++++-- plugin/plugins/filesystem/nodes/pinfs.go | 9 +++++---- plugin/plugins/filesystem/nodes/root.go | 6 ++++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 0b390d611b9..68eb643cde2 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -44,9 +44,10 @@ func IPFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A func (id *IPFS) Attach() (p9.File, error) { id.Logger.Debugf("Attach") - newFid := new(IPFS) - *newFid = *id - + newFid := &IPFS{IPFSBase: id.IPFSBase.clone()} + if err := newFid.forkFilesystem(); err != nil { + return nil, err + } return newFid, nil } diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 19ca6d350ed..a79557c8067 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -47,8 +47,10 @@ func (kd *KeyFS) Attach() (p9.File, error) { newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change // set new fs context - err := newFid.forkFilesystem() - return newFid, err + if err := newFid.forkFilesystem(); err != nil { + return nil, err + } + return newFid, nil } func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return *kd.qid, 0, nil } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index e17f9048710..e6845916606 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -49,10 +49,11 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. func (pd *PinFS) Attach() (p9.File, error) { pd.Logger.Debugf("Attach") - newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change - // set new fs context - err := newFid.forkFilesystem() - return newFid, err + newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} + if err := newFid.forkFilesystem(); err != nil { + return nil, err + } + return newFid, nil } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 740505b7cb5..9a79fb0d165 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -128,8 +128,10 @@ func (ri *RootIndex) Attach() (p9.File, error) { } // set new fs context - err := newFid.forkFilesystem() - return newFid, err + if err := newFid.forkFilesystem(); err != nil { + return nil, err + } + return newFid, nil } func (ri *RootIndex) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return *ri.qid, 0, nil } From ec9f8ea033e0fbf47623bf241ba7350b3bbd5173 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 22 Oct 2019 07:05:00 -0400 Subject: [PATCH 090/102] unbreak root attrs --- plugin/plugins/filesystem/filesystem_common_test.go | 3 +++ plugin/plugins/filesystem/nodes/base.go | 8 +++++++- plugin/plugins/filesystem/nodes/ipfs.go | 3 +++ plugin/plugins/filesystem/nodes/keyfs.go | 4 ++++ plugin/plugins/filesystem/nodes/pinfs.go | 4 ++++ plugin/plugins/filesystem/nodes/root.go | 4 ++++ 6 files changed, 25 insertions(+), 1 deletion(-) diff --git a/plugin/plugins/filesystem/filesystem_common_test.go b/plugin/plugins/filesystem/filesystem_common_test.go index 37aa412ffe7..f1870dd8dd9 100644 --- a/plugin/plugins/filesystem/filesystem_common_test.go +++ b/plugin/plugins/filesystem/filesystem_common_test.go @@ -33,6 +33,9 @@ func baseLine(ctx context.Context, t *testing.T, core coreiface.CoreAPI, attachF } t.Run("walk", func(t *testing.T) { testClones(ctx, t, root) }) + if _, _, _, err = root.GetAttr(p9.AttrMaskAll); err != nil { + t.Fatal(err) + } } func testAttacher(ctx context.Context, t *testing.T, attacher p9.Attacher) { diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index 952fee4a9c2..a2975c6d3ae 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -160,8 +160,8 @@ func (ib *IPFSBase) close() error { } lastErr = err } - */ ib.filesystemCancel() + */ } if ib.operationsCancel != nil { @@ -171,6 +171,12 @@ func (ib *IPFSBase) close() error { return lastErr } +func (b *Base) getAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + //b.Logger.Debugf("GetAttr {%d}:%q", b.qid.Path, b.String()) + + return *b.qid, *b.metaMask, *b.meta, nil +} + /* WalkRef relevant */ func (b *Base) checkWalk() error { diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 68eb643cde2..708b775e8ac 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -53,6 +53,9 @@ func (id *IPFS) Attach() (p9.File, error) { func (id *IPFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { id.Logger.Debugf("GetAttr") + if len(id.Trail) == 0 { + return id.Base.getAttr(req) + } // TODO: we need to re use storage on id // merge existing members with request members fetched via request diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index a79557c8067..dd5aa9b0a19 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -93,3 +93,7 @@ func (kd *KeyFS) Walk(names []string) ([]p9.QID, p9.File, error) { return fsutils.Walker(kd, names) } + +func (kd *KeyFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + return kd.Base.getAttr(req) +} diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index e6845916606..cfa9b392541 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -147,3 +147,7 @@ func (pd *PinFS) Walk(names []string) ([]p9.QID, p9.File, error) { return fsutils.Walker(pd, names) } + +func (pd *PinFS) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + return pd.Base.getAttr(req) +} diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 9a79fb0d165..6e595b03dcb 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -210,3 +210,7 @@ func (ri *RootIndex) Walk(names []string) ([]p9.QID, p9.File, error) { return fsutils.Walker(ri, names) } + +func (ri *RootIndex) GetAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { + return ri.Base.getAttr(req) +} From c744a005e95fa5e507d969c2d9b800fc13698f1d Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Tue, 22 Oct 2019 07:26:25 -0400 Subject: [PATCH 091/102] cleanup attachers --- plugin/plugins/filesystem/nodes/keyfs.go | 17 +++++++++-------- plugin/plugins/filesystem/nodes/pinfs.go | 16 +++++++++------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index dd5aa9b0a19..2eefb6369de 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -42,10 +42,9 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return kd } -func (kd *KeyFS) Attach() (p9.File, error) { - kd.Logger.Debugf("Attach") - - newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change +// this root has no paths to walk; forking anythign besides the fs doesn't make sense for us +func (kd *KeyFS) clone() (fsutils.WalkRef, error) { + newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // set new fs context if err := newFid.forkFilesystem(); err != nil { return nil, err @@ -53,6 +52,11 @@ func (kd *KeyFS) Attach() (p9.File, error) { return newFid, nil } +func (kd *KeyFS) Attach() (p9.File, error) { + kd.Logger.Debugf("Attach") + return kd.clone() +} + func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return *kd.qid, 0, nil } func (kd *KeyFS) Close() error { return kd.IPFSBase.close() } @@ -65,10 +69,7 @@ func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { /* WalkRef relevant */ func (kd *KeyFS) Fork() (fsutils.WalkRef, error) { - newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change - // set new operations context - err := newFid.forkOperations() - return newFid, err + return kd.clone() } // KeyFS forks the IPFS root that was set during construction diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index cfa9b392541..c9a8a96a193 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -46,9 +46,8 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return pd } -func (pd *PinFS) Attach() (p9.File, error) { - pd.Logger.Debugf("Attach") - +// this root has no paths to walk; forking anythign besides the fs doesn't make sense for us +func (pd *PinFS) clone() (fsutils.WalkRef, error) { newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} if err := newFid.forkFilesystem(); err != nil { return nil, err @@ -56,6 +55,11 @@ func (pd *PinFS) Attach() (p9.File, error) { return newFid, nil } +func (pd *PinFS) Attach() (p9.File, error) { + pd.Logger.Debugf("Attach") + return pd.clone() +} + func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { pd.Logger.Debugf("Open") @@ -119,10 +123,8 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { /* WalkRef relevant */ func (pd *PinFS) Fork() (fsutils.WalkRef, error) { - newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} // root has no paths to walk; don't set node up for change - // set new operations context - err := newFid.forkOperations() - return newFid, err + // root has no paths to walk; don't set node up for change + return pd.clone() } // PinFS forks the IPFS root that was set during construction From 65b677f976adc9e314504a22feaa1d0012db5c0a Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 23 Oct 2019 19:31:04 -0400 Subject: [PATCH 092/102] slightly saner test strings --- .../filesystem/filesystem_common_test.go | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_common_test.go b/plugin/plugins/filesystem/filesystem_common_test.go index f1870dd8dd9..dc773c4c2b3 100644 --- a/plugin/plugins/filesystem/filesystem_common_test.go +++ b/plugin/plugins/filesystem/filesystem_common_test.go @@ -21,16 +21,25 @@ import ( corepath "github.com/ipfs/interface-go-ipfs-core/path" ) +const ( + errFmtRoot = "Failed to attach to 9P root resource: %s\n" + errFmtRootSecond = "Failed to attach to 9P root resource a second time: %s\n" + errFmtClose = "Close errored: %s\n" + errFmtClone = "Failed to clone ref: %s\n" +) + type baseAttacher func(context.Context, coreiface.CoreAPI, ...nodeopts.AttachOption) p9.Attacher func baseLine(ctx context.Context, t *testing.T, core coreiface.CoreAPI, attachFn baseAttacher) { attacher := attachFn(ctx, core) t.Run("attacher", func(t *testing.T) { testAttacher(ctx, t, attacher) }) + root, err := attacher.Attach() if err != nil { t.Fatalf("Attach test passed but attach failed: %s\n", err) } + t.Run("walk", func(t *testing.T) { testClones(ctx, t, root) }) if _, _, _, err = root.GetAttr(p9.AttrMaskAll); err != nil { @@ -42,86 +51,87 @@ func testAttacher(ctx context.Context, t *testing.T, attacher p9.Attacher) { // 2 individual instances, one after another nineRoot, err := attacher.Attach() if err != nil { - t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + t.Fatalf(errFmtRoot, err) } if err = nineRoot.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } nineRootTheRevenge, err := attacher.Attach() if err != nil { - t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) + t.Fatalf(errFmtRootSecond, err) } if err = nineRootTheRevenge.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } // 2 instances at the same time nineRoot, err = attacher.Attach() if err != nil { - t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + t.Fatalf(errFmtRoot, err) } nineRootTheRevenge, err = attacher.Attach() if err != nil { - t.Fatalf("Failed to attach to 9P root resource a second time: %s\n", err) + t.Fatalf(errFmtRootSecond, err) } if err = nineRootTheRevenge.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } if err = nineRoot.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } // final instance nineRoot, err = attacher.Attach() if err != nil { - t.Fatalf("Failed to attach to 9P root resource: %s\n", err) + t.Fatalf(errFmtRoot, err) } if err = nineRoot.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } } func testClones(ctx context.Context, t *testing.T, nineRef p9.File) { + // clone the node we were passed; 1st generation _, newRef, err := nineRef.Walk(nil) if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) + t.Fatalf(errFmtClone, err) } // this `Close` shouldn't affect the parent it's derived from // only descendants if err = newRef.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } // remake the clone from the original; 1st generation again _, gen1, err := nineRef.Walk(nil) if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) + t.Fatalf(errFmtClone, err) } // clone a 2nd generation from the 1st _, gen2, err := gen1.Walk(nil) if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) + t.Fatalf(errFmtClone, err) } // 3rd from the 2nd _, gen3, err := gen2.Walk(nil) if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) + t.Fatalf(errFmtClone, err) } // close the 2nd reference if err = gen2.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } // try to clone from the 2nd reference @@ -135,33 +145,33 @@ func testClones(ctx context.Context, t *testing.T, nineRef p9.File) { // should still succeed regardless of 2's state _, gen4, err := gen3.Walk(nil) if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) + t.Fatalf(errFmtClone, err) } // close the 3rd reference if err = gen3.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } // close the 4th reference if err = gen4.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } // clone a 2nd generation from the 1st again _, gen2, err = gen1.Walk(nil) if err != nil { - t.Fatalf("Failed to clone root: %s\n", err) + t.Fatalf(errFmtClone, err) } // close the 1st if err = gen1.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } // close the 2nd if err = gen2.Close(); err != nil { - t.Fatalf("Close errored: %s\n", err) + t.Fatalf(errFmtClose, err) } } From 7ce0e9bf6c55a19c9cbcd2d09e64dcbfaaa79032 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 23 Oct 2019 19:31:24 -0400 Subject: [PATCH 093/102] test open --- .../filesystem/filesystem_common_test.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/plugin/plugins/filesystem/filesystem_common_test.go b/plugin/plugins/filesystem/filesystem_common_test.go index dc773c4c2b3..4c0b242874c 100644 --- a/plugin/plugins/filesystem/filesystem_common_test.go +++ b/plugin/plugins/filesystem/filesystem_common_test.go @@ -41,6 +41,7 @@ func baseLine(ctx context.Context, t *testing.T, core coreiface.CoreAPI, attachF } t.Run("walk", func(t *testing.T) { testClones(ctx, t, root) }) + t.Run("open", func(t *testing.T) { testOpen(ctx, t, root) }) if _, _, _, err = root.GetAttr(p9.AttrMaskAll); err != nil { t.Fatal(err) @@ -175,6 +176,40 @@ func testClones(ctx context.Context, t *testing.T, nineRef p9.File) { } } +func testOpen(ctx context.Context, t *testing.T, nineRef p9.File) { + _, newRef, err := nineRef.Walk(nil) + if err != nil { + t.Fatalf(errFmtClone, err) + } + + _, thing1, err := nineRef.Walk(nil) + if err != nil { + t.Fatalf(errFmtClone, err) + } + _, thing2, err := nineRef.Walk(nil) + if err != nil { + t.Fatalf(errFmtClone, err) + } + + // a close of one reference should not affect the operation context of another + if err = thing1.Close(); err != nil { + t.Fatalf(errFmtClose, err) + } + + if _, _, err = thing2.Open(0); err != nil { + t.Fatalf("could not open reference after unrelated reference was closed: %s\n", err) + } + + // cleanup + if err = thing2.Close(); err != nil { + t.Fatalf(errFmtClose, err) + } + + if err = newRef.Close(); err != nil { + t.Fatalf(errFmtClose, err) + } +} + func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { var expand func(p9.File) (map[string]p9.Attr, error) expand = func(nineRef p9.File) (map[string]p9.Attr, error) { From 4c6b8ee73e8a497bfcd15758a4e396f6286c7d1c Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 23 Oct 2019 19:32:39 -0400 Subject: [PATCH 094/102] allow ipfs root to be read --- plugin/plugins/filesystem/nodes/ipfs.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 708b775e8ac..66f74bfc8f3 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -99,6 +99,13 @@ func (id *IPFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { // handle directories if qid.Type == p9.TypeDir { + + // handle the root itself (empty) + if len(id.Trail) == 0 { + id.directory = &directoryStream{} + return qid, 0, nil + } + c, err := id.core.Unixfs().Ls(id.operationsCtx, id.CorePath()) if err != nil { //id.operationsCancel() @@ -166,6 +173,11 @@ func (id *IPFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { return nil, id.directory.err } + // special case for root + if len(id.Trail) == 0 { + return nil, nil + } + if offset < id.directory.cursor { return nil, fmt.Errorf("read offset %d is behind current entry %d, seeking backwards in directory streams is not supported", offset, id.directory.cursor) } From 6e654db37e8674a8752f4095ceb564238be5c91d Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 23 Oct 2019 19:36:06 -0400 Subject: [PATCH 095/102] fix context for proxy roots --- plugin/plugins/filesystem/nodes/keyfs.go | 16 ++++++++-------- plugin/plugins/filesystem/nodes/pinfs.go | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 2eefb6369de..77ca354528c 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -42,8 +42,9 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return kd } -// this root has no paths to walk; forking anythign besides the fs doesn't make sense for us -func (kd *KeyFS) clone() (fsutils.WalkRef, error) { +func (kd *KeyFS) Attach() (p9.File, error) { + kd.Logger.Debugf("Attach") + newFid := &KeyFS{IPFSBase: kd.IPFSBase.clone()} // set new fs context if err := newFid.forkFilesystem(); err != nil { @@ -52,11 +53,6 @@ func (kd *KeyFS) clone() (fsutils.WalkRef, error) { return newFid, nil } -func (kd *KeyFS) Attach() (p9.File, error) { - kd.Logger.Debugf("Attach") - return kd.clone() -} - func (kd *KeyFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { return *kd.qid, 0, nil } func (kd *KeyFS) Close() error { return kd.IPFSBase.close() } @@ -69,7 +65,11 @@ func (kd *KeyFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { /* WalkRef relevant */ func (kd *KeyFS) Fork() (fsutils.WalkRef, error) { - return kd.clone() + base, err := kd.IPFSBase.fork() + if err != nil { + return nil, err + } + return &KeyFS{IPFSBase: base}, nil } // KeyFS forks the IPFS root that was set during construction diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index c9a8a96a193..d5c2ec47b29 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -46,18 +46,16 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. return pd } -// this root has no paths to walk; forking anythign besides the fs doesn't make sense for us -func (pd *PinFS) clone() (fsutils.WalkRef, error) { +func (pd *PinFS) Attach() (p9.File, error) { + pd.Logger.Debugf("Attach") + newFid := &PinFS{IPFSBase: pd.IPFSBase.clone()} + if err := newFid.forkFilesystem(); err != nil { return nil, err } - return newFid, nil -} -func (pd *PinFS) Attach() (p9.File, error) { - pd.Logger.Debugf("Attach") - return pd.clone() + return newFid, nil } func (pd *PinFS) Open(mode p9.OpenFlags) (p9.QID, uint32, error) { @@ -123,8 +121,11 @@ func (pd *PinFS) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { /* WalkRef relevant */ func (pd *PinFS) Fork() (fsutils.WalkRef, error) { - // root has no paths to walk; don't set node up for change - return pd.clone() + base, err := pd.IPFSBase.fork() + if err != nil { + return nil, err + } + return &PinFS{IPFSBase: base}, nil } // PinFS forks the IPFS root that was set during construction From 3501592e9a8b042292ef761edfe5503713d378f8 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 23 Oct 2019 19:37:25 -0400 Subject: [PATCH 096/102] fix --- plugin/plugins/filesystem/nodes/root.go | 30 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 6e595b03dcb..d6584bc6c0e 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -119,18 +119,24 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A return ri } +func (ri *RootIndex) clone() *RootIndex { + return &RootIndex{ + IPFSBase: ri.IPFSBase.clone(), + subsystems: ri.subsystems, // share the same subsystem reference across all instances + } +} + func (ri *RootIndex) Attach() (p9.File, error) { ri.Logger.Debugf("Attach") - newFid := &RootIndex{ - IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change - subsystems: ri.subsystems, - } + // this root has no paths to walk, so don't allocate anything new + newFid := ri.clone() - // set new fs context + // new instance, new context if err := newFid.forkFilesystem(); err != nil { return nil, err } + return newFid, nil } @@ -174,14 +180,16 @@ func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { /* WalkRef relevant */ func (ri *RootIndex) Fork() (fsutils.WalkRef, error) { - newFid := &RootIndex{ - IPFSBase: ri.IPFSBase.clone(), // root has no paths to walk; don't set node up for change - subsystems: ri.subsystems, - } + // this root has no paths to walk, so don't allocate anything new + newFid := ri.clone() - // set new operations context + // new instance, new context err := newFid.forkOperations() - return newFid, err + if err != nil { + return nil, err + } + + return newFid, nil } // The RootIndex checks if it has attached to "name" From 07db2b9badd1644968c58066fe6fe9821481301f Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Wed, 23 Oct 2019 19:39:23 -0400 Subject: [PATCH 097/102] doc pass WIP --- plugin/plugins/filesystem/nodes/base.go | 35 ++++++++---------------- plugin/plugins/filesystem/nodes/doc.go | 20 ++++++++------ plugin/plugins/filesystem/nodes/root.go | 14 +++++----- plugin/plugins/filesystem/utils/utils.go | 14 +++++----- 4 files changed, 38 insertions(+), 45 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index a2975c6d3ae..f0880c99a12 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -22,7 +22,7 @@ const ( //device - attempts to comply with standard multicodec table type p9Path = uint64 // Base provides a foundation to build file system nodes which contain file meta data -// as well as some base methods +// as well as some base methods. type Base struct { Trail []string // FS "breadcrumb" trail from node's root @@ -49,14 +49,14 @@ func newBase(ops ...nodeopts.AttachOption) Base { } } -// IPFSBase is much like Base but extends it to hold IPFS specific metadata +// IPFSBase is much like Base but extends it to hold IPFS specific metadata. type IPFSBase struct { Base OverlayFileMeta - /* The filesystem context should be set prior to `Attach` + /* The filesystem context should be set prior to `Attach`. During `fs.Attach`, a new reference to `fs` should be created - with its fs-context + fs-cancel populated with one derived from the existing fs context + with its fs-context + fs-cancel populated with one derived from the existing fs context. This context is expected to be valid for the lifetime of the file system and canceled on `Close` by references returned from `Attach` only. @@ -64,32 +64,23 @@ type IPFSBase struct { filesystemCtx context.Context filesystemCancel context.CancelFunc - /* During `file.Walk` a new reference to `file` should be created - with its op-context + op-cancel populated with one derived from the existing fs context + /* The operations context should be unique to the file reference. + References derived from `file.Walk` should contain a new context and cancel. + (Likely populated with one derived from the existing fs context) This context is expected to be valid as long as the file is being referenced by a particular FID - it should be canceled during `Close` - - for the lifetime of the file system - to be used during operations, such as `Open`, `Read` etc. - and canceled on `Close` by references returned from `Attach` only. + and should be canceled during `Close`. */ - operationsCtx context.Context operationsCancel context.CancelFunc - // Typically you'll want to derive a context from the fs ctx within one operation (like Open) - // use it with the CoreAPI for something (like Get) - // and cancel it in another operation (like Close) - // those pointers should be stored here between operation calls - - // Format the namespace as if it were a rooted directory, sans trailing slash - // e.g. `/ipfs` - // the base relative path is appended to the namespace for core requests upon calling `.CorePath()` + /* Format the namespace as if it were a rooted directory, sans trailing slash. + e.g. `/ipfs` + The base relative path is appended to the namespace for core requests upon calling `.CorePath()`. + */ coreNamespace string core coreiface.CoreAPI } -//func newIPFSBase(ctx context.Context, path corepath.Resolved, kind p9.FileMode, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { func newIPFSBase(ctx context.Context, coreNamespace string, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) IPFSBase { options := nodeopts.AttachOps(ops...) return IPFSBase{ @@ -172,8 +163,6 @@ func (ib *IPFSBase) close() error { } func (b *Base) getAttr(req p9.AttrMask) (p9.QID, p9.AttrMask, p9.Attr, error) { - //b.Logger.Debugf("GetAttr {%d}:%q", b.qid.Path, b.String()) - return *b.qid, *b.metaMask, *b.meta, nil } diff --git a/plugin/plugins/filesystem/nodes/doc.go b/plugin/plugins/filesystem/nodes/doc.go index edf14c78239..98a70fbe745 100644 --- a/plugin/plugins/filesystem/nodes/doc.go +++ b/plugin/plugins/filesystem/nodes/doc.go @@ -1,16 +1,20 @@ /* Package fsnodes provides constructors and interfaces, for composing various 9P -file systems implementations and wrappers. +file systems implementations and wrappers of them. The default RootIndex provided by RootAttacher, is a file system of itself -which relays request to various IPFS subsystems. -Attaching to and utilizing P9.File implementations themselves, in the same way a client program could use them independently. +which wraps the other `p9.File` implementations, and relays request to them. + +It does so by simply utilizing these subsystems in the same way a client program would if used independently. +Linking them together in a slash delimited hierarchy. The current mapping looks like this: - - mountpoint - associated filesystem - - / - points back to itself, returning the root index - - /ipfs - PinFS - - /ipfs/* - IPFS - - /ipns/* - IPNS +Table key: +mountpoint - associated file system - purpose + - / - points back to itself, returning the root index - Maintains the hierarchy and dispatches requests + - /ipfs - PinFS - Exposes the node's pins as a series of files + - /ipfs/* - IPFS - Relays requests to the IPFS namespace, translating UnixFS objects into 9P constructs + - /ipns/ - KeyFS - Exposes the node's keys as a series of files + - /ipns/* - IPNS - Another relay, but for the IPNS namespace */ package fsnodes diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index d6584bc6c0e..017ca274024 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -35,14 +35,14 @@ func (rp rootPath) Cid() cid.Cid { return c } +// pair a filesystem implementation with directory entry metadata about it type systemTuple struct { file fsutils.WalkRef dirent p9.Dirent } //TODO: rename, while this is likely to be the root, it doesn't have to be; maybe "IPFSOverlay" -// RootIndex is a virtual directory file system, that maps a set of file system implementations to a hierarchy -// Currently: "/ipfs":PinFS, "/ipfs/*:IPFS +// RootIndex is a file system, that maps a set of other file system implementations to a hierarchy. type RootIndex struct { unimplfs.NoopFile p9.DefaultWalkGetAttr @@ -61,15 +61,15 @@ type OverlayFileMeta struct { proxy fsutils.WalkRef } -// RootAttacher constructs the default RootIndex file system, providing a means to Attach() to it +// RootAttacher constructs the default RootIndex file system, and all of its dependants, providing a means to Attach() to it func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.AttachOption) p9.Attacher { - // construct root node + // construct root node actual ri := &RootIndex{IPFSBase: newIPFSBase(ctx, "/", core, ops...)} ri.qid.Type = p9.TypeDir ri.meta.Mode, ri.metaMask.Mode = p9.ModeDirectory|IRXA, true // attach to subsystems - // used for proxying walk requests to other filesystems + // used for proxying walk requests to other file systems type subattacher func(context.Context, coreiface.CoreAPI, ...nodeopts.AttachOption) p9.Attacher type attachTuple struct { string @@ -77,7 +77,7 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A logging.EventLogger } - // 9P Access names mapped to IPFS attacher functions + // 9P "Access names" mapped to IPFS attacher functions subsystems := [...]attachTuple{ {"ipfs", PinFSAttacher, logging.Logger("PinFS")}, {"ipns", KeyFSAttacher, logging.Logger("KeyFS")}, @@ -163,7 +163,7 @@ func (ri *RootIndex) Readdir(offset uint64, count uint32) ([]p9.Dirent, error) { ents = make([]p9.Dirent, relativeEnd) } - // use ents from map within request bounds to populate slice + // use ents from map within request bounds to populate slice slots for _, pair := range ri.subsystems { if count == 0 { break diff --git a/plugin/plugins/filesystem/utils/utils.go b/plugin/plugins/filesystem/utils/utils.go index d64907f46b3..2c8ab4151a4 100644 --- a/plugin/plugins/filesystem/utils/utils.go +++ b/plugin/plugins/filesystem/utils/utils.go @@ -2,14 +2,14 @@ package fsutils import "github.com/hugelgupf/p9/p9" -// WalkRef is used to generalize 9P `Walk` operations within filesystem boundries +// WalkRef is used to generalize 9P `Walk` operations within file system boundaries // and allows for traversal across those boundaries if intended by the implementation // -// The reference root node implementation links filesystems together using a parent/child relation -// sending the appropriate node back to the caller by using the linakge between nodes +// The reference root node implementation links file systems together using a parent/child relation +// sending the appropriate node back to the caller by using the linkage between nodes // combined with inspection of a nodes current path // it tries its best to avoid copying by modifying a node directly where possible -// falling back to derived copies when crossing filesystem boundaries during "movement" +// falling back to derived copies when crossing file system boundaries during "movement" type WalkRef interface { p9.File @@ -27,9 +27,9 @@ type WalkRef interface { The returned reference node must "stand" beside the existing `WalkRef` Meaning the node must be "at"/contain the same location/path as the existing reference. - The returned node must also adhear to 'walk(5)' `newfid` semantics + The returned node must also adhere to 'walk(5)' `newfid` semantics Meaning that... - `newfid` must be allowed to `Close` seperatley from the original reference + `newfid` must be allowed to `Close` separately from the original reference `newfid`'s path may be modified during `Walk` without affecting the original `WalkRef` `Open` must flag all references within the same system, at the same path, as open etc. in compliance with 'walk(5)' @@ -51,7 +51,7 @@ type WalkRef interface { Step(name string) (WalkRef, error) /* Backtrack is the handler for `..` requests - it is effectivley the inverse of `Step` + it is effectively the inverse of `Step` if called on the root node, the node should return itself */ Backtrack() (parentRef WalkRef, err error) From 984f1557fc324bf41bdda8d39d17fda8cf3a875b Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 25 Oct 2019 09:34:38 -0400 Subject: [PATCH 098/102] make sure test defers gets called --- .../filesystem/filesystem_common_test.go | 117 ++++++++++++------ plugin/plugins/filesystem/filesystem_test.go | 34 +++-- plugin/plugins/filesystem/ipfs_test.go | 24 ++-- plugin/plugins/filesystem/pinfs_test.go | 21 ++-- plugin/plugins/filesystem/root_test.go | 15 ++- 5 files changed, 140 insertions(+), 71 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem_common_test.go b/plugin/plugins/filesystem/filesystem_common_test.go index 4c0b242874c..459c2002fee 100644 --- a/plugin/plugins/filesystem/filesystem_common_test.go +++ b/plugin/plugins/filesystem/filesystem_common_test.go @@ -37,14 +37,16 @@ func baseLine(ctx context.Context, t *testing.T, core coreiface.CoreAPI, attachF root, err := attacher.Attach() if err != nil { - t.Fatalf("Attach test passed but attach failed: %s\n", err) + t.Logf("Attach test passed but attach failed: %s\n", err) + t.FailNow() } t.Run("walk", func(t *testing.T) { testClones(ctx, t, root) }) t.Run("open", func(t *testing.T) { testOpen(ctx, t, root) }) if _, _, _, err = root.GetAttr(p9.AttrMaskAll); err != nil { - t.Fatal(err) + t.Log(err) + t.FailNow() } } @@ -52,49 +54,59 @@ func testAttacher(ctx context.Context, t *testing.T, attacher p9.Attacher) { // 2 individual instances, one after another nineRoot, err := attacher.Attach() if err != nil { - t.Fatalf(errFmtRoot, err) + t.Logf(errFmtRoot, err) + t.FailNow() } if err = nineRoot.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } nineRootTheRevenge, err := attacher.Attach() if err != nil { - t.Fatalf(errFmtRootSecond, err) + t.Logf(errFmtRootSecond, err) + t.FailNow() } if err = nineRootTheRevenge.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } // 2 instances at the same time nineRoot, err = attacher.Attach() if err != nil { - t.Fatalf(errFmtRoot, err) + t.Logf(errFmtRoot, err) + t.FailNow() } nineRootTheRevenge, err = attacher.Attach() if err != nil { - t.Fatalf(errFmtRootSecond, err) + t.Logf(errFmtRootSecond, err) + t.FailNow() } if err = nineRootTheRevenge.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } if err = nineRoot.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } // final instance nineRoot, err = attacher.Attach() if err != nil { - t.Fatalf(errFmtRoot, err) + t.Logf(errFmtRoot, err) + t.FailNow() } if err = nineRoot.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } } @@ -103,110 +115,130 @@ func testClones(ctx context.Context, t *testing.T, nineRef p9.File) { // clone the node we were passed; 1st generation _, newRef, err := nineRef.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } // this `Close` shouldn't affect the parent it's derived from // only descendants if err = newRef.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } // remake the clone from the original; 1st generation again _, gen1, err := nineRef.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } // clone a 2nd generation from the 1st _, gen2, err := gen1.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } // 3rd from the 2nd _, gen3, err := gen2.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } // close the 2nd reference if err = gen2.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } // try to clone from the 2nd reference // this should fail since we closed it _, undead, err := gen2.Walk(nil) if err == nil { - t.Fatalf("Clone (%p)%q succeeded when parent (%p)%q was closed\n", undead, undead, gen2, gen2) + t.Logf("Clone (%p)%q succeeded when parent (%p)%q was closed\n", undead, undead, gen2, gen2) + t.FailNow() } // 4th from the 3rd // should still succeed regardless of 2's state _, gen4, err := gen3.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } // close the 3rd reference if err = gen3.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } // close the 4th reference if err = gen4.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } // clone a 2nd generation from the 1st again _, gen2, err = gen1.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } // close the 1st if err = gen1.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } // close the 2nd if err = gen2.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } } func testOpen(ctx context.Context, t *testing.T, nineRef p9.File) { _, newRef, err := nineRef.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } _, thing1, err := nineRef.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } _, thing2, err := nineRef.Walk(nil) if err != nil { - t.Fatalf(errFmtClone, err) + t.Logf(errFmtClone, err) + t.FailNow() } // a close of one reference should not affect the operation context of another if err = thing1.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } if _, _, err = thing2.Open(0); err != nil { - t.Fatalf("could not open reference after unrelated reference was closed: %s\n", err) + t.Logf("could not open reference after unrelated reference was closed: %s\n", err) + t.FailNow() } // cleanup if err = thing2.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } if err = newRef.Close(); err != nil { - t.Fatalf(errFmtClose, err) + t.Logf(errFmtClose, err) + t.FailNow() } } @@ -249,12 +281,14 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { f1Map, err := expand(f1) if err != nil { - t.Fatal(err) + t.Log(err) + t.FailNow() } f2Map, err := expand(f2) if err != nil { - t.Fatal(err) + t.Log(err) + t.FailNow() } same := func(permissionContains p9.FileMode, base, target map[string]p9.Attr) bool { @@ -269,7 +303,8 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { targetNames = append(targetNames, name) } - t.Fatalf("map lengths don't match:\nbase:%v\ntarget:%v\n", baseNames, targetNames) + t.Logf("map lengths don't match:\nbase:%v\ntarget:%v\n", baseNames, targetNames) + t.FailNow() return false } @@ -278,16 +313,18 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { tMode := target[path].Mode if bMode.FileType() != tMode.FileType() { - t.Fatalf("type for %q don't match:\nbase:%v\ntarget:%v\n", path, bMode, tMode) + t.Logf("type for %q don't match:\nbase:%v\ntarget:%v\n", path, bMode, tMode) + t.FailNow() return false } if ((bMode.Permissions() & permissionContains) & (tMode.Permissions() & permissionContains)) == 0 { - t.Fatalf("permissions for %q don't match\n(unfiltered)\nbase:%v\ntarget:%v\n(filtered)\nbase:%v\ntarget:%v\n", + t.Logf("permissions for %q don't match\n(unfiltered)\nbase:%v\ntarget:%v\n(filtered)\nbase:%v\ntarget:%v\n", path, bMode.Permissions(), tMode.Permissions(), bMode.Permissions()&permissionContains, tMode.Permissions()&permissionContains, ) + t.FailNow() return false } @@ -296,17 +333,19 @@ func testCompareTreeAttrs(t *testing.T, f1, f2 p9.File) { tSize := target[path].Size if bSize != tSize { - t.Fatalf("size for %q doesn't match\nbase:%d\ntarget:%d\n", + t.Logf("size for %q doesn't match\nbase:%d\ntarget:%d\n", path, bSize, tSize) + t.FailNow() } } } return true } if !same(p9.Read, f1Map, f2Map) { - t.Fatalf("contents don't match \nf1:%v\nf2:%v\n", f1Map, f2Map) + t.Logf("contents don't match \nf1:%v\nf2:%v\n", f1Map, f2Map) + t.FailNow() } } diff --git a/plugin/plugins/filesystem/filesystem_test.go b/plugin/plugins/filesystem/filesystem_test.go index 7b2555ee982..82a13a81026 100644 --- a/plugin/plugins/filesystem/filesystem_test.go +++ b/plugin/plugins/filesystem/filesystem_test.go @@ -19,7 +19,8 @@ func TestAll(t *testing.T) { core, err := InitCore(ctx) if err != nil { - t.Fatalf("Failed to construct CoreAPI: %s\n", err) + t.Logf("Failed to construct CoreAPI: %s\n", err) + t.FailNow() } t.Run("RootFS", func(t *testing.T) { testRootFS(ctx, t, core) }) @@ -39,54 +40,63 @@ func testPlugin(t *testing.T, pluginEnv *plugin.Environment, core coreiface.Core // close and start before init are NOT allowed if err = module.Close(); err == nil { - t.Fatal("plugin was not initialized but Close succeeded") + t.Logf("plugin was not initialized but Close succeeded") + t.FailNow() // also should not hang } if err = module.Start(core); err == nil { - t.Fatal("plugin was not initialized but Start succeeded") + t.Logf("plugin was not initialized but Start succeeded") + t.FailNow() // also should not hang } // initialize the module if err = module.Init(pluginEnv); err != nil { - t.Fatal("Plugin couldn't be initialized: ", err) + t.Logf("Plugin couldn't be initialized: %s", err) + t.FailNow() } // double init is NOT allowed if err = module.Init(pluginEnv); err == nil { - t.Fatal("init isn't intended to succeed twice") + t.Logf("init isn't intended to succeed twice") + t.FailNow() } // close before start is allowed if err = module.Close(); err != nil { - t.Fatal("plugin isn't busy, but it can't close: ", err) + t.Logf("plugin isn't busy, but it can't close: %s", err) + t.FailNow() // also should not hang } // double close is allowed if err = module.Close(); err != nil { - t.Fatal("plugin couldn't close twice: ", err) + t.Logf("plugin couldn't close twice: %s", err) + t.FailNow() } // start the module if err = module.Start(core); err != nil { - t.Fatal("module could not start: ", err) + t.Logf("module could not start: %s", err) + t.FailNow() } // double start is NOT allowed if err = module.Start(core); err == nil { - t.Fatal("module is intended to be exclusive but was allowed to start twice") + t.Logf("module is intended to be exclusive but was allowed to start twice") + t.FailNow() } // actual close if err = module.Close(); err != nil { - t.Fatalf("plugin isn't busy, but it can't close: %#v", err) - t.Fatal("plugin isn't busy, but it can't close: ", err) + t.Logf("plugin isn't busy, but it can't close: %s", err) + t.FailNow() } // another redundant close if err = module.Close(); err != nil { - t.Fatal("plugin isn't busy, but it can't close: ", err) + t.Logf("plugin isn't busy, but it can't close: %s", err) + t.FailNow() // also should not hang } } diff --git a/plugin/plugins/filesystem/ipfs_test.go b/plugin/plugins/filesystem/ipfs_test.go index 795d27fd775..452fbb526bd 100644 --- a/plugin/plugins/filesystem/ipfs_test.go +++ b/plugin/plugins/filesystem/ipfs_test.go @@ -17,27 +17,32 @@ func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { rootRef, err := fsnodes.IPFSAttacher(ctx, core).Attach() if err != nil { - t.Fatalf("Baseline test passed but attach failed: %s\n", err) + t.Logf("Baseline test passed but attach failed: %s\n", err) + t.FailNow() } env, iEnv, err := initEnv(ctx, core) if err != nil { - t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + t.Logf("Failed to construct IPFS test environment: %s\n", err) + t.FailNow() } defer os.RemoveAll(env) localEnv, err := localfs.Attacher(env).Attach() if err != nil { - t.Fatalf("Failed to attach to local resource %q: %s\n", env, err) + t.Logf("Failed to attach to local resource %q: %s\n", env, err) + t.FailNow() } _, ipfsEnv, err := rootRef.Walk([]string{gopath.Base(iEnv.String())}) if err != nil { - t.Fatalf("Failed to walk to IPFS test environment: %s\n", err) + t.Logf("Failed to walk to IPFS test environment: %s\n", err) + t.FailNow() } _, envClone, err := ipfsEnv.Walk(nil) if err != nil { - t.Fatalf("Failed to clone IPFS environment handle: %s\n", err) + t.Logf("Failed to clone IPFS environment handle: %s\n", err) + t.FailNow() } testCompareTreeAttrs(t, localEnv, ipfsEnv) @@ -46,13 +51,16 @@ func testIPFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { //TODO: compare against a table, not just lengths _, _, err = envClone.Open(p9.ReadOnly) if err != nil { - t.Fatalf("Failed to open IPFS test directory: %s\n", err) + t.Logf("Failed to open IPFS test directory: %s\n", err) + t.FailNow() } ents, err := envClone.Readdir(2, 2) // start at ent 2, return max 2 if err != nil { - t.Fatalf("Failed to read IPFS test directory: %s\n", err) + t.Logf("Failed to read IPFS test directory: %s\n", err) + t.FailNow() } if l := len(ents); l == 0 || l > 2 { - t.Fatalf("IPFS test directory contents don't match read request: %v\n", ents) + t.Logf("IPFS test directory contents don't match read request: %v\n", ents) + t.FailNow() } } diff --git a/plugin/plugins/filesystem/pinfs_test.go b/plugin/plugins/filesystem/pinfs_test.go index 72b82d6ca4f..f724de3863d 100644 --- a/plugin/plugins/filesystem/pinfs_test.go +++ b/plugin/plugins/filesystem/pinfs_test.go @@ -15,7 +15,8 @@ func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { pinRoot, err := fsnodes.PinFSAttacher(ctx, core).Attach() if err != nil { - t.Fatalf("Failed to attach to 9P Pin resource: %s\n", err) + t.Logf("Failed to attach to 9P Pin resource: %s\n", err) + t.FailNow() } same := func(base, target []string) bool { @@ -36,15 +37,18 @@ func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { shallowCompare := func() { basePins, err := pinNames(ctx, core) if err != nil { - t.Fatalf("Failed to list IPFS pins: %s\n", err) + t.Logf("Failed to list IPFS pins: %s\n", err) + t.FailNow() } p9Pins, err := p9PinNames(pinRoot) if err != nil { - t.Fatalf("Failed to list 9P pins: %s\n", err) + t.Logf("Failed to list 9P pins: %s\n", err) + t.FailNow() } if !same(basePins, p9Pins) { - t.Fatalf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + t.Logf("Pinsets differ\ncore: %v\n9P: %v\n", basePins, p9Pins) + t.FailNow() } } @@ -54,17 +58,20 @@ func testPinFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { // test modifying pinset +1; initEnv pins its IPFS environment env, _, err := initEnv(ctx, core) if err != nil { - t.Fatalf("Failed to construct IPFS test environment: %s\n", err) + t.Logf("Failed to construct IPFS test environment: %s\n", err) + t.FailNow() } defer os.RemoveAll(env) shallowCompare() // test modifying pinset +1 again; generate garbage and pin it if err := generateGarbage(env); err != nil { - t.Fatalf("Failed to generate test data: %s\n", err) + t.Logf("Failed to generate test data: %s\n", err) + t.FailNow() } if _, err = pinAddDir(ctx, core, env); err != nil { - t.Fatalf("Failed to add directory to IPFS: %s\n", err) + t.Logf("Failed to add directory to IPFS: %s\n", err) + t.FailNow() } shallowCompare() } diff --git a/plugin/plugins/filesystem/root_test.go b/plugin/plugins/filesystem/root_test.go index ad5a5f6c63f..2e571bc0a3e 100644 --- a/plugin/plugins/filesystem/root_test.go +++ b/plugin/plugins/filesystem/root_test.go @@ -34,11 +34,13 @@ func testRootFS(ctx context.Context, t *testing.T, core coreiface.CoreAPI) { rootRef, err := fsnodes.RootAttacher(ctx, core).Attach() if err != nil { - t.Fatalf("Baseline test passed but attach failed: %s\n", err) + t.Logf("Baseline test passed but attach failed: %s\n", err) + t.FailNow() } _, root, err := rootRef.Walk(nil) if err != nil { - t.Fatalf("Baseline test passed but walk failed: %s\n", err) + t.Logf("Baseline test passed but walk failed: %s\n", err) + t.FailNow() } t.Run("Root directory entries", func(t *testing.T) { testRootDir(ctx, t, root) }) @@ -49,11 +51,13 @@ func testRootDir(ctx context.Context, t *testing.T, root p9.File) { ents, err := root.Readdir(0, uint32(len(rootSubsystems))) if err != nil { - t.Fatal(err) + t.Log(err) + t.FailNow() } if _, err = root.Readdir(uint64(len(ents)), ^uint32(0)); err != nil { - t.Fatal(errors.New("entry count mismatch")) + t.Log(errors.New("entry count mismatch")) + t.FailNow() } for i, ent := range ents { @@ -62,7 +66,8 @@ func testRootDir(ctx context.Context, t *testing.T, root p9.File) { rootSubsystems[i].QID.Path = ent.QID.Path if ent != rootSubsystems[i] { - t.Fatal(fmt.Errorf("ent %v != expected %v", ent, rootSubsystems[i])) + t.Log(fmt.Errorf("ent %v != expected %v", ent, rootSubsystems[i])) + t.FailNow() } } From 0fa59833378c03651a49e28f2398a56a9b9659b5 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 25 Oct 2019 09:51:55 -0400 Subject: [PATCH 099/102] try not to leak --- plugin/plugins/filesystem/nodes/base.go | 8 -------- plugin/plugins/filesystem/nodes/keyfs.go | 6 ++++++ plugin/plugins/filesystem/nodes/pinfs.go | 6 ++++++ plugin/plugins/filesystem/nodes/root.go | 8 ++++++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/base.go b/plugin/plugins/filesystem/nodes/base.go index f0880c99a12..4d801068ff4 100644 --- a/plugin/plugins/filesystem/nodes/base.go +++ b/plugin/plugins/filesystem/nodes/base.go @@ -144,15 +144,7 @@ func (ib *IPFSBase) close() error { } if ib.filesystemCancel != nil { - /* TODO: only do this on the last close to the root - if ib.proxy != nil { - if err := ib.proxy.Close(); err != nil { - ib.Logger.Error(err) - } - lastErr = err - } ib.filesystemCancel() - */ } if ib.operationsCancel != nil { diff --git a/plugin/plugins/filesystem/nodes/keyfs.go b/plugin/plugins/filesystem/nodes/keyfs.go index 77ca354528c..102f365ae63 100644 --- a/plugin/plugins/filesystem/nodes/keyfs.go +++ b/plugin/plugins/filesystem/nodes/keyfs.go @@ -2,6 +2,7 @@ package fsnodes import ( "context" + "runtime" "github.com/hugelgupf/p9/p9" "github.com/hugelgupf/p9/unimplfs" @@ -39,6 +40,11 @@ func KeyFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. kd.proxy = subsystem.(fsutils.WalkRef) + // detach from our proxied system when we fall out of memory + runtime.SetFinalizer(kd, func(keyRoot *KeyFS) { + keyRoot.proxy.Close() + }) + return kd } diff --git a/plugin/plugins/filesystem/nodes/pinfs.go b/plugin/plugins/filesystem/nodes/pinfs.go index d5c2ec47b29..4ad3641249e 100644 --- a/plugin/plugins/filesystem/nodes/pinfs.go +++ b/plugin/plugins/filesystem/nodes/pinfs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" gopath "path" + "runtime" "github.com/hugelgupf/p9/p9" "github.com/hugelgupf/p9/unimplfs" @@ -43,6 +44,11 @@ func PinFSAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts. pd.proxy = subsystem.(fsutils.WalkRef) + // detach from our proxied system when we fall out of memory + runtime.SetFinalizer(pd, func(pinRoot *PinFS) { + pinRoot.proxy.Close() + }) + return pd } diff --git a/plugin/plugins/filesystem/nodes/root.go b/plugin/plugins/filesystem/nodes/root.go index 017ca274024..4a21b63229b 100644 --- a/plugin/plugins/filesystem/nodes/root.go +++ b/plugin/plugins/filesystem/nodes/root.go @@ -2,6 +2,7 @@ package fsnodes import ( "context" + "runtime" "github.com/hugelgupf/p9/p9" "github.com/hugelgupf/p9/unimplfs" @@ -116,6 +117,13 @@ func RootAttacher(ctx context.Context, core coreiface.CoreAPI, ops ...nodeopts.A } } + // detach from our proxied systems when we fall out of memory + runtime.SetFinalizer(ri, func(root *RootIndex) { + for _, ss := range ri.subsystems { + ss.file.Close() + } + }) + return ri } From c9faaa7ce9cba9f7c17767d38e638982ec8b5475 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 25 Oct 2019 10:19:01 -0400 Subject: [PATCH 100/102] change Plugin Close procedure --- plugin/plugins/filesystem/filesystem.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index 804e88bff40..f2b08956a16 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -42,7 +42,6 @@ type FileSystemPlugin struct { addr multiaddr.Multiaddr listener manet.Listener closed chan struct{} - closing bool //TODO: p9 lib should probably have a `server.Close` that closes the listener and swallows the final `Accept` error instead of us managing it in this pkg serverErr error } @@ -143,7 +142,7 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { // store error on the fs object then close our syncing channel (see use in `Close` below) fs.serverErr = server.Serve(manet.NetListener(fs.listener)) - if fs.closing { //[async] we expect `Accept` to fail while `Close` is in progress + if fs.ctx.Err() != nil { // [async] we expect `Accept` to fail only if the filesystem is canceled var opErr *net.OpError if errors.As(fs.serverErr, &opErr) && opErr.Op == "accept" { fs.serverErr = nil @@ -165,14 +164,12 @@ func (fs *FileSystemPlugin) Close() error { // synchronization between plugin interface <-> fs server if fs.closed != nil { // implies `Start` was called prior - fs.closing = true // lets the goroutine know that accept is now allowed to fail - fs.listener.Close() // stop accepting actually - fs.cancel() // stop any lingering fs operations + fs.cancel() // stop and prevent all fs operations, signifies "closing" intent + fs.listener.Close() // stop accepting new clients <-fs.closed // wait for the server thread to set the error value - fs.closing = false // reset our async condition fs.listener = nil // reset `Start` conditions fs.closed = nil } // otherwise we were never started to begin with; default/initial value will be returned return fs.serverErr -} \ No newline at end of file +} From c7cfd7125abb4b749f21502bc9aee142f982ae81 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 25 Oct 2019 10:30:50 -0400 Subject: [PATCH 101/102] change Plugin Close procedure part 2 --- plugin/plugins/filesystem/filesystem.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/plugin/plugins/filesystem/filesystem.go b/plugin/plugins/filesystem/filesystem.go index f2b08956a16..a576bbac141 100644 --- a/plugin/plugins/filesystem/filesystem.go +++ b/plugin/plugins/filesystem/filesystem.go @@ -140,13 +140,19 @@ func (fs *FileSystemPlugin) Start(core coreiface.CoreAPI) error { go func() { // run the server until the listener closes // store error on the fs object then close our syncing channel (see use in `Close` below) - fs.serverErr = server.Serve(manet.NetListener(fs.listener)) - if fs.ctx.Err() != nil { // [async] we expect `Accept` to fail only if the filesystem is canceled + err := server.Serve(manet.NetListener(fs.listener)) + + // [async] we expect `net.Accept` to fail when the filesystem has been canceled + if fs.ctx.Err() != nil { + // non-'accept' ops are not expected to fail, so their error is preserved var opErr *net.OpError - if errors.As(fs.serverErr, &opErr) && opErr.Op == "accept" { - fs.serverErr = nil + if errors.As(fs.serverErr, &opErr) && opErr.Op != "accept" { + fs.serverErr = err } + } else { + // unexpected failure during operation + fs.serverErr = err } close(fs.closed) From c294ac8da68d983caa3ac6b8fec9c9032a983b33 Mon Sep 17 00:00:00 2001 From: Dominic Della Valle Date: Fri, 25 Oct 2019 11:32:14 -0400 Subject: [PATCH 102/102] don't swallow Read return value --- plugin/plugins/filesystem/nodes/ipfs.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugin/plugins/filesystem/nodes/ipfs.go b/plugin/plugins/filesystem/nodes/ipfs.go index 66f74bfc8f3..73309383ae2 100644 --- a/plugin/plugins/filesystem/nodes/ipfs.go +++ b/plugin/plugins/filesystem/nodes/ipfs.go @@ -249,8 +249,6 @@ func (id *IPFS) ReadAt(p []byte, offset uint64) (int, error) { readBytes, err := id.file.Read(p) if err != nil && err != io.EOF { id.Logger.Errorf(readAtFmtErr, offset, id.meta.Size, id.String(), err) - //id.operationsCancel() - return 0, err } return readBytes, err