From b6b91e1947b36b558be9ffc0b260d5697aafe102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Sat, 19 Sep 2015 10:49:55 +0200 Subject: [PATCH 1/5] add a ipfs repo ls that list the top hash (hash, type, size) available locally Currently use the recursive pin to find the top hash, so it doesn't list everything. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Michael Muré --- core/commands/repo.go | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/core/commands/repo.go b/core/commands/repo.go index 3b634e6311c..68adbe40387 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -8,6 +8,7 @@ import ( cmds "github.com/ipfs/go-ipfs/commands" corerepo "github.com/ipfs/go-ipfs/core/corerepo" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" + "github.com/ipfs/go-ipfs/unixfs" ) var RepoCmd = &cmds.Command{ @@ -20,6 +21,7 @@ var RepoCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "gc": repoGcCmd, + "ls": repoLsCmd, }, } @@ -95,3 +97,82 @@ order to reclaim hard disk space. }, }, } + +type RepoLsOutput struct { + Hash string + Size uint64 + Type string +} + +var repoLsCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Totally need a documentation", + ShortDescription: ` + I might agree on that + `, + }, + + Run: func(req cmds.Request, res cmds.Response) { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + outChan := make(chan interface{}) + res.SetOutput((<-chan interface{})(outChan)) + + go func() { + defer close(outChan) + for _, k := range n.Pinning.RecursiveKeys() { + node, err := n.DAG.Get(req.Context(), k) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + unixFSNode, err := unixfs.FromBytes(node.Data); + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + outChan <- &RepoLsOutput{ + Hash: k.Pretty(), + Size: unixFSNode.GetFilesize(), + Type: unixFSNode.GetType().String(), + } + } + }() + }, + + Type: RepoLsOutput{}, + + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + outChan, ok := res.Output().(<-chan interface{}) + if !ok { + return nil, u.ErrCast() + } + + marshal := func(v interface{}) (io.Reader, error) { + obj, ok := v.(*RepoLsOutput) + if !ok { + return nil, u.ErrCast() + } + + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%s\t%s \t%v\n", obj.Hash, obj.Type, obj.Size) + + return buf, nil + } + + return &cmds.ChannelMarshaler{ + Channel: outChan, + Marshaler: marshal, + Res: res, + }, nil + }, + }, +} From a152dfad7c29cc2bd573ba7a27f9ddce7dcb50f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Thu, 3 Mar 2016 21:39:08 +0100 Subject: [PATCH 2/5] make the pinner clean its old root node when flushing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Michael Muré --- core/builder.go | 4 ++-- merkledag/merkledag_test.go | 2 +- pin/pin.go | 22 ++++++++++++++++++++-- pin/pin_test.go | 10 +++++----- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/core/builder.go b/core/builder.go index 107eeffd29f..1ae08063aa3 100644 --- a/core/builder.go +++ b/core/builder.go @@ -149,13 +149,13 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { n.Blocks = bserv.New(n.Blockstore, n.Exchange) n.DAG = dag.NewDAGService(n.Blocks) - n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG) + n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.Blockstore, n.DAG) if err != nil { // TODO: we should move towards only running 'NewPinner' explicity on // node init instead of implicitly here as a result of the pinner keys // not being found in the datastore. // this is kinda sketchy and could cause data loss - n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG) + n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.Blockstore, n.DAG) } n.Resolver = &path.Resolver{DAG: n.DAG} diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 91bc2c0f770..b3b04142293 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -36,7 +36,7 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner { bs := bstore.NewBlockstore(db) blockserv := bserv.New(bs, offline.Exchange(bs)) dserv := NewDAGService(blockserv) - mpin := pin.NewPinner(db, dserv) + mpin := pin.NewPinner(db, bs, dserv) return dagservAndPinner{ ds: dserv, mp: mpin, diff --git a/pin/pin.go b/pin/pin.go index a7f62417f64..aac673e6c78 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -13,6 +13,7 @@ import ( mdag "github.com/ipfs/go-ipfs/merkledag" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" ) var log = logging.Logger("pin") @@ -64,12 +65,14 @@ type pinner struct { // Track the keys used for storing the pinning state, so gc does // not delete them. internalPin map[key.Key]struct{} + rootNode *mdag.Node dserv mdag.DAGService + bstore bs.Blockstore dstore ds.Datastore } // NewPinner creates a new pinner using the given datastore as a backend -func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner { +func NewPinner(dstore ds.Datastore, bstore bs.Blockstore, serv mdag.DAGService) Pinner { // Load set from given datastore... rcset := set.NewSimpleBlockSet() @@ -80,6 +83,7 @@ func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner { recursePin: rcset, directPin: dirset, dserv: serv, + bstore: bstore, dstore: dstore, } } @@ -234,7 +238,7 @@ func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) { } // LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) { +func LoadPinner(d ds.Datastore, bstore bs.Blockstore, dserv mdag.DAGService) (Pinner, error) { p := new(pinner) rootKeyI, err := d.Get(pinDatastoreKey) @@ -279,10 +283,12 @@ func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) { p.directPin = set.SimpleSetFromKeys(directKeys) } + p.rootNode = root p.internalPin = internalPin // assign services p.dserv = dserv + p.bstore = bstore p.dstore = d return p, nil @@ -346,6 +352,18 @@ func (p *pinner) Flush() error { if err := p.dstore.Put(pinDatastoreKey, []byte(k)); err != nil { return fmt.Errorf("cannot store pin state: %v", err) } + + // cleanup old root node + if p.rootNode != nil { + for _, link := range p.rootNode.Links { + if child, err := link.GetNode(ctx, p.dserv); err == nil { + p.dserv.Remove(child) + } + } + p.dserv.Remove(p.rootNode) + } + + p.rootNode = root p.internalPin = internalPin return nil } diff --git a/pin/pin_test.go b/pin/pin_test.go index 9eb61acefe0..85b1d2784f3 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -45,7 +45,7 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) a, ak := randNode() _, err := dserv.Add(a) @@ -129,7 +129,7 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv) + np, err := LoadPinner(dstore, bstore, dserv) if err != nil { t.Fatal(err) } @@ -150,7 +150,7 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) a, _ := randNode() _, err := dserv.Add(a) @@ -183,7 +183,7 @@ func TestFlush(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) _, k := randNode() p.PinWithMode(k, Recursive) @@ -200,7 +200,7 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) a, _ := randNode() b, _ := randNode() From 6e0b0925463d0f5b0d9e0e3b22eb4be58b1e3d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Thu, 3 Mar 2016 21:42:42 +0100 Subject: [PATCH 3/5] repo ls: find the DAG roots to display the top nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Michael Muré --- core/commands/repo.go | 102 +++++++++++++++++++++++++++++++++--------- pin/pin.go | 2 +- 2 files changed, 83 insertions(+), 21 deletions(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 68adbe40387..a68db2bc2ca 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -5,10 +5,13 @@ import ( "fmt" "io" + humanize "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize" + "github.com/ipfs/go-ipfs/blocks/key" + "github.com/ipfs/go-ipfs/blocks/set" cmds "github.com/ipfs/go-ipfs/commands" corerepo "github.com/ipfs/go-ipfs/core/corerepo" - u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" "github.com/ipfs/go-ipfs/unixfs" + u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) var RepoCmd = &cmds.Command{ @@ -20,8 +23,8 @@ var RepoCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "gc": repoGcCmd, - "ls": repoLsCmd, + "gc": repoGcCmd, + "ls-roots": repoLsRootsCmd, }, } @@ -98,18 +101,20 @@ order to reclaim hard disk space. }, } -type RepoLsOutput struct { - Hash string - Size uint64 - Type string +type RepoLsRootsOutput struct { + Hash string + Size uint64 + Type string + Pinned string } -var repoLsCmd = &cmds.Command{ +var repoLsRootsCmd = &cmds.Command{ Helptext: cmds.HelpText{ - Tagline: "Totally need a documentation", + Tagline: "Display the top nodes stored locally", ShortDescription: ` - I might agree on that - `, +'ipfs repo ls-roots' will display the top-level files or directory +that are stored locally as well a some of their properties. +`, }, Run: func(req cmds.Request, res cmds.Response) { @@ -119,34 +124,87 @@ var repoLsCmd = &cmds.Command{ return } + // Force pinner flush as we ask the blockstore directly + n.Pinning.Flush() + outChan := make(chan interface{}) res.SetOutput((<-chan interface{})(outChan)) + keychan, err := n.Blockstore.AllKeysChan(req.Context()) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + go func() { defer close(outChan) - for _, k := range n.Pinning.RecursiveKeys() { + roots := set.NewSimpleBlockSet() + childs := set.NewSimpleBlockSet() + + // find the roots of the DAG + KeyLoop: + for k := range keychan { + + // skip internal pinning node + for _, pinnedKey := range n.Pinning.InternalPins() { + if pinnedKey == k { + continue KeyLoop + } + } + + node, err := n.DAG.Get(req.Context(), k) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + if !childs.HasKey(k) { + roots.AddBlock(k) + } + + for _, child := range node.Links { + childKey := key.Key(child.Hash) + roots.RemoveBlock(childKey) + childs.AddBlock(childKey) + } + } + + for _, k := range roots.GetKeys() { node, err := n.DAG.Get(req.Context(), k) if err != nil { res.SetError(err, cmds.ErrNormal) return } - unixFSNode, err := unixfs.FromBytes(node.Data); + unixFSNode, err := unixfs.FromBytes(node.Data) + if err != nil { + // ignore nodes that are not unixFs + continue + } + + pinned, _, err := n.Pinning.IsPinned(k) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + size, err := node.Size() if err != nil { res.SetError(err, cmds.ErrNormal) return } - outChan <- &RepoLsOutput{ - Hash: k.Pretty(), - Size: unixFSNode.GetFilesize(), - Type: unixFSNode.GetType().String(), + outChan <- &RepoLsRootsOutput{ + Hash: k.B58String(), + Size: size, + Type: unixFSNode.Type.String(), + Pinned: pinned, } } }() }, - Type: RepoLsOutput{}, + Type: RepoLsRootsOutput{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { @@ -156,14 +214,18 @@ var repoLsCmd = &cmds.Command{ } marshal := func(v interface{}) (io.Reader, error) { - obj, ok := v.(*RepoLsOutput) + obj, ok := v.(*RepoLsRootsOutput) if !ok { return nil, u.ErrCast() } buf := new(bytes.Buffer) - fmt.Fprintf(buf, "%s\t%s \t%v\n", obj.Hash, obj.Type, obj.Size) + fmt.Fprintf(buf, "%s\t%s\t%s \t%s\n", + obj.Hash, + humanize.Bytes(obj.Size), + obj.Type, + obj.Pinned) return buf, nil } diff --git a/pin/pin.go b/pin/pin.go index aac673e6c78..35bfc029370 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -8,12 +8,12 @@ import ( "time" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" key "github.com/ipfs/go-ipfs/blocks/key" "github.com/ipfs/go-ipfs/blocks/set" mdag "github.com/ipfs/go-ipfs/merkledag" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" - bs "github.com/ipfs/go-ipfs/blocks/blockstore" ) var log = logging.Logger("pin") From 9cd411590c57f3ee0bb447ce5fe02785d6ae0f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Sun, 6 Mar 2016 03:02:11 +0100 Subject: [PATCH 4/5] add basic sharness test for ipfs repo ls-roots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Michael Muré --- test/sharness/t0080-repo.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 01ef79b0ab3..9ae86f7d2bf 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -219,6 +219,15 @@ test_expect_success "'ipfs refs --unique --recursive (bigger)'" ' test_sort_cmp expected actual || test_fsh cat refs_output ' +test_expect_success "'ipfs repo ls-roots' succeeds" ' + ipfs repo ls-roots > ls-roots +' + +test_expect_success "'ipfs repo ls-roots' looks good" ' + grep "$HASH_WELCOME_DOCS" ls-roots && + grep "$EMPTY_DIR" ls-roots +' + test_kill_ipfs_daemon test_done From 56f556f4a2fdecf0b92aa2245dc2d2af647ba611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Tue, 8 Mar 2016 00:08:45 +0100 Subject: [PATCH 5/5] fix a typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Michael Muré --- core/commands/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index a68db2bc2ca..953ceaf9c21 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -113,7 +113,7 @@ var repoLsRootsCmd = &cmds.Command{ Tagline: "Display the top nodes stored locally", ShortDescription: ` 'ipfs repo ls-roots' will display the top-level files or directory -that are stored locally as well a some of their properties. +that are stored locally as well as some of their properties. `, },