diff --git a/core/builder.go b/core/builder.go index 107eeffd29f..1ae08063aa3 100644 --- a/core/builder.go +++ b/core/builder.go @@ -149,13 +149,13 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { n.Blocks = bserv.New(n.Blockstore, n.Exchange) n.DAG = dag.NewDAGService(n.Blocks) - n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG) + n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.Blockstore, n.DAG) if err != nil { // TODO: we should move towards only running 'NewPinner' explicity on // node init instead of implicitly here as a result of the pinner keys // not being found in the datastore. // this is kinda sketchy and could cause data loss - n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG) + n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.Blockstore, n.DAG) } n.Resolver = &path.Resolver{DAG: n.DAG} diff --git a/core/commands/repo.go b/core/commands/repo.go index 3b634e6311c..953ceaf9c21 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -5,8 +5,12 @@ import ( "fmt" "io" + humanize "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize" + "github.com/ipfs/go-ipfs/blocks/key" + "github.com/ipfs/go-ipfs/blocks/set" cmds "github.com/ipfs/go-ipfs/commands" corerepo "github.com/ipfs/go-ipfs/core/corerepo" + "github.com/ipfs/go-ipfs/unixfs" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) @@ -19,7 +23,8 @@ var RepoCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "gc": repoGcCmd, + "gc": repoGcCmd, + "ls-roots": repoLsRootsCmd, }, } @@ -95,3 +100,141 @@ order to reclaim hard disk space. }, }, } + +type RepoLsRootsOutput struct { + Hash string + Size uint64 + Type string + Pinned string +} + +var repoLsRootsCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Display the top nodes stored locally", + ShortDescription: ` +'ipfs repo ls-roots' will display the top-level files or directory +that are stored locally as well as some of their properties. +`, + }, + + Run: func(req cmds.Request, res cmds.Response) { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + // Force pinner flush as we ask the blockstore directly + n.Pinning.Flush() + + outChan := make(chan interface{}) + res.SetOutput((<-chan interface{})(outChan)) + + keychan, err := n.Blockstore.AllKeysChan(req.Context()) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + go func() { + defer close(outChan) + roots := set.NewSimpleBlockSet() + childs := set.NewSimpleBlockSet() + + // find the roots of the DAG + KeyLoop: + for k := range keychan { + + // skip internal pinning node + for _, pinnedKey := range n.Pinning.InternalPins() { + if pinnedKey == k { + continue KeyLoop + } + } + + node, err := n.DAG.Get(req.Context(), k) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + if !childs.HasKey(k) { + roots.AddBlock(k) + } + + for _, child := range node.Links { + childKey := key.Key(child.Hash) + roots.RemoveBlock(childKey) + childs.AddBlock(childKey) + } + } + + for _, k := range roots.GetKeys() { + node, err := n.DAG.Get(req.Context(), k) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + unixFSNode, err := unixfs.FromBytes(node.Data) + if err != nil { + // ignore nodes that are not unixFs + continue + } + + pinned, _, err := n.Pinning.IsPinned(k) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + size, err := node.Size() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + outChan <- &RepoLsRootsOutput{ + Hash: k.B58String(), + Size: size, + Type: unixFSNode.Type.String(), + Pinned: pinned, + } + } + }() + }, + + Type: RepoLsRootsOutput{}, + + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + outChan, ok := res.Output().(<-chan interface{}) + if !ok { + return nil, u.ErrCast() + } + + marshal := func(v interface{}) (io.Reader, error) { + obj, ok := v.(*RepoLsRootsOutput) + if !ok { + return nil, u.ErrCast() + } + + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%s\t%s\t%s \t%s\n", + obj.Hash, + humanize.Bytes(obj.Size), + obj.Type, + obj.Pinned) + + return buf, nil + } + + return &cmds.ChannelMarshaler{ + Channel: outChan, + Marshaler: marshal, + Res: res, + }, nil + }, + }, +} diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 91bc2c0f770..b3b04142293 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -36,7 +36,7 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner { bs := bstore.NewBlockstore(db) blockserv := bserv.New(bs, offline.Exchange(bs)) dserv := NewDAGService(blockserv) - mpin := pin.NewPinner(db, dserv) + mpin := pin.NewPinner(db, bs, dserv) return dagservAndPinner{ ds: dserv, mp: mpin, diff --git a/pin/pin.go b/pin/pin.go index a7f62417f64..35bfc029370 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -8,6 +8,7 @@ import ( "time" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" key "github.com/ipfs/go-ipfs/blocks/key" "github.com/ipfs/go-ipfs/blocks/set" mdag "github.com/ipfs/go-ipfs/merkledag" @@ -64,12 +65,14 @@ type pinner struct { // Track the keys used for storing the pinning state, so gc does // not delete them. internalPin map[key.Key]struct{} + rootNode *mdag.Node dserv mdag.DAGService + bstore bs.Blockstore dstore ds.Datastore } // NewPinner creates a new pinner using the given datastore as a backend -func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner { +func NewPinner(dstore ds.Datastore, bstore bs.Blockstore, serv mdag.DAGService) Pinner { // Load set from given datastore... rcset := set.NewSimpleBlockSet() @@ -80,6 +83,7 @@ func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner { recursePin: rcset, directPin: dirset, dserv: serv, + bstore: bstore, dstore: dstore, } } @@ -234,7 +238,7 @@ func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) { } // LoadPinner loads a pinner and its keysets from the given datastore -func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) { +func LoadPinner(d ds.Datastore, bstore bs.Blockstore, dserv mdag.DAGService) (Pinner, error) { p := new(pinner) rootKeyI, err := d.Get(pinDatastoreKey) @@ -279,10 +283,12 @@ func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) { p.directPin = set.SimpleSetFromKeys(directKeys) } + p.rootNode = root p.internalPin = internalPin // assign services p.dserv = dserv + p.bstore = bstore p.dstore = d return p, nil @@ -346,6 +352,18 @@ func (p *pinner) Flush() error { if err := p.dstore.Put(pinDatastoreKey, []byte(k)); err != nil { return fmt.Errorf("cannot store pin state: %v", err) } + + // cleanup old root node + if p.rootNode != nil { + for _, link := range p.rootNode.Links { + if child, err := link.GetNode(ctx, p.dserv); err == nil { + p.dserv.Remove(child) + } + } + p.dserv.Remove(p.rootNode) + } + + p.rootNode = root p.internalPin = internalPin return nil } diff --git a/pin/pin_test.go b/pin/pin_test.go index 9eb61acefe0..85b1d2784f3 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -45,7 +45,7 @@ func TestPinnerBasic(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) a, ak := randNode() _, err := dserv.Add(a) @@ -129,7 +129,7 @@ func TestPinnerBasic(t *testing.T) { t.Fatal(err) } - np, err := LoadPinner(dstore, dserv) + np, err := LoadPinner(dstore, bstore, dserv) if err != nil { t.Fatal(err) } @@ -150,7 +150,7 @@ func TestDuplicateSemantics(t *testing.T) { dserv := mdag.NewDAGService(bserv) // TODO does pinner need to share datastore with blockservice? - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) a, _ := randNode() _, err := dserv.Add(a) @@ -183,7 +183,7 @@ func TestFlush(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) _, k := randNode() p.PinWithMode(k, Recursive) @@ -200,7 +200,7 @@ func TestPinRecursiveFail(t *testing.T) { bserv := bs.New(bstore, offline.Exchange(bstore)) dserv := mdag.NewDAGService(bserv) - p := NewPinner(dstore, dserv) + p := NewPinner(dstore, bstore, dserv) a, _ := randNode() b, _ := randNode() diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 01ef79b0ab3..9ae86f7d2bf 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -219,6 +219,15 @@ test_expect_success "'ipfs refs --unique --recursive (bigger)'" ' test_sort_cmp expected actual || test_fsh cat refs_output ' +test_expect_success "'ipfs repo ls-roots' succeeds" ' + ipfs repo ls-roots > ls-roots +' + +test_expect_success "'ipfs repo ls-roots' looks good" ' + grep "$HASH_WELCOME_DOCS" ls-roots && + grep "$EMPTY_DIR" ls-roots +' + test_kill_ipfs_daemon test_done