add a ipfs repo ls-roots command

core/builder.go

@@ -149,13 +149,13 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 
 	n.Blocks = bserv.New(n.Blockstore, n.Exchange)
 	n.DAG = dag.NewDAGService(n.Blocks)
-	n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG)
+	n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.Blockstore, n.DAG)
 	if err != nil {
 		// TODO: we should move towards only running 'NewPinner' explicity on
 		// node init instead of implicitly here as a result of the pinner keys
 		// not being found in the datastore.
 		// this is kinda sketchy and could cause data loss
-		n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG)
+		n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.Blockstore, n.DAG)
 	}
 	n.Resolver = &path.Resolver{DAG: n.DAG}
 

core/commands/repo.go

@@ -5,8 +5,12 @@ import (
 	"fmt"
 	"io"
 
+	humanize "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize"
+	"github.com/ipfs/go-ipfs/blocks/key"
+	"github.com/ipfs/go-ipfs/blocks/set"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
+	"github.com/ipfs/go-ipfs/unixfs"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 )
 
@@ -19,7 +23,8 @@ var RepoCmd = &cmds.Command{
 	},
 
 	Subcommands: map[string]*cmds.Command{
-		"gc": repoGcCmd,
+		"gc":       repoGcCmd,
+		"ls-roots": repoLsRootsCmd,
 	},
 }
 
@@ -95,3 +100,141 @@ order to reclaim hard disk space.
 		},
 	},
 }
+
+type RepoLsRootsOutput struct {
+	Hash   string
+	Size   uint64
+	Type   string
+	Pinned string
+}
+
+var repoLsRootsCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Display the top nodes stored locally",
+		ShortDescription: `
+'ipfs repo ls-roots' will display the top-level files or directory
+that are stored locally as well as some of their properties.
+`,
+	},
+
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		// Force pinner flush as we ask the blockstore directly
+		n.Pinning.Flush()
+
+		outChan := make(chan interface{})
+		res.SetOutput((<-chan interface{})(outChan))
+
+		keychan, err := n.Blockstore.AllKeysChan(req.Context())
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		go func() {
+			defer close(outChan)
+			roots := set.NewSimpleBlockSet()
+			childs := set.NewSimpleBlockSet()
+
+			// find the roots of the DAG
+		KeyLoop:
+			for k := range keychan {
+
+				// skip internal pinning node
+				for _, pinnedKey := range n.Pinning.InternalPins() {
+					if pinnedKey == k {
+						continue KeyLoop
+					}
+				}
+
+				node, err := n.DAG.Get(req.Context(), k)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+
+				if !childs.HasKey(k) {
+					roots.AddBlock(k)
+				}
+
+				for _, child := range node.Links {
+					childKey := key.Key(child.Hash)
+					roots.RemoveBlock(childKey)
+					childs.AddBlock(childKey)
+				}
+			}
+
+			for _, k := range roots.GetKeys() {
+				node, err := n.DAG.Get(req.Context(), k)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+
+				unixFSNode, err := unixfs.FromBytes(node.Data)
+				if err != nil {
+					// ignore nodes that are not unixFs
+					continue
+				}
+
+				pinned, _, err := n.Pinning.IsPinned(k)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+
+				size, err := node.Size()
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+
+				outChan <- &RepoLsRootsOutput{
+					Hash:   k.B58String(),
+					Size:   size,
+					Type:   unixFSNode.Type.String(),
+					Pinned: pinned,
+				}
+			}
+		}()
+	},
+
+	Type: RepoLsRootsOutput{},
+
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			outChan, ok := res.Output().(<-chan interface{})
+			if !ok {
+				return nil, u.ErrCast()
+			}
+
+			marshal := func(v interface{}) (io.Reader, error) {
+				obj, ok := v.(*RepoLsRootsOutput)
+				if !ok {
+					return nil, u.ErrCast()
+				}
+
+				buf := new(bytes.Buffer)
+
+				fmt.Fprintf(buf, "%s\t%s\t%s    \t%s\n",
+					obj.Hash,
+					humanize.Bytes(obj.Size),
+					obj.Type,
+					obj.Pinned)
+
+				return buf, nil
+			}
+
+			return &cmds.ChannelMarshaler{
+				Channel:   outChan,
+				Marshaler: marshal,
+				Res:       res,
+			}, nil
+		},
+	},
+}

merkledag/merkledag_test.go

@@ -36,7 +36,7 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner {
 	bs := bstore.NewBlockstore(db)
 	blockserv := bserv.New(bs, offline.Exchange(bs))
 	dserv := NewDAGService(blockserv)
-	mpin := pin.NewPinner(db, dserv)
+	mpin := pin.NewPinner(db, bs, dserv)
 	return dagservAndPinner{
 		ds: dserv,
 		mp: mpin,

pin/pin.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
+	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
 	key "github.com/ipfs/go-ipfs/blocks/key"
 	"github.com/ipfs/go-ipfs/blocks/set"
 	mdag "github.com/ipfs/go-ipfs/merkledag"
@@ -64,12 +65,14 @@ type pinner struct {
 	// Track the keys used for storing the pinning state, so gc does
 	// not delete them.
 	internalPin map[key.Key]struct{}
+	rootNode    *mdag.Node
 	dserv       mdag.DAGService
+	bstore      bs.Blockstore
 	dstore      ds.Datastore
 }
 
 // NewPinner creates a new pinner using the given datastore as a backend
-func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner {
+func NewPinner(dstore ds.Datastore, bstore bs.Blockstore, serv mdag.DAGService) Pinner {
 
 	// Load set from given datastore...
 	rcset := set.NewSimpleBlockSet()
@@ -80,6 +83,7 @@ func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner {
 		recursePin: rcset,
 		directPin:  dirset,
 		dserv:      serv,
+		bstore:     bstore,
 		dstore:     dstore,
 	}
 }
@@ -234,7 +238,7 @@ func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) {
 }
 
 // LoadPinner loads a pinner and its keysets from the given datastore
-func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
+func LoadPinner(d ds.Datastore, bstore bs.Blockstore, dserv mdag.DAGService) (Pinner, error) {
 	p := new(pinner)
 
 	rootKeyI, err := d.Get(pinDatastoreKey)
@@ -279,10 +283,12 @@ func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
 		p.directPin = set.SimpleSetFromKeys(directKeys)
 	}
 
+	p.rootNode = root
 	p.internalPin = internalPin
 
 	// assign services
 	p.dserv = dserv
+	p.bstore = bstore
 	p.dstore = d
 
 	return p, nil
@@ -346,6 +352,18 @@ func (p *pinner) Flush() error {
 	if err := p.dstore.Put(pinDatastoreKey, []byte(k)); err != nil {
 		return fmt.Errorf("cannot store pin state: %v", err)
 	}
+
+	// cleanup old root node
+	if p.rootNode != nil {
+		for _, link := range p.rootNode.Links {
+			if child, err := link.GetNode(ctx, p.dserv); err == nil {
+				p.dserv.Remove(child)
+			}
+		}
+		p.dserv.Remove(p.rootNode)
+	}
+
+	p.rootNode = root
 	p.internalPin = internalPin
 	return nil
 }

pin/pin_test.go

@@ -45,7 +45,7 @@ func TestPinnerBasic(t *testing.T) {
 	dserv := mdag.NewDAGService(bserv)
 
 	// TODO does pinner need to share datastore with blockservice?
-	p := NewPinner(dstore, dserv)
+	p := NewPinner(dstore, bstore, dserv)
 
 	a, ak := randNode()
 	_, err := dserv.Add(a)
@@ -129,7 +129,7 @@ func TestPinnerBasic(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	np, err := LoadPinner(dstore, dserv)
+	np, err := LoadPinner(dstore, bstore, dserv)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -150,7 +150,7 @@ func TestDuplicateSemantics(t *testing.T) {
 	dserv := mdag.NewDAGService(bserv)
 
 	// TODO does pinner need to share datastore with blockservice?
-	p := NewPinner(dstore, dserv)
+	p := NewPinner(dstore, bstore, dserv)
 
 	a, _ := randNode()
 	_, err := dserv.Add(a)
@@ -183,7 +183,7 @@ func TestFlush(t *testing.T) {
 	bserv := bs.New(bstore, offline.Exchange(bstore))
 
 	dserv := mdag.NewDAGService(bserv)
-	p := NewPinner(dstore, dserv)
+	p := NewPinner(dstore, bstore, dserv)
 	_, k := randNode()
 
 	p.PinWithMode(k, Recursive)
@@ -200,7 +200,7 @@ func TestPinRecursiveFail(t *testing.T) {
 	bserv := bs.New(bstore, offline.Exchange(bstore))
 	dserv := mdag.NewDAGService(bserv)
 
-	p := NewPinner(dstore, dserv)
+	p := NewPinner(dstore, bstore, dserv)
 
 	a, _ := randNode()
 	b, _ := randNode()

test/sharness/t0080-repo.sh

@@ -219,6 +219,15 @@ test_expect_success "'ipfs refs --unique --recursive (bigger)'" '
 	test_sort_cmp expected actual || test_fsh cat refs_output
 '
 
+test_expect_success "'ipfs repo ls-roots' succeeds" '
+    ipfs repo ls-roots > ls-roots
+'
+
+test_expect_success "'ipfs repo ls-roots' looks good" '
+    grep "$HASH_WELCOME_DOCS" ls-roots &&
+    grep "$EMPTY_DIR" ls-roots
+'
+
 test_kill_ipfs_daemon
 
 test_done