flushing and shallow list names

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

core/commands/files/files.go

@@ -41,6 +41,7 @@ Files is an API for manipulating ipfs objects as if they were a unix filesystem.
 		"mkdir": FilesMkdirCmd,
 		"stat":  FilesStatCmd,
 		"rm":    FilesRmCmd,
+		"flush": FilesFlushCmd,
 	},
 }
 
@@ -244,17 +245,10 @@ Examples:
 		switch fsn := fsn.(type) {
 		case *mfs.Directory:
 			if !long {
-				mdnd, err := fsn.GetNode()
-				if err != nil {
-					res.SetError(err, cmds.ErrNormal)
-					return
-				}
-
 				var output []mfs.NodeListing
-				for _, lnk := range mdnd.Links {
+				for _, name := range fsn.ListNames() {
 					output = append(output, mfs.NodeListing{
-						Name: lnk.Name,
-						Hash: lnk.Hash.B58String(),
+						Name: name,
 					})
 				}
 				res.SetOutput(&FilesLsOutput{output})
@@ -501,7 +495,14 @@ Warning:
 		}
 
 		if flush {
-			defer fi.Close()
+			defer func() {
+				fi.Close()
+				err := mfs.FlushPath(nd.FilesRoot, path)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+			}()
 		} else {
 			defer fi.Sync()
 		}
@@ -600,6 +601,40 @@ Examples:
 	},
 }
 
+var FilesFlushCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "flush a given path's data to disk",
+		ShortDescription: `
+flush a given path to disk. This is only useful when other commands
+are run with the '--flush=false'.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("path", false, false, "path to flush (default '/')"),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		nd, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		// take the lock and defer the unlock
+		defer nd.Blockstore.PinLock()()
+
+		path := "/"
+		if len(req.Arguments()) > 0 {
+			path = req.Arguments()[0]
+		}
+
+		err = mfs.FlushPath(nd.FilesRoot, path)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+	},
+}
+
 var FilesRmCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "remove a file",

core/core.go

@@ -474,6 +474,20 @@ func (n *IpfsNode) loadBootstrapPeers() ([]peer.PeerInfo, error) {
 func (n *IpfsNode) loadFilesRoot() error {
 	dsk := ds.NewKey("/local/filesroot")
 	pf := func(ctx context.Context, k key.Key) error {
+		ds := n.Repo.Datastore()
+		if old, err := ds.Get(dsk); err == nil {
+			_ = n.Pinning.Unpin(n.Context(), key.Key(old.([]byte)), true)
+		}
+		nnd, err := n.DAG.Get(n.Context(), k)
+		if err != nil {
+			return err
+		}
+
+		err = n.Pinning.Pin(n.Context(), nnd, true)
+		if err != nil {
+			return err
+		}
+
 		return n.Repo.Datastore().Put(dsk, []byte(k))
 	}
 

mfs/dir.go

@@ -175,6 +175,30 @@ type NodeListing struct {
 	Hash string
 }
 
+func (d *Directory) ListNames() []string {
+	d.Lock()
+	defer d.Unlock()
+
+	names := make(map[string]struct{})
+	for n, _ := range d.childDirs {
+		names[n] = struct{}{}
+	}
+	for n, _ := range d.files {
+		names[n] = struct{}{}
+	}
+
+	for _, l := range d.node.Links {
+		names[l.Name] = struct{}{}
+	}
+
+	var out []string
+	for n, _ := range names {
+		out = append(out, n)
+	}
+
+	return out
+}
+
 func (d *Directory) List() ([]NodeListing, error) {
 	d.lock.Lock()
 	defer d.lock.Unlock()

mfs/mfs_test.go

@@ -675,3 +675,53 @@ func TestMfsStress(t *testing.T) {
 		}
 	}
 }
+
+func TestFlushing(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	_, rt := setupRoot(ctx, t)
+
+	dir := rt.GetValue().(*Directory)
+	c := mkdirP(t, dir, "a/b/c")
+	d := mkdirP(t, dir, "a/b/d")
+	e := mkdirP(t, dir, "a/b/e")
+
+	data := []byte("this is a test\n")
+	nd1 := &dag.Node{Data: ft.FilePBData(data, uint64(len(data)))}
+
+	if err := c.AddChild("TEST", nd1); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.AddChild("TEST", nd1); err != nil {
+		t.Fatal(err)
+	}
+	if err := e.AddChild("TEST", nd1); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := FlushPath(rt, "/a/b/c/TEST"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := FlushPath(rt, "/a/b/d/TEST"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := FlushPath(rt, "/a/b/e/TEST"); err != nil {
+		t.Fatal(err)
+	}
+
+	rnd, err := dir.GetNode()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rnk, err := rnd.Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if rnk.B58String() != "QmWcvrHUFk7LQRrA4WqKjqy7ZyRGFLVagtgNxbEodTEzQ4" {
+		t.Fatal("dag looks wrong")
+	}
+}

mfs/ops.go

@@ -194,3 +194,64 @@ func DirLookup(d *Directory, pth string) (FSNode, error) {
 	}
 	return cur, nil
 }
+
+func FlushPath(r *Root, pth string) error {
+	parts := path.SplitList(strings.Trim(pth, "/"))
+
+	d, ok := r.GetValue().(*Directory)
+	if !ok {
+		return errors.New("mfs root somehow didnt point to a directory")
+	}
+
+	nd, err := flushPathRec(d, parts)
+	if err != nil {
+		return err
+	}
+
+	k, err := nd.Key()
+	if err != nil {
+		return err
+	}
+
+	r.repub.Update(k)
+	return nil
+}
+
+func flushPathRec(d *Directory, parts []string) (*dag.Node, error) {
+	if len(parts) == 0 {
+		return d.GetNode()
+	}
+
+	d.Lock()
+	defer d.Unlock()
+
+	next, err := d.childUnsync(parts[0])
+	if err != nil {
+		log.Errorf("childnode: %q %q", parts[0], err)
+		return nil, err
+	}
+
+	switch next := next.(type) {
+	case *Directory:
+		nd, err := flushPathRec(next, parts[1:])
+		if err != nil {
+			return nil, err
+		}
+
+		newnode, err := d.node.UpdateNodeLink(parts[0], nd)
+		if err != nil {
+			return nil, err
+		}
+
+		d.node = newnode
+		return newnode, nil
+	case *File:
+		if len(parts) > 1 {
+			return nil, fmt.Errorf("%s is a file, not a directory", parts[0])
+		}
+
+		return next.GetNode()
+	default:
+		return nil, fmt.Errorf("unrecognized FSNode type: %#v", next)
+	}
+}

test/sharness/t0250-files-api.sh

@@ -331,10 +331,16 @@ test_files_api() {
 	'
 
 	test_expect_success "root hash looks good" '
-		echo "QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt" > root_hash_exp &&
+		export EXP_ROOT_HASH="QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt" &&
+		echo $EXP_ROOT_HASH > root_hash_exp &&
 		test_cmp root_hash_exp root_hash
 	'
 
+	test_expect_success "root hash is pinned" '
+		ipfs pin ls
+		return 1
+	'
+
 	# test mv
 	test_expect_success "can mv dir" '
 		ipfs files mv /cats/this/is /cats/