Filestore: Refactor fsutil.List

License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
ipfs · May 13, 2016 · 283746b · 283746b
1 parent 07aec44
commit 283746b
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 21 deletions.
diff --git a/core/commands/filestore.go b/core/commands/filestore.go
@@ -64,8 +64,13 @@ represents the whole file.
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
-		ch, _ := fsutil.List(fs, quiet)
-		res.SetOutput(&chanWriter{ch, "", 0, false})
+		if (quiet) {
+			ch, _ := fsutil.ListKeys(fs)
+			res.SetOutput(&chanWriter{ch, "", 0, false})
+		} else {
+			ch, _ := fsutil.ListAll(fs)
+			res.SetOutput(&chanWriter{ch, "", 0, false})
+		}
 	},
 	Marshalers: cmds.MarshalerMap{
 		cmds.Text: func(res cmds.Response) (io.Reader, error) {

diff --git a/filestore/util/common.go b/filestore/util/common.go
@@ -107,17 +107,33 @@ func (r *ListRes) Format() string {
 	}
 }
 
-func List(d *Datastore, keysOnly bool) (<-chan ListRes, error) {
+func ListKeys(d *Datastore) (<-chan ListRes, error) {
 	qr, err := d.Query(query.Query{KeysOnly: true})
 	if err != nil {
 		return nil, err
 	}
 
-	bufSize := 128
-	if keysOnly {
-		bufSize = 1024
+	out := make(chan ListRes, 1024)
+
+	go func() {
+		defer close(out)
+		for r := range qr.Next() {
+			if r.Error != nil {
+				return // FIXME
+			}
+			out <- ListRes{ds.NewKey(r.Key), nil, 0}
+		}
+	}()
+	return out, nil
+}
+
+func List(d *Datastore, filter func(ListRes) bool) (<-chan ListRes, error) {
+	qr, err := d.Query(query.Query{KeysOnly: true})
+	if err != nil {
+		return nil, err
 	}
-	out := make(chan ListRes, bufSize)
+
+	out := make(chan ListRes, 128)
 
 	go func() {
 		defer close(out)
@@ -126,17 +142,21 @@ func List(d *Datastore, keysOnly bool) (<-chan ListRes, error) {
 				return // FIXME
 			}
 			key := ds.NewKey(r.Key)
-			if keysOnly {
-				out <- ListRes{key, nil, 0}
-			} else {
-				val, _ := d.GetDirect(key)
-				out <- ListRes{key, val, 0}
-			}
+			val, _ := d.GetDirect(key)
+			out <- ListRes{key, val, 0}
 		}
 	}()
 	return out, nil
 }
 
+func ListAll(d *Datastore) (<-chan ListRes, error) {
+	return List(d, func(_ ListRes) bool { return true })
+}
+
+func ListWholeFile(d *Datastore) (<-chan ListRes, error) {
+	return List(d, func(r ListRes) bool { return r.WholeFile() })
+}
+
 func verify(d *Datastore, key ds.Key, val *DataObj, level int) int {
 	status := 0
 	_, err := d.GetData(key, val, level, true)

diff --git a/filestore/util/misc.go b/filestore/util/misc.go
@@ -11,7 +11,7 @@ import (
 )
 
 func RmDups(wtr io.Writer, fs *Datastore, bs b.Blockstore) error {
-	ls, err := List(fs, true)
+	ls, err := ListKeys(fs)
 	if err != nil {return err}
 	for res := range ls {
 		key := k.KeyFromDsKey(res.Key)

diff --git a/filestore/util/pin.go b/filestore/util/pin.go
@@ -144,7 +144,7 @@ func verifyRecPin(key bk.Key, good *[]bk.Key, fs *Datastore, bs b.Blockstore) (b
 }
 
 func Unpinned(n *core.IpfsNode, fs *Datastore, wtr io.Writer) error {
-	ls, err := List(fs, false)
+	ls, err := ListWholeFile(fs)
 	if err != nil {
 		return err
 	}

diff --git a/filestore/util/verify.go b/filestore/util/verify.go
@@ -13,7 +13,7 @@ import (
 )
 
 func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error) {
-	in, err := List(fs, false)
+	in, err := List(fs, func(r ListRes) bool { return r.NoBlockData() })
 	if err != nil {
 		return nil, err
 	}
@@ -25,9 +25,6 @@ func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error)
 	go func() {
 		defer close(out)
 		for res := range in {
-			if !res.NoBlockData() {
-				continue
-			}
 			res.Status = verify(fs, res.Key, res.DataObj, verifyWhat)
 			if verbose >= 3 || OfInterest(res.Status) {
 				out <- res
@@ -39,7 +36,7 @@ func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error)
 
 func VerifyFull(node *core.IpfsNode, fs *Datastore, level int, verbose int, skipOrphans bool) (<-chan ListRes, error) {
 	p := verifyParams{make(chan ListRes, 16), node, fs, level, verbose, skipOrphans, nil}
-	ch, err := List(p.fs, true)
+	ch, err := ListKeys(p.fs)
 	if err != nil {
 		return nil, err
 	}
@@ -85,10 +82,10 @@ func (p *verifyParams) verify(ch <-chan ListRes) {
 	unsafeToCont := false
 	for res := range ch {
 		dagNode, dataObj, r := p.get(res.Key)
-		res.DataObj = dataObj
 		if dataObj == nil {
 			r = StatusError
 		}
+		res.DataObj = dataObj
 		if AnError(r) {
 			/* nothing to do */
 		} else if res.FileRoot() {