diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 6491beb43c4..5b177eaf967 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -64,8 +64,13 @@ represents the whole file. res.SetError(err, cmds.ErrNormal) return } - ch, _ := fsutil.List(fs, quiet) - res.SetOutput(&chanWriter{ch, "", 0, false}) + if (quiet) { + ch, _ := fsutil.ListKeys(fs) + res.SetOutput(&chanWriter{ch, "", 0, false}) + } else { + ch, _ := fsutil.ListAll(fs) + res.SetOutput(&chanWriter{ch, "", 0, false}) + } }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { diff --git a/filestore/util/common.go b/filestore/util/common.go index 74a4b5d4d1d..55268271e90 100644 --- a/filestore/util/common.go +++ b/filestore/util/common.go @@ -107,17 +107,33 @@ func (r *ListRes) Format() string { } } -func List(d *Datastore, keysOnly bool) (<-chan ListRes, error) { +func ListKeys(d *Datastore) (<-chan ListRes, error) { qr, err := d.Query(query.Query{KeysOnly: true}) if err != nil { return nil, err } - bufSize := 128 - if keysOnly { - bufSize = 1024 + out := make(chan ListRes, 1024) + + go func() { + defer close(out) + for r := range qr.Next() { + if r.Error != nil { + return // FIXME + } + out <- ListRes{ds.NewKey(r.Key), nil, 0} + } + }() + return out, nil +} + +func List(d *Datastore, filter func(ListRes) bool) (<-chan ListRes, error) { + qr, err := d.Query(query.Query{KeysOnly: true}) + if err != nil { + return nil, err } - out := make(chan ListRes, bufSize) + + out := make(chan ListRes, 128) go func() { defer close(out) @@ -126,17 +142,21 @@ func List(d *Datastore, keysOnly bool) (<-chan ListRes, error) { return // FIXME } key := ds.NewKey(r.Key) - if keysOnly { - out <- ListRes{key, nil, 0} - } else { - val, _ := d.GetDirect(key) - out <- ListRes{key, val, 0} - } + val, _ := d.GetDirect(key) + out <- ListRes{key, val, 0} } }() return out, nil } +func ListAll(d *Datastore) (<-chan ListRes, error) { + return List(d, func(_ ListRes) bool { return true }) +} + +func ListWholeFile(d *Datastore) (<-chan ListRes, error) { + return List(d, func(r ListRes) bool { return r.WholeFile() }) +} + func verify(d *Datastore, key ds.Key, val *DataObj, level int) int { status := 0 _, err := d.GetData(key, val, level, true) diff --git a/filestore/util/misc.go b/filestore/util/misc.go index 76617bd196e..5cfb1a9ebc6 100644 --- a/filestore/util/misc.go +++ b/filestore/util/misc.go @@ -11,7 +11,7 @@ import ( ) func RmDups(wtr io.Writer, fs *Datastore, bs b.Blockstore) error { - ls, err := List(fs, true) + ls, err := ListKeys(fs) if err != nil {return err} for res := range ls { key := k.KeyFromDsKey(res.Key) diff --git a/filestore/util/pin.go b/filestore/util/pin.go index f63cadd4648..263b294a34a 100644 --- a/filestore/util/pin.go +++ b/filestore/util/pin.go @@ -144,7 +144,7 @@ func verifyRecPin(key bk.Key, good *[]bk.Key, fs *Datastore, bs b.Blockstore) (b } func Unpinned(n *core.IpfsNode, fs *Datastore, wtr io.Writer) error { - ls, err := List(fs, false) + ls, err := ListWholeFile(fs) if err != nil { return err } diff --git a/filestore/util/verify.go b/filestore/util/verify.go index 5152fee664a..4abf744b7d7 100644 --- a/filestore/util/verify.go +++ b/filestore/util/verify.go @@ -13,7 +13,7 @@ import ( ) func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error) { - in, err := List(fs, false) + in, err := List(fs, func(r ListRes) bool { return r.NoBlockData() }) if err != nil { return nil, err } @@ -25,9 +25,6 @@ func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error) go func() { defer close(out) for res := range in { - if !res.NoBlockData() { - continue - } res.Status = verify(fs, res.Key, res.DataObj, verifyWhat) if verbose >= 3 || OfInterest(res.Status) { out <- res @@ -39,7 +36,7 @@ func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error) func VerifyFull(node *core.IpfsNode, fs *Datastore, level int, verbose int, skipOrphans bool) (<-chan ListRes, error) { p := verifyParams{make(chan ListRes, 16), node, fs, level, verbose, skipOrphans, nil} - ch, err := List(p.fs, true) + ch, err := ListKeys(p.fs) if err != nil { return nil, err } @@ -85,10 +82,10 @@ func (p *verifyParams) verify(ch <-chan ListRes) { unsafeToCont := false for res := range ch { dagNode, dataObj, r := p.get(res.Key) - res.DataObj = dataObj if dataObj == nil { r = StatusError } + res.DataObj = dataObj if AnError(r) { /* nothing to do */ } else if res.FileRoot() {