diff --git a/core/commands/filestore.go b/core/commands/filestore.go new file mode 100644 index 00000000000..029ba51ba3d --- /dev/null +++ b/core/commands/filestore.go @@ -0,0 +1,100 @@ +package commands + +import ( + "errors" + "io" + + cmds "github.com/ipfs/go-ipfs/commands" + "github.com/ipfs/go-ipfs/filestore" + "github.com/ipfs/go-ipfs/repo/fsrepo" +) + +type chanWriter struct { + ch <-chan *filestore.ListRes + buf string + offset int +} + +func (w *chanWriter) Read(p []byte) (int, error) { + if w.offset >= len(w.buf) { + w.offset = 0 + res, more := <-w.ch + if !more { + return 0, io.EOF + } + w.buf = res.Format() + } + sz := copy(p, w.buf[w.offset:]) + w.offset += sz + return sz, nil +} + +var FileStoreCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Interact with filestore objects", + }, + Subcommands: map[string]*cmds.Command{ + "ls": lsFileStore, + "verify": verifyFileStore, + }, +} + +var lsFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List objects on filestore", + }, + + Run: func(req cmds.Request, res cmds.Response) { + node, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + fsrepo, ok := node.Repo.Self().(*fsrepo.FSRepo) + if !ok { + res.SetError(errors.New("Not a FSRepo"), cmds.ErrNormal) + return + } + ch := make(chan *filestore.ListRes) + go func() { + defer close(ch) + filestore.List(fsrepo.Filestore(), ch) + }() + res.SetOutput(&chanWriter{ch, "", 0}) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + return res.(io.Reader), nil + }, + }, +} + +var verifyFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Verify objects in filestore", + }, + + Run: func(req cmds.Request, res cmds.Response) { + node, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + fsrepo, ok := node.Repo.Self().(*fsrepo.FSRepo) + if !ok { + res.SetError(errors.New("Not a FSRepo"), cmds.ErrNormal) + return + } + ch := make(chan *filestore.ListRes) + go func() { + defer close(ch) + filestore.Verify(fsrepo.Filestore(), ch) + }() + res.SetOutput(&chanWriter{ch, "", 0}) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + return res.(io.Reader), nil + }, + }, +} diff --git a/core/commands/root.go b/core/commands/root.go index 00cea12c083..deaa5bc6781 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -123,6 +123,7 @@ var rootSubcommands = map[string]*cmds.Command{ "update": ExternalBinary(), "version": VersionCmd, "bitswap": BitswapCmd, + "filestore": FileStoreCmd, } // RootRO is the readonly version of Root diff --git a/filestore/dataobj.go b/filestore/dataobj.go index aa00b9b073d..e9c485b6d2c 100644 --- a/filestore/dataobj.go +++ b/filestore/dataobj.go @@ -1,6 +1,7 @@ package filestore import ( + "fmt" pb "github.com/ipfs/go-ipfs/filestore/pb" ) @@ -34,6 +35,23 @@ type DataObj struct { Data []byte } +func (d *DataObj) StripData() DataObj { + return DataObj{ + d.NoBlockData, d.WholeFile, d.FileRoot, + d.FilePath, d.Offset, d.Size, nil, + } +} + +func (d *DataObj) Format() string { + if d.NoBlockData { + return fmt.Sprintf("block %s %d %d", d.FilePath, d.Offset, d.Size) + } else if d.FileRoot { + return fmt.Sprintf("root %s %d %d", d.FilePath, d.Offset, d.Size) + } else { + return fmt.Sprintf("other %s %d %d", d.FilePath, d.Offset, d.Size) + } +} + func (d *DataObj) Marshal() ([]byte, error) { pd := new(pb.DataObj) diff --git a/filestore/datastore.go b/filestore/datastore.go index ee88f4781c9..53fff28475d 100644 --- a/filestore/datastore.go +++ b/filestore/datastore.go @@ -91,9 +91,17 @@ func (d *Datastore) decode(dataObj interface{}) (*DataObj, error) { return val, nil } +type InvalidBlock struct{} + +func (e InvalidBlock) Error() string { + return "Datastore: Block Verification Failed" +} + // Get the orignal data out of the DataObj func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, error) { - if val.NoBlockData { + if val == nil { + return nil, errors.New("Nil DataObj") + } else if val.NoBlockData { file, err := os.Open(val.FilePath) if err != nil { return nil, err @@ -114,7 +122,7 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, erro if verify { newKey := k.Key(u.Hash(data)).DsKey() if newKey != key { - return nil, errors.New("Datastore: Block Verification Failed") + return nil, InvalidBlock{} } } return data, nil diff --git a/filestore/util.go b/filestore/util.go new file mode 100644 index 00000000000..cd4db3b406d --- /dev/null +++ b/filestore/util.go @@ -0,0 +1,79 @@ +package filestore + +import ( + "fmt" + "io" + "os" + + ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query" + b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" +) + +const ( + StatusOk = 1 + StatusMissing = 2 + StatusInvalid = 3 + StatusError = 4 +) + +func statusStr(status int) string { + switch status { + case 0: + return "" + case 1: + return "ok " + case 2: + return "missing " + case 3: + return "invalid " + case 4: + return "error " + default: + return "?? " + } +} + +type ListRes struct { + Key []byte + DataObj + Status int +} + +func (r *ListRes) Format() string { + mhash := b58.Encode(r.Key) + return fmt.Sprintf("%s%s %s\n", statusStr(r.Status), mhash, r.DataObj.Format()) +} + +func list(d *Datastore, out chan<- *ListRes, verify bool) error { + qr, err := d.Query(query.Query{KeysOnly: true}) + if err != nil { + return err + } + for r := range qr.Next() { + if r.Error != nil { + return r.Error + } + key := ds.NewKey(r.Key) + val, _ := d.GetDirect(key) + status := 0 + if verify { + _, err := d.GetData(key, val, true) + if err == nil { + status = StatusOk + } else if os.IsNotExist(err) { + status = StatusMissing + } else if _, ok := err.(InvalidBlock); ok || err == io.EOF || err == io.ErrUnexpectedEOF { + status = StatusInvalid + } else { + status = StatusError + } + } + out <- &ListRes{key.Bytes()[1:], val.StripData(), status} + } + return nil +} + +func List(d *Datastore, out chan<- *ListRes) error { return list(d, out, false) } + +func Verify(d *Datastore, out chan<- *ListRes) error { return list(d, out, true) }