diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go index 77062e9e275..671ae2c2541 100644 --- a/blocks/blockstore/blockstore.go +++ b/blocks/blockstore/blockstore.go @@ -12,7 +12,6 @@ import ( dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query" blocks "github.com/ipfs/go-ipfs/blocks" key "github.com/ipfs/go-ipfs/blocks/key" - "github.com/ipfs/go-ipfs/filestore" mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" logging "gx/ipfs/QmaDNZ4QMdBdku1YZWBysufYyoQt1negQGNav6PLYarbY8/go-log" @@ -93,11 +92,12 @@ func (bs *blockstore) Get(k key.Key) (blocks.Block, error) { func (bs *blockstore) Put(block blocks.Block) error { k := block.Key().DsKey() - data := bs.prepareBlock(k, block) - if data == nil { - return nil + // Has is cheaper than Put, so see if we already have it + exists, err := bs.datastore.Has(k) + if err == nil && exists { + return nil // already stored. } - return bs.datastore.Put(k, data) + return bs.datastore.Put(k, block.Data()) } func (bs *blockstore) PutMany(blocks []blocks.Block) error { @@ -107,11 +107,12 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error { } for _, b := range blocks { k := b.Key().DsKey() - data := bs.prepareBlock(k, b) - if data == nil { + exists, err := bs.datastore.Has(k) + if err == nil && exists { continue } - err = t.Put(k, data) + + err = t.Put(k, b.Data()) if err != nil { return err } @@ -119,33 +120,6 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error { return t.Commit() } -func (bs *blockstore) prepareBlock(k ds.Key, block blocks.Block) interface{} { - if fsBlock, ok := block.(*blocks.FilestoreBlock); !ok { - // Has is cheaper than Put, so see if we already have it - exists, err := bs.datastore.Has(k) - if err == nil && exists { - return nil // already stored. - } - return block.Data() - } else { - d := &filestore.DataObj{ - FilePath: fsBlock.FilePath, - Offset: fsBlock.Offset, - Size: fsBlock.Size, - ModTime: filestore.FromTime(fsBlock.ModTime), - } - if fsBlock.AltData == nil { - d.Flags |= filestore.WholeFile | filestore.FileRoot - d.Data = block.Data() - } else { - d.Flags |= filestore.NoBlockData - d.Data = fsBlock.AltData - } - return &filestore.DataWOpts{d, fsBlock.AddOpts} - } - -} - func (bs *blockstore) Has(k key.Key) (bool, error) { return bs.datastore.Has(k.DsKey()) } diff --git a/blocks/blockstore/write_cache.go b/blocks/blockstore/write_cache.go index cbe61755378..f7c2caf4567 100644 --- a/blocks/blockstore/write_cache.go +++ b/blocks/blockstore/write_cache.go @@ -39,31 +39,23 @@ func (w *writecache) Get(k key.Key) (blocks.Block, error) { } func (w *writecache) Put(b blocks.Block) error { - // Don't cache "advance" blocks - if _, ok := b.(*blocks.BasicBlock); ok { - k := b.Key() - if _, ok := w.cache.Get(k); ok { - return nil - } - defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done() - - w.cache.Add(b.Key(), struct{}{}) + k := b.Key() + if _, ok := w.cache.Get(k); ok { + return nil } + defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done() + + w.cache.Add(b.Key(), struct{}{}) return w.blockstore.Put(b) } func (w *writecache) PutMany(bs []blocks.Block) error { var good []blocks.Block for _, b := range bs { - // Don't cache "advance" blocks - if _, ok := b.(*blocks.BasicBlock); ok { - if _, ok := w.cache.Get(b.Key()); !ok { - good = append(good, b) - k := b.Key() - defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done() - } - } else { + if _, ok := w.cache.Get(b.Key()); !ok { good = append(good, b) + k := b.Key() + defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done() } } return w.blockstore.PutMany(good) diff --git a/core/commands/add.go b/core/commands/add.go index 30cdce2a471..55f33ef673e 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -9,11 +9,14 @@ import ( "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" "github.com/ipfs/go-ipfs/core/coreunix" "github.com/ipfs/go-ipfs/filestore" + "github.com/ipfs/go-ipfs/filestore/support" + bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" cli "github.com/ipfs/go-ipfs/commands/cli" files "github.com/ipfs/go-ipfs/commands/files" core "github.com/ipfs/go-ipfs/core" + dag "github.com/ipfs/go-ipfs/merkledag" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) @@ -160,7 +163,14 @@ You can now refer to the added file in a gateway, like so: outChan := make(chan interface{}, 8) res.SetOutput((<-chan interface{})(outChan)) - fileAdder, err := coreunix.NewAdder(n.DataServices(req.Context()), outChan) + ds := n.DataServices(req.Context()) + if nocopy || link { + ds.Blockstore = filestore_support.NewBlockstore(ds.Blockstore, n.Repo.Datastore()) + blockService := bserv.New(ds.Blockstore, n.Exchange) + ds.DAG = dag.NewDAGService(blockService) + } + + fileAdder, err := coreunix.NewAdder(ds, outChan) if err != nil { res.SetError(err, cmds.ErrNormal) return diff --git a/filestore/support/blockstore.go b/filestore/support/blockstore.go new file mode 100644 index 00000000000..7a293f37318 --- /dev/null +++ b/filestore/support/blockstore.go @@ -0,0 +1,80 @@ +// package blockstore implements a thin wrapper over a datastore, giving a +// clean interface for Getting and Putting block objects. +package filestore_support + +import ( + ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore" + dsns "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/namespace" + blocks "github.com/ipfs/go-ipfs/blocks" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" + fs "github.com/ipfs/go-ipfs/filestore" +) + +type blockstore struct { + bs.GCBlockstore + //filestore fs.Datastore + datastore ds.Batching +} + +func NewBlockstore(b bs.GCBlockstore, d ds.Batching) bs.GCBlockstore { + return &blockstore{b, dsns.Wrap(d, bs.BlockPrefix)} +} + +func (bs *blockstore) Put(block blocks.Block) error { + k := block.Key().DsKey() + println("putting...") + + data := bs.prepareBlock(k, block) + if data == nil { + return nil + } + return bs.datastore.Put(k, data) +} + +func (bs *blockstore) PutMany(blocks []blocks.Block) error { + println("put many...") + t, err := bs.datastore.Batch() + if err != nil { + return err + } + for _, b := range blocks { + k := b.Key().DsKey() + data := bs.prepareBlock(k, b) + if data == nil { + continue + } + err = t.Put(k, data) + if err != nil { + return err + } + } + return t.Commit() +} + +func (bs *blockstore) prepareBlock(k ds.Key, block blocks.Block) interface{} { + if fsBlock, ok := block.(*blocks.FilestoreBlock); !ok { + // Has is cheaper than Put, so see if we already have it + exists, err := bs.datastore.Has(k) + if err == nil && exists { + return nil // already stored. + } + return block.Data() + } else { + println("DataObj") + d := &fs.DataObj{ + FilePath: fsBlock.FilePath, + Offset: fsBlock.Offset, + Size: fsBlock.Size, + ModTime: fs.FromTime(fsBlock.ModTime), + } + if fsBlock.AltData == nil { + d.Flags |= fs.WholeFile | fs.FileRoot + d.Data = block.Data() + } else { + d.Flags |= fs.NoBlockData + d.Data = fsBlock.AltData + } + return &fs.DataWOpts{d, fsBlock.AddOpts} + } + +}