Skip to content

Commit

Permalink
Create an alternative blockstore for use with the filestore.
Browse files Browse the repository at this point in the history
Restore blockstore, write_cache to original unmodified versions.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>
  • Loading branch information
kevina committed May 23, 2016
1 parent f7fcefd commit e6bfbfa
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 53 deletions.
44 changes: 9 additions & 35 deletions blocks/blockstore/blockstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
blocks "github.com/ipfs/go-ipfs/blocks"
key "github.com/ipfs/go-ipfs/blocks/key"
"github.com/ipfs/go-ipfs/filestore"
mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
logging "gx/ipfs/QmaDNZ4QMdBdku1YZWBysufYyoQt1negQGNav6PLYarbY8/go-log"
Expand Down Expand Up @@ -93,11 +92,12 @@ func (bs *blockstore) Get(k key.Key) (blocks.Block, error) {
func (bs *blockstore) Put(block blocks.Block) error {
k := block.Key().DsKey()

data := bs.prepareBlock(k, block)
if data == nil {
return nil
// Has is cheaper than Put, so see if we already have it
exists, err := bs.datastore.Has(k)
if err == nil && exists {
return nil // already stored.
}
return bs.datastore.Put(k, data)
return bs.datastore.Put(k, block.Data())
}

func (bs *blockstore) PutMany(blocks []blocks.Block) error {
Expand All @@ -107,45 +107,19 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
}
for _, b := range blocks {
k := b.Key().DsKey()
data := bs.prepareBlock(k, b)
if data == nil {
exists, err := bs.datastore.Has(k)
if err == nil && exists {
continue
}
err = t.Put(k, data)

err = t.Put(k, b.Data())
if err != nil {
return err
}
}
return t.Commit()
}

func (bs *blockstore) prepareBlock(k ds.Key, block blocks.Block) interface{} {
if fsBlock, ok := block.(*blocks.FilestoreBlock); !ok {
// Has is cheaper than Put, so see if we already have it
exists, err := bs.datastore.Has(k)
if err == nil && exists {
return nil // already stored.
}
return block.Data()
} else {
d := &filestore.DataObj{
FilePath: fsBlock.FilePath,
Offset: fsBlock.Offset,
Size: fsBlock.Size,
ModTime: filestore.FromTime(fsBlock.ModTime),
}
if fsBlock.AltData == nil {
d.Flags |= filestore.WholeFile | filestore.FileRoot
d.Data = block.Data()
} else {
d.Flags |= filestore.NoBlockData
d.Data = fsBlock.AltData
}
return &filestore.DataWOpts{d, fsBlock.AddOpts}
}

}

func (bs *blockstore) Has(k key.Key) (bool, error) {
return bs.datastore.Has(k.DsKey())
}
Expand Down
26 changes: 9 additions & 17 deletions blocks/blockstore/write_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,31 +39,23 @@ func (w *writecache) Get(k key.Key) (blocks.Block, error) {
}

func (w *writecache) Put(b blocks.Block) error {
// Don't cache "advance" blocks
if _, ok := b.(*blocks.BasicBlock); ok {
k := b.Key()
if _, ok := w.cache.Get(k); ok {
return nil
}
defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done()

w.cache.Add(b.Key(), struct{}{})
k := b.Key()
if _, ok := w.cache.Get(k); ok {
return nil
}
defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done()

w.cache.Add(b.Key(), struct{}{})
return w.blockstore.Put(b)
}

func (w *writecache) PutMany(bs []blocks.Block) error {
var good []blocks.Block
for _, b := range bs {
// Don't cache "advance" blocks
if _, ok := b.(*blocks.BasicBlock); ok {
if _, ok := w.cache.Get(b.Key()); !ok {
good = append(good, b)
k := b.Key()
defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done()
}
} else {
if _, ok := w.cache.Get(b.Key()); !ok {
good = append(good, b)
k := b.Key()
defer log.EventBegin(context.TODO(), "writecache.BlockAdded", &k).Done()
}
}
return w.blockstore.PutMany(good)
Expand Down
12 changes: 11 additions & 1 deletion core/commands/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@ import (
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
"github.com/ipfs/go-ipfs/core/coreunix"
"github.com/ipfs/go-ipfs/filestore"
"github.com/ipfs/go-ipfs/filestore/support"

bserv "github.com/ipfs/go-ipfs/blockservice"
cmds "github.com/ipfs/go-ipfs/commands"
cli "github.com/ipfs/go-ipfs/commands/cli"
files "github.com/ipfs/go-ipfs/commands/files"
core "github.com/ipfs/go-ipfs/core"
dag "github.com/ipfs/go-ipfs/merkledag"
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

Expand Down Expand Up @@ -160,7 +163,14 @@ You can now refer to the added file in a gateway, like so:
outChan := make(chan interface{}, 8)
res.SetOutput((<-chan interface{})(outChan))

fileAdder, err := coreunix.NewAdder(n.DataServices(req.Context()), outChan)
ds := n.DataServices(req.Context())
if nocopy || link {
ds.Blockstore = filestore_support.NewBlockstore(ds.Blockstore, n.Repo.Datastore())
blockService := bserv.New(ds.Blockstore, n.Exchange)
ds.DAG = dag.NewDAGService(blockService)
}

fileAdder, err := coreunix.NewAdder(ds, outChan)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
Expand Down
80 changes: 80 additions & 0 deletions filestore/support/blockstore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// package blockstore implements a thin wrapper over a datastore, giving a
// clean interface for Getting and Putting block objects.
package filestore_support

import (
ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
dsns "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/namespace"
blocks "github.com/ipfs/go-ipfs/blocks"
bs "github.com/ipfs/go-ipfs/blocks/blockstore"
fs "github.com/ipfs/go-ipfs/filestore"
)

type blockstore struct {
bs.GCBlockstore
//filestore fs.Datastore
datastore ds.Batching
}

func NewBlockstore(b bs.GCBlockstore, d ds.Batching) bs.GCBlockstore {
return &blockstore{b, dsns.Wrap(d, bs.BlockPrefix)}
}

func (bs *blockstore) Put(block blocks.Block) error {
k := block.Key().DsKey()
println("putting...")

data := bs.prepareBlock(k, block)
if data == nil {
return nil
}
return bs.datastore.Put(k, data)
}

func (bs *blockstore) PutMany(blocks []blocks.Block) error {
println("put many...")
t, err := bs.datastore.Batch()
if err != nil {
return err
}
for _, b := range blocks {
k := b.Key().DsKey()
data := bs.prepareBlock(k, b)
if data == nil {
continue
}
err = t.Put(k, data)
if err != nil {
return err
}
}
return t.Commit()
}

func (bs *blockstore) prepareBlock(k ds.Key, block blocks.Block) interface{} {
if fsBlock, ok := block.(*blocks.FilestoreBlock); !ok {
// Has is cheaper than Put, so see if we already have it
exists, err := bs.datastore.Has(k)
if err == nil && exists {
return nil // already stored.
}
return block.Data()
} else {
println("DataObj")
d := &fs.DataObj{
FilePath: fsBlock.FilePath,
Offset: fsBlock.Offset,
Size: fsBlock.Size,
ModTime: fs.FromTime(fsBlock.ModTime),
}
if fsBlock.AltData == nil {
d.Flags |= fs.WholeFile | fs.FileRoot
d.Data = block.Data()
} else {
d.Flags |= fs.NoBlockData
d.Data = fsBlock.AltData
}
return &fs.DataWOpts{d, fsBlock.AddOpts}
}

}

0 comments on commit e6bfbfa

Please sign in to comment.