In merkledag.Node and blocks.Block maintain a DataPtr

The DataPtr points to the location of the data within a file on the
file system.  It the node is a leaf it also contains an alternative
serialization of the Node or Block that does not contain the data.

Required for #875.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

blocks/blocks.go

@@ -15,6 +15,15 @@ import (
 type Block struct {
 	Multihash mh.Multihash
 	Data      []byte
+	DataPtr   *DataPtr
+}
+
+// This DataPtr had different AltData than the node DataPtr
+type DataPtr struct {
+	AltData  []byte
+	FilePath string
+	Offset   uint64
+	Size     uint64
 }
 
 // NewBlock creates a Block object from opaque data. It will hash the data.

importer/balanced/builder.go

@@ -2,6 +2,7 @@ package balanced
 
 import (
 	"errors"
+	//"fmt"
 
 	h "github.com/ipfs/go-ipfs/importer/helpers"
 	dag "github.com/ipfs/go-ipfs/merkledag"
@@ -31,6 +32,7 @@ func BalancedLayout(db *h.DagBuilderHelper) (*dag.Node, error) {
 		root = h.NewUnixfsNode()
 	}
 
+	db.SetAsRoot(root)
 	out, err := db.Add(root)
 	if err != nil {
 		return nil, err

importer/helpers/dagbuilder.go

@@ -100,7 +100,7 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error {
 }
 
 func (db *DagBuilderHelper) FillNodeWithData(node *UnixfsNode) error {
-	data, _ /*offset*/ := db.Next()
+	data, offset := db.Next()
 	if data == nil { // we're done!
 		return nil
 	}
@@ -110,9 +110,20 @@ func (db *DagBuilderHelper) FillNodeWithData(node *UnixfsNode) error {
 	}
 
 	node.SetData(data)
+	if db.absPath != "" {
+		node.SetDataPtr(db.absPath, offset)
+	}
+
 	return nil
 }
 
+func (db *DagBuilderHelper) SetAsRoot(node *UnixfsNode) {
+	//fmt.Println("SetAsRoot!")
+	if db.absPath != "" {
+		node.SetDataPtr(db.absPath, 0)
+	}
+}
+
 func (db *DagBuilderHelper) Add(node *UnixfsNode) (*dag.Node, error) {
 	dn, err := node.GetDagNode()
 	if err != nil {

importer/helpers/helpers.go

@@ -2,6 +2,7 @@ package helpers
 
 import (
 	"fmt"
+	//"runtime/debug"
 
 	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
@@ -37,8 +38,10 @@ var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
 // UnixfsNode is a struct created to aid in the generation
 // of unixfs DAG trees
 type UnixfsNode struct {
-	node *dag.Node
-	ufmt *ft.FSNode
+	node     *dag.Node
+	ufmt     *ft.FSNode
+	filePath string
+	offset   int64
 }
 
 // NewUnixfsNode creates a new Unixfs node to represent a file
@@ -118,14 +121,34 @@ func (n *UnixfsNode) RemoveChild(index int, dbh *DagBuilderHelper) {
 func (n *UnixfsNode) SetData(data []byte) {
 	n.ufmt.Data = data
 }
+func (n *UnixfsNode) SetDataPtr(filePath string, offset int64) {
+	//fmt.Println("SetDataPtr: ", filePath, offset)
+	//debug.PrintStack()
+	n.filePath = filePath
+	n.offset = offset
+}
 
 // getDagNode fills out the proper formatting for the unixfs node
 // inside of a DAG node and returns the dag node
 func (n *UnixfsNode) GetDagNode() (*dag.Node, error) {
+	//fmt.Println("GetDagNode")
 	data, err := n.ufmt.GetBytes()
 	if err != nil {
 		return nil, err
 	}
 	n.node.Data = data
+	if n.filePath != "" {
+		if n.ufmt.NumChildren() == 0 && (n.ufmt.Type == ft.TFile || n.ufmt.Type == ft.TRaw) {
+			//fmt.Println("We have a block.")
+			// We have a block
+			d, _ := n.ufmt.GetBytesNoData()
+			n.node.DataPtr = &dag.DataPtr{d, n.filePath, uint64(n.offset), uint64(len(n.ufmt.Data))}
+		} else if n.ufmt.Type == ft.TFile {
+			//fmt.Println("We have a root.")
+			// We have a root
+			n.node.DataPtr = &dag.DataPtr{nil, n.filePath, 0, n.ufmt.FileSize()}
+		}
+	}
+
 	return n.node, nil
 }

importer/trickle/trickledag.go

@@ -8,6 +8,8 @@ import (
 	h "github.com/ipfs/go-ipfs/importer/helpers"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
+
+	//ds2 "github.com/ipfs/go-ipfs/datastore"
 )
 
 // layerRepeat specifies how many times to append a child tree of a
@@ -32,6 +34,7 @@ func TrickleLayout(db *h.DagBuilderHelper) (*dag.Node, error) {
 		}
 	}
 
+	db.SetAsRoot(root)
 	out, err := db.Add(root)
 	if err != nil {
 		return nil, err

merkledag/coding.go

@@ -6,6 +6,7 @@ import (
 
 	mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
 
+	blocks "github.com/ipfs/go-ipfs/blocks"
 	pb "github.com/ipfs/go-ipfs/merkledag/pb"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 )
@@ -40,15 +41,24 @@ func (n *Node) unmarshal(encoded []byte) error {
 // Marshal encodes a *Node instance into a new byte slice.
 // The conversion uses an intermediate PBNode.
 func (n *Node) Marshal() ([]byte, error) {
-	pbn := n.getPBNode()
+	pbn := n.getPBNode(true)
 	data, err := pbn.Marshal()
 	if err != nil {
 		return data, fmt.Errorf("Marshal failed. %v", err)
 	}
 	return data, nil
 }
 
-func (n *Node) getPBNode() *pb.PBNode {
+func (n *Node) MarshalNoData() ([]byte, error) {
+	pbn := n.getPBNode(false)
+	data, err := pbn.Marshal()
+	if err != nil {
+		return data, fmt.Errorf("Marshal failed. %v", err)
+	}
+	return data, nil
+}
+
+func (n *Node) getPBNode(useData bool) *pb.PBNode {
 	pbn := &pb.PBNode{}
 	if len(n.Links) > 0 {
 		pbn.Links = make([]*pb.PBLink, len(n.Links))
@@ -62,8 +72,16 @@ func (n *Node) getPBNode() *pb.PBNode {
 		pbn.Links[i].Hash = []byte(l.Hash)
 	}
 
-	if len(n.Data) > 0 {
-		pbn.Data = n.Data
+	if useData {
+		if len(n.Data) > 0 {
+			pbn.Data = n.Data
+		}
+	} else {
+		if n.DataPtr != nil && len(n.DataPtr.AltData) > 0 {
+			pbn.Data = n.DataPtr.AltData
+		} else if len(n.Data) > 0 {
+			pbn.Data = n.Data
+		}
 	}
 	return pbn
 }
@@ -84,6 +102,27 @@ func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
 	return n.encoded, nil
 }
 
+// Converts the node DataPtr to a block DataPtr, must be called after
+// EncodeProtobuf
+func (n *Node) EncodeDataPtr() (*blocks.DataPtr, error) {
+	if n.DataPtr == nil {
+		return nil, nil
+	}
+	bl := &blocks.DataPtr{
+		FilePath: n.DataPtr.FilePath,
+		Offset:   n.DataPtr.Offset,
+		Size:     n.DataPtr.Size}
+	if n.DataPtr.AltData == nil {
+		return bl, nil
+	}
+	d, err := n.MarshalNoData()
+	if err != nil {
+		return nil, err
+	}
+	bl.AltData = d
+	return bl, nil
+}
+
 // Decoded decodes raw data and returns a new Node instance.
 func DecodeProtobuf(encoded []byte) (*Node, error) {
 	n := new(Node)

merkledag/merkledag.go

@@ -65,6 +65,10 @@ func (n *dagService) AddWOpts(nd *Node, addOpts interface{}) (key.Key, error) {
 	if err != nil {
 		return "", err
 	}
+	b.DataPtr, err = nd.EncodeDataPtr()
+	if err != nil {
+		return "", err
+	}
 
 	return n.Blocks.AddBlock(b, addOpts)
 }
@@ -341,6 +345,10 @@ func (t *Batch) Add(nd *Node) (key.Key, error) {
 	if err != nil {
 		return "", err
 	}
+	b.DataPtr, err = nd.EncodeDataPtr()
+	if err != nil {
+		return "", err
+	}
 
 	k := key.Key(b.Multihash)
 

merkledag/node.go

@@ -21,6 +21,15 @@ type Node struct {
 	encoded []byte
 
 	cached mh.Multihash
+
+	DataPtr *DataPtr
+}
+
+type DataPtr struct {
+	AltData  []byte
+	FilePath string
+	Offset   uint64
+	Size     uint64
 }
 
 // NodeStat is a statistics object for a Node. Mostly sizes.

unixfs/format.go

@@ -160,15 +160,25 @@ func (n *FSNode) RemoveBlockSize(i int) {
 	n.blocksizes = append(n.blocksizes[:i], n.blocksizes[i+1:]...)
 }
 
-func (n *FSNode) GetBytes() ([]byte, error) {
+func (n *FSNode) newPB() *pb.Data {
 	pbn := new(pb.Data)
 	pbn.Type = &n.Type
 	pbn.Filesize = proto.Uint64(uint64(len(n.Data)) + n.subtotal)
 	pbn.Blocksizes = n.blocksizes
+	return pbn
+}
+
+func (n *FSNode) GetBytes() ([]byte, error) {
+	pbn := n.newPB()
 	pbn.Data = n.Data
 	return proto.Marshal(pbn)
 }
 
+func (n *FSNode) GetBytesNoData() ([]byte, error) {
+	pbn := n.newPB()
+	return proto.Marshal(pbn)
+}
+
 func (n *FSNode) FileSize() uint64 {
 	return uint64(len(n.Data)) + n.subtotal
 }