Merge pull request #3307 from ipfs/feat/raw-nodes

Implement 'Raw Node' node type

blocks/blocks.go

@@ -37,8 +37,11 @@ func NewBlock(data []byte) *BasicBlock {
 // we are able to be confident that the data is correct
 func NewBlockWithCid(data []byte, c *cid.Cid) (*BasicBlock, error) {
 	if u.Debug {
-		// TODO: fix assumptions
-		chkc := cid.NewCidV0(u.Hash(data))
+		chkc, err := c.Prefix().Sum(data)
+		if err != nil {
+			return nil, err
+		}
+
 		if !chkc.Equals(c) {
 			return nil, ErrWrongHash
 		}

blocks/blockstore/blockstore.go

@@ -100,12 +100,16 @@ func (bs *blockstore) Get(k *cid.Cid) (blocks.Block, error) {
 	}
 
 	if bs.rehash {
-		rb := blocks.NewBlock(bdata)
-		if !rb.Cid().Equals(k) {
+		rbcid, err := k.Prefix().Sum(bdata)
+		if err != nil {
+			return nil, err
+		}
+
+		if !rbcid.Equals(k) {
 			return nil, ErrHashMismatch
-		} else {
-			return rb, nil
 		}
+
+		return blocks.NewBlockWithCid(bdata, rbcid)
 	} else {
 		return blocks.NewBlockWithCid(bdata, k)
 	}

core/commands/add.go

@@ -23,15 +23,16 @@ import (
 var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")
 
 const (
-	quietOptionName    = "quiet"
-	silentOptionName   = "silent"
-	progressOptionName = "progress"
-	trickleOptionName  = "trickle"
-	wrapOptionName     = "wrap-with-directory"
-	hiddenOptionName   = "hidden"
-	onlyHashOptionName = "only-hash"
-	chunkerOptionName  = "chunker"
-	pinOptionName      = "pin"
+	quietOptionName     = "quiet"
+	silentOptionName    = "silent"
+	progressOptionName  = "progress"
+	trickleOptionName   = "trickle"
+	wrapOptionName      = "wrap-with-directory"
+	hiddenOptionName    = "hidden"
+	onlyHashOptionName  = "only-hash"
+	chunkerOptionName   = "chunker"
+	pinOptionName       = "pin"
+	rawLeavesOptionName = "raw-leaves"
 )
 
 var AddCmd = &cmds.Command{
@@ -78,6 +79,7 @@ You can now refer to the added file in a gateway, like so:
 		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add.").Default(false),
 		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
 		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
+		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
 	},
 	PreRun: func(req cmds.Request) error {
 		if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
@@ -135,6 +137,7 @@ You can now refer to the added file in a gateway, like so:
 		silent, _, _ := req.Option(silentOptionName).Bool()
 		chunker, _, _ := req.Option(chunkerOptionName).String()
 		dopin, _, _ := req.Option(pinOptionName).Bool()
+		rawblks, _, _ := req.Option(rawLeavesOptionName).Bool()
 
 		if hash {
 			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
@@ -174,6 +177,7 @@ You can now refer to the added file in a gateway, like so:
 		fileAdder.Wrap = wrap
 		fileAdder.Pin = dopin
 		fileAdder.Silent = silent
+		fileAdder.RawLeaves = rawblks
 
 		if hash {
 			md := dagtest.Mock()

core/commands/files/files.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -16,8 +17,8 @@ import (
 	path "github.com/ipfs/go-ipfs/path"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 
-	context "context"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 )
 
 var log = logging.Logger("cmds/files")
@@ -160,7 +161,12 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
 
 	c := nd.Cid()
 
-	d, err := ft.FromBytes(nd.Data())
+	pbnd, ok := nd.(*dag.ProtoNode)
+	if !ok {
+		return nil, dag.ErrNotProtobuf
+	}
+
+	d, err := ft.FromBytes(pbnd.Data())
 	if err != nil {
 		return nil, err
 	}
@@ -245,7 +251,7 @@ var FilesCpCmd = &cmds.Command{
 	},
 }
 
-func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (*dag.ProtoNode, error) {
+func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (node.Node, error) {
 	switch {
 	case strings.HasPrefix(p, "/ipfs/"):
 		np, err := path.ParsePath(p)

core/corehttp/gateway_handler.go

@@ -1,6 +1,7 @@
 package corehttp
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -18,10 +19,10 @@ import (
 	path "github.com/ipfs/go-ipfs/path"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
 
-	"context"
 	routing "gx/ipfs/QmNUgVQTYnXQVrGT2rajZYsuKV8GYdiL91cdZSQDKNPNgE/go-libp2p-routing"
 	humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
 	cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 )
 
 const (
@@ -45,7 +46,7 @@ func newGatewayHandler(node *core.IpfsNode, conf GatewayConfig) *gatewayHandler
 }
 
 // TODO(cryptix):  find these helpers somewhere else
-func (i *gatewayHandler) newDagFromReader(r io.Reader) (*dag.ProtoNode, error) {
+func (i *gatewayHandler) newDagFromReader(r io.Reader) (node.Node, error) {
 	// TODO(cryptix): change and remove this helper once PR1136 is merged
 	// return ufs.AddFromReader(i.node, r.Body)
 	return importer.BuildDagFromReader(
@@ -353,7 +354,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var newnode *dag.ProtoNode
+	var newnode node.Node
 	if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" {
 		newnode = uio.NewEmptyDirectory()
 	} else {
@@ -417,8 +418,14 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
+		pbnewnode, ok := newnode.(*dag.ProtoNode)
+		if !ok {
+			webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
+			return
+		}
+
 		// object set-data case
-		pbnd.SetData(newnode.Data())
+		pbnd.SetData(pbnewnode.Data())
 
 		newcid, err = i.node.DAG.Add(pbnd)
 		if err != nil {

core/corehttp/gateway_test.go

@@ -1,6 +1,7 @@
 package corehttp
 
 import (
+	"context"
 	"errors"
 	"io/ioutil"
 	"net/http"
@@ -9,14 +10,15 @@ import (
 	"testing"
 	"time"
 
-	context "context"
 	core "github.com/ipfs/go-ipfs/core"
 	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
+	dag "github.com/ipfs/go-ipfs/merkledag"
 	namesys "github.com/ipfs/go-ipfs/namesys"
 	path "github.com/ipfs/go-ipfs/path"
 	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	testutil "github.com/ipfs/go-ipfs/thirdparty/testutil"
+
 	id "gx/ipfs/QmcRa2qn6iCmap9bjp8jAwkvYAq13AUfxdY3rrYiaJbLum/go-libp2p/p2p/protocol/identify"
 	ci "gx/ipfs/QmfWDLQjGjVe4fr5CoztYW2DYYjRysMJrFe1RCsXLPTf46/go-libp2p-crypto"
 )
@@ -178,11 +180,13 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	_, dagn2, err := coreunix.AddWrapped(n, strings.NewReader("_"), "index.html")
 	if err != nil {
 		t.Fatal(err)
 	}
-	dagn1.AddNodeLink("foo", dagn2)
+
+	dagn1.(*dag.ProtoNode).AddNodeLink("foo", dagn2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -197,7 +201,7 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	k := dagn1.Key()
+	k := dagn1.Cid()
 	t.Logf("k: %s\n", k)
 	ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
 
@@ -268,8 +272,8 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	dagn2.AddNodeLink("bar", dagn3)
-	dagn1.AddNodeLink("foo? #<'", dagn2)
+	dagn2.(*dag.ProtoNode).AddNodeLink("bar", dagn3)
+	dagn1.(*dag.ProtoNode).AddNodeLink("foo? #<'", dagn2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -287,7 +291,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	k := dagn1.Key()
+	k := dagn1.Cid()
 	t.Logf("k: %s\n", k)
 	ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
 

core/coreunix/add.go

@@ -1,6 +1,7 @@
 package coreunix
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,16 +14,18 @@ import (
 	"github.com/ipfs/go-ipfs/commands/files"
 	core "github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/exchange/offline"
-	importer "github.com/ipfs/go-ipfs/importer"
+	balanced "github.com/ipfs/go-ipfs/importer/balanced"
 	"github.com/ipfs/go-ipfs/importer/chunk"
+	ihelper "github.com/ipfs/go-ipfs/importer/helpers"
+	trickle "github.com/ipfs/go-ipfs/importer/trickle"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	mfs "github.com/ipfs/go-ipfs/mfs"
 	"github.com/ipfs/go-ipfs/pin"
 	unixfs "github.com/ipfs/go-ipfs/unixfs"
 
-	context "context"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
 	cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
 	syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
 )
@@ -97,10 +100,11 @@ type Adder struct {
 	Hidden     bool
 	Pin        bool
 	Trickle    bool
+	RawLeaves  bool
 	Silent     bool
 	Wrap       bool
 	Chunker    string
-	root       *dag.ProtoNode
+	root       node.Node
 	mr         *mfs.Root
 	unlocker   bs.Unlocker
 	tempRoot   *cid.Cid
@@ -111,25 +115,25 @@ func (adder *Adder) SetMfsRoot(r *mfs.Root) {
 }
 
 // Perform the actual add & pin locally, outputting results to reader
-func (adder Adder) add(reader io.Reader) (*dag.ProtoNode, error) {
+func (adder Adder) add(reader io.Reader) (node.Node, error) {
 	chnk, err := chunk.FromString(reader, adder.Chunker)
 	if err != nil {
 		return nil, err
 	}
+	params := ihelper.DagBuilderParams{
+		Dagserv:   adder.dagService,
+		RawLeaves: adder.RawLeaves,
+		Maxlinks:  ihelper.DefaultLinksPerBlock,
+	}
 
 	if adder.Trickle {
-		return importer.BuildTrickleDagFromReader(
-			adder.dagService,
-			chnk,
-		)
-	}
-	return importer.BuildDagFromReader(
-		adder.dagService,
-		chnk,
-	)
+		return trickle.TrickleLayout(params.New(chnk))
+	}
+
+	return balanced.BalancedLayout(params.New(chnk))
 }
 
-func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
+func (adder *Adder) RootNode() (node.Node, error) {
 	// for memoizing
 	if adder.root != nil {
 		return adder.root, nil
@@ -147,12 +151,7 @@ func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
 			return nil, err
 		}
 
-		pbnd, ok := nd.(*dag.ProtoNode)
-		if !ok {
-			return nil, dag.ErrNotProtobuf
-		}
-
-		root = pbnd
+		root = nd
 	}
 
 	adder.root = root
@@ -185,7 +184,7 @@ func (adder *Adder) PinRoot() error {
 	return adder.pinning.Flush()
 }
 
-func (adder *Adder) Finalize() (*dag.ProtoNode, error) {
+func (adder *Adder) Finalize() (node.Node, error) {
 	root := adder.mr.GetValue()
 
 	// cant just call adder.RootNode() here as we need the name for printing
@@ -307,7 +306,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
 // to preserve the filename.
 // Returns the path of the added file ("<dir hash>/filename"), the DAG node of
 // the directory, and and error if any.
-func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.ProtoNode, error) {
+func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, node.Node, error) {
 	file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
 	fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG)
 	if err != nil {
@@ -331,7 +330,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Pr
 	return gopath.Join(c.String(), filename), dagnode, nil
 }
 
-func (adder *Adder) addNode(node *dag.ProtoNode, path string) error {
+func (adder *Adder) addNode(node node.Node, path string) error {
 	// patch it into the root
 	if path == "" {
 		path = node.Cid().String()
@@ -456,7 +455,7 @@ func (adder *Adder) maybePauseForGC() error {
 }
 
 // outputDagnode sends dagnode info over the output channel
-func outputDagnode(out chan interface{}, name string, dn *dag.ProtoNode) error {
+func outputDagnode(out chan interface{}, name string, dn node.Node) error {
 	if out == nil {
 		return nil
 	}
@@ -482,7 +481,7 @@ func NewMemoryDagService() dag.DAGService {
 }
 
 // from core/commands/object.go
-func getOutput(dagnode *dag.ProtoNode) (*Object, error) {
+func getOutput(dagnode node.Node) (*Object, error) {
 	c := dagnode.Cid()
 
 	output := &Object{