Initial version for review

core/commands/dag/dag.go

@@ -1,14 +1,17 @@
 package dagcmd
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"math"
 	"strings"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	"github.com/ipfs/go-ipfs/core/coredag"
 
+	gocar "github.com/ipfs/go-car"
 	cid "github.com/ipfs/go-cid"
 	cidenc "github.com/ipfs/go-cidutil/cidenc"
 	cmds "github.com/ipfs/go-ipfs-cmds"
@@ -19,6 +22,13 @@ import (
 	mh "github.com/multiformats/go-multihash"
 )
 
+const (
+	//progressOptionName = "progress"
+	//quietOptionName    = "quiet"
+	silentOptionName   = "silent"
+	pinRootsOptionName = "pin-roots"
+)
+
 var DagCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Interact with ipld dag objects.",
@@ -33,10 +43,11 @@ to deprecate and replace the existing 'ipfs object' command moving forward.
 		"put":     DagPutCmd,
 		"get":     DagGetCmd,
 		"resolve": DagResolveCmd,
+		"import":  DagImportCmd,
 	},
 }
 
-// OutputObject is the output type of 'dag put' command
+// OutputObject is the output type of 'dag put'/'dag import'command
 type OutputObject struct {
 	Cid cid.Cid
 }
@@ -241,3 +252,203 @@ var DagResolveCmd = &cmds.Command{
 	},
 	Type: ResolveOutput{},
 }
+
+var DagImportCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Import the contents of .car files",
+		ShortDescription: `
+'ipfs dag import' parses .car files and adds all contained objects to the blockstore.
+`,
+	},
+
+	Arguments: []cmds.Argument{
+		cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
+	},
+	Options: []cmds.Option{
+		// FIXME - need to copy the progress bar from core/commands/add.go
+		// Base it on a proxy-reader
+		//cmds.BoolOption(progressOptionName, "p", "Stream progress data.").WithDefault(true),
+		//cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
+		cmds.BoolOption(silentOptionName, "Write no output."),
+		cmds.BoolOption(pinRootsOptionName, "Atomically pin (optional) roots listed in the .car headers after importing.").WithDefault(true),
+	},
+	// FIXME - progress bar option disabled above
+	// PreRun: func(req *cmds.Request, env cmds.Environment) error {
+	// 	quiet, _ := req.Options[quietOptionName].(bool)
+	// 	silent, _ := req.Options[silentOptionName].(bool)
+
+	// 	if quiet || silent {
+	// 		return nil
+	// 	}
+
+	// 	// ipfs cli progress bar defaults to true unless quiet or silent is used
+	// 	_, found := req.Options[progressOptionName].(bool)
+	// 	if !found {
+	// 		req.Options[progressOptionName] = true
+	// 	}
+
+	// 	return nil
+	// },
+	Type: OutputObject{},
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		api, err := cmdenv.GetApi(env, req)
+		if err != nil {
+			return err
+		}
+
+		doPinRoots, _ := req.Options[pinRootsOptionName].(bool)
+		silent, _ := req.Options[silentOptionName].(bool)
+
+		keepTrackOfRoots := doPinRoots || !silent
+
+		// It is not guaranteed that a root in a header is actually present in the same ( or any .car )
+		// Accumulate any root CID seen in a header, and supplement its actual node if/when encountered
+		// We will pin *only* at the end in case the entire operation is successful
+		// Q: - way to do above better?
+		// Q: - can I *not* keep a full copy of the node and still pin it anyway?
+		var expectRoots map[cid.Cid]*ipld.Node
+
+		if keepTrackOfRoots {
+			expectRoots = make(map[cid.Cid]*ipld.Node, 32)
+		}
+
+		// If more than one file - parse all headers first
+		// ( due to misdesign we do "reparse" each car's header twice, but that's cheap )
+		// Q: there seems to be no management of file closures... ANYWHERE
+		argCount, _ := req.Files.Size()
+		if keepTrackOfRoots && argCount > 1 {
+
+			it := req.Files.Entries()
+			for it.Next() {
+
+				file := files.FileFromEntry(it)
+				if file == nil {
+					return errors.New("expected a file")
+				}
+
+				buf := make([]byte, 1)
+				file.Read(buf)
+				file.Seek(0, 0)
+				spew.Dump(buf)
+
+				continue
+
+				car, err := gocar.NewCarReader(file)
+				if err != nil {
+					return err
+				}
+
+				for _, cid := range car.Header.Roots {
+					if _, exists := expectRoots[cid]; !exists {
+						expectRoots[cid] = nil
+					}
+				}
+			}
+		}
+
+		// Q: How do I combine this with pinnig?
+		adder := api.Dag()
+		b := ipld.NewBatch(req.Context, adder)
+
+		it := req.Files.Entries()
+		for it.Next() {
+
+			file := files.FileFromEntry(it)
+			if file == nil {
+				return errors.New("expected a file")
+			}
+			defer func() { file.Close() }()
+
+			car, err := gocar.NewCarReader(file)
+			if err != nil {
+				return err
+			}
+
+			if car.Header.Version != 1 {
+				return errors.New("only car files version 1 supported at present")
+			}
+
+			// duplicate of a block from above: in case we were streaming and never pre-read the file
+			if keepTrackOfRoots {
+				for _, cid := range car.Header.Roots {
+					if _, exists := expectRoots[cid]; !exists {
+						expectRoots[cid] = nil
+					}
+				}
+			}
+
+			for {
+				block, err := car.Next()
+				if err != nil && err != io.EOF {
+					return err
+				} else if block == nil {
+					break
+				}
+
+				// Q: this dual-pass is so odd, but nothing woks with blocks.Block...
+				nd, err := ipld.Decode(block)
+				if err != nil {
+					return err
+				}
+
+				if err := b.Add(req.Context, nd); err != nil {
+					return err
+				}
+
+				if keepTrackOfRoots {
+					val, exists := expectRoots[nd.Cid()]
+
+					// encountered something known to be a root, for the first time
+					if exists && val == nil {
+						expectRoots[nd.Cid()] = &nd
+
+						if !silent {
+							// Q: everything makes thing async - should I?
+							if err := res.Emit(&OutputObject{Cid: nd.Cid()}); err != nil {
+								return err
+							}
+						}
+					}
+				}
+			}
+		}
+
+		if err := it.Err(); err != nil {
+			return err
+		}
+
+		if doPinRoots && len(expectRoots) > 0 {
+			// run through everything making sure we found all roots
+			toPin := make([]ipld.Node, 0, len(expectRoots))
+			for cid, nd := range expectRoots {
+				if nd == nil {
+					return fmt.Errorf("the CID '%s' named as root, but never encountered in any .car", cid)
+				}
+				toPin = append(toPin, *nd)
+			}
+
+			// all found - pin them down
+			// ( this does in effect call b.Commit()... it seems )
+			if err := api.Dag().Pinning().AddMany(req.Context, toPin); err != nil {
+				return err
+			}
+		}
+
+		// well... here goes nothing
+		if err := b.Commit(); err != nil {
+			return err
+		}
+
+		return nil
+	},
+	Encoders: cmds.EncoderMap{
+		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OutputObject) error {
+			enc, err := cmdenv.GetLowLevelCidEncoder(req)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(w, enc.Encode(out.Cid))
+			return nil
+		}),
+	},
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d
 	github.com/coreos/go-systemd/v22 v22.0.0
+	github.com/davecgh/go-spew v1.1.1
 	github.com/dustin/go-humanize v1.0.0
 	github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302
 	github.com/fatih/color v1.9.0 // indirect
@@ -17,6 +18,7 @@ require (
 	github.com/ipfs/go-bitswap v0.2.3
 	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-blockservice v0.1.2
+	github.com/ipfs/go-car v0.0.2
 	github.com/ipfs/go-cid v0.0.5
 	github.com/ipfs/go-cidutil v0.0.2
 	github.com/ipfs/go-datastore v0.4.4

go.sum

@@ -206,6 +206,8 @@ github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7s
 github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
 github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc=
 github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
+github.com/ipfs/go-car v0.0.2 h1:j02lzgeijorstzoMl3nQmvvb8wjJUVCiOAl8XEwYMCQ=
+github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5JwQ=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms=
@@ -336,6 +338,7 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy
 github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE=
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
+github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0=
 github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
 github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ=
@@ -901,6 +904,7 @@ github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=