creation of car from file / directory

cmd/car/car.go

@@ -15,6 +15,20 @@ func main1() int {
 		Name:  "car",
 		Usage: "Utility for working with car files",
 		Commands: []*cli.Command{
+			{
+				Name:    "create",
+				Usage:   "Create a car file",
+				Aliases: []string{"c"},
+				Action:  CreateCar,
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:      "file",
+						Aliases:   []string{"f", "output", "o"},
+						Usage:     "The car file to write to",
+						TakesFile: true,
+					},
+				},
+			},
 			{
 				Name:   "detach-index",
 				Usage:  "Detach an index to a detached file",
@@ -72,13 +86,28 @@ func main1() int {
 						Usage:   "The type of index to write",
 						Value:   multicodec.CarMultihashIndexSorted.String(),
 					},
+					&cli.BoolFlag{
+						Name:  "v1",
+						Usage: "Write out only the carV1 file. Implies codec of 'none'",
+					},
 				},
 			},
 			{
 				Name:    "list",
 				Aliases: []string{"l"},
 				Usage:   "List the CIDs in a car",
 				Action:  ListCar,
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:    "verbose",
+						Aliases: []string{"v"},
+						Usage:   "Include verbose information about contained blocks",
+					},
+					&cli.BoolFlag{
+						Name:  "unixfs",
+						Usage: "List unixfs filesystem from the root of the car",
+					},
+				},
 			},
 			{
 				Name:    "verify",

cmd/car/create.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"path"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-unixfsnode/data/builder"
+	"github.com/ipld/go-car/v2"
+	"github.com/ipld/go-car/v2/blockstore"
+	dagpb "github.com/ipld/go-codec-dagpb"
+	"github.com/ipld/go-ipld-prime"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/multiformats/go-multicodec"
+	"github.com/multiformats/go-multihash"
+	"github.com/urfave/cli/v2"
+)
+
+// CreateCar creates a car
+func CreateCar(c *cli.Context) error {
+	var err error
+	if c.Args().Len() == 0 {
+		return fmt.Errorf("a source location to build the car from must be specified")
+	}
+
+	if !c.IsSet("file") {
+		return fmt.Errorf("a file destination must be specified")
+	}
+
+	// make a cid with the right length that we eventually will patch with the root.
+	hasher, err := multihash.GetHasher(multihash.SHA2_256)
+	if err != nil {
+		return err
+	}
+	digest := hasher.Sum([]byte{})
+	hash, err := multihash.Encode(digest, multihash.SHA2_256)
+	if err != nil {
+		return err
+	}
+	proxyRoot := cid.NewCidV1(uint64(multicodec.DagPb), hash)
+
+	cdest, err := blockstore.OpenReadWrite(c.String("file"), []cid.Cid{proxyRoot})
+	if err != nil {
+		return err
+	}
+
+	// Write the unixfs blocks into the store.
+	root, err := writeFiles(c.Context, cdest, c.Args().Slice()...)
+	if err != nil {
+		return err
+	}
+
+	if err := cdest.Finalize(); err != nil {
+		return err
+	}
+	// re-open/finalize with the final root.
+	return car.ReplaceRootsInFile(c.String("file"), []cid.Cid{root})
+}
+
+func writeFiles(ctx context.Context, bs *blockstore.ReadWrite, paths ...string) (cid.Cid, error) {
+	ls := cidlink.DefaultLinkSystem()
+	ls.TrustedStorage = true
+	ls.StorageReadOpener = func(_ ipld.LinkContext, l ipld.Link) (io.Reader, error) {
+		cl, ok := l.(cidlink.Link)
+		if !ok {
+			return nil, fmt.Errorf("not a cidlink")
+		}
+		blk, err := bs.Get(cl.Cid)
+		if err != nil {
+			return nil, err
+		}
+		return bytes.NewBuffer(blk.RawData()), nil
+	}
+	ls.StorageWriteOpener = func(_ ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) {
+		buf := bytes.NewBuffer(nil)
+		return buf, func(l ipld.Link) error {
+			cl, ok := l.(cidlink.Link)
+			if !ok {
+				return fmt.Errorf("not a cidlink")
+			}
+			blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid)
+			if err != nil {
+				return err
+			}
+			bs.Put(blk)
+			return nil
+		}, nil
+	}
+
+	topLevel := make([]dagpb.PBLink, 0, len(paths))
+	for _, p := range paths {
+		l, size, err := builder.BuildUnixFSRecursive(p, &ls)
+		if err != nil {
+			return cid.Undef, err
+		}
+		name := path.Base(p)
+		entry, err := builder.BuildUnixFSDirectoryEntry(name, int64(size), l)
+		if err != nil {
+			return cid.Undef, err
+		}
+		topLevel = append(topLevel, entry)
+	}
+
+	// make a directory for the file(s).
+
+	root, err := builder.BuildUnixFSDirectory(topLevel, &ls)
+	if err != nil {
+		return cid.Undef, nil
+	}
+	rcl, ok := root.(cidlink.Link)
+	if !ok {
+		return cid.Undef, fmt.Errorf("could not interpret %s", root)
+	}
+
+	return rcl.Cid, nil
+}

cmd/car/get.go

@@ -85,8 +85,9 @@ func GetCarDag(c *cli.Context) error {
 	}
 
 	ls := cidlink.DefaultLinkSystem()
+	ls.TrustedStorage = true
 	ls.StorageReadOpener = func(_ linking.LinkContext, l datamodel.Link) (io.Reader, error) {
-		if cl, ok := l.(*cidlink.Link); ok {
+		if cl, ok := l.(cidlink.Link); ok {
 			blk, err := bs.Get(cl.Cid)
 			if err != nil {
 				if err == ipfsbs.ErrNotFound {
@@ -104,7 +105,7 @@ func GetCarDag(c *cli.Context) error {
 	ls.StorageWriteOpener = func(_ linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) {
 		buf := bytes.NewBuffer(nil)
 		return buf, func(l datamodel.Link) error {
-			if cl, ok := l.(*cidlink.Link); ok {
+			if cl, ok := l.(cidlink.Link); ok {
 				blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid)
 				if err != nil {
 					return err
@@ -124,7 +125,7 @@ func GetCarDag(c *cli.Context) error {
 	}
 
 	// selector traversal
-	s := selectorParser.CommonSelector_MatchAllRecursively
+	s, _ := selector.CompileSelector(selectorParser.CommonSelector_MatchAllRecursively)
 	if c.IsSet("selector") {
 		sn, err := selectorParser.ParseJSONSelector(c.String("selector"))
 		if err != nil {
@@ -141,7 +142,7 @@ func GetCarDag(c *cli.Context) error {
 	}
 	err = traversal.WalkMatching(node, s, func(p traversal.Progress, n datamodel.Node) error {
 		if p.LastBlock.Link != nil {
-			if cl, ok := p.LastBlock.Link.(*cidlink.Link); ok {
+			if cl, ok := p.LastBlock.Link.(cidlink.Link); ok {
 				lnkProto = cidlink.LinkPrototype{
 					Prefix: cl.Prefix(),
 				}

cmd/car/index.go

@@ -23,6 +23,23 @@ func IndexCar(c *cli.Context) error {
 	}
 	defer r.Close()
 
+	if c.Bool("v1") {
+		if c.IsSet("codec") && c.String("codec") != "none" {
+			return fmt.Errorf("only supported codec for a v1 car is 'none'")
+		}
+		outStream := os.Stdout
+		if c.Args().Len() >= 2 {
+			outStream, err = os.Create(c.Args().Get(1))
+			if err != nil {
+				return err
+			}
+		}
+		defer outStream.Close()
+
+		_, err := io.Copy(outStream, r.DataReader())
+		return err
+	}
+
 	var idx index.Index
 	if c.String("codec") != "none" {
 		var mc multicodec.Code