diff --git a/cmd/car/car.go b/cmd/car/car.go index aea9f4c2..850673c6 100644 --- a/cmd/car/car.go +++ b/cmd/car/car.go @@ -72,6 +72,11 @@ func main1() int { Name: "strict", Usage: "Fail if the selector finds links to blocks not in the original car", }, + &cli.IntFlag{ + Name: "version", + Value: 2, + Usage: "Write output as a v1 or v2 format car", + }, }, }, { @@ -86,9 +91,10 @@ func main1() int { Usage: "The type of index to write", Value: multicodec.CarMultihashIndexSorted.String(), }, - &cli.BoolFlag{ - Name: "v1", - Usage: "Write out only the carV1 file. Implies codec of 'none'", + &cli.IntFlag{ + Name: "version", + Value: 2, + Usage: "Write output as a v1 or v2 format car", }, }, }, diff --git a/cmd/car/get.go b/cmd/car/get.go index f1373313..324f050a 100644 --- a/cmd/car/get.go +++ b/cmd/car/get.go @@ -2,20 +2,22 @@ package main import ( "bytes" + "context" "fmt" "io" "os" - _ "github.com/ipld/go-codec-dagpb" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" _ "github.com/ipld/go-ipld-prime/codec/cbor" _ "github.com/ipld/go-ipld-prime/codec/dagcbor" _ "github.com/ipld/go-ipld-prime/codec/dagjson" _ "github.com/ipld/go-ipld-prime/codec/json" _ "github.com/ipld/go-ipld-prime/codec/raw" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipfsbs "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipld/go-car" "github.com/ipld/go-car/v2/blockstore" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" @@ -68,18 +70,45 @@ func GetCarDag(c *cli.Context) error { return fmt.Errorf("usage: car get-dag [-s selector] ") } - bs, err := blockstore.OpenReadOnly(c.Args().Get(0)) + // string to CID for the root of the DAG to extract + rootCid, err := cid.Parse(c.Args().Get(1)) if err != nil { return err } - // string to CID - blkCid, err := cid.Parse(c.Args().Get(1)) + bs, err := blockstore.OpenReadOnly(c.Args().Get(0)) if err != nil { return err } - outStore, err := blockstore.OpenReadWrite(c.Args().Get(2), []cid.Cid{blkCid}) + output := c.Args().Get(2) + strict := c.Bool("strict") + + // selector traversal, default to ExploreAllRecursively which only explores the DAG blocks + // because we only care about the blocks loaded during the walk, not the nodes matched + sel := selectorParser.CommonSelector_ExploreAllRecursively + if c.IsSet("selector") { + sel, err = selectorParser.ParseJSONSelector(c.String("selector")) + if err != nil { + return err + } + } + linkVisitOnlyOnce := !c.IsSet("selector") // if using a custom selector, this isn't as safe + + switch c.Int("version") { + case 2: + return writeCarV2(rootCid, output, bs, strict, sel, linkVisitOnlyOnce) + case 1: + return writeCarV1(rootCid, output, bs, strict, sel, linkVisitOnlyOnce) + default: + return fmt.Errorf("invalid CAR version %d", c.Int("version")) + } +} + +func writeCarV2(rootCid cid.Cid, output string, bs *blockstore.ReadOnly, strict bool, sel datamodel.Node, linkVisitOnlyOnce bool) error { + _ = os.Remove(output) + + outStore, err := blockstore.OpenReadWrite(output, []cid.Cid{rootCid}, blockstore.AllowDuplicatePuts(false)) if err != nil { return err } @@ -91,7 +120,7 @@ func GetCarDag(c *cli.Context) error { blk, err := bs.Get(cl.Cid) if err != nil { if err == ipfsbs.ErrNotFound { - if c.Bool("strict") { + if strict { return nil, err } return nil, traversal.SkipMe{} @@ -102,61 +131,53 @@ func GetCarDag(c *cli.Context) error { } return nil, fmt.Errorf("unknown link type: %T", l) } - ls.StorageWriteOpener = func(_ linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { - buf := bytes.NewBuffer(nil) - return buf, func(l datamodel.Link) error { - if cl, ok := l.(cidlink.Link); ok { - blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid) - if err != nil { - return err - } - return outStore.Put(blk) - } - return fmt.Errorf("unknown link type: %T", l) - }, nil - } - rootlnk := cidlink.Link{ - Cid: blkCid, + nsc := func(lnk datamodel.Link, lctx ipld.LinkContext) (datamodel.NodePrototype, error) { + if lnk, ok := lnk.(cidlink.Link); ok && lnk.Cid.Prefix().Codec == 0x70 { + return dagpb.Type.PBNode, nil + } + return basicnode.Prototype.Any, nil } - node, err := ls.Load(linking.LinkContext{}, rootlnk, basicnode.Prototype.Any) + + rootLink := cidlink.Link{Cid: rootCid} + ns, _ := nsc(rootLink, ipld.LinkContext{}) + rootNode, err := ls.Load(ipld.LinkContext{}, rootLink, ns) if err != nil { return err } - // selector traversal - s, _ := selector.CompileSelector(selectorParser.CommonSelector_MatchAllRecursively) - if c.IsSet("selector") { - sn, err := selectorParser.ParseJSONSelector(c.String("selector")) - if err != nil { - return err - } - s, err = selector.CompileSelector(sn) - if err != nil { - return err - } + traversalProgress := traversal.Progress{ + Cfg: &traversal.Config{ + LinkSystem: ls, + LinkTargetNodePrototypeChooser: nsc, + LinkVisitOnlyOnce: linkVisitOnlyOnce, + }, } - lnkProto := cidlink.LinkPrototype{ - Prefix: blkCid.Prefix(), + s, err := selector.CompileSelector(sel) + if err != nil { + return err } - err = traversal.WalkMatching(node, s, func(p traversal.Progress, n datamodel.Node) error { - if p.LastBlock.Link != nil { - if cl, ok := p.LastBlock.Link.(cidlink.Link); ok { - lnkProto = cidlink.LinkPrototype{ - Prefix: cl.Prefix(), - } - } - } - _, err = ls.Store(linking.LinkContext{}, lnkProto, n) - if err != nil { - return err - } - return nil - }) + + err = traversalProgress.WalkMatching(rootNode, s, func(p traversal.Progress, n datamodel.Node) error { return nil }) if err != nil { return err } return outStore.Finalize() } + +func writeCarV1(rootCid cid.Cid, output string, bs *blockstore.ReadOnly, strict bool, sel datamodel.Node, linkVisitOnlyOnce bool) error { + opts := make([]car.Option, 0) + if linkVisitOnlyOnce { + opts = append(opts, car.TraverseLinksOnlyOnce()) + } + sc := car.NewSelectiveCar(context.Background(), bs, []car.Dag{{Root: rootCid, Selector: sel}}, opts...) + f, err := os.Create(output) + if err != nil { + return err + } + defer f.Close() + + return sc.Write(f) +} diff --git a/cmd/car/index.go b/cmd/car/index.go index ac368886..153c0b7c 100644 --- a/cmd/car/index.go +++ b/cmd/car/index.go @@ -23,9 +23,9 @@ func IndexCar(c *cli.Context) error { } defer r.Close() - if c.Bool("v1") { + if c.Int("version") == 1 { if c.IsSet("codec") && c.String("codec") != "none" { - return fmt.Errorf("only supported codec for a v1 car is 'none'") + return fmt.Errorf("'none' is the only supported codec for a v1 car") } outStream := os.Stdout if c.Args().Len() >= 2 { @@ -40,6 +40,10 @@ func IndexCar(c *cli.Context) error { return err } + if c.Int("version") != 2 { + return fmt.Errorf("invalid CAR version %d", c.Int("version")) + } + var idx index.Index if c.String("codec") != "none" { var mc multicodec.Code