From b270be5a04b00bdf3b036a6a591d67631f67e116 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 21 Oct 2021 21:36:04 +1100 Subject: [PATCH] feat: fix get-dag and add version=1 option --- cmd/car/car.go | 5 ++ cmd/car/get.go | 128 ++++++++++++++++++++++++++++++------------------- 2 files changed, 84 insertions(+), 49 deletions(-) diff --git a/cmd/car/car.go b/cmd/car/car.go index aea9f4c2..76a1ed49 100644 --- a/cmd/car/car.go +++ b/cmd/car/car.go @@ -72,6 +72,11 @@ func main1() int { Name: "strict", Usage: "Fail if the selector finds links to blocks not in the original car", }, + &cli.IntFlag{ + Name: "version", + Value: 2, + Usage: "Write output as a v1 or v2 format car", + }, }, }, { diff --git a/cmd/car/get.go b/cmd/car/get.go index f1373313..60fbd4e4 100644 --- a/cmd/car/get.go +++ b/cmd/car/get.go @@ -2,11 +2,13 @@ package main import ( "bytes" + "context" "fmt" "io" "os" - _ "github.com/ipld/go-codec-dagpb" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" _ "github.com/ipld/go-ipld-prime/codec/cbor" _ "github.com/ipld/go-ipld-prime/codec/dagcbor" _ "github.com/ipld/go-ipld-prime/codec/dagjson" @@ -16,6 +18,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipfsbs "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipld/go-car" "github.com/ipld/go-car/v2/blockstore" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" @@ -68,18 +71,45 @@ func GetCarDag(c *cli.Context) error { return fmt.Errorf("usage: car get-dag [-s selector] ") } - bs, err := blockstore.OpenReadOnly(c.Args().Get(0)) + // string to CID for the root of the DAG to extract + rootCid, err := cid.Parse(c.Args().Get(1)) if err != nil { return err } - // string to CID - blkCid, err := cid.Parse(c.Args().Get(1)) + bs, err := blockstore.OpenReadOnly(c.Args().Get(0)) if err != nil { return err } - outStore, err := blockstore.OpenReadWrite(c.Args().Get(2), []cid.Cid{blkCid}) + output := c.Args().Get(2) + strict := c.Bool("strict") + + // selector traversal, default to ExploreAllRecursively which only explores the DAG blocks + // because we only care about the blocks loaded during the walk, not the nodes matched + sel := selectorParser.CommonSelector_ExploreAllRecursively + if c.IsSet("selector") { + sel, err = selectorParser.ParseJSONSelector(c.String("selector")) + if err != nil { + return err + } + } + linkVisitOnlyOnce := !c.IsSet("selector") // if using a custom selector, this isn't as safe + + switch c.Int("version") { + case 2: + return writeCarV2(rootCid, output, bs, strict, sel, linkVisitOnlyOnce) + case 1: + return writeCarV1(rootCid, output, bs, strict, sel, linkVisitOnlyOnce) + default: + return fmt.Errorf("invalid CAR version %d", c.Int("version")) + } +} + +func writeCarV2(rootCid cid.Cid, output string, bs *blockstore.ReadOnly, strict bool, sel datamodel.Node, linkVisitOnlyOnce bool) error { + _ = os.Remove(output) + + outStore, err := blockstore.OpenReadWrite(output, []cid.Cid{rootCid}, blockstore.AllowDuplicatePuts(false)) if err != nil { return err } @@ -91,7 +121,7 @@ func GetCarDag(c *cli.Context) error { blk, err := bs.Get(cl.Cid) if err != nil { if err == ipfsbs.ErrNotFound { - if c.Bool("strict") { + if strict { return nil, err } return nil, traversal.SkipMe{} @@ -102,61 +132,61 @@ func GetCarDag(c *cli.Context) error { } return nil, fmt.Errorf("unknown link type: %T", l) } - ls.StorageWriteOpener = func(_ linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { - buf := bytes.NewBuffer(nil) - return buf, func(l datamodel.Link) error { - if cl, ok := l.(cidlink.Link); ok { - blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid) - if err != nil { - return err - } - return outStore.Put(blk) - } - return fmt.Errorf("unknown link type: %T", l) - }, nil - } - rootlnk := cidlink.Link{ - Cid: blkCid, + nsc := func(lnk datamodel.Link, lctx ipld.LinkContext) (datamodel.NodePrototype, error) { + if lnk, ok := lnk.(cidlink.Link); ok && lnk.Cid.Prefix().Codec == 0x70 { + return dagpb.Type.PBNode, nil + } + return basicnode.Prototype.Any, nil } - node, err := ls.Load(linking.LinkContext{}, rootlnk, basicnode.Prototype.Any) + + rootLink := cidlink.Link{Cid: rootCid} + ns, _ := nsc(rootLink, ipld.LinkContext{}) + rootNode, err := ls.Load(ipld.LinkContext{}, rootLink, ns) if err != nil { return err } - // selector traversal - s, _ := selector.CompileSelector(selectorParser.CommonSelector_MatchAllRecursively) - if c.IsSet("selector") { - sn, err := selectorParser.ParseJSONSelector(c.String("selector")) - if err != nil { - return err - } - s, err = selector.CompileSelector(sn) - if err != nil { - return err - } + traversalProgress := traversal.Progress{ + Cfg: &traversal.Config{ + LinkSystem: ls, + LinkTargetNodePrototypeChooser: nsc, + LinkVisitOnlyOnce: linkVisitOnlyOnce, + }, } - lnkProto := cidlink.LinkPrototype{ - Prefix: blkCid.Prefix(), + s, err := selector.CompileSelector(sel) + if err != nil { + return err } - err = traversal.WalkMatching(node, s, func(p traversal.Progress, n datamodel.Node) error { - if p.LastBlock.Link != nil { - if cl, ok := p.LastBlock.Link.(cidlink.Link); ok { - lnkProto = cidlink.LinkPrototype{ - Prefix: cl.Prefix(), - } - } - } - _, err = ls.Store(linking.LinkContext{}, lnkProto, n) - if err != nil { - return err - } - return nil - }) + + err = traversalProgress.WalkMatching(rootNode, s, func(p traversal.Progress, n datamodel.Node) error { return nil }) if err != nil { return err } return outStore.Finalize() } + +func writeCarV1(rootCid cid.Cid, output string, bs *blockstore.ReadOnly, strict bool, sel datamodel.Node, linkVisitOnlyOnce bool) error { + opts := make([]car.Option, 0) + if linkVisitOnlyOnce { + opts = append(opts, car.TraverseLinksOnlyOnce()) + } + sc := car.NewSelectiveCar(context.TODO(), &readStore{bs}, []car.Dag{car.Dag{Root: rootCid, Selector: sel}}, opts...) + f, err := os.Create(output) + if err != nil { + return err + } + defer f.Close() + + return sc.Write(f) +} + +type readStore struct { + bs *blockstore.ReadOnly +} + +func (rs *readStore) Get(c cid.Cid) (blocks.Block, error) { + return rs.bs.Get(c) +}