Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: fix get-dag and add version=1 option #260

Merged
merged 2 commits into from
Oct 22, 2021
Merged

feat: fix get-dag and add version=1 option #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
644c8e2
feat: fix get-dag and add version=1 option
rvagg Oct 21, 2021
3f9a436
fix!: use -version=n instead of -v1 for index command
rvagg Oct 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions cmd/car/car.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ func main1() int {
Name: "strict",
Usage: "Fail if the selector finds links to blocks not in the original car",
},
&cli.IntFlag{
Name: "version",
Value: 2,
Usage: "Write output as a v1 or v2 format car",
},
},
},
{
Expand All @@ -86,9 +91,10 @@ func main1() int {
Usage: "The type of index to write",
Value: multicodec.CarMultihashIndexSorted.String(),
},
&cli.BoolFlag{
Name: "v1",
Usage: "Write out only the carV1 file. Implies codec of 'none'",
&cli.IntFlag{
Name: "version",
Value: 2,
Usage: "Write output as a v1 or v2 format car",
},
},
},
Expand Down
121 changes: 71 additions & 50 deletions cmd/car/get.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,22 @@ package main

import (
"bytes"
"context"
"fmt"
"io"
"os"

_ "github.com/ipld/go-codec-dagpb"
dagpb "github.com/ipld/go-codec-dagpb"
"github.com/ipld/go-ipld-prime"
_ "github.com/ipld/go-ipld-prime/codec/cbor"
_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
_ "github.com/ipld/go-ipld-prime/codec/dagjson"
_ "github.com/ipld/go-ipld-prime/codec/json"
_ "github.com/ipld/go-ipld-prime/codec/raw"

blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipfsbs "github.com/ipfs/go-ipfs-blockstore"
"github.com/ipld/go-car"
"github.com/ipld/go-car/v2/blockstore"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/linking"
Expand Down Expand Up @@ -68,18 +70,45 @@ func GetCarDag(c *cli.Context) error {
return fmt.Errorf("usage: car get-dag [-s selector] <file.car> <root cid> <output file>")
}

bs, err := blockstore.OpenReadOnly(c.Args().Get(0))
// string to CID for the root of the DAG to extract
rootCid, err := cid.Parse(c.Args().Get(1))
if err != nil {
return err
}

// string to CID
blkCid, err := cid.Parse(c.Args().Get(1))
bs, err := blockstore.OpenReadOnly(c.Args().Get(0))
if err != nil {
return err
}

outStore, err := blockstore.OpenReadWrite(c.Args().Get(2), []cid.Cid{blkCid})
output := c.Args().Get(2)
strict := c.Bool("strict")

// selector traversal, default to ExploreAllRecursively which only explores the DAG blocks
// because we only care about the blocks loaded during the walk, not the nodes matched
sel := selectorParser.CommonSelector_ExploreAllRecursively
if c.IsSet("selector") {
sel, err = selectorParser.ParseJSONSelector(c.String("selector"))
if err != nil {
return err
}
}
linkVisitOnlyOnce := !c.IsSet("selector") // if using a custom selector, this isn't as safe

switch c.Int("version") {
case 2:
return writeCarV2(rootCid, output, bs, strict, sel, linkVisitOnlyOnce)
case 1:
return writeCarV1(rootCid, output, bs, strict, sel, linkVisitOnlyOnce)
default:
return fmt.Errorf("invalid CAR version %d", c.Int("version"))
}
}

func writeCarV2(rootCid cid.Cid, output string, bs *blockstore.ReadOnly, strict bool, sel datamodel.Node, linkVisitOnlyOnce bool) error {
_ = os.Remove(output)

outStore, err := blockstore.OpenReadWrite(output, []cid.Cid{rootCid}, blockstore.AllowDuplicatePuts(false))
if err != nil {
return err
}
Expand All @@ -91,7 +120,7 @@ func GetCarDag(c *cli.Context) error {
blk, err := bs.Get(cl.Cid)
if err != nil {
if err == ipfsbs.ErrNotFound {
if c.Bool("strict") {
if strict {
return nil, err
}
return nil, traversal.SkipMe{}
Expand All @@ -102,61 +131,53 @@ func GetCarDag(c *cli.Context) error {
}
return nil, fmt.Errorf("unknown link type: %T", l)
}
ls.StorageWriteOpener = func(_ linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) {
buf := bytes.NewBuffer(nil)
return buf, func(l datamodel.Link) error {
if cl, ok := l.(cidlink.Link); ok {
blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid)
if err != nil {
return err
}
return outStore.Put(blk)
}
return fmt.Errorf("unknown link type: %T", l)
}, nil
}

rootlnk := cidlink.Link{
Cid: blkCid,
nsc := func(lnk datamodel.Link, lctx ipld.LinkContext) (datamodel.NodePrototype, error) {
if lnk, ok := lnk.(cidlink.Link); ok && lnk.Cid.Prefix().Codec == 0x70 {
return dagpb.Type.PBNode, nil
}
return basicnode.Prototype.Any, nil
}
node, err := ls.Load(linking.LinkContext{}, rootlnk, basicnode.Prototype.Any)

rootLink := cidlink.Link{Cid: rootCid}
ns, _ := nsc(rootLink, ipld.LinkContext{})
rootNode, err := ls.Load(ipld.LinkContext{}, rootLink, ns)
if err != nil {
return err
}

// selector traversal
s, _ := selector.CompileSelector(selectorParser.CommonSelector_MatchAllRecursively)
if c.IsSet("selector") {
sn, err := selectorParser.ParseJSONSelector(c.String("selector"))
if err != nil {
return err
}
s, err = selector.CompileSelector(sn)
if err != nil {
return err
}
traversalProgress := traversal.Progress{
Cfg: &traversal.Config{
LinkSystem: ls,
LinkTargetNodePrototypeChooser: nsc,
LinkVisitOnlyOnce: linkVisitOnlyOnce,
},
}

lnkProto := cidlink.LinkPrototype{
Prefix: blkCid.Prefix(),
s, err := selector.CompileSelector(sel)
if err != nil {
return err
}
err = traversal.WalkMatching(node, s, func(p traversal.Progress, n datamodel.Node) error {
if p.LastBlock.Link != nil {
if cl, ok := p.LastBlock.Link.(cidlink.Link); ok {
lnkProto = cidlink.LinkPrototype{
Prefix: cl.Prefix(),
}
}
}
_, err = ls.Store(linking.LinkContext{}, lnkProto, n)
if err != nil {
return err
}
return nil
})

err = traversalProgress.WalkMatching(rootNode, s, func(p traversal.Progress, n datamodel.Node) error { return nil })
if err != nil {
return err
}

return outStore.Finalize()
}

func writeCarV1(rootCid cid.Cid, output string, bs *blockstore.ReadOnly, strict bool, sel datamodel.Node, linkVisitOnlyOnce bool) error {
opts := make([]car.Option, 0)
if linkVisitOnlyOnce {
opts = append(opts, car.TraverseLinksOnlyOnce())
}
sc := car.NewSelectiveCar(context.Background(), bs, []car.Dag{{Root: rootCid, Selector: sel}}, opts...)
f, err := os.Create(output)
if err != nil {
return err
}
defer f.Close()

return sc.Write(f)
}
8 changes: 6 additions & 2 deletions cmd/car/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ func IndexCar(c *cli.Context) error {
}
defer r.Close()

if c.Bool("v1") {
if c.Int("version") == 1 {
if c.IsSet("codec") && c.String("codec") != "none" {
return fmt.Errorf("only supported codec for a v1 car is 'none'")
return fmt.Errorf("'none' is the only supported codec for a v1 car")
}
outStream := os.Stdout
if c.Args().Len() >= 2 {
Expand All @@ -40,6 +40,10 @@ func IndexCar(c *cli.Context) error {
return err
}

if c.Int("version") != 2 {
return fmt.Errorf("invalid CAR version %d", c.Int("version"))
}

var idx index.Index
if c.String("codec") != "none" {
var mc multicodec.Code
Expand Down