From 5633dab1d6d791e217e6cd6e97242eb076454d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 May 2018 00:27:40 +0200 Subject: [PATCH 1/3] pin cmd: rewrite pin ls to new cmd lib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/pin.go | 87 +++++++++++++++++++------------------------ core/commands/root.go | 2 +- 2 files changed, 39 insertions(+), 50 deletions(-) diff --git a/core/commands/pin.go b/core/commands/pin.go index 1829ac04272..f168cc8f6d3 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -8,7 +8,8 @@ import ( "time" bserv "github.com/ipfs/go-ipfs/blockservice" - cmds "github.com/ipfs/go-ipfs/commands" + oldcmds "github.com/ipfs/go-ipfs/commands" + lgc "github.com/ipfs/go-ipfs/commands/legacy" core "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" corerepo "github.com/ipfs/go-ipfs/core/corerepo" @@ -23,6 +24,7 @@ import ( offline "gx/ipfs/QmPf114DXfa6TqGKYhBGR7EtXRho4rCJgwyA1xkuMY5vwF/go-ipfs-exchange-offline" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" + cmds "gx/ipfs/QmaFrNcnXHp579hUixbcTH1TNtNwsMogtBCwUUUwzBwYoM/go-ipfs-cmds" ) var PinCmd = &cmds.Command{ @@ -31,11 +33,11 @@ var PinCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "add": addPinCmd, - "rm": rmPinCmd, + "add": lgc.NewCommand(addPinCmd), + "rm": lgc.NewCommand(rmPinCmd), "ls": listPinCmd, - "verify": verifyPinCmd, - "update": updatePinCmd, + "verify": lgc.NewCommand(verifyPinCmd), + "update": lgc.NewCommand(updatePinCmd), }, } @@ -48,7 +50,7 @@ type AddPinOutput struct { Progress int `json:",omitempty"` } -var addPinCmd = &cmds.Command{ +var addPinCmd = &oldcmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Pin objects to local storage.", ShortDescription: "Stores an IPFS object(s) from a given path locally to disk.", @@ -62,7 +64,7 @@ var addPinCmd = &cmds.Command{ cmdkit.BoolOption("progress", "Show progress"), }, Type: AddPinOutput{}, - Run: func(req cmds.Request, res cmds.Response) { + Run: func(req oldcmds.Request, res oldcmds.Response) { n, err := req.InvocContext().GetNode() if err != nil { res.SetError(err, cmdkit.ErrNormal) @@ -129,8 +131,8 @@ var addPinCmd = &cmds.Command{ } } }, - Marshalers: cmds.MarshalerMap{ - cmds.Text: func(res cmds.Response) (io.Reader, error) { + Marshalers: oldcmds.MarshalerMap{ + oldcmds.Text: func(res oldcmds.Response) (io.Reader, error) { v, err := unwrapOutput(res.Output()) if err != nil { return nil, err @@ -171,7 +173,7 @@ var addPinCmd = &cmds.Command{ }, } -var rmPinCmd = &cmds.Command{ +var rmPinCmd = &oldcmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Remove pinned objects from local storage.", ShortDescription: ` @@ -187,7 +189,7 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) cmdkit.BoolOption("recursive", "r", "Recursively unpin the object linked to by the specified object(s).").WithDefault(true), }, Type: PinOutput{}, - Run: func(req cmds.Request, res cmds.Response) { + Run: func(req oldcmds.Request, res oldcmds.Response) { n, err := req.InvocContext().GetNode() if err != nil { res.SetError(err, cmdkit.ErrNormal) @@ -209,8 +211,8 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) res.SetOutput(&PinOutput{cidsToStrings(removed)}) }, - Marshalers: cmds.MarshalerMap{ - cmds.Text: func(res cmds.Response) (io.Reader, error) { + Marshalers: oldcmds.MarshalerMap{ + oldcmds.Text: func(res oldcmds.Response) (io.Reader, error) { v, err := unwrapOutput(res.Output()) if err != nil { return nil, err @@ -281,18 +283,14 @@ Example: cmdkit.StringOption("type", "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"), cmdkit.BoolOption("quiet", "q", "Write just hashes of objects."), }, - Run: func(req cmds.Request, res cmds.Response) { - n, err := req.InvocContext().GetNode() + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) { + n, err := GetNode(env) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } - typeStr, _, err := req.Option("type").String() - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } + typeStr, _ := req.Options["type"].(string) switch typeStr { case "all", "direct", "indirect", "recursive": @@ -304,49 +302,40 @@ Example: var keys map[string]RefKeyObject - if len(req.Arguments()) > 0 { - keys, err = pinLsKeys(req.Context(), req.Arguments(), typeStr, n) + if len(req.Arguments) > 0 { + keys, err = pinLsKeys(req.Context, req.Arguments, typeStr, n) } else { - keys, err = pinLsAll(req.Context(), typeStr, n) + keys, err = pinLsAll(req.Context, typeStr, n) } if err != nil { res.SetError(err, cmdkit.ErrNormal) } else { - res.SetOutput(&RefKeyList{Keys: keys}) + res.Emit(&RefKeyList{Keys: keys}) } }, Type: RefKeyList{}, - Marshalers: cmds.MarshalerMap{ - cmds.Text: func(res cmds.Response) (io.Reader, error) { - v, err := unwrapOutput(res.Output()) - if err != nil { - return nil, err - } - - quiet, _, err := res.Request().Option("quiet").Bool() - if err != nil { - return nil, err - } + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { + quiet, _ := req.Options["quiet"].(bool) keys, ok := v.(*RefKeyList) if !ok { - return nil, e.TypeErr(keys, v) + return e.TypeErr(keys, v) } - out := new(bytes.Buffer) for k, v := range keys.Keys { if quiet { - fmt.Fprintf(out, "%s\n", k) + fmt.Fprintf(w, "%s\n", k) } else { - fmt.Fprintf(out, "%s %s\n", k, v.Type) + fmt.Fprintf(w, "%s %s\n", k, v.Type) } } - return out, nil - }, + return nil + }), }, } -var updatePinCmd = &cmds.Command{ +var updatePinCmd = &oldcmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Update a recursive pin", ShortDescription: ` @@ -364,7 +353,7 @@ new pin and removing the old one. cmdkit.BoolOption("unpin", "Remove the old pin.").WithDefault(true), }, Type: PinOutput{}, - Run: func(req cmds.Request, res cmds.Response) { + Run: func(req oldcmds.Request, res oldcmds.Response) { n, err := req.InvocContext().GetNode() if err != nil { res.SetError(err, cmdkit.ErrNormal) @@ -414,8 +403,8 @@ new pin and removing the old one. res.SetOutput(&PinOutput{Pins: []string{from.String(), to.String()}}) }, - Marshalers: cmds.MarshalerMap{ - cmds.Text: func(res cmds.Response) (io.Reader, error) { + Marshalers: oldcmds.MarshalerMap{ + oldcmds.Text: func(res oldcmds.Response) (io.Reader, error) { added, ok := res.Output().(*PinOutput) if !ok { return nil, u.ErrCast() @@ -428,7 +417,7 @@ new pin and removing the old one. }, } -var verifyPinCmd = &cmds.Command{ +var verifyPinCmd = &oldcmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Verify that recursive pins are complete.", }, @@ -436,7 +425,7 @@ var verifyPinCmd = &cmds.Command{ cmdkit.BoolOption("verbose", "Also write the hashes of non-broken pins."), cmdkit.BoolOption("quiet", "q", "Write just hashes of broken pins."), }, - Run: func(req cmds.Request, res cmds.Response) { + Run: func(req oldcmds.Request, res oldcmds.Response) { n, err := req.InvocContext().GetNode() if err != nil { res.SetError(err, cmdkit.ErrNormal) @@ -459,8 +448,8 @@ var verifyPinCmd = &cmds.Command{ res.SetOutput(out) }, Type: PinVerifyRes{}, - Marshalers: cmds.MarshalerMap{ - cmds.Text: func(res cmds.Response) (io.Reader, error) { + Marshalers: oldcmds.MarshalerMap{ + oldcmds.Text: func(res oldcmds.Response) (io.Reader, error) { quiet, _, _ := res.Request().Option("quiet").Bool() out, err := unwrapOutput(res.Output()) diff --git a/core/commands/root.go b/core/commands/root.go index 6f9c87adce8..f9a13d93f74 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -127,7 +127,7 @@ var rootSubcommands = map[string]*cmds.Command{ "mount": lgc.NewCommand(MountCmd), "name": lgc.NewCommand(NameCmd), "object": ocmd.ObjectCmd, - "pin": lgc.NewCommand(PinCmd), + "pin": PinCmd, "ping": lgc.NewCommand(PingCmd), "p2p": lgc.NewCommand(P2PCmd), "refs": lgc.NewCommand(RefsCmd), From abf45ff9157a4706905d9529f58e8ff239eb56ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 May 2018 01:32:31 +0200 Subject: [PATCH 2/3] pin cmd: stream recursive pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/pin.go | 80 +++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/core/commands/pin.go b/core/commands/pin.go index f168cc8f6d3..f5a81ce7f15 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -300,35 +300,29 @@ Example: return } - var keys map[string]RefKeyObject - if len(req.Arguments) > 0 { - keys, err = pinLsKeys(req.Context, req.Arguments, typeStr, n) + err = pinLsKeys(req.Context, req.Arguments, typeStr, n, res.Emit) } else { - keys, err = pinLsAll(req.Context, typeStr, n) + err = pinLsAll(req.Context, typeStr, n, res.Emit) } if err != nil { res.SetError(err, cmdkit.ErrNormal) - } else { - res.Emit(&RefKeyList{Keys: keys}) } }, - Type: RefKeyList{}, + Type: RefKeyObject{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { quiet, _ := req.Options["quiet"].(bool) - keys, ok := v.(*RefKeyList) + obj, ok := v.(*RefKeyObject) if !ok { - return e.TypeErr(keys, v) + return e.TypeErr(obj, v) } - for k, v := range keys.Keys { - if quiet { - fmt.Fprintf(w, "%s\n", k) - } else { - fmt.Fprintf(w, "%s %s\n", k, v.Type) - } + if quiet { + fmt.Fprintf(w, "%s\n", obj.Cid) + } else { + fmt.Fprintf(w, "%s %s\n", obj.Cid, obj.Type) } return nil }), @@ -474,21 +468,18 @@ var verifyPinCmd = &oldcmds.Command{ } type RefKeyObject struct { + Cid string Type string } -type RefKeyList struct { - Keys map[string]RefKeyObject -} - -func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsNode) (map[string]RefKeyObject, error) { +func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsNode, emit func(value interface{}) error) error { mode, ok := pin.StringToMode(typeStr) if !ok { - return nil, fmt.Errorf("invalid pin mode '%s'", typeStr) + return fmt.Errorf("invalid pin mode '%s'", typeStr) } - keys := make(map[string]RefKeyObject) + keys := make(map[string]struct{}) r := &resolver.Resolver{ DAG: n.DAG, @@ -498,21 +489,21 @@ func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsN for _, p := range args { pth, err := path.ParsePath(p) if err != nil { - return nil, err + return err } c, err := core.ResolveToCid(ctx, n.Namesys, r, pth) if err != nil { - return nil, err + return err } pinType, pinned, err := n.Pinning.IsPinnedWithType(c, mode) if err != nil { - return nil, err + return err } if !pinned { - return nil, fmt.Errorf("path '%s' is not pinned", p) + return fmt.Errorf("path '%s' is not pinned", p) } switch pinType { @@ -520,22 +511,28 @@ func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsN default: pinType = "indirect through " + pinType } - keys[c.String()] = RefKeyObject{ + keys[c.String()] = struct{}{} + + emit(&RefKeyObject{ Type: pinType, - } + Cid: c.String(), + }) } - return keys, nil + return nil } -func pinLsAll(ctx context.Context, typeStr string, n *core.IpfsNode) (map[string]RefKeyObject, error) { +func pinLsAll(ctx context.Context, typeStr string, n *core.IpfsNode, emit func(value interface{}) error) error { - keys := make(map[string]RefKeyObject) + keys := cid.NewSet() AddToResultKeys := func(keyList []*cid.Cid, typeStr string) { for _, c := range keyList { - keys[c.String()] = RefKeyObject{ - Type: typeStr, + if keys.Visit(c) { + emit(&RefKeyObject{ + Type: typeStr, + Cid: c.String(), + }) } } } @@ -544,20 +541,27 @@ func pinLsAll(ctx context.Context, typeStr string, n *core.IpfsNode) (map[string AddToResultKeys(n.Pinning.DirectKeys(), "direct") } if typeStr == "indirect" || typeStr == "all" { - set := cid.NewSet() for _, k := range n.Pinning.RecursiveKeys() { - err := dag.EnumerateChildren(ctx, dag.GetLinksWithDAG(n.DAG), k, set.Visit) + err := dag.EnumerateChildren(ctx, dag.GetLinksWithDAG(n.DAG), k, func(c *cid.Cid) bool { + r := keys.Visit(c) + if r { + emit(&RefKeyObject{ + Type: "indirect", + Cid: c.String(), + }) + } + return r + }) if err != nil { - return nil, err + return err } } - AddToResultKeys(set.Keys(), "indirect") } if typeStr == "recursive" || typeStr == "all" { AddToResultKeys(n.Pinning.RecursiveKeys(), "recursive") } - return keys, nil + return nil } // PinVerifyRes is the result returned for each pin checked in "pin verify" From ccbded69229728c7f8ade677a2b90f652125b6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Jun 2018 00:27:08 +0200 Subject: [PATCH 3/3] WIP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/pin.go | 64 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/core/commands/pin.go b/core/commands/pin.go index f5a81ce7f15..d657f3a1a28 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -22,9 +22,9 @@ import ( u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util" offline "gx/ipfs/QmPf114DXfa6TqGKYhBGR7EtXRho4rCJgwyA1xkuMY5vwF/go-ipfs-exchange-offline" + cmds "gx/ipfs/QmaFrNcnXHp579hUixbcTH1TNtNwsMogtBCwUUUwzBwYoM/go-ipfs-cmds" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" - cmds "gx/ipfs/QmaFrNcnXHp579hUixbcTH1TNtNwsMogtBCwUUUwzBwYoM/go-ipfs-cmds" ) var PinCmd = &cmds.Command{ @@ -282,6 +282,7 @@ Example: Options: []cmdkit.Option{ cmdkit.StringOption("type", "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"), cmdkit.BoolOption("quiet", "q", "Write just hashes of objects."), + cmdkit.BoolOption("stream", "Don't buffer pins before sending."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) { n, err := GetNode(env) @@ -291,6 +292,7 @@ Example: } typeStr, _ := req.Options["type"].(string) + stream, _ := req.Options["stream"].(bool) switch typeStr { case "all", "direct", "indirect", "recursive": @@ -300,30 +302,61 @@ Example: return } + emit := res.Emit + lgcList := map[string]RefObject{} + if !stream { + emit = func(v interface{}) error { + obj := v.(*RefKeyObject) + lgcList[obj.Cid] = RefObject{Type: obj.Type} + return nil + } + } + if len(req.Arguments) > 0 { - err = pinLsKeys(req.Context, req.Arguments, typeStr, n, res.Emit) + err = pinLsKeys(req.Context, req.Arguments, typeStr, n, emit) } else { - err = pinLsAll(req.Context, typeStr, n, res.Emit) + err = pinLsAll(req.Context, typeStr, n, emit) + } + + if !stream { + res.Emit(&RefKeyList{lgcList}) } if err != nil { res.SetError(err, cmdkit.ErrNormal) } }, - Type: RefKeyObject{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { quiet, _ := req.Options["quiet"].(bool) + stream, _ := req.Options["stream"].(bool) + + if stream { + obj, ok := v.(*RefKeyObject) + if !ok { + return e.TypeErr(obj, v) + } + if quiet { + fmt.Fprintf(w, "%s\n", obj.Cid) + } else { + fmt.Fprintf(w, "%s %s\n", obj.Cid, obj.Type) + } + return nil + } - obj, ok := v.(*RefKeyObject) + keys, ok := v.(*RefKeyList) if !ok { - return e.TypeErr(obj, v) + return e.TypeErr(keys, v) } - if quiet { - fmt.Fprintf(w, "%s\n", obj.Cid) - } else { - fmt.Fprintf(w, "%s %s\n", obj.Cid, obj.Type) + + for k, v := range keys.Keys { + if quiet { + fmt.Fprintf(w, "%s\n", k) + } else { + fmt.Fprintf(w, "%s %s\n", k, v.Type) + } } + return nil }), }, @@ -472,6 +505,14 @@ type RefKeyObject struct { Type string } +type RefObject struct { + Type string +} + +type RefKeyList struct { + Keys map[string]RefObject +} + func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsNode, emit func(value interface{}) error) error { mode, ok := pin.StringToMode(typeStr) @@ -479,8 +520,6 @@ func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsN return fmt.Errorf("invalid pin mode '%s'", typeStr) } - keys := make(map[string]struct{}) - r := &resolver.Resolver{ DAG: n.DAG, ResolveOnce: uio.ResolveUnixfsOnce, @@ -511,7 +550,6 @@ func pinLsKeys(ctx context.Context, args []string, typeStr string, n *core.IpfsN default: pinType = "indirect through " + pinType } - keys[c.String()] = struct{}{} emit(&RefKeyObject{ Type: pinType,