From 51235eeed36629d0c88276202876e11c373cb0bc Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 8 Sep 2020 15:05:00 -0700 Subject: [PATCH 001/127] first draft: ipfs pin remote add --- core/commands/pin.go | 1 + core/commands/remotepin.go | 97 ++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 4 ++ 4 files changed, 103 insertions(+) create mode 100644 core/commands/remotepin.go diff --git a/core/commands/pin.go b/core/commands/pin.go index 9d209040a59..affa2ef155d 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -35,6 +35,7 @@ var PinCmd = &cmds.Command{ "ls": listPinCmd, "verify": verifyPinCmd, "update": updatePinCmd, + "remote": remotePinCmd, }, } diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go new file mode 100644 index 00000000000..c9e74444d4b --- /dev/null +++ b/core/commands/remotepin.go @@ -0,0 +1,97 @@ +package commands + +import ( + "context" + "fmt" + "io" + "os" + + cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/go-ipfs/core/commands/cmdenv" + pinclient "github.com/ipfs/go-pinning-service-http-client" + path "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/multiformats/go-multiaddr" +) + +var remotePinURL = os.Getenv("IPFS_REMOTE_PIN_SERVICE") +var remotePinKey = os.Getenv("IPFS_REMOTE_PIN_KEY") + +var remotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Pin (and unpin) objects to remote pinning service.", + }, + + Subcommands: map[string]*cmds.Command{ + "add": addRemotePinCmd, + }, +} + +const pinNameOptionName = "name" + +type AddRemotePinOutput struct { + ID string + Name string + Delegates []multiaddr.Multiaddr +} + +var addRemotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Pin objects to remote storage.", + ShortDescription: "Stores an IPFS object(s) from a given path to a remote pinning service.", + }, + + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be pinned.").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.StringOption(pinNameOptionName, "An optional name for the pin."), + }, + Type: AddRemotePinOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opts := []pinclient.AddOption{} + if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { + opts = append(opts, pinclient.PinOpts.WithName(name)) + } + + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + + if len(req.Arguments) != 1 { + return fmt.Errorf("expecting one CID argument") + } + rp, err := api.ResolvePath(ctx, path.New(req.Arguments[0])) + if err != nil { + return err + } + + c := pinclient.NewClient(remotePinURL, remotePinKey) + + ps, err := c.Add(ctx, rp.Cid(), opts...) + + return res.Emit(&AddRemotePinOutput{ + ID: ps.GetId(), + Name: ps.GetPin().GetName(), + Delegates: ps.GetDelegates(), + }) + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddRemotePinOutput) error { + fmt.Printf("pin_id=%v\n", out.ID) + fmt.Printf("pin_name=%q\n", out.Name) + for _, d := range out.Delegates { + fmt.Printf("pin_delegate=%v\n", d.String()) + } + return nil + }), + }, + // PostRun: cmds.PostRunMap{ + // cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { + // XXX + // }, + // }, +} diff --git a/go.mod b/go.mod index 21081465235..bfcb756dab2 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 + github.com/ipfs/go-pinning-service-http-client v0.0.0-20200901171249-d3c88f6ac4ce github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 4ed782816aa..1d226f1ce71 100644 --- a/go.sum +++ b/go.sum @@ -454,6 +454,8 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20200901171249-d3c88f6ac4ce h1:T9SW+smyw1Wj07rDMFVgPmcPtx7HuJeaaGyXn4QuTSc= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20200901171249-d3c88f6ac4ce/go.mod h1:CnOlD0wFt1FJtL2colQy3WINJjhj0Ve7VzDWqjqDQiE= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= @@ -1356,6 +1358,7 @@ golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1486,6 +1489,7 @@ google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= From 7931918c86bae5af27cfc8b5e45653b2605c84e9 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 9 Sep 2020 08:53:17 -0700 Subject: [PATCH 002/127] first draft ls command --- core/commands/remotepin.go | 78 +++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index c9e74444d4b..577f644eb99 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -6,6 +6,7 @@ import ( "io" "os" + cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/go-ipfs/core/commands/cmdenv" pinclient "github.com/ipfs/go-pinning-service-http-client" @@ -23,10 +24,12 @@ var remotePinCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "add": addRemotePinCmd, + "ls": listRemotePinCmd, }, } const pinNameOptionName = "name" +const pinCIDsOptionName = "cid" type AddRemotePinOutput struct { ID string @@ -89,9 +92,74 @@ var addRemotePinCmd = &cmds.Command{ return nil }), }, - // PostRun: cmds.PostRunMap{ - // cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { - // XXX - // }, - // }, +} + +var listRemotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List objects pinned to remote pinning service.", + ShortDescription: ` +Returns a list of objects that are pinned to a remote pinning service. +`, + LongDescription: ` +Returns a list of objects that are pinned to a remote pinning service. +`, + }, + + Arguments: []cmds.Argument{}, + Options: []cmds.Option{ + cmds.StringOption(pinNameOptionName, "An optional name for the pin to be listed."), + cmds.StringsOption(pinCIDsOptionName, "An optional list of CIDs to be listed."), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opts := []pinclient.LsOption{} + if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { + opts = append(opts, pinclient.PinOpts.FilterName(name)) + } + if cidsRaw, cidsFound := req.Options[pinNameOptionName].([]string); cidsFound { + parsedCIDs := []cid.Cid{} + for _, rawCID := range cidsRaw { + parsedCID, err := cid.Decode(rawCID) + if err != nil { + return fmt.Errorf("CID %s cannot be parsed (%v)", rawCID, err) + } + parsedCIDs = append(parsedCIDs, parsedCID) + } + opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) + } + + c := pinclient.NewClient(remotePinURL, remotePinKey) + + psCh, errCh := c.Ls(ctx, opts...) + + for { + select { + case ps := <-psCh: + if err := res.Emit(&AddRemotePinOutput{ + ID: ps.GetId(), + Name: ps.GetPin().GetName(), + Delegates: ps.GetDelegates(), + }); err != nil { + return err + } + case err := <-errCh: + return err + case <-ctx.Done(): + return fmt.Errorf("interrupted") + } + } + }, + Type: &AddRemotePinOutput{}, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddRemotePinOutput) error { + fmt.Printf("pin_id=%v\n", out.ID) + fmt.Printf("pin_name=%q\n", out.Name) + for _, d := range out.Delegates { + fmt.Printf("pin_delegate=%v\n", d.String()) + } + return nil + }), + }, } From cd7bdc1164e15b713d59e6ac60f6de2fa6f8d00a Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 9 Sep 2020 10:24:04 -0700 Subject: [PATCH 003/127] first draft rm cmd --- core/commands/remotepin.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 577f644eb99..bb9bd4929f1 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -25,6 +25,7 @@ var remotePinCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "add": addRemotePinCmd, "ls": listRemotePinCmd, + "rm": rmRemotePinCmd, }, } @@ -75,6 +76,9 @@ var addRemotePinCmd = &cmds.Command{ c := pinclient.NewClient(remotePinURL, remotePinKey) ps, err := c.Add(ctx, rp.Cid(), opts...) + if err != nil { + return err + } return res.Emit(&AddRemotePinOutput{ ID: ps.GetId(), @@ -163,3 +167,30 @@ Returns a list of objects that are pinned to a remote pinning service. }), }, } + +var rmRemotePinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove pinned objects from remote pinning service.", + ShortDescription: ` +Removes the pin from the given object allowing it to be garbage +collected if needed. +`, + }, + + Arguments: []cmds.Argument{ + cmds.StringArg("pin-id", true, true, "ID of the pin to be removed.").EnableStdin(), + }, + Options: []cmds.Option{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if len(req.Arguments) == 0 { + return fmt.Errorf("missing a pin ID argument") + } + + c := pinclient.NewClient(remotePinURL, remotePinKey) + + return c.DeleteByID(ctx, req.Arguments[0]) + }, +} From 4f717f9861b60fb870152806f618552f4f980d81 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 10 Sep 2020 10:04:52 -0700 Subject: [PATCH 004/127] connect to pin delegates after add --- core/commands/remotepin.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index bb9bd4929f1..44cf62864a2 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -11,6 +11,7 @@ import ( "github.com/ipfs/go-ipfs/core/commands/cmdenv" pinclient "github.com/ipfs/go-pinning-service-http-client" path "github.com/ipfs/interface-go-ipfs-core/path" + peer "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" ) @@ -80,6 +81,18 @@ var addRemotePinCmd = &cmds.Command{ return err } + n, err := cmdenv.GetNode(env) + if err != nil { + return err + } + for _, d := range ps.GetDelegates() { + p, err := peer.AddrInfoFromP2pAddr(d) + if err != nil { + return err + } + n.PeerHost.Connect(ctx, *p) + } + return res.Emit(&AddRemotePinOutput{ ID: ps.GetId(), Name: ps.GetPin().GetName(), From 0eacb463e54cf68473e8d955dbfb5e1d388a8628 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 11 Sep 2020 12:02:14 -0400 Subject: [PATCH 005/127] fix: handle errors in remote pins --- core/commands/remotepin.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 44cf62864a2..7f684a07ea3 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -90,7 +90,10 @@ var addRemotePinCmd = &cmds.Command{ if err != nil { return err } - n.PeerHost.Connect(ctx, *p) + if err := n.PeerHost.Connect(ctx, *p); err != nil { + log.Infof("error connecting to remote pin delegate %v : %w", d, err) + } + } return res.Emit(&AddRemotePinOutput{ @@ -151,22 +154,17 @@ Returns a list of objects that are pinned to a remote pinning service. psCh, errCh := c.Ls(ctx, opts...) - for { - select { - case ps := <-psCh: - if err := res.Emit(&AddRemotePinOutput{ - ID: ps.GetId(), - Name: ps.GetPin().GetName(), - Delegates: ps.GetDelegates(), - }); err != nil { - return err - } - case err := <-errCh: + for ps := range psCh { + if err := res.Emit(&AddRemotePinOutput{ + ID: ps.GetId(), + Name: ps.GetPin().GetName(), + Delegates: ps.GetDelegates(), + }); err != nil { return err - case <-ctx.Done(): - return fmt.Errorf("interrupted") } } + + return <-errCh }, Type: &AddRemotePinOutput{}, Encoders: cmds.EncoderMap{ From ad925992daf0636f58d55d803b44c19fec78edd5 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 11 Sep 2020 12:04:59 -0400 Subject: [PATCH 006/127] bump go-pinning-service --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bfcb756dab2..ba5b19212ba 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client v0.0.0-20200901171249-d3c88f6ac4ce + github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 1d226f1ce71..7826a3806d2 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,8 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20200901171249-d3c88f6ac4ce h1:T9SW+smyw1Wj07rDMFVgPmcPtx7HuJeaaGyXn4QuTSc= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20200901171249-d3c88f6ac4ce/go.mod h1:CnOlD0wFt1FJtL2colQy3WINJjhj0Ve7VzDWqjqDQiE= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6 h1:mzOITuLX1cp3slCdrYwqcythh9en1yYRuSyi4WGjHB0= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6/go.mod h1:CnOlD0wFt1FJtL2colQy3WINJjhj0Ve7VzDWqjqDQiE= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= From 23a49e823f293f72cb780386aa70c0076dcfc1cd Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 11 Sep 2020 09:12:32 -0700 Subject: [PATCH 007/127] fix --- core/commands/remotepin.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 7f684a07ea3..217ac32abf8 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -81,16 +81,12 @@ var addRemotePinCmd = &cmds.Command{ return err } - n, err := cmdenv.GetNode(env) - if err != nil { - return err - } for _, d := range ps.GetDelegates() { p, err := peer.AddrInfoFromP2pAddr(d) if err != nil { return err } - if err := n.PeerHost.Connect(ctx, *p); err != nil { + if err := api.Swarm().Connect(ctx, *p); err != nil { log.Infof("error connecting to remote pin delegate %v : %w", d, err) } From ba49da36b91d319305d1902f815c8589fe58e427 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 11 Sep 2020 09:27:19 -0700 Subject: [PATCH 008/127] fix --- core/commands/remotepin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 217ac32abf8..1d881279608 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -51,7 +51,7 @@ var addRemotePinCmd = &cmds.Command{ Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), }, - Type: AddRemotePinOutput{}, + Type: &AddRemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 952efa2c0b4dcff279da40dc6bc6af7b865e71e2 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 11 Sep 2020 09:29:46 -0700 Subject: [PATCH 009/127] fix --- core/commands/remotepin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 1d881279608..066ff741353 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -51,7 +51,7 @@ var addRemotePinCmd = &cmds.Command{ Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), }, - Type: &AddRemotePinOutput{}, + Type: AddRemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -162,7 +162,7 @@ Returns a list of objects that are pinned to a remote pinning service. return <-errCh }, - Type: &AddRemotePinOutput{}, + Type: AddRemotePinOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddRemotePinOutput) error { fmt.Printf("pin_id=%v\n", out.ID) From a89c16a7a3100b9202bfa1fa54bfa73152d7e9e1 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 18 Sep 2020 09:28:56 -0700 Subject: [PATCH 010/127] add pinning api service to sharness circle ci --- .circleci/config.yml | 14 ++++++++++++++ core/commands/remotepin.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fe87124e89..397e91d5821 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,6 +93,17 @@ jobs: gotest: executor: golang steps: + # start remote pinning service + - checkout ~/ipfs-shipyard + # install rails + - run: + git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && + sudo apt install ruby ruby-dev zlib1g-dev && + sudo gem install rails + - run: + cd rb-pinning-service-api + docker-compose up -d + - checkout - *make_out_dirs - *restore_gomod @@ -131,6 +142,9 @@ jobs: when: always command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile + # prerequisite for docker compose + - setup_remote_docker + - run: mv "test/sharness/test-results/sharness.xml" /tmp/circleci-test-results/sharness # make sure we fail if there are test failures - run: find test/sharness/test-results -name 't*-*.sh.*.counts' | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0' diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 066ff741353..d57fa82e8a8 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -46,7 +46,7 @@ var addRemotePinCmd = &cmds.Command{ }, Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be pinned.").EnableStdin(), + cmds.StringArg("ipfs-path", true, false, "Path to object(s) to be pinned.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), From e29c51c067216cbbd183ca9cb354345cb628430f Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 18 Sep 2020 09:32:12 -0700 Subject: [PATCH 011/127] ok --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 397e91d5821..2e80ce9f91f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,7 +94,7 @@ jobs: executor: golang steps: # start remote pinning service - - checkout ~/ipfs-shipyard + - checkout # install rails - run: git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && From 82abe0b779069e2eacf0fcf9ed4b4503718366d1 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 18 Sep 2020 09:39:56 -0700 Subject: [PATCH 012/127] fix --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e80ce9f91f..fb2747a8b15 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,7 +101,7 @@ jobs: sudo apt install ruby ruby-dev zlib1g-dev && sudo gem install rails - run: - cd rb-pinning-service-api + cd rb-pinning-service-api && docker-compose up -d - checkout From bfd5b6b1aa4d0f4efd2e5d46d0608afe8f9eddb4 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 18 Sep 2020 09:51:32 -0700 Subject: [PATCH 013/127] ok --- .circleci/config.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb2747a8b15..3c4d38f2be0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,17 +93,6 @@ jobs: gotest: executor: golang steps: - # start remote pinning service - - checkout - # install rails - - run: - git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && - sudo apt install ruby ruby-dev zlib1g-dev && - sudo gem install rails - - run: - cd rb-pinning-service-api && - docker-compose up -d - - checkout - *make_out_dirs - *restore_gomod @@ -133,6 +122,16 @@ jobs: steps: - run: sudo apt install socat - checkout + + - run: + git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && + sudo apt install ruby ruby-dev zlib1g-dev && + sudo gem install rails + - setup_remote_docker + - run: + cd rb-pinning-service-api && + docker-compose up -d + - *make_out_dirs - *restore_gomod @@ -142,9 +141,6 @@ jobs: when: always command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile - # prerequisite for docker compose - - setup_remote_docker - - run: mv "test/sharness/test-results/sharness.xml" /tmp/circleci-test-results/sharness # make sure we fail if there are test failures - run: find test/sharness/test-results -name 't*-*.sh.*.counts' | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0' From 885c962079faf64e3c8d6c52ed16670b8656d5c6 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 18 Sep 2020 12:52:09 -0700 Subject: [PATCH 014/127] ok --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c4d38f2be0..0868457a2b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,6 +130,7 @@ jobs: - setup_remote_docker - run: cd rb-pinning-service-api && + echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && docker-compose up -d - *make_out_dirs From a086d7c49c172fdaa54257bf0ee2310e9a470da5 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 18 Sep 2020 13:03:36 -0700 Subject: [PATCH 015/127] Update core/commands/remotepin.go Co-authored-by: Marcin Rataj --- core/commands/remotepin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index d57fa82e8a8..a977e585066 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -124,7 +124,7 @@ Returns a list of objects that are pinned to a remote pinning service. Arguments: []cmds.Argument{}, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin to be listed."), - cmds.StringsOption(pinCIDsOptionName, "An optional list of CIDs to be listed."), + cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) From af1994bf71570c076a058b018189b5bd2f6b5a3b Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 18 Sep 2020 13:04:23 -0700 Subject: [PATCH 016/127] Update core/commands/remotepin.go Co-authored-by: Marcin Rataj --- core/commands/remotepin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index a977e585066..89f64a5ac96 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -123,7 +123,7 @@ Returns a list of objects that are pinned to a remote pinning service. Arguments: []cmds.Argument{}, Options: []cmds.Option{ - cmds.StringOption(pinNameOptionName, "An optional name for the pin to be listed."), + cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-insensitive, partial or full match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { From 27e5dad67a3dd329a024cf6e3a228539408390b7 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 21 Sep 2020 16:56:16 -0700 Subject: [PATCH 017/127] upgrades --- core/commands/remotepin.go | 4 ++-- go.mod | 2 +- go.sum | 2 ++ test/sharness/t0700-remotepin.sh | 39 ++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100755 test/sharness/t0700-remotepin.sh diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 89f64a5ac96..78333232e9d 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -93,7 +93,7 @@ var addRemotePinCmd = &cmds.Command{ } return res.Emit(&AddRemotePinOutput{ - ID: ps.GetId(), + ID: ps.GetRequestId(), Name: ps.GetPin().GetName(), Delegates: ps.GetDelegates(), }) @@ -152,7 +152,7 @@ Returns a list of objects that are pinned to a remote pinning service. for ps := range psCh { if err := res.Emit(&AddRemotePinOutput{ - ID: ps.GetId(), + ID: ps.GetRequestId(), Name: ps.GetPin().GetName(), Delegates: ps.GetDelegates(), }); err != nil { diff --git a/go.mod b/go.mod index ba5b19212ba..3ecae3e1382 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6 + github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 7826a3806d2..ce27d7656e2 100644 --- a/go.sum +++ b/go.sum @@ -456,6 +456,8 @@ github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6 h1:mzOITuLX1cp3slCdrYwqcythh9en1yYRuSyi4WGjHB0= github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6/go.mod h1:CnOlD0wFt1FJtL2colQy3WINJjhj0Ve7VzDWqjqDQiE= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073 h1:lb2nhLNWyQAU0wSWqMUFxp4IgVXigzVFHubFZxyLUzw= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh new file mode 100755 index 00000000000..ac5909c59a0 --- /dev/null +++ b/test/sharness/t0700-remotepin.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +test_description="Test ipfs remote pinning operations" + +. lib/test-lib.sh + +test_remote_pins() { + PIN_ARGS="$1" + LS_ARGS="$2" + BASE=$3 + if [ -n "$BASE" ]; then + BASE_ARGS="--cid-base=$BASE" + fi + + test_expect_success "create some hashes using base $BASE" ' + HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) && + HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && + HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) + ' + test_expect_success "'ipfs pin remote add $PIN_ARGS' via stdin" ' + ipfs pin remote add --enc=json $PIN_ARGS $BASE_ARGS $HASH_A + ' + +} + +test_init_ipfs + +# create user on pinning service +# curl -X POST http://163.172.146.161:5000/api/v1/users -d email=sharness@ipfs.io +# returns {"email":"sharness@ipfs.io","access_token":"b79694866904060dc21a4ddcced4f8f1"} + +IPFS_REMOTE_PIN_SERVICE=http://163.172.146.161:5000/api/v1 +IPFS_REMOTE_PIN_KEY="b79694866904060dc21a4ddcced4f8f1" + +test_remote_pins "" "" "" + +test_kill_ipfs_daemon + +test_done From 8e07f26d4a3fd23037bda0b54015d01fe10bea5f Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 22 Sep 2020 09:47:21 -0700 Subject: [PATCH 018/127] improvements --- core/commands/remotepin.go | 28 +++++++++++++++++++++++----- test/sharness/t0700-remotepin.sh | 28 +++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 78333232e9d..fdf078c0dff 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -36,7 +36,9 @@ const pinCIDsOptionName = "cid" type AddRemotePinOutput struct { ID string Name string - Delegates []multiaddr.Multiaddr + Delegates []string // multiaddr + Status string + Cid string } var addRemotePinCmd = &cmds.Command{ @@ -95,7 +97,9 @@ var addRemotePinCmd = &cmds.Command{ return res.Emit(&AddRemotePinOutput{ ID: ps.GetRequestId(), Name: ps.GetPin().GetName(), - Delegates: ps.GetDelegates(), + Delegates: multiaddrsToStrings(ps.GetDelegates()), + Status: ps.GetStatus().String(), + Cid: ps.GetPin().GetCid().String(), }) }, Encoders: cmds.EncoderMap{ @@ -103,13 +107,23 @@ var addRemotePinCmd = &cmds.Command{ fmt.Printf("pin_id=%v\n", out.ID) fmt.Printf("pin_name=%q\n", out.Name) for _, d := range out.Delegates { - fmt.Printf("pin_delegate=%v\n", d.String()) + fmt.Printf("pin_delegate=%v\n", d) } + fmt.Printf("pin_status=%v\n", out.Status) + fmt.Printf("pin_cid=%v\n", out.Cid) return nil }), }, } +func multiaddrsToStrings(m []multiaddr.Multiaddr) []string { + r := make([]string, len(m)) + for i := range m { + r[i] = m[i].String() + } + return r +} + var listRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects pinned to remote pinning service.", @@ -154,7 +168,9 @@ Returns a list of objects that are pinned to a remote pinning service. if err := res.Emit(&AddRemotePinOutput{ ID: ps.GetRequestId(), Name: ps.GetPin().GetName(), - Delegates: ps.GetDelegates(), + Delegates: multiaddrsToStrings(ps.GetDelegates()), + Status: ps.GetStatus().String(), + Cid: ps.GetPin().GetCid().String(), }); err != nil { return err } @@ -168,8 +184,10 @@ Returns a list of objects that are pinned to a remote pinning service. fmt.Printf("pin_id=%v\n", out.ID) fmt.Printf("pin_name=%q\n", out.Name) for _, d := range out.Delegates { - fmt.Printf("pin_delegate=%v\n", d.String()) + fmt.Printf("pin_delegate=%v\n", d) } + fmt.Printf("pin_status=%v\n", out.Status) + fmt.Printf("pin_cid=%v\n", out.Cid) return nil }), }, diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index ac5909c59a0..0d389c7e7da 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -17,16 +17,38 @@ test_remote_pins() { HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) ' - test_expect_success "'ipfs pin remote add $PIN_ARGS' via stdin" ' - ipfs pin remote add --enc=json $PIN_ARGS $BASE_ARGS $HASH_A + test_expect_success "'ipfs pin remote add $PIN_ARGS'" ' + ID_A=$(ipfs pin remote add --enc=json $PIN_ARGS $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .ID) ' + test_expect_success "'ipfs pin remote ls $PIN_ARGS' for existing pins by name" ' + FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a) | jq --raw-output .ID && + if [ $ID_A = $FOUND_ID_A ]; then + return 0 + else + return 1 + fi + ' + + test_expect_success "'ipfs pin remote rm $PIN_ARGS' an existing pin by ID" ' + ipfs pin remote rm --enc=json $ID_A) | jq --raw-output .ID + ' + + test_expect_success "'ipfs pin remote ls $PIN_ARGS' for deleted pin" ' + FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a) | jq --raw-output .ID && + if [ "" = $FOUND_ID_A ]; then + return 0 + else + return 1 + fi + ' } test_init_ipfs # create user on pinning service -# curl -X POST http://163.172.146.161:5000/api/v1/users -d email=sharness@ipfs.io +# IPFS_REMOTE_PIN_SERVICE=XXX +# IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output '.access_token') # returns {"email":"sharness@ipfs.io","access_token":"b79694866904060dc21a4ddcced4f8f1"} IPFS_REMOTE_PIN_SERVICE=http://163.172.146.161:5000/api/v1 From c58e2960353adc8c0f7cbe25fdaf7c929dbd3e9a Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 22 Sep 2020 10:00:44 -0700 Subject: [PATCH 019/127] ok --- test/sharness/t0700-remotepin.sh | 39 ++++++++++++++------------------ 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 0d389c7e7da..71f42d2c987 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -5,9 +5,8 @@ test_description="Test ipfs remote pinning operations" . lib/test-lib.sh test_remote_pins() { - PIN_ARGS="$1" - LS_ARGS="$2" - BASE=$3 + LS_ARGS="$1" + BASE=$2 if [ -n "$BASE" ]; then BASE_ARGS="--cid-base=$BASE" fi @@ -17,30 +16,26 @@ test_remote_pins() { HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) ' - test_expect_success "'ipfs pin remote add $PIN_ARGS'" ' - ID_A=$(ipfs pin remote add --enc=json $PIN_ARGS $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .ID) + test_expect_success "'ipfs pin remote add'" ' + ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .ID) ' - test_expect_success "'ipfs pin remote ls $PIN_ARGS' for existing pins by name" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a) | jq --raw-output .ID && - if [ $ID_A = $FOUND_ID_A ]; then - return 0 - else - return 1 - fi + test_expect_success "'ipfs pin remote ls' for existing pins by name" ' + FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .ID) && + echo $ID_A > expected + echo $FOUND_ID_A > actual + test_cmp expected actual ' - test_expect_success "'ipfs pin remote rm $PIN_ARGS' an existing pin by ID" ' - ipfs pin remote rm --enc=json $ID_A) | jq --raw-output .ID + test_expect_success "'ipfs pin remote rm' an existing pin by ID" ' + ipfs pin remote rm --enc=json $ID_A ' - test_expect_success "'ipfs pin remote ls $PIN_ARGS' for deleted pin" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a) | jq --raw-output .ID && - if [ "" = $FOUND_ID_A ]; then - return 0 - else - return 1 - fi + test_expect_success "'ipfs pin remote ls' for deleted pin" ' + FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .ID) && + echo "" > expected + echo $FOUND_ID_A > actual + test_cmp expected actual ' } @@ -54,7 +49,7 @@ test_init_ipfs IPFS_REMOTE_PIN_SERVICE=http://163.172.146.161:5000/api/v1 IPFS_REMOTE_PIN_KEY="b79694866904060dc21a4ddcced4f8f1" -test_remote_pins "" "" "" +test_remote_pins "" "" test_kill_ipfs_daemon From c8f07434cabee931023a7a00f86e9aa96a4ffaf3 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 22 Sep 2020 14:48:21 -0700 Subject: [PATCH 020/127] sharness ready for a CI spin --- test/sharness/t0700-remotepin.sh | 35 +++++++++++++------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 71f42d2c987..1c6e1681c21 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -5,25 +5,22 @@ test_description="Test ipfs remote pinning operations" . lib/test-lib.sh test_remote_pins() { - LS_ARGS="$1" - BASE=$2 + BASE=$1 if [ -n "$BASE" ]; then BASE_ARGS="--cid-base=$BASE" fi test_expect_success "create some hashes using base $BASE" ' - HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) && - HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && - HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) + HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) ' test_expect_success "'ipfs pin remote add'" ' ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .ID) ' test_expect_success "'ipfs pin remote ls' for existing pins by name" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .ID) && - echo $ID_A > expected - echo $FOUND_ID_A > actual + FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a --cid=$HASH_A | jq --raw-output .ID | grep $ID_A) && + echo $ID_A > expected && + echo $FOUND_ID_A > actual && test_cmp expected actual ' @@ -32,25 +29,21 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote ls' for deleted pin" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .ID) && - echo "" > expected - echo $FOUND_ID_A > actual - test_cmp expected actual + ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .ID > list + test_expect_code 1 grep $ID_A list ' } test_init_ipfs -# create user on pinning service -# IPFS_REMOTE_PIN_SERVICE=XXX -# IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output '.access_token') -# returns {"email":"sharness@ipfs.io","access_token":"b79694866904060dc21a4ddcced4f8f1"} - -IPFS_REMOTE_PIN_SERVICE=http://163.172.146.161:5000/api/v1 -IPFS_REMOTE_PIN_KEY="b79694866904060dc21a4ddcced4f8f1" +if [ -z "$IPFS_REMOTE_PIN_SERVICE" && -z "$IPFS_REMOTE_PIN_KEY" ]; then + # create user on pinning service + IPFS_REMOTE_PIN_SERVICE=localhost:5000 + IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output '.access_token') +fi -test_remote_pins "" "" +test_remote_pins "" -test_kill_ipfs_daemon +# test_kill_ipfs_daemon test_done From 4bb8dd6b0466a08f0863f2e274b0412f813ed200 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 22 Sep 2020 15:16:15 -0700 Subject: [PATCH 021/127] ok --- core/commands/commands_test.go | 6 +++++- test/sharness/t0700-remotepin.sh | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 50545418b76..0b99f89a390 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -177,11 +177,15 @@ func TestCommands(t *testing.T) { "/p2p/stream/ls", "/pin", "/pin/add", - "/ping", "/pin/ls", + "/pin/remote", + "/pin/remote/add", + "/pin/remote/ls", + "/pin/remote/rm", "/pin/rm", "/pin/update", "/pin/verify", + "/ping", "/pubsub", "/pubsub/ls", "/pubsub/peers", diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 1c6e1681c21..0239361b354 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -13,6 +13,11 @@ test_remote_pins() { test_expect_success "create some hashes using base $BASE" ' HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) ' + + test_expect_success "make sure we are connected to pinning service" ' + ipfs pin remote ls --enc=json + ' + test_expect_success "'ipfs pin remote add'" ' ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .ID) ' @@ -36,10 +41,13 @@ test_remote_pins() { test_init_ipfs -if [ -z "$IPFS_REMOTE_PIN_SERVICE" && -z "$IPFS_REMOTE_PIN_KEY" ]; then +if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] && [ -z "$IPFS_REMOTE_PIN_KEY" ]; then # create user on pinning service + echo "Creating test user on remote pinning service" IPFS_REMOTE_PIN_SERVICE=localhost:5000 IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output '.access_token') +else + echo "Using remote pinning service from environment" fi test_remote_pins "" From 74a00b131f0d7320b9fe1c30bc3a441e50dcffcd Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 22 Sep 2020 15:22:00 -0700 Subject: [PATCH 022/127] ok --- test/sharness/t0700-remotepin.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 0239361b354..4da0f5a642c 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -41,13 +41,12 @@ test_remote_pins() { test_init_ipfs -if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] && [ -z "$IPFS_REMOTE_PIN_KEY" ]; then +if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] || [ -z "$IPFS_REMOTE_PIN_KEY" ]; then # create user on pinning service - echo "Creating test user on remote pinning service" - IPFS_REMOTE_PIN_SERVICE=localhost:5000 - IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output '.access_token') -else - echo "Using remote pinning service from environment" + test_expect_success "creating test user on remote pinning service" ' + IPFS_REMOTE_PIN_SERVICE=localhost:5000 && + IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) + ' fi test_remote_pins "" From a17a1703ead44c18cdffd032e44b38985feec8de Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 23 Sep 2020 08:30:15 -0700 Subject: [PATCH 023/127] ok --- test/sharness/t0700-remotepin.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 4da0f5a642c..f6152e1ce85 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -14,7 +14,7 @@ test_remote_pins() { HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) ' - test_expect_success "make sure we are connected to pinning service" ' + test_expect_success "check connection to pinning service" ' ipfs pin remote ls --enc=json ' @@ -44,11 +44,15 @@ test_init_ipfs if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] || [ -z "$IPFS_REMOTE_PIN_KEY" ]; then # create user on pinning service test_expect_success "creating test user on remote pinning service" ' - IPFS_REMOTE_PIN_SERVICE=localhost:5000 && + IPFS_REMOTE_PIN_SERVICE=http://localhost:5000/api/v1 && IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) ' fi +test_expect_success "verify the pin service is reachable" ' + curl $IPFS_REMOTE_PIN_SERVICE +' + test_remote_pins "" # test_kill_ipfs_daemon From 3383e81864a548923565fc18ef88bf056a7639ba Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 23 Sep 2020 09:21:49 -0700 Subject: [PATCH 024/127] ok --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0868457a2b0..3fc0442f252 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,7 +131,8 @@ jobs: - run: cd rb-pinning-service-api && echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && - docker-compose up -d + docker-compose up -d && + docker ps - *make_out_dirs - *restore_gomod From e7d3d477f952308ac2b2902bb8d861d7d298f01e Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 23 Sep 2020 16:36:42 -0700 Subject: [PATCH 025/127] ok --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fc0442f252..ca75531c30c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,7 +132,8 @@ jobs: cd rb-pinning-service-api && echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && docker-compose up -d && - docker ps + docker ps && + curl http://localhost:5000/api/v1 - *make_out_dirs - *restore_gomod From cf4ae65a43ddc7ad75abbfa07b0c188ab2445834 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 23 Sep 2020 21:37:51 -0700 Subject: [PATCH 026/127] ok --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ca75531c30c..dd12a97a073 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,6 +133,7 @@ jobs: echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && docker-compose up -d && docker ps && + sleep 5 && curl http://localhost:5000/api/v1 - *make_out_dirs From 125bd8507e3ba3c754cf504cdad550ff607c79c0 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 25 Sep 2020 14:42:48 -0700 Subject: [PATCH 027/127] add ipfs pin remote service add --- .circleci/config.yml | 5 +-- core/commands/remotepin.go | 81 ++++++++++++++++++++++++++++++++++++-- go.mod | 2 +- go.sum | 2 + 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd12a97a073..0868457a2b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,10 +131,7 @@ jobs: - run: cd rb-pinning-service-api && echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && - docker-compose up -d && - docker ps && - sleep 5 && - curl http://localhost:5000/api/v1 + docker-compose up -d - *make_out_dirs - *restore_gomod diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index fdf078c0dff..98cdcff240b 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -8,7 +8,9 @@ import ( cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" + config "github.com/ipfs/go-ipfs-config" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" pinclient "github.com/ipfs/go-pinning-service-http-client" path "github.com/ipfs/interface-go-ipfs-core/path" peer "github.com/libp2p/go-libp2p-core/peer" @@ -24,9 +26,23 @@ var remotePinCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "add": addRemotePinCmd, - "ls": listRemotePinCmd, - "rm": rmRemotePinCmd, + "add": addRemotePinCmd, + "ls": listRemotePinCmd, + "rm": rmRemotePinCmd, + "service": remotePinServiceCmd, + }, +} + +var remotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Configure remote pinning services.", + }, + + Subcommands: map[string]*cmds.Command{ + "add": addRemotePinServiceCmd, + // "rename": renameRemotePinServiceCmd, + // "update": updateRemotePinServiceCmd, + // "rm": rmRemotePinServiceCmd, }, } @@ -58,6 +74,16 @@ var addRemotePinCmd = &cmds.Command{ ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // cfgRoot, err := cmdenv.GetConfigRoot(env) + // if err != nil { + // return err + // } + // repo, err := fsrepo.Open(cfgRoot) + // if err != nil { + // return err + // } + // defer repo.Close() + opts := []pinclient.AddOption{} if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { opts = append(opts, pinclient.PinOpts.WithName(name)) @@ -219,3 +245,52 @@ collected if needed. return c.DeleteByID(ctx, req.Arguments[0]) }, } + +var addRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Add remote pinning service.", + ShortDescription: "Add a credentials for access to a remote pinning service.", + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, true, "Name, URL and key (in that order) for a remote pinning service.").EnableStdin(), + }, + Options: []cmds.Option{}, + Type: nil, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + if len(req.Arguments) != 3 { + return fmt.Errorf("expecting three argument: name, url and key") + } + name := req.Arguments[0] + url := req.Arguments[1] + key := req.Arguments[2] + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.RemotePinServices.Services != nil { + if _, present := cfg.RemotePinServices.Services[name]; present { + return fmt.Errorf("service already present") + } + } else { + cfg.RemotePinServices.Services = map[string]config.RemotePinService{} + } + cfg.RemotePinServices.Services[name] = config.RemotePinService{ + Name: name, + URL: url, + Key: key, + } + + return repo.SetConfig(cfg) + }, +} diff --git a/go.mod b/go.mod index 3ecae3e1382..fd16cf3abfa 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.4.0 - github.com/ipfs/go-ipfs-config v0.9.0 + github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 diff --git a/go.sum b/go.sum index ce27d7656e2..5ba228538a2 100644 --- a/go.sum +++ b/go.sum @@ -360,6 +360,8 @@ github.com/ipfs/go-ipfs-cmds v0.4.0 h1:xUavIxA9Ts8U6PAHmQBvDGMlGfUrQ13Rymd+5t8LI github.com/ipfs/go-ipfs-cmds v0.4.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= github.com/ipfs/go-ipfs-config v0.9.0 h1:qTXJ9CyOyQv1LFJUMysxz8fi6RxxnP9QqcmiobuANvw= github.com/ipfs/go-ipfs-config v0.9.0/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= +github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3 h1:7EwzMoBhktnUjdgukBmOm+L0J8pwVlHdXEOJa5Ct6KA= +github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From 0c639f8b9f98043171df670b97d09a5bc714ba27 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 25 Sep 2020 14:58:55 -0700 Subject: [PATCH 028/127] ok --- core/commands/remotepin.go | 78 +++++++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 98cdcff240b..a6c8b80fd98 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -48,6 +48,7 @@ var remotePinServiceCmd = &cmds.Command{ const pinNameOptionName = "name" const pinCIDsOptionName = "cid" +const pinServiceNameOptionName = "service" type AddRemotePinOutput struct { ID string @@ -68,22 +69,13 @@ var addRemotePinCmd = &cmds.Command{ }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), + cmds.StringsOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Type: AddRemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // cfgRoot, err := cmdenv.GetConfigRoot(env) - // if err != nil { - // return err - // } - // repo, err := fsrepo.Open(cfgRoot) - // if err != nil { - // return err - // } - // defer repo.Close() - opts := []pinclient.AddOption{} if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { opts = append(opts, pinclient.PinOpts.WithName(name)) @@ -102,7 +94,15 @@ var addRemotePinCmd = &cmds.Command{ return err } - c := pinclient.NewClient(remotePinURL, remotePinKey) + service, serviceFound := req.Options[pinServiceNameOptionName].(string) + if !serviceFound { + return fmt.Errorf("remote pinning service name not specified") + } + url, key, err := getRemotePinServiceOrEnv(env, service) + if err != nil { + return err + } + c := pinclient.NewClient(url, key) ps, err := c.Add(ctx, rp.Cid(), opts...) if err != nil { @@ -165,6 +165,7 @@ Returns a list of objects that are pinned to a remote pinning service. Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-insensitive, partial or full match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), + cmds.StringsOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) @@ -186,7 +187,15 @@ Returns a list of objects that are pinned to a remote pinning service. opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) } - c := pinclient.NewClient(remotePinURL, remotePinKey) + service, serviceFound := req.Options[pinServiceNameOptionName].(string) + if !serviceFound { + return fmt.Errorf("remote pinning service name not specified") + } + url, key, err := getRemotePinServiceOrEnv(env, service) + if err != nil { + return err + } + c := pinclient.NewClient(url, key) psCh, errCh := c.Ls(ctx, opts...) @@ -231,7 +240,9 @@ collected if needed. Arguments: []cmds.Argument{ cmds.StringArg("pin-id", true, true, "ID of the pin to be removed.").EnableStdin(), }, - Options: []cmds.Option{}, + Options: []cmds.Option{ + cmds.StringsOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -240,7 +251,15 @@ collected if needed. return fmt.Errorf("missing a pin ID argument") } - c := pinclient.NewClient(remotePinURL, remotePinKey) + service, serviceFound := req.Options[pinServiceNameOptionName].(string) + if !serviceFound { + return fmt.Errorf("remote pinning service name not specified") + } + url, key, err := getRemotePinServiceOrEnv(env, service) + if err != nil { + return err + } + c := pinclient.NewClient(url, key) return c.DeleteByID(ctx, req.Arguments[0]) }, @@ -294,3 +313,34 @@ var addRemotePinServiceCmd = &cmds.Command{ return repo.SetConfig(cfg) }, } + +func getRemotePinServiceOrEnv(env cmds.Environment, name string) (url, key string, err error) { + if remotePinURL != "" && remotePinKey != "" { + return remotePinURL, remotePinKey, nil + } + return getRemotePinService(env, name) +} + +func getRemotePinService(env cmds.Environment, name string) (url, key string, err error) { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return "", "", err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return "", "", err + } + defer repo.Close() + cfg, err := repo.Config() + if err != nil { + return "", "", err + } + if cfg.RemotePinServices.Services == nil { + return "", "", fmt.Errorf("service not known") + } + service, present := cfg.RemotePinServices.Services[name] + if !present { + return "", "", fmt.Errorf("service not known") + } + return service.URL, service.Key, nil +} From d974b63fc637b515b38e6af4994b0d4bdb19378f Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 28 Sep 2020 09:26:54 -0700 Subject: [PATCH 029/127] add ipfs pin remote servce add|update|rm|rename --- core/commands/remotepin.go | 138 +++++++++++++++++++++++++++++++++++-- 1 file changed, 134 insertions(+), 4 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index a6c8b80fd98..298ce000858 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -39,10 +39,10 @@ var remotePinServiceCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "add": addRemotePinServiceCmd, - // "rename": renameRemotePinServiceCmd, - // "update": updateRemotePinServiceCmd, - // "rm": rmRemotePinServiceCmd, + "add": addRemotePinServiceCmd, + "rename": renameRemotePinServiceCmd, + "update": updateRemotePinServiceCmd, + "rm": rmRemotePinServiceCmd, }, } @@ -314,6 +314,136 @@ var addRemotePinServiceCmd = &cmds.Command{ }, } +var updateRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Update a remote pinning service.", + ShortDescription: "Update credentials for access to a remote pinning service.", + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, true, "Name, new URL and new key (in that order) for a remote pinning service.").EnableStdin(), + }, + Options: []cmds.Option{}, + Type: nil, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + if len(req.Arguments) != 3 { + return fmt.Errorf("expecting three argument: name, url and key") + } + name := req.Arguments[0] + url := req.Arguments[1] + key := req.Arguments[2] + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.RemotePinServices.Services == nil { + return fmt.Errorf("service not found") + } + if _, present := cfg.RemotePinServices.Services[name]; !present { + return fmt.Errorf("service not found") + } + cfg.RemotePinServices.Services[name] = config.RemotePinService{ + Name: name, + URL: url, + Key: key, + } + + return repo.SetConfig(cfg) + }, +} + +var rmRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove remote pinning service.", + ShortDescription: "Remove credentials for access to a remote pinning service.", + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, false, "Name of remote pinning service to remove.").EnableStdin(), + }, + Options: []cmds.Option{}, + Type: nil, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + if len(req.Arguments) != 1 { + return fmt.Errorf("expecting one argument: name") + } + name := req.Arguments[0] + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.RemotePinServices.Services != nil { + delete(cfg.RemotePinServices.Services, name) + } + return repo.SetConfig(cfg) + }, +} + +var renameRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Rename a remote pinning service.", + ShortDescription: "Rename a remote pinning service.", + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, true, "Old and new name of remote pinning service to rename.").EnableStdin(), + }, + Options: []cmds.Option{}, + Type: nil, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + if len(req.Arguments) != 2 { + return fmt.Errorf("expecting two arguments: old name and new name") + } + oldName := req.Arguments[0] + newName := req.Arguments[1] + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.RemotePinServices.Services == nil { + return fmt.Errorf("remote pinning service not found") + } + s, present := cfg.RemotePinServices.Services[oldName] + if !present { + return fmt.Errorf("remote pinning service not found") + } + delete(cfg.RemotePinServices.Services, oldName) + s.Name = newName + cfg.RemotePinServices.Services[newName] = s + return repo.SetConfig(cfg) + }, +} + func getRemotePinServiceOrEnv(env cmds.Environment, name string) (url, key string, err error) { if remotePinURL != "" && remotePinKey != "" { return remotePinURL, remotePinKey, nil From 4dd2806ada649ecfa46df531ff5f7c95b1a84ca4 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 29 Sep 2020 08:35:20 -0700 Subject: [PATCH 030/127] flag fixes --- core/commands/remotepin.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 298ce000858..e8cc4852d00 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -58,6 +58,8 @@ type AddRemotePinOutput struct { Cid string } +// remote pin commands + var addRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin objects to remote storage.", @@ -69,7 +71,7 @@ var addRemotePinCmd = &cmds.Command{ }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), - cmds.StringsOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Type: AddRemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -165,7 +167,7 @@ Returns a list of objects that are pinned to a remote pinning service. Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-insensitive, partial or full match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), - cmds.StringsOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) @@ -175,7 +177,7 @@ Returns a list of objects that are pinned to a remote pinning service. if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { opts = append(opts, pinclient.PinOpts.FilterName(name)) } - if cidsRaw, cidsFound := req.Options[pinNameOptionName].([]string); cidsFound { + if cidsRaw, cidsFound := req.Options[pinCIDsOptionName].([]string); cidsFound { parsedCIDs := []cid.Cid{} for _, rawCID := range cidsRaw { parsedCID, err := cid.Decode(rawCID) @@ -241,7 +243,7 @@ collected if needed. cmds.StringArg("pin-id", true, true, "ID of the pin to be removed.").EnableStdin(), }, Options: []cmds.Option{ - cmds.StringsOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) @@ -265,6 +267,8 @@ collected if needed. }, } +// remote service commands + var addRemotePinServiceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add remote pinning service.", From d51f78c6d133974901fcfc022d9a03a428c6e6f3 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 14 Oct 2020 14:30:24 -0700 Subject: [PATCH 031/127] execute ci using machine --- .circleci/config.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0868457a2b0..14af1198459 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,18 @@ jobs: - store_artifacts: path: /tmp/circleci-test-results sharness: - executor: golang + resource_class: xlarge +# executor: golang + machine: + image: ubuntu-1604:202007-01 + working_directory: ~/ipfs/go-ipfs + environment: + <<: *default_environment + GO111MODULE: "on" + TEST_NO_DOCKER: 1 + TEST_NO_FUSE: 1 + GOPATH: /home/circleci/go + TEST_VERBOSE: 1 steps: - run: sudo apt install socat - checkout @@ -131,7 +142,8 @@ jobs: - run: cd rb-pinning-service-api && echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && - docker-compose up -d + docker-compose up -d && + cd .. - *make_out_dirs - *restore_gomod From 82714497fb24553ffbfd77069cd58ff91b14ca2a Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 14 Oct 2020 14:32:05 -0700 Subject: [PATCH 032/127] rm setup_remote_docker --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 14af1198459..ad852ef9128 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,7 +138,7 @@ jobs: git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && sudo apt install ruby ruby-dev zlib1g-dev && sudo gem install rails - - setup_remote_docker +# - setup_remote_docker - run: cd rb-pinning-service-api && echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && From 87babfef8efd299165f4b7254bac76a77ee2bef3 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 14 Oct 2020 14:33:17 -0700 Subject: [PATCH 033/127] rm ruby install --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ad852ef9128..4bb95614040 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,8 +136,7 @@ jobs: - run: git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && - sudo apt install ruby ruby-dev zlib1g-dev && - sudo gem install rails + gem install rails # - setup_remote_docker - run: cd rb-pinning-service-api && From 9a33faa4ad0374d05f2c7ae16c15700834989d2a Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 14 Oct 2020 14:53:56 -0700 Subject: [PATCH 034/127] point remote pin test to service in ci --- .circleci/config.yml | 4 +++- test/sharness/t0700-remotepin.sh | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4bb95614040..c37e5614fad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,7 +147,9 @@ jobs: - *make_out_dirs - *restore_gomod - - run: make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 + - run: + export CI_HOST_IP=$(ip route|awk '/172/ { print $9 }') # get ip of circle ci host machine within the docker net + make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 - run: when: always diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index f6152e1ce85..45fef8314de 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -44,7 +44,8 @@ test_init_ipfs if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] || [ -z "$IPFS_REMOTE_PIN_KEY" ]; then # create user on pinning service test_expect_success "creating test user on remote pinning service" ' - IPFS_REMOTE_PIN_SERVICE=http://localhost:5000/api/v1 && + echo CI host IP address ${CI_HOST_IP} + IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) ' fi From ce4e47c787d38aefaff3be36fc919c70227c150f Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 14 Oct 2020 14:59:05 -0700 Subject: [PATCH 035/127] ok --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c37e5614fad..5f5c9192cc0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: - *restore_gomod - run: - export CI_HOST_IP=$(ip route|awk '/172/ { print $9 }') # get ip of circle ci host machine within the docker net + export CI_HOST_IP=$(ip route|awk '/172/ { print $9 }') && # get ip of circle ci host machine within the docker net make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 - run: From a723323a8ae39c27b591eb0ca29e2ba99b1a753a Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 14 Oct 2020 15:01:15 -0700 Subject: [PATCH 036/127] ok --- .circleci/config.yml | 2 +- test/sharness/t0700-remotepin.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f5c9192cc0..6922781a2f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: - *restore_gomod - run: - export CI_HOST_IP=$(ip route|awk '/172/ { print $9 }') && # get ip of circle ci host machine within the docker net + export CI_HOST_IP=$(ip route|awk '/172/ { print $9 }') && make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 - run: diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 45fef8314de..1cf8bacc1ce 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -44,7 +44,7 @@ test_init_ipfs if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] || [ -z "$IPFS_REMOTE_PIN_KEY" ]; then # create user on pinning service test_expect_success "creating test user on remote pinning service" ' - echo CI host IP address ${CI_HOST_IP} + echo CI host IP address ${CI_HOST_IP} && IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) ' From 2014d40cf213927cdeafd6f4616afae9d3d0d687 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 14 Oct 2020 15:04:14 -0700 Subject: [PATCH 037/127] ok --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6922781a2f2..4508dc4ca31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: - *restore_gomod - run: - export CI_HOST_IP=$(ip route|awk '/172/ { print $9 }') && + CI_HOST_IP=`ip route|awk "/172/ { print $9 }"` && make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 - run: From f940ce29f0b5be931da1387544aa04998e8b939b Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 07:24:22 -0700 Subject: [PATCH 038/127] ok --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4508dc4ca31..7b6f84b9ff4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,9 +147,13 @@ jobs: - *make_out_dirs - *restore_gomod + - run: + name: Setup Environment Variables + command: | + echo "export CI_HOST_IP=`ip route|awk '/172/ { print $9 }'`" >> $BASH_ENV + echo CI_HOST_IP=$CI_HOST_IP - run: - CI_HOST_IP=`ip route|awk "/172/ { print $9 }"` && - make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 + make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: when: always From 512d28a2edfd18d02854d116b5a93798048f9d86 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 07:40:55 -0700 Subject: [PATCH 039/127] ok --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b6f84b9ff4..ef9c0a3a822 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -150,9 +150,10 @@ jobs: - run: name: Setup Environment Variables command: | - echo "export CI_HOST_IP=`ip route|awk '/172/ { print $9 }'`" >> $BASH_ENV + echo 'export CI_HOST_IP=$(ip route|awk "/172/ { print $9 }")' >> $BASH_ENV echo CI_HOST_IP=$CI_HOST_IP - run: + echo CI_HOST_IP=$CI_HOST_IP && make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: From e5df8d45b55f886eeefc388e5b0957acdcc25bc6 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 07:46:34 -0700 Subject: [PATCH 040/127] fixes --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef9c0a3a822..202525b3795 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -149,9 +149,7 @@ jobs: - run: name: Setup Environment Variables - command: | - echo 'export CI_HOST_IP=$(ip route|awk "/172/ { print $9 }")' >> $BASH_ENV - echo CI_HOST_IP=$CI_HOST_IP + command: echo 'export CI_HOST_IP=$(ip route | grep docker | cut -d " " -f 12)' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP From 6af2be2d6af2e3312f86cc1da2f49609a0850e31 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 08:34:14 -0700 Subject: [PATCH 041/127] ok --- .circleci/config.yml | 4 +--- core/commands/remotepin.go | 18 ++++++------------ test/sharness/t0700-remotepin.sh | 4 ---- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 202525b3795..c5596b006d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,7 +119,6 @@ jobs: path: /tmp/circleci-test-results sharness: resource_class: xlarge -# executor: golang machine: image: ubuntu-1604:202007-01 working_directory: ~/ipfs/go-ipfs @@ -136,8 +135,7 @@ jobs: - run: git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && - gem install rails -# - setup_remote_docker + # gem install rails - run: cd rb-pinning-service-api && echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index e8cc4852d00..c45893955c7 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -96,10 +96,7 @@ var addRemotePinCmd = &cmds.Command{ return err } - service, serviceFound := req.Options[pinServiceNameOptionName].(string) - if !serviceFound { - return fmt.Errorf("remote pinning service name not specified") - } + service, _ := req.Options[pinServiceNameOptionName].(string) url, key, err := getRemotePinServiceOrEnv(env, service) if err != nil { return err @@ -189,10 +186,7 @@ Returns a list of objects that are pinned to a remote pinning service. opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) } - service, serviceFound := req.Options[pinServiceNameOptionName].(string) - if !serviceFound { - return fmt.Errorf("remote pinning service name not specified") - } + service, _ := req.Options[pinServiceNameOptionName].(string) url, key, err := getRemotePinServiceOrEnv(env, service) if err != nil { return err @@ -253,10 +247,7 @@ collected if needed. return fmt.Errorf("missing a pin ID argument") } - service, serviceFound := req.Options[pinServiceNameOptionName].(string) - if !serviceFound { - return fmt.Errorf("remote pinning service name not specified") - } + service, _ := req.Options[pinServiceNameOptionName].(string) url, key, err := getRemotePinServiceOrEnv(env, service) if err != nil { return err @@ -452,6 +443,9 @@ func getRemotePinServiceOrEnv(env cmds.Environment, name string) (url, key strin if remotePinURL != "" && remotePinKey != "" { return remotePinURL, remotePinKey, nil } + if name == "" { + return "", "", fmt.Errorf("remote pinning service name not specified") + } return getRemotePinService(env, name) } diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 1cf8bacc1ce..991bc669abc 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -50,10 +50,6 @@ if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] || [ -z "$IPFS_REMOTE_PIN_KEY" ]; then ' fi -test_expect_success "verify the pin service is reachable" ' - curl $IPFS_REMOTE_PIN_SERVICE -' - test_remote_pins "" # test_kill_ipfs_daemon From db63b3d5eda5618b0866e7ea1f083f771b8366ae Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 08:35:29 -0700 Subject: [PATCH 042/127] ok --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c5596b006d1..b2b37708060 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,7 +134,7 @@ jobs: - checkout - run: - git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git && + git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git # gem install rails - run: cd rb-pinning-service-api && From 482d0c6e4ba59f0ae23b589b507e81c3c13475f9 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 08:52:04 -0700 Subject: [PATCH 043/127] fix go test --- core/commands/commands_test.go | 5 +++++ core/commands/remotepin.go | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 0b99f89a390..8514c7f3265 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -182,6 +182,11 @@ func TestCommands(t *testing.T) { "/pin/remote/add", "/pin/remote/ls", "/pin/remote/rm", + "/pin/remote/service/rename", + "/pin/remote/service/rm", + "/pin/remote/service/add", + "/pin/remote/service/update", + "/pin/remote/service", "/pin/rm", "/pin/update", "/pin/verify", diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index c45893955c7..cf23849960e 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -50,8 +50,8 @@ const pinNameOptionName = "name" const pinCIDsOptionName = "cid" const pinServiceNameOptionName = "service" -type AddRemotePinOutput struct { - ID string +type RemotePinOutput struct { + RequestID string Name string Delegates []string // multiaddr Status string @@ -73,7 +73,7 @@ var addRemotePinCmd = &cmds.Command{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, - Type: AddRemotePinOutput{}, + Type: RemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -119,8 +119,8 @@ var addRemotePinCmd = &cmds.Command{ } - return res.Emit(&AddRemotePinOutput{ - ID: ps.GetRequestId(), + return res.Emit(&RemotePinOutput{ + RequestID: ps.GetRequestId(), Name: ps.GetPin().GetName(), Delegates: multiaddrsToStrings(ps.GetDelegates()), Status: ps.GetStatus().String(), @@ -128,8 +128,8 @@ var addRemotePinCmd = &cmds.Command{ }) }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddRemotePinOutput) error { - fmt.Printf("pin_id=%v\n", out.ID) + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { + fmt.Printf("pin_id=%v\n", out.RequestID) fmt.Printf("pin_name=%q\n", out.Name) for _, d := range out.Delegates { fmt.Printf("pin_delegate=%v\n", d) @@ -196,8 +196,8 @@ Returns a list of objects that are pinned to a remote pinning service. psCh, errCh := c.Ls(ctx, opts...) for ps := range psCh { - if err := res.Emit(&AddRemotePinOutput{ - ID: ps.GetRequestId(), + if err := res.Emit(&RemotePinOutput{ + RequestID: ps.GetRequestId(), Name: ps.GetPin().GetName(), Delegates: multiaddrsToStrings(ps.GetDelegates()), Status: ps.GetStatus().String(), @@ -209,10 +209,10 @@ Returns a list of objects that are pinned to a remote pinning service. return <-errCh }, - Type: AddRemotePinOutput{}, + Type: RemotePinOutput{}, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddRemotePinOutput) error { - fmt.Printf("pin_id=%v\n", out.ID) + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { + fmt.Printf("pin_id=%v\n", out.RequestID) fmt.Printf("pin_name=%q\n", out.Name) for _, d := range out.Delegates { fmt.Printf("pin_delegate=%v\n", d) @@ -234,7 +234,7 @@ collected if needed. }, Arguments: []cmds.Argument{ - cmds.StringArg("pin-id", true, true, "ID of the pin to be removed.").EnableStdin(), + cmds.StringArg("request-id", true, true, "Request ID of the pin to be removed.").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), @@ -244,7 +244,7 @@ collected if needed. defer cancel() if len(req.Arguments) == 0 { - return fmt.Errorf("missing a pin ID argument") + return fmt.Errorf("missing a pin request ID argument") } service, _ := req.Options[pinServiceNameOptionName].(string) From ae295f2f80c48b975e77ba78c36de52dbe0a09dc Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 08:55:32 -0700 Subject: [PATCH 044/127] ok --- test/sharness/t0700-remotepin.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 991bc669abc..c8446081e73 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -10,6 +10,9 @@ test_remote_pins() { BASE_ARGS="--cid-base=$BASE" fi + echo Using IPFS_REMOTE_PIN_SERVICE=$IPFS_REMOTE_PIN_SERVICE + echo Using IPFS_REMOTE_PIN_KEY=$IPFS_REMOTE_PIN_KEY + test_expect_success "create some hashes using base $BASE" ' HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) ' @@ -41,14 +44,12 @@ test_remote_pins() { test_init_ipfs -if [ -z "$IPFS_REMOTE_PIN_SERVICE" ] || [ -z "$IPFS_REMOTE_PIN_KEY" ]; then - # create user on pinning service - test_expect_success "creating test user on remote pinning service" ' - echo CI host IP address ${CI_HOST_IP} && - IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && - IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) - ' -fi +# create user on pinning service +test_expect_success "creating test user on remote pinning service" ' + echo CI host IP address ${CI_HOST_IP} && + IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && + IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) +' test_remote_pins "" From 95511887a40fdbf0bd206cdb7cd743c5e6811fb2 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 09:28:49 -0700 Subject: [PATCH 045/127] fixes --- test/sharness/t0700-remotepin.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index c8446081e73..22679e3d549 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -22,11 +22,11 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote add'" ' - ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .ID) + ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) ' test_expect_success "'ipfs pin remote ls' for existing pins by name" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a --cid=$HASH_A | jq --raw-output .ID | grep $ID_A) && + FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && echo $ID_A > expected && echo $FOUND_ID_A > actual && test_cmp expected actual @@ -37,7 +37,7 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote ls' for deleted pin" ' - ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .ID > list + ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .RequestID > list test_expect_code 1 grep $ID_A list ' } @@ -47,8 +47,8 @@ test_init_ipfs # create user on pinning service test_expect_success "creating test user on remote pinning service" ' echo CI host IP address ${CI_HOST_IP} && - IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && - IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) + export IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && + export IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) ' test_remote_pins "" From b0b79368ade5f7a77df22bccb5e279ad6cdaa3b3 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 10:42:29 -0700 Subject: [PATCH 046/127] ok --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2b37708060..9721e3e4a3a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,7 @@ jobs: - store_artifacts: path: /tmp/circleci-test-results sharness: - resource_class: xlarge + resource_class: 2xlarge machine: image: ubuntu-1604:202007-01 working_directory: ~/ipfs/go-ipfs From fd5977c3d74977f4e95d002c44904dbd7acd84c9 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 15 Oct 2020 11:02:25 -0700 Subject: [PATCH 047/127] ok --- test/sharness/t0700-remotepin.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 22679e3d549..bec8d9857d2 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -2,6 +2,7 @@ test_description="Test ipfs remote pinning operations" +set -x . lib/test-lib.sh test_remote_pins() { @@ -27,6 +28,7 @@ test_remote_pins() { test_expect_success "'ipfs pin remote ls' for existing pins by name" ' FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && + echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && echo $ID_A > expected && echo $FOUND_ID_A > actual && test_cmp expected actual From 269559b7713e826cc204a5c887047b2b307315a2 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 16 Oct 2020 12:56:44 -0700 Subject: [PATCH 048/127] update http client --- go.mod | 2 +- go.sum | 2 ++ test/sharness/t0700-remotepin.sh | 5 ++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index fd16cf3abfa..a2c41205f32 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073 + github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 5ba228538a2..6fc847420fb 100644 --- a/go.sum +++ b/go.sum @@ -460,6 +460,8 @@ github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6/go.mod h1:CnOlD0wFt1FJtL2colQy3WINJjhj0Ve7VzDWqjqDQiE= github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073 h1:lb2nhLNWyQAU0wSWqMUFxp4IgVXigzVFHubFZxyLUzw= github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1 h1:s8TGDcpmZRAs4iT4mnilVLA6hLelrXHyhFtO4/MaZZQ= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index bec8d9857d2..9329f355660 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -2,7 +2,6 @@ test_description="Test ipfs remote pinning operations" -set -x . lib/test-lib.sh test_remote_pins() { @@ -15,7 +14,7 @@ test_remote_pins() { echo Using IPFS_REMOTE_PIN_KEY=$IPFS_REMOTE_PIN_KEY test_expect_success "create some hashes using base $BASE" ' - HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) + export HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) ' test_expect_success "check connection to pinning service" ' @@ -23,7 +22,7 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote add'" ' - ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) + export ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) ' test_expect_success "'ipfs pin remote ls' for existing pins by name" ' From 5665beb7c8eba77830f3e9d1fe9b9aec2a9eae81 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 26 Oct 2020 11:52:05 -0700 Subject: [PATCH 049/127] try sleeping after ipfs pin remote add --- test/sharness/t0700-remotepin.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 9329f355660..5274eba3ee9 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -22,7 +22,8 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote add'" ' - export ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) + export ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && + sleep 3 # provide time for the pinning service to download the file ' test_expect_success "'ipfs pin remote ls' for existing pins by name" ' From aa9be939449b8459005ad6a13b1fd298725b2750 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 27 Oct 2020 11:58:12 -0400 Subject: [PATCH 050/127] ci: try later ubuntu sharness image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9721e3e4a3a..caad86f34c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,7 +120,7 @@ jobs: sharness: resource_class: 2xlarge machine: - image: ubuntu-1604:202007-01 + image: ubuntu-2004:202010-01 working_directory: ~/ipfs/go-ipfs environment: <<: *default_environment From 1decef07e5da21bd60803ce014862bdda33670c2 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 27 Oct 2020 09:13:22 -0700 Subject: [PATCH 051/127] reduce make parallelism to 3 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index caad86f34c5..8581e006c8b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -150,7 +150,7 @@ jobs: command: echo 'export CI_HOST_IP=$(ip route | grep docker | cut -d " " -f 12)' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && - make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP + make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: when: always From ef05661e57b1b005b7603ac540b74c8a29b39dc3 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 27 Oct 2020 12:28:15 -0400 Subject: [PATCH 052/127] change cut for CI Host IP, need something better --- .circleci/config.yml | 2 +- go.sum | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8581e006c8b..121c9a31a19 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,7 +147,7 @@ jobs: - run: name: Setup Environment Variables - command: echo 'export CI_HOST_IP=$(ip route | grep docker | cut -d " " -f 12)' >> $BASH_ENV + command: echo 'export CI_HOST_IP=$(ip route | grep docker | cut -d " " -f 9)' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP diff --git a/go.sum b/go.sum index 6fc847420fb..8518e780ff4 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,6 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.4.0 h1:xUavIxA9Ts8U6PAHmQBvDGMlGfUrQ13Rymd+5t8LIF4= github.com/ipfs/go-ipfs-cmds v0.4.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-config v0.9.0 h1:qTXJ9CyOyQv1LFJUMysxz8fi6RxxnP9QqcmiobuANvw= -github.com/ipfs/go-ipfs-config v0.9.0/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3 h1:7EwzMoBhktnUjdgukBmOm+L0J8pwVlHdXEOJa5Ct6KA= github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -456,10 +454,6 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6 h1:mzOITuLX1cp3slCdrYwqcythh9en1yYRuSyi4WGjHB0= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20200911012638-6064981450e6/go.mod h1:CnOlD0wFt1FJtL2colQy3WINJjhj0Ve7VzDWqjqDQiE= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073 h1:lb2nhLNWyQAU0wSWqMUFxp4IgVXigzVFHubFZxyLUzw= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20200921200811-9fbf23e5f073/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1 h1:s8TGDcpmZRAs4iT4mnilVLA6hLelrXHyhFtO4/MaZZQ= github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= From 0484dace3c307e326a583bfc8aad29fa6a0f8aca Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 27 Oct 2020 09:43:45 -0700 Subject: [PATCH 053/127] -j 10 again --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 121c9a31a19..7309e23ffef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -150,7 +150,7 @@ jobs: command: echo 'export CI_HOST_IP=$(ip route | grep docker | cut -d " " -f 9)' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && - make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP + make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: when: always From fba1fa307c671db4f5ec4c5348487aa771e50318 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 27 Oct 2020 10:01:36 -0700 Subject: [PATCH 054/127] hardcode ci host ip --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7309e23ffef..3f6cedaf55c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,10 +147,10 @@ jobs: - run: name: Setup Environment Variables - command: echo 'export CI_HOST_IP=$(ip route | grep docker | cut -d " " -f 9)' >> $BASH_ENV + command: echo 'export CI_HOST_IP=172.17.0.1' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && - make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP + make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: when: always From 21881111c5600330d2183aaa56911e30ff833b42 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 27 Oct 2020 14:59:23 -0700 Subject: [PATCH 055/127] add ipfs pin remote ls --status flag --- core/commands/remotepin.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index cf23849960e..01f214cbb8b 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -48,6 +48,7 @@ var remotePinServiceCmd = &cmds.Command{ const pinNameOptionName = "name" const pinCIDsOptionName = "cid" +const pinStatusOptionName = "status" const pinServiceNameOptionName = "service" type RemotePinOutput struct { @@ -164,6 +165,7 @@ Returns a list of objects that are pinned to a remote pinning service. Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-insensitive, partial or full match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), + cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses; optional, comma separated."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -185,6 +187,16 @@ Returns a list of objects that are pinned to a remote pinning service. } opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) } + if statusRaw, statusFound := req.Options[pinStatusOptionName].([]string); statusFound { + parsedStatuses := []pinclient.Status{} + for _, rawStatus := range statusRaw { + if pinclient.Status(rawStatus).String() == string(pinclient.StatusUnknown) { + return fmt.Errorf("status %s is not valid", rawStatus) + } + parsedStatuses = append(parsedStatuses, pinclient.Status(rawStatus)) + } + opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) + } service, _ := req.Options[pinServiceNameOptionName].(string) url, key, err := getRemotePinServiceOrEnv(env, service) From d96e6d19b3e66d01c36316470f063eebf6296338 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 27 Oct 2020 15:06:53 -0700 Subject: [PATCH 056/127] test pin remote ls --status --- test/sharness/t0700-remotepin.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 5274eba3ee9..a0491d38153 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -34,6 +34,14 @@ test_remote_pins() { test_cmp expected actual ' + test_expect_success "'ipfs pin remote ls' for existing pins by status" ' + FOUND_ID_A=$(ipfs pin remote ls --enc=json --status=pinned | jq --raw-output .RequestID | grep $ID_A) && + echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && + echo $ID_A > expected && + echo $FOUND_ID_A > actual && + test_cmp expected actual + ' + test_expect_success "'ipfs pin remote rm' an existing pin by ID" ' ipfs pin remote rm --enc=json $ID_A ' From ebc3e7d43dfe8bc53350fc2270acdd9af6dbac23 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 28 Oct 2020 09:00:19 -0700 Subject: [PATCH 057/127] add --name --status --cid to pin remote rm --- core/commands/remotepin.go | 103 ++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 37 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 01f214cbb8b..1defe0cfd70 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -169,43 +169,10 @@ Returns a list of objects that are pinned to a remote pinning service. cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - opts := []pinclient.LsOption{} - if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { - opts = append(opts, pinclient.PinOpts.FilterName(name)) - } - if cidsRaw, cidsFound := req.Options[pinCIDsOptionName].([]string); cidsFound { - parsedCIDs := []cid.Cid{} - for _, rawCID := range cidsRaw { - parsedCID, err := cid.Decode(rawCID) - if err != nil { - return fmt.Errorf("CID %s cannot be parsed (%v)", rawCID, err) - } - parsedCIDs = append(parsedCIDs, parsedCID) - } - opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) - } - if statusRaw, statusFound := req.Options[pinStatusOptionName].([]string); statusFound { - parsedStatuses := []pinclient.Status{} - for _, rawStatus := range statusRaw { - if pinclient.Status(rawStatus).String() == string(pinclient.StatusUnknown) { - return fmt.Errorf("status %s is not valid", rawStatus) - } - parsedStatuses = append(parsedStatuses, pinclient.Status(rawStatus)) - } - opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) - } - - service, _ := req.Options[pinServiceNameOptionName].(string) - url, key, err := getRemotePinServiceOrEnv(env, service) + psCh, errCh, err := lsRemote(req, env) if err != nil { return err } - c := pinclient.NewClient(url, key) - - psCh, errCh := c.Ls(ctx, opts...) for ps := range psCh { if err := res.Emit(&RemotePinOutput{ @@ -236,6 +203,48 @@ Returns a list of objects that are pinned to a remote pinning service. }, } +func lsRemote(req *cmds.Request, env cmds.Environment) (chan pinclient.PinStatusGetter, chan error, error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opts := []pinclient.LsOption{} + if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { + opts = append(opts, pinclient.PinOpts.FilterName(name)) + } + if cidsRaw, cidsFound := req.Options[pinCIDsOptionName].([]string); cidsFound { + parsedCIDs := []cid.Cid{} + for _, rawCID := range cidsRaw { + parsedCID, err := cid.Decode(rawCID) + if err != nil { + return nil, nil, fmt.Errorf("CID %s cannot be parsed (%v)", rawCID, err) + } + parsedCIDs = append(parsedCIDs, parsedCID) + } + opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) + } + if statusRaw, statusFound := req.Options[pinStatusOptionName].([]string); statusFound { + parsedStatuses := []pinclient.Status{} + for _, rawStatus := range statusRaw { + if pinclient.Status(rawStatus).String() == string(pinclient.StatusUnknown) { + return nil, nil, fmt.Errorf("status %s is not valid", rawStatus) + } + parsedStatuses = append(parsedStatuses, pinclient.Status(rawStatus)) + } + opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) + } + + service, _ := req.Options[pinServiceNameOptionName].(string) + url, key, err := getRemotePinServiceOrEnv(env, service) + if err != nil { + return nil, nil, err + } + c := pinclient.NewClient(url, key) + + psCh, errCh := c.Ls(ctx, opts...) + + return psCh, errCh, nil +} + var rmRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove pinned objects from remote pinning service.", @@ -246,17 +255,32 @@ collected if needed. }, Arguments: []cmds.Argument{ - cmds.StringArg("request-id", true, true, "Request ID of the pin to be removed.").EnableStdin(), + cmds.StringArg("request-id", false, true, "Request ID of the pin to be removed.").EnableStdin(), }, Options: []cmds.Option{ + cmds.StringOption(pinNameOptionName, "Remove pins objects with names that contain provided value (case-insensitive, partial or full match)."), + cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s); optional, comma separated."), + cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses; optional, comma separated."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + rmIDs := []string{} if len(req.Arguments) == 0 { - return fmt.Errorf("missing a pin request ID argument") + psCh, errCh, err := lsRemote(req, env) + if err != nil { + return err + } + for ps := range psCh { + rmIDs = append(rmIDs, ps.GetRequestId()) + } + if err = <-errCh; err != nil { + return fmt.Errorf("listing remote pin IDs (%v)", err) + } + } else { + rmIDs = append(rmIDs, req.Arguments[0]) } service, _ := req.Options[pinServiceNameOptionName].(string) @@ -266,7 +290,12 @@ collected if needed. } c := pinclient.NewClient(url, key) - return c.DeleteByID(ctx, req.Arguments[0]) + for _, rmID := range rmIDs { + if err = c.DeleteByID(ctx, rmID); err != nil { + return fmt.Errorf("removing pin with request ID %s (%v)", rmID, err) + } + } + return nil }, } From fb57add9ac1fe7df5eca94b0067f9c4402fd25a6 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 29 Oct 2020 08:18:41 -0700 Subject: [PATCH 058/127] add tests for all ls and remove flags --- test/sharness/t0700-remotepin.sh | 43 ++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index a0491d38153..abd81cff78d 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -4,6 +4,15 @@ test_description="Test ipfs remote pinning operations" . lib/test-lib.sh +test_init_ipfs + +# create user on pinning service +test_expect_success "creating test user on remote pinning service" ' + echo CI host IP address ${CI_HOST_IP} && + export IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && + export IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) +' + test_remote_pins() { BASE=$1 if [ -n "$BASE" ]; then @@ -14,7 +23,9 @@ test_remote_pins() { echo Using IPFS_REMOTE_PIN_KEY=$IPFS_REMOTE_PIN_KEY test_expect_success "create some hashes using base $BASE" ' - export HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) + export HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) ' test_expect_success "check connection to pinning service" ' @@ -23,11 +34,21 @@ test_remote_pins() { test_expect_success "'ipfs pin remote add'" ' export ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && + export ID_B=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && + export ID_C=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && sleep 3 # provide time for the pinning service to download the file ' + test_expect_success "'ipfs pin remote ls' for existing pins by ID" ' + FOUND_ID_A=$(ipfs pin remote ls --enc=json --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && + echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && + echo $ID_A > expected && + echo $FOUND_ID_A > actual && + test_cmp expected actual + ' + test_expect_success "'ipfs pin remote ls' for existing pins by name" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && + FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .RequestID | grep $ID_A) && echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && echo $ID_A > expected && echo $FOUND_ID_A > actual && @@ -46,23 +67,23 @@ test_remote_pins() { ipfs pin remote rm --enc=json $ID_A ' + test_expect_success "'ipfs pin remote rm' an existing pin by name" ' + ipfs pin remote rm --enc=json --name=name_b + ' + + test_expect_success "'ipfs pin remote rm' an existing pin by status" ' + ipfs pin remote rm --enc=json --status=pinned + ' + test_expect_success "'ipfs pin remote ls' for deleted pin" ' ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .RequestID > list test_expect_code 1 grep $ID_A list ' } -test_init_ipfs - -# create user on pinning service -test_expect_success "creating test user on remote pinning service" ' - echo CI host IP address ${CI_HOST_IP} && - export IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && - export IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) -' - test_remote_pins "" +# test_launch_ipfs_daemon # test_kill_ipfs_daemon test_done From 1c3d0336614af2046ad3ffeb465b7d7a38800dce Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 30 Oct 2020 09:24:36 -0700 Subject: [PATCH 059/127] update client dep --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a2c41205f32..c05ec2bc4d7 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1 + github.com/ipfs/go-pinning-service-http-client 38f3baeb88780e20a4fe144fa671befcb3985a5f github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 From 2d9607a5871d513651699a81b1a017b6c5513b5e Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 30 Oct 2020 10:27:19 -0700 Subject: [PATCH 060/127] fix context in lsRemote --- core/commands/remotepin.go | 12 ++++++------ go.mod | 2 +- go.sum | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 1defe0cfd70..7c10b52f38f 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -169,7 +169,10 @@ Returns a list of objects that are pinned to a remote pinning service. cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - psCh, errCh, err := lsRemote(req, env) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + psCh, errCh, err := lsRemote(ctx, req, env) if err != nil { return err } @@ -203,10 +206,7 @@ Returns a list of objects that are pinned to a remote pinning service. }, } -func lsRemote(req *cmds.Request, env cmds.Environment) (chan pinclient.PinStatusGetter, chan error, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - +func lsRemote(ctx context.Context, req *cmds.Request, env cmds.Environment) (chan pinclient.PinStatusGetter, chan error, error) { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { opts = append(opts, pinclient.PinOpts.FilterName(name)) @@ -269,7 +269,7 @@ collected if needed. rmIDs := []string{} if len(req.Arguments) == 0 { - psCh, errCh, err := lsRemote(req, env) + psCh, errCh, err := lsRemote(ctx, req, env) if err != nil { return err } diff --git a/go.mod b/go.mod index c05ec2bc4d7..d999924b924 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client 38f3baeb88780e20a4fe144fa671befcb3985a5f + github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 8518e780ff4..0bae74b8344 100644 --- a/go.sum +++ b/go.sum @@ -456,6 +456,8 @@ github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1 h1:s8TGDcpmZRAs4iT4mnilVLA6hLelrXHyhFtO4/MaZZQ= github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878 h1:RQLHEUEGAikm7pFNtxtSSin3TfRQdnhBcCrsaTxxLz0= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= From e8d2d6147ff66e09885522d920467d7c692990cf Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 2 Nov 2020 14:33:05 -0800 Subject: [PATCH 061/127] use remote pin service config in test; demote circle ci machine --- .circleci/config.yml | 2 +- test/sharness/t0700-remotepin.sh | 29 +++++++++++++---------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f6cedaf55c..a912fa725b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,7 @@ jobs: - store_artifacts: path: /tmp/circleci-test-results sharness: - resource_class: 2xlarge +# resource_class: 2xlarge machine: image: ubuntu-2004:202010-01 working_directory: ~/ipfs/go-ipfs diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index abd81cff78d..cc30a2bde6c 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -9,8 +9,8 @@ test_init_ipfs # create user on pinning service test_expect_success "creating test user on remote pinning service" ' echo CI host IP address ${CI_HOST_IP} && - export IPFS_REMOTE_PIN_SERVICE=http://${CI_HOST_IP}:5000/api/v1 && - export IPFS_REMOTE_PIN_KEY=$(curl -X POST $IPFS_REMOTE_PIN_SERVICE/users -d email=sharness@ipfs.io | jq --raw-output .access_token) + export TEST_PIN_SVC=http://${CI_HOST_IP}:5000/api/v1 && + ipfs pin remote service add test_pin_svc $TEST_PIN_SVC $(curl -X POST $TEST_PIN_SVC/users -d email=sharness@ipfs.io | jq --raw-output .access_token) ' test_remote_pins() { @@ -19,9 +19,6 @@ test_remote_pins() { BASE_ARGS="--cid-base=$BASE" fi - echo Using IPFS_REMOTE_PIN_SERVICE=$IPFS_REMOTE_PIN_SERVICE - echo Using IPFS_REMOTE_PIN_KEY=$IPFS_REMOTE_PIN_KEY - test_expect_success "create some hashes using base $BASE" ' export HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) && export HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && @@ -29,18 +26,18 @@ test_remote_pins() { ' test_expect_success "check connection to pinning service" ' - ipfs pin remote ls --enc=json + ipfs pin remote ls --service=test_pin_svc --enc=json ' test_expect_success "'ipfs pin remote add'" ' - export ID_A=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && - export ID_B=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && - export ID_C=$(ipfs pin remote add --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && + export ID_A=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && + export ID_B=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && + export ID_C=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && sleep 3 # provide time for the pinning service to download the file ' test_expect_success "'ipfs pin remote ls' for existing pins by ID" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && + FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && echo $ID_A > expected && echo $FOUND_ID_A > actual && @@ -48,7 +45,7 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote ls' for existing pins by name" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .RequestID | grep $ID_A) && + FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | jq --raw-output .RequestID | grep $ID_A) && echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && echo $ID_A > expected && echo $FOUND_ID_A > actual && @@ -56,7 +53,7 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote ls' for existing pins by status" ' - FOUND_ID_A=$(ipfs pin remote ls --enc=json --status=pinned | jq --raw-output .RequestID | grep $ID_A) && + FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json --status=pinned | jq --raw-output .RequestID | grep $ID_A) && echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && echo $ID_A > expected && echo $FOUND_ID_A > actual && @@ -64,19 +61,19 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote rm' an existing pin by ID" ' - ipfs pin remote rm --enc=json $ID_A + ipfs pin remote rm --service=test_pin_svc --enc=json $ID_A ' test_expect_success "'ipfs pin remote rm' an existing pin by name" ' - ipfs pin remote rm --enc=json --name=name_b + ipfs pin remote rm --service=test_pin_svc --enc=json --name=name_b ' test_expect_success "'ipfs pin remote rm' an existing pin by status" ' - ipfs pin remote rm --enc=json --status=pinned + ipfs pin remote rm --service=test_pin_svc --enc=json --status=pinned ' test_expect_success "'ipfs pin remote ls' for deleted pin" ' - ipfs pin remote ls --enc=json --name=name_a | jq --raw-output .RequestID > list + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | jq --raw-output .RequestID > list test_expect_code 1 grep $ID_A list ' } From 5102a9fc8b9d4aba4a5cab3375f9bf9f2ccd8ac9 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 2 Nov 2020 15:08:40 -0800 Subject: [PATCH 062/127] remove nonce secret key for ruby pinning --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a912fa725b1..e7aabdb4ee9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,6 @@ jobs: - store_artifacts: path: /tmp/circleci-test-results sharness: -# resource_class: 2xlarge machine: image: ubuntu-2004:202010-01 working_directory: ~/ipfs/go-ipfs @@ -135,10 +134,9 @@ jobs: - run: git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git - # gem install rails - run: cd rb-pinning-service-api && - echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && + # echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && docker-compose up -d && cd .. From 8d3f998c628bdd32c6d852f5977946ea93071c51 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 2 Nov 2020 15:14:00 -0800 Subject: [PATCH 063/127] fix comment in config.yml; remove IPFS_* env variables for pinning --- .circleci/config.yml | 4 ++-- core/commands/remotepin.go | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e7aabdb4ee9..475cc2d987d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,9 +134,8 @@ jobs: - run: git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git - - run: + - run: # echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && cd rb-pinning-service-api && - # echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && docker-compose up -d && cd .. @@ -145,6 +144,7 @@ jobs: - run: name: Setup Environment Variables + # 172.17.0.1 is the docker host IP; all ports exported by child containers can be accessed there. command: echo 'export CI_HOST_IP=172.17.0.1' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 7c10b52f38f..a2b5ee12336 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "os" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" @@ -17,9 +16,6 @@ import ( "github.com/multiformats/go-multiaddr" ) -var remotePinURL = os.Getenv("IPFS_REMOTE_PIN_SERVICE") -var remotePinKey = os.Getenv("IPFS_REMOTE_PIN_KEY") - var remotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin (and unpin) objects to remote pinning service.", @@ -481,9 +477,6 @@ var renameRemotePinServiceCmd = &cmds.Command{ } func getRemotePinServiceOrEnv(env cmds.Environment, name string) (url, key string, err error) { - if remotePinURL != "" && remotePinKey != "" { - return remotePinURL, remotePinKey, nil - } if name == "" { return "", "", fmt.Errorf("remote pinning service name not specified") } From fcaed78cc61aed3850c5f6dc05ecbbf909e60869 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 2 Nov 2020 15:42:57 -0800 Subject: [PATCH 064/127] ok --- .circleci/config.yml | 2 +- core/commands/commands_test.go | 4 ++-- core/commands/remotepin.go | 15 ++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 475cc2d987d..4128b91ff71 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,7 +134,7 @@ jobs: - run: git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git - - run: # echo "SECRET_KEY_BASE=28bcaf69b1991b" >> .env && + - run: cd rb-pinning-service-api && docker-compose up -d && cd .. diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 8514c7f3265..a35ba38005d 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -182,11 +182,11 @@ func TestCommands(t *testing.T) { "/pin/remote/add", "/pin/remote/ls", "/pin/remote/rm", + "/pin/remote/service", + "/pin/remote/service/add", "/pin/remote/service/rename", "/pin/remote/service/rm", - "/pin/remote/service/add", "/pin/remote/service/update", - "/pin/remote/service", "/pin/rm", "/pin/update", "/pin/verify", diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index a2b5ee12336..318df1a8a1d 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -72,7 +72,7 @@ var addRemotePinCmd = &cmds.Command{ }, Type: RemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(req.Context) defer cancel() opts := []pinclient.AddOption{} @@ -159,13 +159,13 @@ Returns a list of objects that are pinned to a remote pinning service. Arguments: []cmds.Argument{}, Options: []cmds.Option{ - cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-insensitive, partial or full match)."), + cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses; optional, comma separated."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(req.Context) defer cancel() psCh, errCh, err := lsRemote(ctx, req, env) @@ -221,10 +221,11 @@ func lsRemote(ctx context.Context, req *cmds.Request, env cmds.Environment) (cha if statusRaw, statusFound := req.Options[pinStatusOptionName].([]string); statusFound { parsedStatuses := []pinclient.Status{} for _, rawStatus := range statusRaw { - if pinclient.Status(rawStatus).String() == string(pinclient.StatusUnknown) { + s := pinclient.Status(rawStatus) + if s.String() == string(pinclient.StatusUnknown) { return nil, nil, fmt.Errorf("status %s is not valid", rawStatus) } - parsedStatuses = append(parsedStatuses, pinclient.Status(rawStatus)) + parsedStatuses = append(parsedStatuses, s) } opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) } @@ -254,13 +255,13 @@ collected if needed. cmds.StringArg("request-id", false, true, "Request ID of the pin to be removed.").EnableStdin(), }, Options: []cmds.Option{ - cmds.StringOption(pinNameOptionName, "Remove pins objects with names that contain provided value (case-insensitive, partial or full match)."), + cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s); optional, comma separated."), cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses; optional, comma separated."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(req.Context) defer cancel() rmIDs := []string{} From 4ea4d918f30f974c5091859e4663ef247a490b73 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 2 Nov 2020 16:43:12 -0800 Subject: [PATCH 065/127] add ipfs pin remote service ls --- core/commands/commands_test.go | 1 + core/commands/remotepin.go | 82 ++++++++++++++++++++++++++++---- test/sharness/t0700-remotepin.sh | 4 ++ 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index a35ba38005d..d7447ccf23c 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -184,6 +184,7 @@ func TestCommands(t *testing.T) { "/pin/remote/rm", "/pin/remote/service", "/pin/remote/service/add", + "/pin/remote/service/ls", "/pin/remote/service/rename", "/pin/remote/service/rm", "/pin/remote/service/update", diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 318df1a8a1d..1f69739344b 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "sort" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" @@ -36,9 +37,10 @@ var remotePinServiceCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "add": addRemotePinServiceCmd, + "ls": lsRemotePinServiceCmd, "rename": renameRemotePinServiceCmd, - "update": updateRemotePinServiceCmd, "rm": rmRemotePinServiceCmd, + "update": updateRemotePinServiceCmd, }, } @@ -94,11 +96,10 @@ var addRemotePinCmd = &cmds.Command{ } service, _ := req.Options[pinServiceNameOptionName].(string) - url, key, err := getRemotePinServiceOrEnv(env, service) + c, err := getRemotePinServiceOrEnv(env, service) if err != nil { return err } - c := pinclient.NewClient(url, key) ps, err := c.Add(ctx, rp.Cid(), opts...) if err != nil { @@ -231,11 +232,10 @@ func lsRemote(ctx context.Context, req *cmds.Request, env cmds.Environment) (cha } service, _ := req.Options[pinServiceNameOptionName].(string) - url, key, err := getRemotePinServiceOrEnv(env, service) + c, err := getRemotePinServiceOrEnv(env, service) if err != nil { return nil, nil, err } - c := pinclient.NewClient(url, key) psCh, errCh := c.Ls(ctx, opts...) @@ -281,11 +281,10 @@ collected if needed. } service, _ := req.Options[pinServiceNameOptionName].(string) - url, key, err := getRemotePinServiceOrEnv(env, service) + c, err := getRemotePinServiceOrEnv(env, service) if err != nil { return err } - c := pinclient.NewClient(url, key) for _, rmID := range rmIDs { if err = c.DeleteByID(ctx, rmID); err != nil { @@ -477,11 +476,74 @@ var renameRemotePinServiceCmd = &cmds.Command{ }, } -func getRemotePinServiceOrEnv(env cmds.Environment, name string) (url, key string, err error) { +var lsRemotePinServiceCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List remote pinning services.", + ShortDescription: "List remote pinning services.", + }, + Arguments: []cmds.Argument{}, + Options: []cmds.Option{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cfgRoot, err := cmdenv.GetConfigRoot(env) + if err != nil { + return err + } + repo, err := fsrepo.Open(cfgRoot) + if err != nil { + return err + } + defer repo.Close() + + cfg, err := repo.Config() + if err != nil { + return err + } + if cfg.RemotePinServices.Services == nil { + return nil // no pinning services added yet + } + result := sortedServiceAndURL{} + for svcName, svcConfig := range cfg.RemotePinServices.Services { + result = append(result, PinServiceAndURL{svcName, svcConfig.URL}) + } + sort.Sort(result) + for _, r := range result { + if err := res.Emit(r); err != nil { + return err + } + } + return nil + }, + Type: PinServiceAndURL{}, +} + +type PinServiceAndURL struct { + Service string + URL string +} + +type sortedServiceAndURL []PinServiceAndURL + +func (s sortedServiceAndURL) Len() int { + return len(s) +} + +func (s sortedServiceAndURL) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s sortedServiceAndURL) Less(i, j int) bool { + return s[i].Service < s[j].Service +} + +func getRemotePinServiceOrEnv(env cmds.Environment, name string) (*pinclient.Client, error) { if name == "" { - return "", "", fmt.Errorf("remote pinning service name not specified") + return nil, fmt.Errorf("remote pinning service name not specified") + } + url, key, err := getRemotePinService(env, name) + if err != nil { + return nil, err } - return getRemotePinService(env, name) + return pinclient.NewClient(url, key), nil } func getRemotePinService(env cmds.Environment, name string) (url, key string, err error) { diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index cc30a2bde6c..17f0941b98c 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -13,6 +13,10 @@ test_expect_success "creating test user on remote pinning service" ' ipfs pin remote service add test_pin_svc $TEST_PIN_SVC $(curl -X POST $TEST_PIN_SVC/users -d email=sharness@ipfs.io | jq --raw-output .access_token) ' +test_expect_success "test 'ipfs pin remote service ls'"' + ipfs pin remote service ls | jq --raw-output .Service | grep test_pin_svc +' + test_remote_pins() { BASE=$1 if [ -n "$BASE" ]; then From 12398e8719f9163f1c5cdab3677338f85722daee Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 3 Nov 2020 07:59:25 -0800 Subject: [PATCH 066/127] -j 10 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4128b91ff71..0c57382870c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: command: echo 'export CI_HOST_IP=172.17.0.1' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && - make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP + make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: when: always From 736afdff6b77b7045e1ab462282856c5eaafca5f Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 3 Nov 2020 08:19:31 -0800 Subject: [PATCH 067/127] add --background to pin add --- core/commands/remotepin.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 1f69739344b..75b034b4b79 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "sort" + "time" cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" @@ -48,6 +49,7 @@ const pinNameOptionName = "name" const pinCIDsOptionName = "cid" const pinStatusOptionName = "status" const pinServiceNameOptionName = "service" +const pinBackgroundOptionName = "background" type RemotePinOutput struct { RequestID string @@ -71,6 +73,7 @@ var addRemotePinCmd = &cmds.Command{ Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + cmds.BoolOption(pinBackgroundOptionName, "Add the pins in the background.").WithDefault(true), }, Type: RemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -117,6 +120,28 @@ var addRemotePinCmd = &cmds.Command{ } + if !req.Options[pinBackgroundOptionName].(bool) { + for { + ps, err = c.GetStatusByID(ctx, ps.GetRequestId()) + if err != nil { + return fmt.Errorf("failed to query pin (%v)", err) + } + s := ps.GetStatus() + if s == pinclient.StatusPinned { + break + } + if s == pinclient.StatusFailed { + return fmt.Errorf("failed to pin") + } + tmr := time.NewTimer(time.Second / 2) + select { + case <-tmr.C: + case <-ctx.Done(): + return fmt.Errorf("waiting for pin interrupted") + } + } + } + return res.Emit(&RemotePinOutput{ RequestID: ps.GetRequestId(), Name: ps.GetPin().GetName(), From eb57fbd9b885b5c3d92ffe72e5d5c16207f31d50 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 3 Nov 2020 08:44:23 -0800 Subject: [PATCH 068/127] scrap update and rename --- core/commands/commands_test.go | 2 - core/commands/remotepin.go | 103 ++------------------------------- 2 files changed, 4 insertions(+), 101 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index d7447ccf23c..657ed2f4069 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -185,9 +185,7 @@ func TestCommands(t *testing.T) { "/pin/remote/service", "/pin/remote/service/add", "/pin/remote/service/ls", - "/pin/remote/service/rename", "/pin/remote/service/rm", - "/pin/remote/service/update", "/pin/rm", "/pin/update", "/pin/verify", diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 75b034b4b79..80eb7cb8c3f 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -37,11 +37,9 @@ var remotePinServiceCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "add": addRemotePinServiceCmd, - "ls": lsRemotePinServiceCmd, - "rename": renameRemotePinServiceCmd, - "rm": rmRemotePinServiceCmd, - "update": updateRemotePinServiceCmd, + "add": addRemotePinServiceCmd, + "ls": lsRemotePinServiceCmd, + "rm": rmRemotePinServiceCmd, }, } @@ -328,7 +326,7 @@ var addRemotePinServiceCmd = &cmds.Command{ ShortDescription: "Add a credentials for access to a remote pinning service.", }, Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, true, "Name, URL and key (in that order) for a remote pinning service.").EnableStdin(), + cmds.StringArg("remote-pin-service", true, true, "Name, URL and key (in that order) for a remote pinning service.").EnableStdin(), }, Options: []cmds.Option{}, Type: nil, @@ -371,54 +369,6 @@ var addRemotePinServiceCmd = &cmds.Command{ }, } -var updateRemotePinServiceCmd = &cmds.Command{ - Helptext: cmds.HelpText{ - Tagline: "Update a remote pinning service.", - ShortDescription: "Update credentials for access to a remote pinning service.", - }, - Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, true, "Name, new URL and new key (in that order) for a remote pinning service.").EnableStdin(), - }, - Options: []cmds.Option{}, - Type: nil, - Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - cfgRoot, err := cmdenv.GetConfigRoot(env) - if err != nil { - return err - } - repo, err := fsrepo.Open(cfgRoot) - if err != nil { - return err - } - defer repo.Close() - - if len(req.Arguments) != 3 { - return fmt.Errorf("expecting three argument: name, url and key") - } - name := req.Arguments[0] - url := req.Arguments[1] - key := req.Arguments[2] - - cfg, err := repo.Config() - if err != nil { - return err - } - if cfg.RemotePinServices.Services == nil { - return fmt.Errorf("service not found") - } - if _, present := cfg.RemotePinServices.Services[name]; !present { - return fmt.Errorf("service not found") - } - cfg.RemotePinServices.Services[name] = config.RemotePinService{ - Name: name, - URL: url, - Key: key, - } - - return repo.SetConfig(cfg) - }, -} - var rmRemotePinServiceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove remote pinning service.", @@ -456,51 +406,6 @@ var rmRemotePinServiceCmd = &cmds.Command{ }, } -var renameRemotePinServiceCmd = &cmds.Command{ - Helptext: cmds.HelpText{ - Tagline: "Rename a remote pinning service.", - ShortDescription: "Rename a remote pinning service.", - }, - Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, true, "Old and new name of remote pinning service to rename.").EnableStdin(), - }, - Options: []cmds.Option{}, - Type: nil, - Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - cfgRoot, err := cmdenv.GetConfigRoot(env) - if err != nil { - return err - } - repo, err := fsrepo.Open(cfgRoot) - if err != nil { - return err - } - defer repo.Close() - - if len(req.Arguments) != 2 { - return fmt.Errorf("expecting two arguments: old name and new name") - } - oldName := req.Arguments[0] - newName := req.Arguments[1] - - cfg, err := repo.Config() - if err != nil { - return err - } - if cfg.RemotePinServices.Services == nil { - return fmt.Errorf("remote pinning service not found") - } - s, present := cfg.RemotePinServices.Services[oldName] - if !present { - return fmt.Errorf("remote pinning service not found") - } - delete(cfg.RemotePinServices.Services, oldName) - s.Name = newName - cfg.RemotePinServices.Services[newName] = s - return repo.SetConfig(cfg) - }, -} - var lsRemotePinServiceCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List remote pinning services.", From a463bc9799e553dc90dc02cc5ff145b112aa6e39 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 3 Nov 2020 08:44:59 -0800 Subject: [PATCH 069/127] -j 5 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c57382870c..c249a3a6084 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: command: echo 'export CI_HOST_IP=172.17.0.1' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && - make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP + make -O -j 5 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: when: always From 8dd822053758520373b7e9a3004b5c56d3d2c84b Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 3 Nov 2020 14:37:31 -0800 Subject: [PATCH 070/127] -j 3 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c249a3a6084..4128b91ff71 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: command: echo 'export CI_HOST_IP=172.17.0.1' >> $BASH_ENV - run: echo CI_HOST_IP=$CI_HOST_IP && - make -O -j 5 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP + make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP - run: when: always From 4322a2c7483d701c1d311751b4aa69961d7491e4 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 4 Nov 2020 08:04:56 -0800 Subject: [PATCH 071/127] prevent 'ipfs config' from showing remote pinning service info --- core/commands/config.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/commands/config.go b/core/commands/config.go index 50af4434357..22764e26abe 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -15,8 +15,8 @@ import ( "github.com/ipfs/go-ipfs/repo/fsrepo" "github.com/elgris/jsondiff" - "github.com/ipfs/go-ipfs-cmds" - "github.com/ipfs/go-ipfs-config" + cmds "github.com/ipfs/go-ipfs-cmds" + config "github.com/ipfs/go-ipfs-config" ) // ConfigUpdateOutput is config profile apply command's output @@ -83,6 +83,8 @@ Set the value of the 'Datastore.Path' key: switch strings.ToLower(key) { case "identity", "identity.privkey": return errors.New("cannot show or change private key through API") + case "remotepinservices": + return errors.New("cannot show remote pinning services, use 'ipfs pin remote service ls'") default: } @@ -144,7 +146,7 @@ var configShowCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Output config file contents.", ShortDescription: ` -NOTE: For security reasons, this command will omit your private key. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo. +NOTE: For security reasons, this command will omit your private key and remote services. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo. `, }, Type: map[string]interface{}{}, @@ -170,7 +172,7 @@ NOTE: For security reasons, this command will omit your private key. If you woul return err } - err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag}) + err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag, config.RemotePinServicesTag}) if err != nil { return err } From d56ec1463cf5368497d258aa7d1fa317ab3fae29 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 4 Nov 2020 08:22:35 -0800 Subject: [PATCH 072/127] lsRemote --- core/commands/remotepin.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 80eb7cb8c3f..da4241e43ab 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -192,7 +192,13 @@ Returns a list of objects that are pinned to a remote pinning service. ctx, cancel := context.WithCancel(req.Context) defer cancel() - psCh, errCh, err := lsRemote(ctx, req, env) + service, _ := req.Options[pinServiceNameOptionName].(string) + c, err := getRemotePinServiceOrEnv(env, service) + if err != nil { + return err + } + + psCh, errCh, err := lsRemote(ctx, req, c) if err != nil { return err } @@ -226,7 +232,7 @@ Returns a list of objects that are pinned to a remote pinning service. }, } -func lsRemote(ctx context.Context, req *cmds.Request, env cmds.Environment) (chan pinclient.PinStatusGetter, chan error, error) { +func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { opts = append(opts, pinclient.PinOpts.FilterName(name)) @@ -254,12 +260,6 @@ func lsRemote(ctx context.Context, req *cmds.Request, env cmds.Environment) (cha opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) } - service, _ := req.Options[pinServiceNameOptionName].(string) - c, err := getRemotePinServiceOrEnv(env, service) - if err != nil { - return nil, nil, err - } - psCh, errCh := c.Ls(ctx, opts...) return psCh, errCh, nil @@ -287,9 +287,15 @@ collected if needed. ctx, cancel := context.WithCancel(req.Context) defer cancel() + service, _ := req.Options[pinServiceNameOptionName].(string) + c, err := getRemotePinServiceOrEnv(env, service) + if err != nil { + return err + } + rmIDs := []string{} if len(req.Arguments) == 0 { - psCh, errCh, err := lsRemote(ctx, req, env) + psCh, errCh, err := lsRemote(ctx, req, c) if err != nil { return err } @@ -303,12 +309,6 @@ collected if needed. rmIDs = append(rmIDs, req.Arguments[0]) } - service, _ := req.Options[pinServiceNameOptionName].(string) - c, err := getRemotePinServiceOrEnv(env, service) - if err != nil { - return err - } - for _, rmID := range rmIDs { if err = c.DeleteByID(ctx, rmID); err != nil { return fmt.Errorf("removing pin with request ID %s (%v)", rmID, err) From 39a3539a38cd5a5e827ee6024d08536138506067 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 4 Nov 2020 09:17:32 -0800 Subject: [PATCH 073/127] fix --- core/commands/config.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/commands/config.go b/core/commands/config.go index 22764e26abe..df219b99423 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -172,7 +172,11 @@ NOTE: For security reasons, this command will omit your private key and remote s return err } - err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag, config.RemotePinServicesTag}) + err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag}) + if err != nil { + return err + } + err = scrubValue(cfg, []string{config.RemotePinServicesTag}) if err != nil { return err } From 4283745652d76cfe7c0f27244177677eded16d0f Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 4 Nov 2020 10:59:28 -0800 Subject: [PATCH 074/127] test pinning service registrygu --- test/sharness/t0700-remotepin.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 17f0941b98c..32d8ceacb06 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -10,11 +10,20 @@ test_init_ipfs test_expect_success "creating test user on remote pinning service" ' echo CI host IP address ${CI_HOST_IP} && export TEST_PIN_SVC=http://${CI_HOST_IP}:5000/api/v1 && - ipfs pin remote service add test_pin_svc $TEST_PIN_SVC $(curl -X POST $TEST_PIN_SVC/users -d email=sharness@ipfs.io | jq --raw-output .access_token) + ipfs pin remote service add test_pin_svc $TEST_PIN_SVC $(curl -X POST $TEST_PIN_SVC/users -d email=sharness@ipfs.io | jq --raw-output .access_token) && + ipfs pin remote service add fake_pin_svc http://0.0.0.0:5000 fake_api_key ' test_expect_success "test 'ipfs pin remote service ls'"' - ipfs pin remote service ls | jq --raw-output .Service | grep test_pin_svc + ipfs pin remote service ls | jq --raw-output .Service | grep test_pin_svc && + ipfs pin remote service ls | jq --raw-output .Service | grep fake_pin_svc +' +test_expect_success "check connection to test pinning service" ' + ipfs pin remote ls --service=test_pin_svc --enc=json +' + +test_expect_failure "test fake pinning service calls fail" ' + ipfs pin remote ls --service=fake_pin_svc --enc=json ' test_remote_pins() { @@ -29,10 +38,6 @@ test_remote_pins() { export HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) ' - test_expect_success "check connection to pinning service" ' - ipfs pin remote ls --service=test_pin_svc --enc=json - ' - test_expect_success "'ipfs pin remote add'" ' export ID_A=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && export ID_B=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && From 28d061465e41dd595d4062df953c45f2d699bb1c Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Wed, 4 Nov 2020 10:59:44 -0800 Subject: [PATCH 075/127] test remote pinning service config operations --- core/commands/remotepin.go | 2 +- test/sharness/t0700-remotepin.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index da4241e43ab..195fd46cd4a 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -375,7 +375,7 @@ var rmRemotePinServiceCmd = &cmds.Command{ ShortDescription: "Remove credentials for access to a remote pinning service.", }, Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, false, "Name of remote pinning service to remove.").EnableStdin(), + cmds.StringArg("remote-pin-service", true, false, "Name of remote pinning service to remove.").EnableStdin(), }, Options: []cmds.Option{}, Type: nil, diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 32d8ceacb06..6081e27348a 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -18,6 +18,7 @@ test_expect_success "test 'ipfs pin remote service ls'"' ipfs pin remote service ls | jq --raw-output .Service | grep test_pin_svc && ipfs pin remote service ls | jq --raw-output .Service | grep fake_pin_svc ' + test_expect_success "check connection to test pinning service" ' ipfs pin remote ls --service=test_pin_svc --enc=json ' @@ -26,6 +27,14 @@ test_expect_failure "test fake pinning service calls fail" ' ipfs pin remote ls --service=fake_pin_svc --enc=json ' +test_expect_success "remove fake pinning service" ' + ipfs pin remote service rm fake_pin_svc +' + +test_expect_failure "verify fake pinning service removed"' + ipfs pin remote service ls | jq --raw-output .Service | grep fake_pin_svc +' + test_remote_pins() { BASE=$1 if [ -n "$BASE" ]; then From 0047826b56275800939dd3994a2c34a2cd83af70 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 5 Nov 2020 10:19:38 -0800 Subject: [PATCH 076/127] test --background --- test/sharness/t0700-remotepin.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 6081e27348a..495647b546c 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -44,14 +44,20 @@ test_remote_pins() { test_expect_success "create some hashes using base $BASE" ' export HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) && export HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) + export HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_D=$(echo "D" | ipfs add $BASE_ARGS -q --pin=false) + ' + + test_expect_success "verify background add works" ' + export ID_D=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) && + sleep 3 && + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_d | jq --raw-output .Status | grep pinned ' test_expect_success "'ipfs pin remote add'" ' - export ID_A=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && - export ID_B=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && - export ID_C=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && - sleep 3 # provide time for the pinning service to download the file + export ID_A=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && + export ID_B=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && + export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) ' test_expect_success "'ipfs pin remote ls' for existing pins by ID" ' From ff89659f497e98b99f9238a51ee26c2c3aaf0ecc Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 5 Nov 2020 11:16:36 -0800 Subject: [PATCH 077/127] add --force --- core/commands/remotepin.go | 5 +++++ test/sharness/t0700-remotepin.sh | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 195fd46cd4a..39074c3b864 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -48,6 +48,7 @@ const pinCIDsOptionName = "cid" const pinStatusOptionName = "status" const pinServiceNameOptionName = "service" const pinBackgroundOptionName = "background" +const pinForceOptionName = "force" type RemotePinOutput struct { RequestID string @@ -282,6 +283,7 @@ collected if needed. cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s); optional, comma separated."), cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses; optional, comma separated."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + cmds.BoolOption(pinForceOptionName, "Delete multiple pins.").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(req.Context) @@ -305,6 +307,9 @@ collected if needed. if err = <-errCh; err != nil { return fmt.Errorf("listing remote pin IDs (%v)", err) } + if len(rmIDs) > 0 && !req.Options[pinForceOptionName].(bool) { + return fmt.Errorf("multiple pins may be removed, use --force") + } } else { rmIDs = append(rmIDs, req.Arguments[0]) } diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 495647b546c..3e261ecc962 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -88,12 +88,16 @@ test_remote_pins() { ipfs pin remote rm --service=test_pin_svc --enc=json $ID_A ' - test_expect_success "'ipfs pin remote rm' an existing pin by name" ' + test_expect_failure "'ipfs pin remote rm' needs --force when globbing" ' ipfs pin remote rm --service=test_pin_svc --enc=json --name=name_b ' + test_expect_success "'ipfs pin remote rm' an existing pin by name" ' + ipfs pin remote rm --service=test_pin_svc --enc=json --force --name=name_b + ' + test_expect_success "'ipfs pin remote rm' an existing pin by status" ' - ipfs pin remote rm --service=test_pin_svc --enc=json --status=pinned + ipfs pin remote rm --service=test_pin_svc --enc=json --force --status=pinned ' test_expect_success "'ipfs pin remote ls' for deleted pin" ' From 74260954e57003e033c06b0305a3fc5495ad9358 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 5 Nov 2020 15:14:23 -0800 Subject: [PATCH 078/127] add missing cid --- test/sharness/t0700-remotepin.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 3e261ecc962..00709eef947 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -57,7 +57,17 @@ test_remote_pins() { test_expect_success "'ipfs pin remote add'" ' export ID_A=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && export ID_B=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && - export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) + export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && + export HASH_MISSING="QmNRpQVA5n7osjtyjYaWEQpwYnbV1QoVkrSe7oyccMJh1m" && + export ID_M=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING | jq --raw-output .RequestID) + ' + + test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' + FOUND_ID_M=$(ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinned | jq --raw-output .RequestID | grep $ID_M) && + echo ID_M=$ID_M FOUND_ID_M=$FOUND_ID_M && + echo $ID_M > expected && + echo $FOUND_ID_M > actual && + test_cmp expected actual ' test_expect_success "'ipfs pin remote ls' for existing pins by ID" ' From b2d073ce0e746d4585fb416d2aec62e8da034ca0 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 6 Nov 2020 08:03:49 -0800 Subject: [PATCH 079/127] service add --name --url --key --- core/commands/remotepin.go | 21 ++++++++++++++++----- test/sharness/t0700-remotepin.sh | 4 ++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/commands/remotepin.go b/core/commands/remotepin.go index 39074c3b864..1d956c7671d 100644 --- a/core/commands/remotepin.go +++ b/core/commands/remotepin.go @@ -47,6 +47,8 @@ const pinNameOptionName = "name" const pinCIDsOptionName = "cid" const pinStatusOptionName = "status" const pinServiceNameOptionName = "service" +const pinServiceURLOptionName = "url" +const pinServiceKeyOptionName = "key" const pinBackgroundOptionName = "background" const pinForceOptionName = "force" @@ -332,6 +334,9 @@ var addRemotePinServiceCmd = &cmds.Command{ }, Arguments: []cmds.Argument{ cmds.StringArg("remote-pin-service", true, true, "Name, URL and key (in that order) for a remote pinning service.").EnableStdin(), + cmds.StringArg(pinServiceNameOptionName, true, false, "Service name."), + cmds.StringArg(pinServiceURLOptionName, true, false, "Service URL."), + cmds.StringArg(pinServiceKeyOptionName, true, false, "Service key."), }, Options: []cmds.Option{}, Type: nil, @@ -346,12 +351,18 @@ var addRemotePinServiceCmd = &cmds.Command{ } defer repo.Close() - if len(req.Arguments) != 3 { - return fmt.Errorf("expecting three argument: name, url and key") + name, nameFound := req.Options[pinServiceNameOptionName].(string) + if !nameFound { + return fmt.Errorf("service name not given") + } + url, urlFound := req.Options[pinServiceURLOptionName].(string) + if !urlFound { + return fmt.Errorf("service url not given") + } + key, keyFound := req.Options[pinServiceKeyOptionName].(string) + if !keyFound { + return fmt.Errorf("service key not given") } - name := req.Arguments[0] - url := req.Arguments[1] - key := req.Arguments[2] cfg, err := repo.Config() if err != nil { diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 00709eef947..e4770722717 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -10,8 +10,8 @@ test_init_ipfs test_expect_success "creating test user on remote pinning service" ' echo CI host IP address ${CI_HOST_IP} && export TEST_PIN_SVC=http://${CI_HOST_IP}:5000/api/v1 && - ipfs pin remote service add test_pin_svc $TEST_PIN_SVC $(curl -X POST $TEST_PIN_SVC/users -d email=sharness@ipfs.io | jq --raw-output .access_token) && - ipfs pin remote service add fake_pin_svc http://0.0.0.0:5000 fake_api_key + ipfs pin remote service add --name=test_pin_svc --url=$TEST_PIN_SVC --key=$(curl -X POST $TEST_PIN_SVC/users -d email=sharness@ipfs.io | jq --raw-output .access_token) && + ipfs pin remote service add --name=fake_pin_svc --url=http://0.0.0.0:5000 --key=fake_api_key ' test_expect_success "test 'ipfs pin remote service ls'"' From a3e500c4d01632319eb0abeafa16a992285f27cf Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 6 Nov 2020 08:12:40 -0800 Subject: [PATCH 080/127] pin package --- core/commands/{ => pin}/pin.go | 2 +- core/commands/{ => pin}/remotepin.go | 5 ++++- core/commands/root.go | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) rename core/commands/{ => pin}/pin.go (99%) rename core/commands/{ => pin}/remotepin.go (99%) diff --git a/core/commands/pin.go b/core/commands/pin/pin.go similarity index 99% rename from core/commands/pin.go rename to core/commands/pin/pin.go index affa2ef155d..a951bac950d 100644 --- a/core/commands/pin.go +++ b/core/commands/pin/pin.go @@ -1,4 +1,4 @@ -package commands +package pin import ( "context" diff --git a/core/commands/remotepin.go b/core/commands/pin/remotepin.go similarity index 99% rename from core/commands/remotepin.go rename to core/commands/pin/remotepin.go index 1d956c7671d..58fb656db84 100644 --- a/core/commands/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -1,4 +1,4 @@ -package commands +package pin import ( "context" @@ -12,12 +12,15 @@ import ( config "github.com/ipfs/go-ipfs-config" "github.com/ipfs/go-ipfs/core/commands/cmdenv" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + logging "github.com/ipfs/go-log" pinclient "github.com/ipfs/go-pinning-service-http-client" path "github.com/ipfs/interface-go-ipfs-core/path" peer "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" ) +var log = logging.Logger("core/commands/cmdenv") + var remotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin (and unpin) objects to remote pinning service.", diff --git a/core/commands/root.go b/core/commands/root.go index b9a8dc40909..1440c31b7eb 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -7,6 +7,7 @@ import ( dag "github.com/ipfs/go-ipfs/core/commands/dag" name "github.com/ipfs/go-ipfs/core/commands/name" ocmd "github.com/ipfs/go-ipfs/core/commands/object" + "github.com/ipfs/go-ipfs/core/commands/pin" unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs" cmds "github.com/ipfs/go-ipfs-cmds" @@ -136,7 +137,7 @@ var rootSubcommands = map[string]*cmds.Command{ "mount": MountCmd, "name": name.NameCmd, "object": ocmd.ObjectCmd, - "pin": PinCmd, + "pin": pin.PinCmd, "ping": PingCmd, "p2p": P2PCmd, "refs": RefsCmd, From 6511f423c11d478b88703f5f124b4e452dce3259 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 6 Nov 2020 09:18:29 -0800 Subject: [PATCH 081/127] fix --- core/commands/pin/remotepin.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 58fb656db84..2a178a0d4b1 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -336,7 +336,6 @@ var addRemotePinServiceCmd = &cmds.Command{ ShortDescription: "Add a credentials for access to a remote pinning service.", }, Arguments: []cmds.Argument{ - cmds.StringArg("remote-pin-service", true, true, "Name, URL and key (in that order) for a remote pinning service.").EnableStdin(), cmds.StringArg(pinServiceNameOptionName, true, false, "Service name."), cmds.StringArg(pinServiceURLOptionName, true, false, "Service URL."), cmds.StringArg(pinServiceKeyOptionName, true, false, "Service key."), From 82142ccd52eaff97a4567c03da1ff75fbc6759b6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 12 Nov 2020 17:50:47 +0100 Subject: [PATCH 082/127] fix(ci): use the latest docker images --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4128b91ff71..cdb303ba257 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,9 +133,10 @@ jobs: - checkout - run: - git clone https://github.com/ipfs-shipyard/rb-pinning-service-api.git + git clone --depth 1 https://github.com/ipfs-shipyard/rb-pinning-service-api.git - run: cd rb-pinning-service-api && + docker-compose pull && docker-compose up -d && cd .. From a16e847bdfa7098b023729b9e5c25c34ca7deade Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 12 Nov 2020 21:41:38 +0100 Subject: [PATCH 083/127] fix(ci): deterministic rb-pinning-service-api This ensures sharness tests are not broken when new version of rb-pinning-service-api lands in master --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cdb303ba257..252fb31e6bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,6 +136,7 @@ jobs: git clone --depth 1 https://github.com/ipfs-shipyard/rb-pinning-service-api.git - run: cd rb-pinning-service-api && + git checkout c6ad5c428ec3422bc56131c5150d6b00815abfee && docker-compose pull && docker-compose up -d && cd .. From 23b559e8fd336ac9b93eacc5bcc860714c11f57c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 13 Nov 2020 04:12:04 +0100 Subject: [PATCH 084/127] fix: pin remote improvements - wip support for Pin.origins (commented out) - safeguard against remote service changing remoteid during add with background=false - change default output to follow commands like 'files stat' - add bunch of TODOs for identified problems - 'pin remote service add' - require three arguments - validate that URL is URL and starts with http - refactor relevant tests - add test for invalid URL (two cases: 404 and DNS) - go mod tidy - tests - 'pin remote ls' - switch --status to working notation, add TODO about supporting shorter form - removed 'sleep' from tests (we are testing 'pin remote', not the pinning service itself) - added test for inlined CID --- core/commands/pin/remotepin.go | 108 +++++++++++++++++++++---------- go.sum | 2 - test/sharness/t0700-remotepin.sh | 105 ++++++++++++++++++++++-------- 3 files changed, 153 insertions(+), 62 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 2a178a0d4b1..e3b269c3439 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -5,8 +5,11 @@ import ( "fmt" "io" "sort" + "strings" "time" + neturl "net/url" + cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" config "github.com/ipfs/go-ipfs-config" @@ -84,36 +87,66 @@ var addRemotePinCmd = &cmds.Command{ ctx, cancel := context.WithCancel(req.Context) defer cancel() - opts := []pinclient.AddOption{} - if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { - opts = append(opts, pinclient.PinOpts.WithName(name)) - } - - api, err := cmdenv.GetApi(env, req) + // Get remote service + service, _ := req.Options[pinServiceNameOptionName].(string) + c, err := getRemotePinServiceOrEnv(env, service) if err != nil { return err } + // Prepare value for Pin.cid if len(req.Arguments) != 1 { return fmt.Errorf("expecting one CID argument") } + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } rp, err := api.ResolvePath(ctx, path.New(req.Arguments[0])) if err != nil { return err } - service, _ := req.Options[pinServiceNameOptionName].(string) - c, err := getRemotePinServiceOrEnv(env, service) + // Prepare Pin.name + opts := []pinclient.AddOption{} + if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { + opts = append(opts, pinclient.PinOpts.WithName(name)) + } + + /* TODO + // Prepare Pin.origins + // Add own multiaddrs to the 'origins' array, so Pinning Service can + // use that as a hint and connect back to us (if possible) + node, err := cmdenv.GetNode(env) if err != nil { return err } + if node.PeerHost != nil { + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost)) + if err != nil { + return err + } + var origins []multiaddr.Multiaddr = nil + for _, a := range addrs { + origins = append(origins, a) + } + opts = append(opts, pinclient.PinOpts.WithOrigins(origins)) + } + */ + // Execute remote pin request + // TODO: fix panic when pinning service is down ps, err := c.Add(ctx, rp.Cid(), opts...) if err != nil { return err } + // Act on PinStatus.delegates + // If Pinning Service returned any delegates, proactively try to + // connect to them to facilitate data exchange without waiting for DHT + // lookup for _, d := range ps.GetDelegates() { + // TODO: confirm this works as expected p, err := peer.AddrInfoFromP2pAddr(d) if err != nil { return err @@ -121,15 +154,19 @@ var addRemotePinCmd = &cmds.Command{ if err := api.Swarm().Connect(ctx, *p); err != nil { log.Infof("error connecting to remote pin delegate %v : %w", d, err) } - } + // Block until pinned if background=false if !req.Options[pinBackgroundOptionName].(bool) { + requestId := ps.GetRequestId() for { - ps, err = c.GetStatusByID(ctx, ps.GetRequestId()) + ps, err = c.GetStatusByID(ctx, requestId) if err != nil { return fmt.Errorf("failed to query pin (%v)", err) } + if ps.GetRequestId() != requestId { + return fmt.Errorf("unable to check status for requestId=%q, remote service sent unexpected %q", requestId, ps.GetRequestId()) + } s := ps.GetStatus() if s == pinclient.StatusPinned { break @@ -156,13 +193,11 @@ var addRemotePinCmd = &cmds.Command{ }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { - fmt.Printf("pin_id=%v\n", out.RequestID) - fmt.Printf("pin_name=%q\n", out.Name) - for _, d := range out.Delegates { - fmt.Printf("pin_delegate=%v\n", d) - } - fmt.Printf("pin_status=%v\n", out.Status) - fmt.Printf("pin_cid=%v\n", out.Cid) + fmt.Printf("RequestID: %v\n", out.RequestID) + fmt.Printf("CID: %v\n", out.Cid) + fmt.Printf("Name: %v\n", out.Name) + fmt.Printf("Status: %v\n", out.Status) + fmt.Printf("Delegates: %v\n", strings.Join(out.Delegates, ", ")) return nil }), }, @@ -226,13 +261,11 @@ Returns a list of objects that are pinned to a remote pinning service. Type: RemotePinOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { - fmt.Printf("pin_id=%v\n", out.RequestID) - fmt.Printf("pin_name=%q\n", out.Name) - for _, d := range out.Delegates { - fmt.Printf("pin_delegate=%v\n", d) - } - fmt.Printf("pin_status=%v\n", out.Status) - fmt.Printf("pin_cid=%v\n", out.Cid) + fmt.Printf("RequestID: %v\n", out.RequestID) + fmt.Printf("CID: %v\n", out.Cid) + fmt.Printf("Name: %v\n", out.Name) + fmt.Printf("Status: %v\n", out.Status) + fmt.Printf("Delegates: %v\n", strings.Join(out.Delegates, ", ")) return nil }), }, @@ -243,8 +276,14 @@ func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { opts = append(opts, pinclient.PinOpts.FilterName(name)) } + + // TODO: remove debug after ergonomics below are fixed + // data, _ := json.Marshal(req.Options) + // fmt.Println(string(data)) + if cidsRaw, cidsFound := req.Options[pinCIDsOptionName].([]string); cidsFound { parsedCIDs := []cid.Cid{} + // TODO: improve ergonomics: support both --cid=a --cid=b and --cid=a,b for _, rawCID := range cidsRaw { parsedCID, err := cid.Decode(rawCID) if err != nil { @@ -256,6 +295,7 @@ func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan } if statusRaw, statusFound := req.Options[pinStatusOptionName].([]string); statusFound { parsedStatuses := []pinclient.Status{} + // TODO: improve ergonomics: support both --status=a --status=b and --status=a,b for _, rawStatus := range statusRaw { s := pinclient.Status(rawStatus) if s.String() == string(pinclient.StatusUnknown) { @@ -353,17 +393,17 @@ var addRemotePinServiceCmd = &cmds.Command{ } defer repo.Close() - name, nameFound := req.Options[pinServiceNameOptionName].(string) - if !nameFound { - return fmt.Errorf("service name not given") + if len(req.Arguments) < 3 { + return fmt.Errorf("expecting three arguments: service name, url and key") } - url, urlFound := req.Options[pinServiceURLOptionName].(string) - if !urlFound { - return fmt.Errorf("service url not given") - } - key, keyFound := req.Options[pinServiceKeyOptionName].(string) - if !keyFound { - return fmt.Errorf("service key not given") + + name := req.Arguments[0] + url := req.Arguments[1] + key := req.Arguments[2] + + u, err := neturl.ParseRequestURI(url) + if err != nil || !strings.HasPrefix(u.Scheme, "http") { + return fmt.Errorf("service url must be a valid HTTP URL") } cfg, err := repo.Config() diff --git a/go.sum b/go.sum index 0bae74b8344..0ab54e69a9f 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,6 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1 h1:s8TGDcpmZRAs4iT4mnilVLA6hLelrXHyhFtO4/MaZZQ= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201015193505-957255340bf1/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878 h1:RQLHEUEGAikm7pFNtxtSSin3TfRQdnhBcCrsaTxxLz0= github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index e4770722717..e103de65e3e 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -4,35 +4,70 @@ test_description="Test ipfs remote pinning operations" . lib/test-lib.sh +if [ -z ${CI_HOST_IP+x} ]; then + # TODO: set up instead of skipping? + skip_all='Skipping pinning service integration tests: missing CI_HOST_IP, remote pinning service not available' + test_done +fi + +# daemon running in online mode to ensure Pin.origins/PinStatus.delegates work test_init_ipfs +test_launch_ipfs_daemon + +TEST_PIN_SVC="http://${CI_HOST_IP}:5000/api/v1" +TEST_PIN_SVC_KEY=$(curl -s -X POST "$TEST_PIN_SVC/users" -d email="go-ipfs-sharness@ipfs.example.com" | jq --raw-output .access_token) # create user on pinning service test_expect_success "creating test user on remote pinning service" ' - echo CI host IP address ${CI_HOST_IP} && - export TEST_PIN_SVC=http://${CI_HOST_IP}:5000/api/v1 && - ipfs pin remote service add --name=test_pin_svc --url=$TEST_PIN_SVC --key=$(curl -X POST $TEST_PIN_SVC/users -d email=sharness@ipfs.io | jq --raw-output .access_token) && - ipfs pin remote service add --name=fake_pin_svc --url=http://0.0.0.0:5000 --key=fake_api_key + echo CI host IP address ${TEST_PIN_SVC} && + ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} && + ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key && + ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key && + ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key ' -test_expect_success "test 'ipfs pin remote service ls'"' - ipfs pin remote service ls | jq --raw-output .Service | grep test_pin_svc && - ipfs pin remote service ls | jq --raw-output .Service | grep fake_pin_svc +test_expect_success "test 'ipfs pin remote service ls'" ' + ipfs pin remote service ls | tee > ls_out && + grep test_pin_svc ls_out && + grep test_invalid_key_svc ls_out && + grep test_invalid_url_path_svc ls_out && + grep test_invalid_url_dns_svc ls_out ' test_expect_success "check connection to test pinning service" ' ipfs pin remote ls --service=test_pin_svc --enc=json ' -test_expect_failure "test fake pinning service calls fail" ' - ipfs pin remote ls --service=fake_pin_svc --enc=json +# TODO: improve error returned below +test_expect_failure "unathorized pinning service calls fail" ' + ipfs pin remote ls --service=test_invalid_key_svc +' + +# TODO: improve error returned below +test_expect_failure "misconfigured pinning service calls fail (wrong path)" ' + ipfs pin remote ls --service=test_invalid_url_path_svc +' + +# TODO: improve error returned below (panic when offline mode) +test_expect_failure "misconfigured pinning service calls fail (dns error)" ' + ipfs pin remote ls --service=test_invalid_url_dns_svc +' + +test_expect_success "remove pinning service" ' + ipfs pin remote service rm test_invalid_key_svc && + ipfs pin remote service rm test_invalid_url_path_svc && + ipfs pin remote service rm test_invalid_url_dns_svc ' -test_expect_success "remove fake pinning service" ' - ipfs pin remote service rm fake_pin_svc +test_expect_success "verify pinning service removal works" ' + ipfs pin remote service ls | jq --raw-output .Service | grep -L test_invalid_key_svc ' -test_expect_failure "verify fake pinning service removed"' - ipfs pin remote service ls | jq --raw-output .Service | grep fake_pin_svc +# TODO: why this was not pinned instantly? no bitswap is required for inlined CID +test_expect_success "verify background add works with data inlined in CID" ' + ipfs pin remote add --background=true --service=test_pin_svc --enc=json --name=inlined_null bafkqaaa && + ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=queued --status=pinning --status=pinned > ls_out && + grep -E "queued|pinning|pinned" ls_out ' test_remote_pins() { @@ -41,29 +76,45 @@ test_remote_pins() { BASE_ARGS="--cid-base=$BASE" fi + # time-based CIDs to ensure we test with data that is not on public network test_expect_success "create some hashes using base $BASE" ' - export HASH_A=$(echo "A" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_B=$(echo "B" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_C=$(echo "C" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_D=$(echo "D" | ipfs add $BASE_ARGS -q --pin=false) - ' - - test_expect_success "verify background add works" ' - export ID_D=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) && - sleep 3 && - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_d | jq --raw-output .Status | grep pinned + export HASH_A=$(echo "A @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_B=$(echo "B @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_C=$(echo "C @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_D=$(echo "D @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) ' + # TODO: when running locally, background=false seems to block forever - perhaps due to missing Pin.origins? test_expect_success "'ipfs pin remote add'" ' export ID_A=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && export ID_B=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && + export ID_D=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) + ' + + test_expect_success "verify background add worked" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_d --status=queued --status=pinning --status=pinned > ls_out && + grep name_d ls_out + ' + + test_expect_success "'ipfs pin remote add' with CID that is not available" ' export HASH_MISSING="QmNRpQVA5n7osjtyjYaWEQpwYnbV1QoVkrSe7oyccMJh1m" && export ID_M=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING | jq --raw-output .RequestID) ' test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' - FOUND_ID_M=$(ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinned | jq --raw-output .RequestID | grep $ID_M) && + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued --status=pinning --status=pinned --status=failed | jq --raw-output .RequestID > ls_out && + grep $ID_A ls_out && + grep $ID_B ls_out && + grep $ID_C ls_out && + grep $ID_D ls_out && + grep $ID_M ls_out + ' + + # TODO: fix / verify remaining tests below + + test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' + FOUND_ID_M=$(ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued --status=pinning --status=pinned | jq --raw-output .RequestID | grep $ID_M) && echo ID_M=$ID_M FOUND_ID_M=$FOUND_ID_M && echo $ID_M > expected && echo $FOUND_ID_M > actual && @@ -118,7 +169,9 @@ test_remote_pins() { test_remote_pins "" -# test_launch_ipfs_daemon -# test_kill_ipfs_daemon +# TODO: origin / delegates +# TODO: name search + +test_kill_ipfs_daemon test_done From 576b08590e1c3e4b710f1551924edd98a6762af5 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 13 Nov 2020 15:42:40 +0100 Subject: [PATCH 085/127] fix(pins): pass own multiaddrs in pin.origins --- core/commands/pin/remotepin.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index e3b269c3439..31214e12b6e 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -18,6 +18,7 @@ import ( logging "github.com/ipfs/go-log" pinclient "github.com/ipfs/go-pinning-service-http-client" path "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/libp2p/go-libp2p-core/host" peer "github.com/libp2p/go-libp2p-core/peer" "github.com/multiformats/go-multiaddr" ) @@ -113,7 +114,6 @@ var addRemotePinCmd = &cmds.Command{ opts = append(opts, pinclient.PinOpts.WithName(name)) } - /* TODO // Prepare Pin.origins // Add own multiaddrs to the 'origins' array, so Pinning Service can // use that as a hint and connect back to us (if possible) @@ -130,9 +130,8 @@ var addRemotePinCmd = &cmds.Command{ for _, a := range addrs { origins = append(origins, a) } - opts = append(opts, pinclient.PinOpts.WithOrigins(origins)) + opts = append(opts, pinclient.PinOpts.WithOrigins(origins...)) } - */ // Execute remote pin request // TODO: fix panic when pinning service is down From a152a43d7ba1daf212f828618dc2173f126fbacd Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 13 Nov 2020 21:13:00 +0100 Subject: [PATCH 086/127] fix: output of pin remote service ls - by default outputs two columns separated with tab, just like 'key list' does - when --enc=json is passed, an object with RemoteServices array is returned (mirroring what 'key list' does) --- core/commands/pin/remotepin.go | 40 ++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 31214e12b6e..857577f2ef8 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -6,6 +6,7 @@ import ( "io" "sort" "strings" + "text/tabwriter" "time" neturl "net/url" @@ -488,19 +489,25 @@ var lsRemotePinServiceCmd = &cmds.Command{ if cfg.RemotePinServices.Services == nil { return nil // no pinning services added yet } - result := sortedServiceAndURL{} - for svcName, svcConfig := range cfg.RemotePinServices.Services { - result = append(result, PinServiceAndURL{svcName, svcConfig.URL}) + services := cfg.RemotePinServices.Services + result := PinServicesList{make([]PinServiceAndURL, 0, len(services))} + for svcName, svcConfig := range services { + result.RemoteServices = append(result.RemoteServices, PinServiceAndURL{svcName, svcConfig.URL}) } sort.Sort(result) - for _, r := range result { - if err := res.Emit(r); err != nil { - return err + return cmds.EmitOnce(res, &result) + }, + Type: PinServicesList{}, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *PinServicesList) error { + tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) + for _, s := range list.RemoteServices { + fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.URL) } - } - return nil + tw.Flush() + return nil + }), }, - Type: PinServiceAndURL{}, } type PinServiceAndURL struct { @@ -508,17 +515,22 @@ type PinServiceAndURL struct { URL string } -type sortedServiceAndURL []PinServiceAndURL +// Struct returned by ipfs pin remote service ls --enc=json | jq +type PinServicesList struct { + RemoteServices []PinServiceAndURL +} -func (s sortedServiceAndURL) Len() int { - return len(s) +func (l PinServicesList) Len() int { + return len(l.RemoteServices) } -func (s sortedServiceAndURL) Swap(i, j int) { +func (l PinServicesList) Swap(i, j int) { + s := l.RemoteServices s[i], s[j] = s[j], s[i] } -func (s sortedServiceAndURL) Less(i, j int) bool { +func (l PinServicesList) Less(i, j int) bool { + s := l.RemoteServices return s[i].Service < s[j].Service } From b7e0c08dc16c187866a1bca417cdb5b096085f63 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 14 Nov 2020 01:11:09 +0100 Subject: [PATCH 087/127] feat: ipfs pin remote ls [] This deduplicates code responsible for printing single PinStatus response, and adds support for checking status based on RequestID which is required by ipfs-webui integration. Next step is to support more than one RequestID. --- core/commands/pin/remotepin.go | 81 +++++++++++++++++++++----------- test/sharness/t0700-remotepin.sh | 14 ++++-- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 857577f2ef8..2a0eb42ef56 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -62,10 +62,33 @@ const pinForceOptionName = "force" type RemotePinOutput struct { RequestID string - Name string - Delegates []string // multiaddr Status string Cid string + Name string + Delegates []string // multiaddr +} + +func toRemotePinOutput(ps pinclient.PinStatusGetter) RemotePinOutput { + return RemotePinOutput{ + RequestID: ps.GetRequestId(), + Name: ps.GetPin().GetName(), + Delegates: multiaddrsToStrings(ps.GetDelegates()), + Status: ps.GetStatus().String(), + Cid: ps.GetPin().GetCid().String(), + } +} + +func printRemotePinDetails(w io.Writer, out *RemotePinOutput) { + tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) + defer tw.Flush() + fw := func(k string, v string) { + fmt.Fprintf(tw, "%s:\t%s\n", k, v) + } + fw("RequestID", out.RequestID) + fw("CID", out.Cid) + fw("Name", out.Name) + fw("Status", out.Status) + fw("Delegates", strings.Join(out.Delegates, ", ")) } // remote pin commands @@ -183,21 +206,11 @@ var addRemotePinCmd = &cmds.Command{ } } - return res.Emit(&RemotePinOutput{ - RequestID: ps.GetRequestId(), - Name: ps.GetPin().GetName(), - Delegates: multiaddrsToStrings(ps.GetDelegates()), - Status: ps.GetStatus().String(), - Cid: ps.GetPin().GetCid().String(), - }) + return res.Emit(toRemotePinOutput(ps)) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { - fmt.Printf("RequestID: %v\n", out.RequestID) - fmt.Printf("CID: %v\n", out.Cid) - fmt.Printf("Name: %v\n", out.Name) - fmt.Printf("Status: %v\n", out.Status) - fmt.Printf("Delegates: %v\n", strings.Join(out.Delegates, ", ")) + printRemotePinDetails(w, out) return nil }), }, @@ -222,7 +235,9 @@ Returns a list of objects that are pinned to a remote pinning service. `, }, - Arguments: []cmds.Argument{}, + Arguments: []cmds.Argument{ + cmds.StringArg("request-id", false, false, "Request ID of the pin to be listed.").EnableStdin(), + }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), @@ -239,19 +254,25 @@ Returns a list of objects that are pinned to a remote pinning service. return err } + // If request-id is provided, return PinStatus directly + if len(req.Arguments) == 1 { + // TODO: support more than one? + ps, err := lsRemoteByRequestId(ctx, req, c) + if err != nil { + return err + } + // return single element + return res.Emit(toRemotePinOutput(ps)) + } + + // Else, use filters to find remote pins that match criteria psCh, errCh, err := lsRemote(ctx, req, c) if err != nil { return err } for ps := range psCh { - if err := res.Emit(&RemotePinOutput{ - RequestID: ps.GetRequestId(), - Name: ps.GetPin().GetName(), - Delegates: multiaddrsToStrings(ps.GetDelegates()), - Status: ps.GetStatus().String(), - Cid: ps.GetPin().GetCid().String(), - }); err != nil { + if err := res.Emit(toRemotePinOutput(ps)); err != nil { return err } } @@ -261,16 +282,22 @@ Returns a list of objects that are pinned to a remote pinning service. Type: RemotePinOutput{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { - fmt.Printf("RequestID: %v\n", out.RequestID) - fmt.Printf("CID: %v\n", out.Cid) - fmt.Printf("Name: %v\n", out.Name) - fmt.Printf("Status: %v\n", out.Status) - fmt.Printf("Delegates: %v\n", strings.Join(out.Delegates, ", ")) + // pin remote ls produces a flat output similar to legacy pin ls + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", out.RequestID, out.Cid, out.Status, out.Name) return nil }), }, } +func lsRemoteByRequestId(ctx context.Context, req *cmds.Request, c *pinclient.Client) (pinclient.PinStatusGetter, error) { + requestId := req.Arguments[0] + ps, err := c.GetStatusByID(ctx, requestId) + if err != nil { + return nil, fmt.Errorf("failed to find pin with request ID %s (%v)", requestId, err) + } + return ps, nil +} + func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index e103de65e3e..f1017f3e466 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -93,8 +93,8 @@ test_remote_pins() { ' test_expect_success "verify background add worked" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_d --status=queued --status=pinning --status=pinned > ls_out && - grep name_d ls_out + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_D > ls_out && + grep $ID_D ls_out ' test_expect_success "'ipfs pin remote add' with CID that is not available" ' @@ -121,7 +121,15 @@ test_remote_pins() { test_cmp expected actual ' - test_expect_success "'ipfs pin remote ls' for existing pins by ID" ' + test_expect_success "'ipfs pin remote ls' for existing pins by RequestID" ' + FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A | jq --raw-output .RequestID | grep $ID_A) && + echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && + echo $ID_A > expected && + echo $FOUND_ID_A > actual && + test_cmp expected actual + ' + + test_expect_success "'ipfs pin remote ls' for existing pins by CID" ' FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && echo $ID_A > expected && From bea8056d7b59f75a35080b1f31055db7772af32f Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 14 Nov 2020 03:16:23 +0100 Subject: [PATCH 088/127] feat: ipfs pin remote ls []... This adds support for checking status of multiple remote pins via single command. should save a bunch of time in userland, including ipfs-webui While at it, fixed some tests to be more reliable. This is still WIP (pin remote rm seems to be still buggy) --- core/commands/pin/remotepin.go | 63 ++++++++++------ test/sharness/t0700-remotepin.sh | 120 ++++++++++++++----------------- 2 files changed, 95 insertions(+), 88 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 2a0eb42ef56..c6b735cfb2a 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -100,11 +100,12 @@ var addRemotePinCmd = &cmds.Command{ }, Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, false, "Path to object(s) to be pinned.").EnableStdin(), + cmds.StringArg("ipfs-path", true, false, "Path to object(s) to be pinned."), }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + // TODO: swap default to --background=false cmds.BoolOption(pinBackgroundOptionName, "Add the pins in the background.").WithDefault(true), }, Type: RemotePinOutput{}, @@ -236,7 +237,7 @@ Returns a list of objects that are pinned to a remote pinning service. }, Arguments: []cmds.Argument{ - cmds.StringArg("request-id", false, false, "Request ID of the pin to be listed.").EnableStdin(), + cmds.StringArg("request-id", false, true, "Request ID of the pin to be listed."), }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), @@ -254,18 +255,6 @@ Returns a list of objects that are pinned to a remote pinning service. return err } - // If request-id is provided, return PinStatus directly - if len(req.Arguments) == 1 { - // TODO: support more than one? - ps, err := lsRemoteByRequestId(ctx, req, c) - if err != nil { - return err - } - // return single element - return res.Emit(toRemotePinOutput(ps)) - } - - // Else, use filters to find remote pins that match criteria psCh, errCh, err := lsRemote(ctx, req, c) if err != nil { return err @@ -289,16 +278,44 @@ Returns a list of objects that are pinned to a remote pinning service. }, } -func lsRemoteByRequestId(ctx context.Context, req *cmds.Request, c *pinclient.Client) (pinclient.PinStatusGetter, error) { - requestId := req.Arguments[0] - ps, err := c.GetStatusByID(ctx, requestId) - if err != nil { - return nil, fmt.Errorf("failed to find pin with request ID %s (%v)", requestId, err) +// Returns pin objects with passed RequestIDs or matching filters. +func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { + // If request-ids are provided, do direct lookup for specific objects + if len(req.Arguments) > 0 { + return lsRemoteByRequestId(ctx, req, c) } - return ps, nil + // else, apply filters to find matching pin objects + return lsRemoteWithFilters(ctx, req, c) } -func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { +// Executes GET /pins/{requestid} for each requestid passed as argument. +// Status checks are executed one by one and operation is aborted on first error. +func lsRemoteByRequestId(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { + lsIDs := req.Arguments + res := make(chan pinclient.PinStatusGetter, 1) + errs := make(chan error, 1) + go func() { + defer close(res) + defer close(errs) + for _, requestId := range lsIDs { + ps, err := c.GetStatusByID(ctx, requestId) + if err != nil { + errs <- err + return + } + select { + case res <- ps: + case <-ctx.Done(): + errs <- ctx.Err() + return + } + } + }() + return res, errs, nil +} + +// Executes GET /pins/?query-with-filters +func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { opts = append(opts, pinclient.PinOpts.FilterName(name)) @@ -348,7 +365,7 @@ collected if needed. }, Arguments: []cmds.Argument{ - cmds.StringArg("request-id", false, true, "Request ID of the pin to be removed.").EnableStdin(), + cmds.StringArg("request-id", false, true, "Request ID of the pin to be removed."), }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), @@ -460,7 +477,7 @@ var rmRemotePinServiceCmd = &cmds.Command{ ShortDescription: "Remove credentials for access to a remote pinning service.", }, Arguments: []cmds.Argument{ - cmds.StringArg("remote-pin-service", true, false, "Name of remote pinning service to remove.").EnableStdin(), + cmds.StringArg("remote-pin-service", true, false, "Name of remote pinning service to remove."), }, Options: []cmds.Option{}, Type: nil, diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index f1017f3e466..bbb902bb6a1 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -27,11 +27,11 @@ test_expect_success "creating test user on remote pinning service" ' ' test_expect_success "test 'ipfs pin remote service ls'" ' - ipfs pin remote service ls | tee > ls_out && - grep test_pin_svc ls_out && - grep test_invalid_key_svc ls_out && - grep test_invalid_url_path_svc ls_out && - grep test_invalid_url_dns_svc ls_out + ipfs pin remote service ls | tee ls_out && + grep -q test_pin_svc ls_out && + grep -q test_invalid_key_svc ls_out && + grep -q test_invalid_url_path_svc ls_out && + grep -q test_invalid_url_dns_svc ls_out ' test_expect_success "check connection to test pinning service" ' @@ -39,18 +39,18 @@ test_expect_success "check connection to test pinning service" ' ' # TODO: improve error returned below -test_expect_failure "unathorized pinning service calls fail" ' - ipfs pin remote ls --service=test_invalid_key_svc +test_expect_success "unathorized pinning service calls fail" ' + test_expect_code 1 ipfs pin remote ls --service=test_invalid_key_svc ' # TODO: improve error returned below -test_expect_failure "misconfigured pinning service calls fail (wrong path)" ' - ipfs pin remote ls --service=test_invalid_url_path_svc +test_expect_success "misconfigured pinning service calls fail (wrong path)" ' + test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_path_svc ' # TODO: improve error returned below (panic when offline mode) -test_expect_failure "misconfigured pinning service calls fail (dns error)" ' - ipfs pin remote ls --service=test_invalid_url_dns_svc +test_expect_success "misconfigured pinning service calls fail (dns error)" ' + test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_dns_svc ' test_expect_success "remove pinning service" ' @@ -60,14 +60,14 @@ test_expect_success "remove pinning service" ' ' test_expect_success "verify pinning service removal works" ' - ipfs pin remote service ls | jq --raw-output .Service | grep -L test_invalid_key_svc + ipfs pin remote service ls | grep -L test_invalid_key_svc ' # TODO: why this was not pinned instantly? no bitswap is required for inlined CID test_expect_success "verify background add works with data inlined in CID" ' ipfs pin remote add --background=true --service=test_pin_svc --enc=json --name=inlined_null bafkqaaa && - ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=queued --status=pinning --status=pinned > ls_out && - grep -E "queued|pinning|pinned" ls_out + ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=queued --status=pinning --status=pinned | tee ls_out && + grep -qE "queued|pinning|pinned" ls_out ' test_remote_pins() { @@ -77,88 +77,65 @@ test_remote_pins() { fi # time-based CIDs to ensure we test with data that is not on public network + # note: one hash is not imported to IPFS on purpose, to reliably test 'queued' state test_expect_success "create some hashes using base $BASE" ' export HASH_A=$(echo "A @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && export HASH_B=$(echo "B @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && export HASH_C=$(echo "C @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_D=$(echo "D @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) + export HASH_D=$(echo "D @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date)" | ipfs add $BASE_ARGS -q --only-hash) ' - # TODO: when running locally, background=false seems to block forever - perhaps due to missing Pin.origins? - test_expect_success "'ipfs pin remote add'" ' - export ID_A=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && - export ID_B=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && - export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && - export ID_D=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) + test_expect_success "'ipfs pin remote add --background=true'" ' + export ID_A=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && + export ID_B=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) ' test_expect_success "verify background add worked" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_D > ls_out && - grep $ID_D ls_out + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A $ID_B | tee ls_out && + grep -q $HASH_A ls_out && + grep -q $HASH_B ls_out ' test_expect_success "'ipfs pin remote add' with CID that is not available" ' - export HASH_MISSING="QmNRpQVA5n7osjtyjYaWEQpwYnbV1QoVkrSe7oyccMJh1m" && export ID_M=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING | jq --raw-output .RequestID) ' test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued --status=pinning --status=pinned --status=failed | jq --raw-output .RequestID > ls_out && - grep $ID_A ls_out && - grep $ID_B ls_out && - grep $ID_C ls_out && - grep $ID_D ls_out && - grep $ID_M ls_out + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued --status=pinning --status=pinned --status=failed | tee ls_out && + grep -q $HASH_A ls_out && + grep -q $HASH_B ls_out && + grep -q $HASH_MISSING ls_out ' # TODO: fix / verify remaining tests below - test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' - FOUND_ID_M=$(ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued --status=pinning --status=pinned | jq --raw-output .RequestID | grep $ID_M) && - echo ID_M=$ID_M FOUND_ID_M=$FOUND_ID_M && - echo $ID_M > expected && - echo $FOUND_ID_M > actual && - test_cmp expected actual - ' - test_expect_success "'ipfs pin remote ls' for existing pins by RequestID" ' - FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A | jq --raw-output .RequestID | grep $ID_A) && - echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && - echo $ID_A > expected && - echo $FOUND_ID_A > actual && - test_cmp expected actual + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A | tee ls_out && + grep -q $HASH_A ls_out ' test_expect_success "'ipfs pin remote ls' for existing pins by CID" ' - FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_A | jq --raw-output .RequestID | grep $ID_A) && - echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && - echo $ID_A > expected && - echo $FOUND_ID_A > actual && - test_cmp expected actual + ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_B | tee ls_out && + grep -q $HASH_B ls_out ' test_expect_success "'ipfs pin remote ls' for existing pins by name" ' - FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | jq --raw-output .RequestID | grep $ID_A) && - echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && - echo $ID_A > expected && - echo $FOUND_ID_A > actual && - test_cmp expected actual + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out && + grep -q $HASH_A ls_out ' test_expect_success "'ipfs pin remote ls' for existing pins by status" ' - FOUND_ID_A=$(ipfs pin remote ls --service=test_pin_svc --enc=json --status=pinned | jq --raw-output .RequestID | grep $ID_A) && - echo ID_A=$ID_A FOUND_ID_A=$FOUND_ID_A && - echo $ID_A > expected && - echo $FOUND_ID_A > actual && - test_cmp expected actual + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued | tee ls_out && + grep -q $HASH_MISSING ls_out ' - test_expect_success "'ipfs pin remote rm' an existing pin by ID" ' - ipfs pin remote rm --service=test_pin_svc --enc=json $ID_A + test_expect_success "'ipfs pin remote rm' an existing pin by RequestID" ' + ipfs pin remote rm --service=test_pin_svc $ID_A ' - test_expect_failure "'ipfs pin remote rm' needs --force when globbing" ' - ipfs pin remote rm --service=test_pin_svc --enc=json --name=name_b + test_expect_success "'ipfs pin remote rm --name' fails without --force when globbing" ' + test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --enc=json --name=name_b ' test_expect_success "'ipfs pin remote rm' an existing pin by name" ' @@ -169,10 +146,23 @@ test_remote_pins() { ipfs pin remote rm --service=test_pin_svc --enc=json --force --status=pinned ' - test_expect_success "'ipfs pin remote ls' for deleted pin" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | jq --raw-output .RequestID > list - test_expect_code 1 grep $ID_A list + test_expect_success "'ipfs pin remote ls' returns error for deleted pin" ' + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out + test_expect_code 1 grep -q $HASH_B ls_out + ' + + # TODO: we test this at the end because so far it hangs forever on CI -- needs investigation + test_expect_success "'ipfs pin remote add --background=false'" ' + export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && + export ID_D=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) ' + + test_expect_success "verify foreground add worked" ' + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_C $ID_D | tee ls_out && + grep -q $HASH_C ls_out && + grep -q $HASH_D ls_out + ' + } test_remote_pins "" From 531923b3171c89888c7ec274495154ccffd0ea9c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 14 Nov 2020 03:43:14 +0100 Subject: [PATCH 089/127] chore: golint --- core/commands/pin/remotepin.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index c6b735cfb2a..54b95edcf71 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -151,11 +151,7 @@ var addRemotePinCmd = &cmds.Command{ if err != nil { return err } - var origins []multiaddr.Multiaddr = nil - for _, a := range addrs { - origins = append(origins, a) - } - opts = append(opts, pinclient.PinOpts.WithOrigins(origins...)) + opts = append(opts, pinclient.PinOpts.WithOrigins(addrs...)) } // Execute remote pin request From 75bd021b52719887bd1bf2d1ae3caddb9bfdd6d2 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 16 Nov 2020 08:26:40 -0800 Subject: [PATCH 090/127] use updated config structure --- core/commands/config.go | 2 +- core/commands/pin/remotepin.go | 41 +++++++++++++++++----------------- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/core/commands/config.go b/core/commands/config.go index df219b99423..efa4d75cfc9 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -176,7 +176,7 @@ NOTE: For security reasons, this command will omit your private key and remote s if err != nil { return err } - err = scrubValue(cfg, []string{config.RemotePinServicesTag}) + err = scrubValue(cfg, []string{config.PinningRemoteServicesTag}) if err != nil { return err } diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 54b95edcf71..42c08378d0d 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -450,17 +450,16 @@ var addRemotePinServiceCmd = &cmds.Command{ if err != nil { return err } - if cfg.RemotePinServices.Services != nil { - if _, present := cfg.RemotePinServices.Services[name]; present { + if cfg.Pinning.RemoteServices != nil { + if _, present := cfg.Pinning.RemoteServices[name]; present { return fmt.Errorf("service already present") } } else { - cfg.RemotePinServices.Services = map[string]config.RemotePinService{} + cfg.Pinning.RemoteServices = map[string]config.RemotePinningService{} } - cfg.RemotePinServices.Services[name] = config.RemotePinService{ - Name: name, - URL: url, - Key: key, + cfg.Pinning.RemoteServices[name] = config.RemotePinningService{ + ApiEndpoint: url, + ApiKey: key, } return repo.SetConfig(cfg) @@ -497,8 +496,8 @@ var rmRemotePinServiceCmd = &cmds.Command{ if err != nil { return err } - if cfg.RemotePinServices.Services != nil { - delete(cfg.RemotePinServices.Services, name) + if cfg.Pinning.RemoteServices != nil { + delete(cfg.Pinning.RemoteServices, name) } return repo.SetConfig(cfg) }, @@ -526,13 +525,13 @@ var lsRemotePinServiceCmd = &cmds.Command{ if err != nil { return err } - if cfg.RemotePinServices.Services == nil { + if cfg.Pinning.RemoteServices == nil { return nil // no pinning services added yet } - services := cfg.RemotePinServices.Services - result := PinServicesList{make([]PinServiceAndURL, 0, len(services))} + services := cfg.Pinning.RemoteServices + result := PinServicesList{make([]PinServiceAndEndpoint, 0, len(services))} for svcName, svcConfig := range services { - result.RemoteServices = append(result.RemoteServices, PinServiceAndURL{svcName, svcConfig.URL}) + result.RemoteServices = append(result.RemoteServices, PinServiceAndEndpoint{svcName, svcConfig.ApiEndpoint}) } sort.Sort(result) return cmds.EmitOnce(res, &result) @@ -542,7 +541,7 @@ var lsRemotePinServiceCmd = &cmds.Command{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *PinServicesList) error { tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) for _, s := range list.RemoteServices { - fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.URL) + fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.ApiEndpoint) } tw.Flush() return nil @@ -550,14 +549,14 @@ var lsRemotePinServiceCmd = &cmds.Command{ }, } -type PinServiceAndURL struct { - Service string - URL string +type PinServiceAndEndpoint struct { + Service string + ApiEndpoint string } // Struct returned by ipfs pin remote service ls --enc=json | jq type PinServicesList struct { - RemoteServices []PinServiceAndURL + RemoteServices []PinServiceAndEndpoint } func (l PinServicesList) Len() int { @@ -599,12 +598,12 @@ func getRemotePinService(env cmds.Environment, name string) (url, key string, er if err != nil { return "", "", err } - if cfg.RemotePinServices.Services == nil { + if cfg.Pinning.RemoteServices == nil { return "", "", fmt.Errorf("service not known") } - service, present := cfg.RemotePinServices.Services[name] + service, present := cfg.Pinning.RemoteServices[name] if !present { return "", "", fmt.Errorf("service not known") } - return service.URL, service.Key, nil + return service.ApiEndpoint, service.ApiKey, nil } diff --git a/go.mod b/go.mod index d999924b924..81518df646f 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.4.0 - github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3 + github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 diff --git a/go.sum b/go.sum index 0ab54e69a9f..9b192d0090e 100644 --- a/go.sum +++ b/go.sum @@ -360,6 +360,8 @@ github.com/ipfs/go-ipfs-cmds v0.4.0 h1:xUavIxA9Ts8U6PAHmQBvDGMlGfUrQ13Rymd+5t8LI github.com/ipfs/go-ipfs-cmds v0.4.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3 h1:7EwzMoBhktnUjdgukBmOm+L0J8pwVlHdXEOJa5Ct6KA= github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb h1:cK7DEEHIsYIBVwAaxlQsO3ScvTOOy+CSkeJ53yCxgMY= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From 87d93a3a9574bdd9f63c42c286a5bb08eef9f454 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 17 Nov 2020 13:31:22 +0100 Subject: [PATCH 091/127] style: improved error messages added requestid to error messages, so user can then inspect/resume/remove the queued pin request --- core/commands/pin/remotepin.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 42c08378d0d..955960f5d62 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -106,7 +106,7 @@ var addRemotePinCmd = &cmds.Command{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), // TODO: swap default to --background=false - cmds.BoolOption(pinBackgroundOptionName, "Add the pins in the background.").WithDefault(true), + cmds.BoolOption(pinBackgroundOptionName, "Queue the pin request on the remote service, return RequestID immediately.").WithDefault(true), }, Type: RemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -182,23 +182,23 @@ var addRemotePinCmd = &cmds.Command{ for { ps, err = c.GetStatusByID(ctx, requestId) if err != nil { - return fmt.Errorf("failed to query pin (%v)", err) + return fmt.Errorf("failed to check pin status for requestid=%q due to error: %v", requestId, err) } if ps.GetRequestId() != requestId { - return fmt.Errorf("unable to check status for requestId=%q, remote service sent unexpected %q", requestId, ps.GetRequestId()) + return fmt.Errorf("failed to check pin status for requestid=%q, remote service sent unexpected requestid=%q", requestId, ps.GetRequestId()) } s := ps.GetStatus() if s == pinclient.StatusPinned { break } if s == pinclient.StatusFailed { - return fmt.Errorf("failed to pin") + return fmt.Errorf("remote service failed to pin requestid=%q", requestId) } tmr := time.NewTimer(time.Second / 2) select { case <-tmr.C: case <-ctx.Done(): - return fmt.Errorf("waiting for pin interrupted") + return fmt.Errorf("waiting for pin interrupted, requestid=%q remains on remote service: check its status with 'ls' or abort with 'rm'", requestId) } } } @@ -327,7 +327,7 @@ func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Cl for _, rawCID := range cidsRaw { parsedCID, err := cid.Decode(rawCID) if err != nil { - return nil, nil, fmt.Errorf("CID %s cannot be parsed (%v)", rawCID, err) + return nil, nil, fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) } parsedCIDs = append(parsedCIDs, parsedCID) } @@ -339,7 +339,7 @@ func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Cl for _, rawStatus := range statusRaw { s := pinclient.Status(rawStatus) if s.String() == string(pinclient.StatusUnknown) { - return nil, nil, fmt.Errorf("status %s is not valid", rawStatus) + return nil, nil, fmt.Errorf("status %q is not valid", rawStatus) } parsedStatuses = append(parsedStatuses, s) } @@ -368,7 +368,7 @@ collected if needed. cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s); optional, comma separated."), cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses; optional, comma separated."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), - cmds.BoolOption(pinForceOptionName, "Delete multiple pins.").WithDefault(false), + cmds.BoolOption(pinForceOptionName, "Delete multiple pins without confirmation.").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(req.Context) @@ -390,10 +390,11 @@ collected if needed. rmIDs = append(rmIDs, ps.GetRequestId()) } if err = <-errCh; err != nil { - return fmt.Errorf("listing remote pin IDs (%v)", err) + return fmt.Errorf("error while listing remote pins: %v", err) } + // TODO: shouldn't we require --force only when more than one pin was found? if len(rmIDs) > 0 && !req.Options[pinForceOptionName].(bool) { - return fmt.Errorf("multiple pins may be removed, use --force") + return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") } } else { rmIDs = append(rmIDs, req.Arguments[0]) @@ -401,7 +402,7 @@ collected if needed. for _, rmID := range rmIDs { if err = c.DeleteByID(ctx, rmID); err != nil { - return fmt.Errorf("removing pin with request ID %s (%v)", rmID, err) + return fmt.Errorf("removing pin identified by requestid=%q failed: %v", rmID, err) } } return nil @@ -509,7 +510,9 @@ var lsRemotePinServiceCmd = &cmds.Command{ ShortDescription: "List remote pinning services.", }, Arguments: []cmds.Argument{}, - Options: []cmds.Option{}, + Options: []cmds.Option{ + // TODO: -s --stats that execute ls query for each status and reads pagination hints to return Stats object with the count of pins in each state: {Queued, Pinning, Pinned, Failed} + }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { From 235a0999b26c6494f52df762600b9ef3563e16af Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 17 Nov 2020 13:38:43 +0100 Subject: [PATCH 092/127] chore: latest rb-pinning-service-api latest version has a smaller image shich should speed up CI a bit sidenote: this version hides go-ipfs used by rb-pinning-service-api behind NAT, removing port conflict on local env, and simulating (reverse) NAT setup for data transfer --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 252fb31e6bd..58f55591ef1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,7 @@ jobs: git clone --depth 1 https://github.com/ipfs-shipyard/rb-pinning-service-api.git - run: cd rb-pinning-service-api && - git checkout c6ad5c428ec3422bc56131c5150d6b00815abfee && + git checkout 0a3a50b715b8bd080a7c75a4298b2145c4e6955d && docker-compose pull && docker-compose up -d && cd .. From f75a209ede1d746e11823dd4dbf6417bf6bea9b0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 17 Nov 2020 14:05:58 +0100 Subject: [PATCH 093/127] fix: support --status=a,b and --cid=a,b improve CLI ergonomics by supporting support bothi notations: --status=a --status=b and --status=a,b --cid=a --cid=b and --cid=a,b --- core/commands/pin/remotepin.go | 18 ++++++++++-------- test/sharness/t0700-remotepin.sh | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 955960f5d62..c257b8392b7 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -317,14 +317,9 @@ func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Cl opts = append(opts, pinclient.PinOpts.FilterName(name)) } - // TODO: remove debug after ergonomics below are fixed - // data, _ := json.Marshal(req.Options) - // fmt.Println(string(data)) - if cidsRaw, cidsFound := req.Options[pinCIDsOptionName].([]string); cidsFound { parsedCIDs := []cid.Cid{} - // TODO: improve ergonomics: support both --cid=a --cid=b and --cid=a,b - for _, rawCID := range cidsRaw { + for _, rawCID := range flattenCommaList(cidsRaw) { parsedCID, err := cid.Decode(rawCID) if err != nil { return nil, nil, fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) @@ -335,8 +330,7 @@ func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Cl } if statusRaw, statusFound := req.Options[pinStatusOptionName].([]string); statusFound { parsedStatuses := []pinclient.Status{} - // TODO: improve ergonomics: support both --status=a --status=b and --status=a,b - for _, rawStatus := range statusRaw { + for _, rawStatus := range flattenCommaList(statusRaw) { s := pinclient.Status(rawStatus) if s.String() == string(pinclient.StatusUnknown) { return nil, nil, fmt.Errorf("status %q is not valid", rawStatus) @@ -351,6 +345,14 @@ func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Cl return psCh, errCh, nil } +func flattenCommaList(list []string) []string { + flatList := list[:0] + for _, s := range list { + flatList = append(flatList, strings.Split(s, ",")...) + } + return flatList +} + var rmRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove pinned objects from remote pinning service.", diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index bbb902bb6a1..52ce3a81f72 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -66,7 +66,7 @@ test_expect_success "verify pinning service removal works" ' # TODO: why this was not pinned instantly? no bitswap is required for inlined CID test_expect_success "verify background add works with data inlined in CID" ' ipfs pin remote add --background=true --service=test_pin_svc --enc=json --name=inlined_null bafkqaaa && - ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=queued --status=pinning --status=pinned | tee ls_out && + ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=queued,pinning,pinned | tee ls_out && grep -qE "queued|pinning|pinned" ls_out ' @@ -102,7 +102,7 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued --status=pinning --status=pinned --status=failed | tee ls_out && + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | tee ls_out && grep -q $HASH_A ls_out && grep -q $HASH_B ls_out && grep -q $HASH_MISSING ls_out From cd0a2735f56e1401e7f2a6d3c5b77438203d4e95 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 17 Nov 2020 14:42:16 +0100 Subject: [PATCH 094/127] refactor(ci): dynamic DOCKER_HOST This should protect us against GitHub or Docker changing the default IPs --- .circleci/config.yml | 10 +++++----- test/sharness/t0700-remotepin.sh | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 58f55591ef1..f33e5ac3cd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,11 +146,11 @@ jobs: - run: name: Setup Environment Variables - # 172.17.0.1 is the docker host IP; all ports exported by child containers can be accessed there. - command: echo 'export CI_HOST_IP=172.17.0.1' >> $BASH_ENV - - run: - echo CI_HOST_IP=$CI_HOST_IP && - make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 CI_HOST_IP=$CI_HOST_IP + # we need the docker host IP; all ports exported by child containers can be accessed there. + command: echo "export DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV + - run: + echo DOCKER_HOST=$DOCKER_HOST && + make -O -j 3 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 DOCKER_HOST=$DOCKER_HOST - run: when: always diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 52ce3a81f72..aca0a420f33 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -4,9 +4,9 @@ test_description="Test ipfs remote pinning operations" . lib/test-lib.sh -if [ -z ${CI_HOST_IP+x} ]; then +if [ -z ${DOCKER_HOST+x} ]; then # TODO: set up instead of skipping? - skip_all='Skipping pinning service integration tests: missing CI_HOST_IP, remote pinning service not available' + skip_all='Skipping pinning service integration tests: missing DOCKER_HOST, remote pinning service not available' test_done fi @@ -14,7 +14,7 @@ fi test_init_ipfs test_launch_ipfs_daemon -TEST_PIN_SVC="http://${CI_HOST_IP}:5000/api/v1" +TEST_PIN_SVC="http://${DOCKER_HOST}:5000/api/v1" TEST_PIN_SVC_KEY=$(curl -s -X POST "$TEST_PIN_SVC/users" -d email="go-ipfs-sharness@ipfs.example.com" | jq --raw-output .access_token) # create user on pinning service From 322ec357f441d720196b09d59668e76bd59ca3c7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 17 Nov 2020 15:46:19 +0100 Subject: [PATCH 095/127] fix: 'config show' crash when no pin services - makes scrubing not fail when field missing This still needs: - config replace - tests --- core/commands/config.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/commands/config.go b/core/commands/config.go index efa4d75cfc9..57d5ae64981 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -83,7 +83,7 @@ Set the value of the 'Datastore.Path' key: switch strings.ToLower(key) { case "identity", "identity.privkey": return errors.New("cannot show or change private key through API") - case "remotepinservices": + case "pinning", "remoteservices": return errors.New("cannot show remote pinning services, use 'ipfs pin remote service ls'") default: } @@ -176,7 +176,9 @@ NOTE: For security reasons, this command will omit your private key and remote s if err != nil { return err } - err = scrubValue(cfg, []string{config.PinningRemoteServicesTag}) + + // TODO: needs tests similar to PrivKey ones in test/sharness/t0021-config.sh + err = scrubOptionalValue(cfg, []string{"Pinning", "RemoteServices"}) if err != nil { return err } @@ -197,6 +199,12 @@ NOTE: For security reasons, this command will omit your private key and remote s } func scrubValue(m map[string]interface{}, key []string) error { + return scrub(m, key, false) +} +func scrubOptionalValue(m map[string]interface{}, key []string) error { + return scrub(m, key, true) +} +func scrub(m map[string]interface{}, key []string, okIfMissing bool) error { find := func(m map[string]interface{}, k string) (string, interface{}, bool) { lckey := strings.ToLower(k) for mkey, val := range m { @@ -211,7 +219,7 @@ func scrubValue(m map[string]interface{}, key []string) error { cur := m for _, k := range key[:len(key)-1] { foundk, val, ok := find(cur, k) - if !ok { + if !ok && !okIfMissing { return errors.New("failed to find specified key") } @@ -229,7 +237,7 @@ func scrubValue(m map[string]interface{}, key []string) error { } todel, _, ok := find(cur, key[len(key)-1]) - if !ok { + if !ok && !okIfMissing { return fmt.Errorf("%s, not found", strings.Join(key, ".")) } @@ -488,5 +496,7 @@ func replaceConfig(r repo.Repo, file io.Reader) error { cfg.Identity.PrivKey = pkstr + // TODO: preserve Pinning.RemoteServices (if present in original config) + return r.SetConfig(&cfg) } From c66f1dd30ad34ae1154e142eee0148341d6ea0e0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 17 Nov 2020 21:24:07 +0100 Subject: [PATCH 096/127] feat: switch to --background=false by default We want the default to be blocking by default, so the behaviour on the CLI is the same as the old local `pin add` and we can add `pin local add` later that behaves the same. Another argument is that we don't want to surprise user that the item is not pinned when they dont read the output of `pin remote add` and assume success as "pinned" and not "queued" --- core/commands/pin/remotepin.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index c257b8392b7..75c76374b95 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -105,8 +105,7 @@ var addRemotePinCmd = &cmds.Command{ Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), - // TODO: swap default to --background=false - cmds.BoolOption(pinBackgroundOptionName, "Queue the pin request on the remote service, return RequestID immediately.").WithDefault(true), + cmds.BoolOption(pinBackgroundOptionName, "Add to the queue on the remote service and return RequestID immediately.").WithDefault(false), }, Type: RemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { From 807b79083077de9b7a1e09e7e04164e98f20ca90 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 18 Nov 2020 00:28:22 +0100 Subject: [PATCH 097/127] refactor: background=false by default and status This switches the default to be blocking, just like the old 'pin add` is. This will make future `pin local add` easier to add in a backward-compatible manner. Pinning Services return only pinned objects by default, which may be confusing. This adds explicit default and list of available options, so users are not surprised when non-pinned (eg. queued) pins are ommited from default output. --- core/commands/pin/remotepin.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 75c76374b95..a39dfec353b 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -175,7 +175,7 @@ var addRemotePinCmd = &cmds.Command{ } } - // Block until pinned if background=false + // Block unless --background=true is passed if !req.Options[pinBackgroundOptionName].(bool) { requestId := ps.GetRequestId() for { @@ -237,7 +237,7 @@ Returns a list of objects that are pinned to a remote pinning service. Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), - cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses; optional, comma separated."), + cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses (queued,pinning,pinned,failed)").WithDefault("pinned"), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -365,11 +365,11 @@ collected if needed. cmds.StringArg("request-id", false, true, "Request ID of the pin to be removed."), }, Options: []cmds.Option{ - cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), - cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s); optional, comma separated."), - cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses; optional, comma separated."), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), - cmds.BoolOption(pinForceOptionName, "Delete multiple pins without confirmation.").WithDefault(false), + cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), + cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s)."), + cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses (queued,pinning,pinned,failed).").WithDefault("pinned"), + cmds.BoolOption(pinForceOptionName, "Remove multiple pins without confirmation.").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(req.Context) @@ -393,8 +393,8 @@ collected if needed. if err = <-errCh; err != nil { return fmt.Errorf("error while listing remote pins: %v", err) } - // TODO: shouldn't we require --force only when more than one pin was found? - if len(rmIDs) > 0 && !req.Options[pinForceOptionName].(bool) { + + if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") } } else { From eb992b41cb58dea610e91e797dfeb42d79773ea1 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 18 Nov 2020 19:43:15 +0100 Subject: [PATCH 098/127] chore: remove delegate list from plain-text output We already hide delegate list inb Text output in `pin remote ls`, this aligns `pin remote add` output to do the same. Delegates list is still returned when `--enc=json` is passed (or on HTTP API) --- core/commands/pin/remotepin.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index a39dfec353b..41bc9b6ad73 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -88,7 +88,6 @@ func printRemotePinDetails(w io.Writer, out *RemotePinOutput) { fw("CID", out.Cid) fw("Name", out.Name) fw("Status", out.Status) - fw("Delegates", strings.Join(out.Delegates, ", ")) } // remote pin commands From 0bf0314fa48eca50ca4926d6355e50006ca5db36 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 18 Nov 2020 20:44:15 +0100 Subject: [PATCH 099/127] refactor: switch remote pin tests to inlined CIDs this is in preparation for https://github.com/ipfs-shipyard/rb-pinning-service-api/issues/8 when that lands, we will leverage the fact that inlined CID can be pinned instantly on the remote service --- test/sharness/t0700-remotepin.sh | 70 ++++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index aca0a420f33..fe3ccf43dd4 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -63,11 +63,13 @@ test_expect_success "verify pinning service removal works" ' ipfs pin remote service ls | grep -L test_invalid_key_svc ' -# TODO: why this was not pinned instantly? no bitswap is required for inlined CID -test_expect_success "verify background add works with data inlined in CID" ' - ipfs pin remote add --background=true --service=test_pin_svc --enc=json --name=inlined_null bafkqaaa && - ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=queued,pinning,pinned | tee ls_out && - grep -qE "queued|pinning|pinned" ls_out +# we leverage the fact that inlined CID can be pinned instantly on the remote service +# (https://github.com/ipfs-shipyard/rb-pinning-service-api/issues/8) +# below test ensures that assumption is correct (before we proceed to actual tests) +test_expect_success "verify that default add (implicit --background=false) works with data inlined in CID" ' + ipfs pin remote add --service=test_pin_svc --name=inlined_null bafkqaaa && + ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=pinned | jq --raw-output .Status | tee ls_out && + grep -q "pinned" ls_out ' test_remote_pins() { @@ -76,13 +78,12 @@ test_remote_pins() { BASE_ARGS="--cid-base=$BASE" fi - # time-based CIDs to ensure we test with data that is not on public network - # note: one hash is not imported to IPFS on purpose, to reliably test 'queued' state + # note: HAS_MISSING is not inlined nor imported to IPFS on purpose, to reliably test 'queued' state test_expect_success "create some hashes using base $BASE" ' - export HASH_A=$(echo "A @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_B=$(echo "B @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_C=$(echo "C @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && - export HASH_D=$(echo "D @ $(date)" | ipfs add $BASE_ARGS -q --pin=false) && + export HASH_A=$(echo "A @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && + export HASH_B=$(echo "B @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && + export HASH_C=$(echo "C @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && + export HASH_D=$(echo "D @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date)" | ipfs add $BASE_ARGS -q --only-hash) ' @@ -97,6 +98,18 @@ test_remote_pins() { grep -q $HASH_B ls_out ' + test_expect_success "'ipfs pin remote add --background=false'" ' + export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && + export ID_D=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) + ' + + test_expect_success "verify foreground add worked" ' + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_C $ID_D | tee ls_out && + grep -q $HASH_C ls_out && + grep -q $HASH_D ls_out + ' + + test_expect_success "'ipfs pin remote add' with CID that is not available" ' export ID_M=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING | jq --raw-output .RequestID) ' @@ -108,8 +121,6 @@ test_remote_pins() { grep -q $HASH_MISSING ls_out ' - # TODO: fix / verify remaining tests below - test_expect_success "'ipfs pin remote ls' for existing pins by RequestID" ' ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A | tee ls_out && grep -q $HASH_A ls_out @@ -126,7 +137,7 @@ test_remote_pins() { ' test_expect_success "'ipfs pin remote ls' for existing pins by status" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued | tee ls_out && + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning | tee ls_out && grep -q $HASH_MISSING ls_out ' @@ -134,42 +145,31 @@ test_remote_pins() { ipfs pin remote rm --service=test_pin_svc $ID_A ' - test_expect_success "'ipfs pin remote rm --name' fails without --force when globbing" ' + # --force is required only when more than a single match is found, + # so we add second pin with the same name to simulate that scenario + test_expect_success "'ipfs pin remote rm --name' fails without --force when matching multiple pins" ' + export ID_B2=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --enc=json --name=name_b ' - test_expect_success "'ipfs pin remote rm' an existing pin by name" ' - ipfs pin remote rm --service=test_pin_svc --enc=json --force --name=name_b + test_expect_success "'ipfs pin remote rm --name' without --force did not remove matching pins" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .RequestID | tee ls_out && + grep -q $ID_B && + grep -q $ID_B2 ' - test_expect_success "'ipfs pin remote rm' an existing pin by status" ' + test_expect_success "'ipfs pin remote rm --force' removes all pinned items" ' ipfs pin remote rm --service=test_pin_svc --enc=json --force --status=pinned ' test_expect_success "'ipfs pin remote ls' returns error for deleted pin" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out && test_expect_code 1 grep -q $HASH_B ls_out ' - # TODO: we test this at the end because so far it hangs forever on CI -- needs investigation - test_expect_success "'ipfs pin remote add --background=false'" ' - export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && - export ID_D=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) - ' - - test_expect_success "verify foreground add worked" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_C $ID_D | tee ls_out && - grep -q $HASH_C ls_out && - grep -q $HASH_D ls_out - ' - } test_remote_pins "" - -# TODO: origin / delegates -# TODO: name search - test_kill_ipfs_daemon test_done From fb31dd307553b23ad3266d2dd6166c51e578babd Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 20 Nov 2020 18:34:16 +0100 Subject: [PATCH 100/127] feat: pin remote service ls --pin-count This adds --pin-count opt-in flag that returns number of 'queued/pinning/pinned/failed' pins at remote service or 'offline' if fetch of pin counts was not possible JSON representation uses PinCount object (missing when 'offline') Requires LsBatchSync from https://github.com/ipfs/go-pinning-service-http-client/pull/4 --- core/commands/pin/remotepin.go | 78 +++++++++++++++++++++++++++++--- go.mod | 2 +- go.sum | 6 +-- test/sharness/t0700-remotepin.sh | 8 ++++ 4 files changed, 82 insertions(+), 12 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 41bc9b6ad73..42e61098500 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -57,6 +57,7 @@ const pinStatusOptionName = "status" const pinServiceNameOptionName = "service" const pinServiceURLOptionName = "url" const pinServiceKeyOptionName = "key" +const pinServicePinCountOptionName = "pin-count" const pinBackgroundOptionName = "background" const pinForceOptionName = "force" @@ -510,10 +511,13 @@ var lsRemotePinServiceCmd = &cmds.Command{ ShortDescription: "List remote pinning services.", }, Arguments: []cmds.Argument{}, - Options: []cmds.Option{ - // TODO: -s --stats that execute ls query for each status and reads pagination hints to return Stats object with the count of pins in each state: {Queued, Pinning, Pinned, Failed} + Options: []cmds.Option{ + cmds.BoolOption(pinServicePinCountOptionName, "Try to fetch and display current pin count on remote service (queued/pinning/pinned/failed).").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err @@ -532,9 +536,52 @@ var lsRemotePinServiceCmd = &cmds.Command{ return nil // no pinning services added yet } services := cfg.Pinning.RemoteServices - result := PinServicesList{make([]PinServiceAndEndpoint, 0, len(services))} + result := PinServicesList{make([]ServiceDetails, 0, len(services))} for svcName, svcConfig := range services { - result.RemoteServices = append(result.RemoteServices, PinServiceAndEndpoint{svcName, svcConfig.ApiEndpoint}) + svcDetails := ServiceDetails{svcName, svcConfig.ApiEndpoint, nil} + + // if --pin-count is passed, we try to fetch pin numbers from remote service + if req.Options[pinServicePinCountOptionName].(bool) { + lsRemotePinCount := func(ctx context.Context, env cmds.Environment, svcName string) (*PinCount, error) { + c, err := getRemotePinServiceOrEnv(env, svcName) + if err != nil { + return nil, err + } + // we only care about total count, so requesting smallest batch + batch := pinclient.PinOpts.Limit(1) + fs := pinclient.PinOpts.FilterStatus + + // TODO run in parallel? + _, queued, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusQueued)) + if err != nil { + return nil, err + } + _, pinning, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusPinning)) + if err != nil { + return nil, err + } + _, pinned, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusPinned)) + if err != nil { + return nil, err + } + _, failed, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusFailed)) + if err != nil { + return nil, err + } + + return &PinCount{queued, pinning, pinned, failed}, nil + } + + pinCount, err := lsRemotePinCount(ctx, env, svcName) + + // PinCount is present only if we were able to fetch counts. + // We don't want to break listing of services so this is best-effort. + // (verbose err is returned by 'pin remote ls', if needed) + if err == nil { + svcDetails.PinCount = pinCount + } + } + result.RemoteServices = append(result.RemoteServices, svcDetails) } sort.Sort(result) return cmds.EmitOnce(res, &result) @@ -543,8 +590,17 @@ var lsRemotePinServiceCmd = &cmds.Command{ Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *PinServicesList) error { tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) + withPinCount := req.Options[pinServicePinCountOptionName].(bool) for _, s := range list.RemoteServices { - fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.ApiEndpoint) + if withPinCount { + pinCount := "offline" + if s.PinCount != nil { + pinCount = fmt.Sprintf("%d/%d/%d/%d", s.PinCount.Queued, s.PinCount.Pinning, s.PinCount.Pinned, s.PinCount.Failed) + } + fmt.Fprintf(tw, "%s\t%s\t%s\n", s.Service, s.ApiEndpoint, pinCount) + } else { + fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.ApiEndpoint) + } } tw.Flush() return nil @@ -552,14 +608,22 @@ var lsRemotePinServiceCmd = &cmds.Command{ }, } -type PinServiceAndEndpoint struct { +type ServiceDetails struct { Service string ApiEndpoint string + PinCount *PinCount `json:",omitempty"` // missing when --pin-service is passed means service is offline +} + +type PinCount struct { + Queued int + Pinning int + Pinned int + Failed int } // Struct returned by ipfs pin remote service ls --enc=json | jq type PinServicesList struct { - RemoteServices []PinServiceAndEndpoint + RemoteServices []ServiceDetails } func (l PinServicesList) Len() int { diff --git a/go.mod b/go.mod index 81518df646f..322628a8074 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878 + github.com/ipfs/go-pinning-service-http-client v0.0.0-20201119193934-34aa0b8a7959 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 9b192d0090e..da60abba9b5 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,6 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.4.0 h1:xUavIxA9Ts8U6PAHmQBvDGMlGfUrQ13Rymd+5t8LIF4= github.com/ipfs/go-ipfs-cmds v0.4.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3 h1:7EwzMoBhktnUjdgukBmOm+L0J8pwVlHdXEOJa5Ct6KA= -github.com/ipfs/go-ipfs-config v0.9.1-0.20200925164234-711def0cc4d3/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb h1:cK7DEEHIsYIBVwAaxlQsO3ScvTOOy+CSkeJ53yCxgMY= github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -456,8 +454,8 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878 h1:RQLHEUEGAikm7pFNtxtSSin3TfRQdnhBcCrsaTxxLz0= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201030162340-38f3baeb8878/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201119193934-34aa0b8a7959 h1:+7p/zpQWfElnHKzbjzks7VRuQtnUg2+8FhX43hyf8c8= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201119193934-34aa0b8a7959/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index fe3ccf43dd4..2efea488349 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -34,6 +34,14 @@ test_expect_success "test 'ipfs pin remote service ls'" ' grep -q test_invalid_url_dns_svc ls_out ' +test_expect_success "pin remote service ls --pin-count' returns numbers for a valid service" ' + ipfs pin remote service ls --pin-count | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$" +' + +test_expect_success "pin remote service ls --pin-count' returns 'offline' for invalid service" ' + ipfs pin remote service ls --pin-count | grep -E "^test_invalid_url_path_svc.+offline$" +' + test_expect_success "check connection to test pinning service" ' ipfs pin remote ls --service=test_pin_svc --enc=json ' From 3293d1ac8e3540b2ce4dbf88832ffc4936df5644 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 24 Nov 2020 21:01:54 +0100 Subject: [PATCH 101/127] fix: partial config sanitization and tests - basic sanitization for Pinning.RemoteServices - TODO: 'config replace' still needs a fix - tests mostly green thanks to inlined CIDs - TODO: filtering by status may have a bug, needs a closer look --- .circleci/config.yml | 2 +- core/commands/config.go | 24 +++++++- test/sharness/t0700-remotepin.sh | 101 +++++++++++++++++++++---------- 3 files changed, 92 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f33e5ac3cd3..4c9beaa1515 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,7 @@ jobs: git clone --depth 1 https://github.com/ipfs-shipyard/rb-pinning-service-api.git - run: cd rb-pinning-service-api && - git checkout 0a3a50b715b8bd080a7c75a4298b2145c4e6955d && + git checkout 3a48a62027a19b365e19b49d1f65ee14b87d0754 && docker-compose pull && docker-compose up -d && cd .. diff --git a/core/commands/config.go b/core/commands/config.go index 57d5ae64981..47c21308edb 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -83,11 +83,15 @@ Set the value of the 'Datastore.Path' key: switch strings.ToLower(key) { case "identity", "identity.privkey": return errors.New("cannot show or change private key through API") - case "pinning", "remoteservices": - return errors.New("cannot show remote pinning services, use 'ipfs pin remote service ls'") + case "pinning", "pinning.remoteservices": + return errors.New("cannot show or change pinning services through API, use 'ipfs pin remote service' instead") default: } + if blocked := inBlockedScope(key, "Pinning.RemoteServices"); blocked { + return errors.New("cannot show or change pinning services through API, use 'ipfs pin remote service' instead") + } + cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return err @@ -142,6 +146,22 @@ Set the value of the 'Datastore.Path' key: Type: ConfigField{}, } +// Returns bool to indicate if tested key is in the blocked scope. +// (scope includes parent, direct, and child match) +func inBlockedScope(testKey string, blockedScope string) bool { + blockedScope = strings.ToLower(blockedScope) + roots := strings.Split(strings.ToLower(testKey), ".") + var scope []string + for _, name := range roots { + scope := append(scope, name) + impactedKey := strings.Join(scope, ".") + if strings.HasPrefix(impactedKey, blockedScope) { + return true + } + } + return false +} + var configShowCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Output config file contents.", diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 2efea488349..1abf07de1e7 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -14,10 +14,13 @@ fi test_init_ipfs test_launch_ipfs_daemon +# create user on pinning service TEST_PIN_SVC="http://${DOCKER_HOST}:5000/api/v1" TEST_PIN_SVC_KEY=$(curl -s -X POST "$TEST_PIN_SVC/users" -d email="go-ipfs-sharness@ipfs.example.com" | jq --raw-output .access_token) -# create user on pinning service +# pin remote service add|ls|rm + +# add valid and invalid services test_expect_success "creating test user on remote pinning service" ' echo CI host IP address ${TEST_PIN_SVC} && ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} && @@ -34,6 +37,28 @@ test_expect_success "test 'ipfs pin remote service ls'" ' grep -q test_invalid_url_dns_svc ls_out ' +# SECURITY: prevent exposing the ApiKey on the network +test_expect_success "'ipfs config Pinning' fails" ' + test_expect_code 1 ipfs config Pinning 2> config_out +' + +test_expect_success "output includes meaningful error" " + echo \"Error: cannot show or change pinning services through API, use 'ipfs pin remote service' instead\" > config_exp && + test_cmp config_exp config_out +" + +test_expect_success "'ipfs config Pinning.RemoteServices' fails" ' + test_expect_code 1 ipfs config Pinning.RemoteServices 2> config_out +' + +test_expect_success "output includes meaningful error" ' + test_cmp config_exp config_out +' + +# TODO: test ipfs config replace + +# /SECURITY + test_expect_success "pin remote service ls --pin-count' returns numbers for a valid service" ' ipfs pin remote service ls --pin-count | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$" ' @@ -61,6 +86,8 @@ test_expect_success "misconfigured pinning service calls fail (dns error)" ' test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_dns_svc ' +# pin remote service rm + test_expect_success "remove pinning service" ' ipfs pin remote service rm test_invalid_key_svc && ipfs pin remote service rm test_invalid_url_path_svc && @@ -71,6 +98,8 @@ test_expect_success "verify pinning service removal works" ' ipfs pin remote service ls | grep -L test_invalid_key_svc ' +# pin remote add + # we leverage the fact that inlined CID can be pinned instantly on the remote service # (https://github.com/ipfs-shipyard/rb-pinning-service-api/issues/8) # below test ensures that assumption is correct (before we proceed to actual tests) @@ -90,67 +119,72 @@ test_remote_pins() { test_expect_success "create some hashes using base $BASE" ' export HASH_A=$(echo "A @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && export HASH_B=$(echo "B @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && - export HASH_C=$(echo "C @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && - export HASH_D=$(echo "D @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && - export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date)" | ipfs add $BASE_ARGS -q --only-hash) + export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date)" | ipfs add $BASE_ARGS -q --only-hash) && + echo "A: $HASH_A" && + echo "B: $HASH_B" && + echo "M: $HASH_MISSING" ' test_expect_success "'ipfs pin remote add --background=true'" ' - export ID_A=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) && - export ID_B=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) + export ID_A=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) ' - test_expect_success "verify background add worked" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A $ID_B | tee ls_out && - grep -q $HASH_A ls_out && - grep -q $HASH_B ls_out + test_expect_success "'ipfs pin remote add --background=true' with CID that is not available" ' + export ID_M=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING | jq --raw-output .RequestID) ' - test_expect_success "'ipfs pin remote add --background=false'" ' - export ID_C=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_c $HASH_C | jq --raw-output .RequestID) && - export ID_D=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_d $HASH_D | jq --raw-output .RequestID) + test_expect_success "'ipfs pin remote ls' includes queued CID that is not available" ' + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_M ' - test_expect_success "verify foreground add worked" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_C $ID_D | tee ls_out && - grep -q $HASH_C ls_out && - grep -q $HASH_D ls_out + test_expect_success "verify background add worked" ' + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A $ID_M | tee ls_out && + test_expect_code 0 grep -q $HASH_A ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out ' + test_expect_success "'ipfs pin remote add --background=false'" ' + export ID_B=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) + ' - test_expect_success "'ipfs pin remote add' with CID that is not available" ' - export ID_M=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING | jq --raw-output .RequestID) + test_expect_success "verify foreground add worked" ' + ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out && + test_expect_code 0 grep -q $HASH_B ls_out ' + # TODO: this does not seem to find $HASH_MISSING test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | tee ls_out && - grep -q $HASH_A ls_out && - grep -q $HASH_B ls_out && - grep -q $HASH_MISSING ls_out + test_expect_code 0 grep -q $HASH_A ls_out && + test_expect_code 0 grep -q $HASH_B ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out ' test_expect_success "'ipfs pin remote ls' for existing pins by RequestID" ' ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A | tee ls_out && - grep -q $HASH_A ls_out + test_expect_code 0 grep -q $HASH_A ls_out ' test_expect_success "'ipfs pin remote ls' for existing pins by CID" ' ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_B | tee ls_out && - grep -q $HASH_B ls_out + test_expect_code 0 grep -q $HASH_B ls_out ' test_expect_success "'ipfs pin remote ls' for existing pins by name" ' ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out && - grep -q $HASH_A ls_out + test_expect_code 0 grep -q $HASH_A ls_out ' - test_expect_success "'ipfs pin remote ls' for existing pins by status" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning | tee ls_out && - grep -q $HASH_MISSING ls_out + # TODO: this does not seem to find $HASH_MISSING + test_expect_success "'ipfs pin remote ls' for ongoing pins by status" ' + ipfs pin remote ls --service=test_pin_svc --status=queued,pinning | tee ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out ' test_expect_success "'ipfs pin remote rm' an existing pin by RequestID" ' - ipfs pin remote rm --service=test_pin_svc $ID_A + test_expect_code 0 ipfs pin remote ls --service=test_pin_svc $ID_A && + ipfs pin remote rm --service=test_pin_svc $ID_A && + test_expect_code 1 ipfs pin remote ls --service=test_pin_svc $ID_A ' # --force is required only when more than a single match is found, @@ -162,12 +196,15 @@ test_remote_pins() { test_expect_success "'ipfs pin remote rm --name' without --force did not remove matching pins" ' ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .RequestID | tee ls_out && - grep -q $ID_B && - grep -q $ID_B2 + test_expect_code 0 grep -q $ID_B ls_out && + test_expect_code 0 grep -q $ID_B2 ls_out ' test_expect_success "'ipfs pin remote rm --force' removes all pinned items" ' - ipfs pin remote rm --service=test_pin_svc --enc=json --force --status=pinned + ipfs pin remote rm --service=test_pin_svc --enc=json --force && + ipfs pin remote ls --service=test_pin_svc --enc=json --status=pinned | tee ls_out && + test_expect_code 1 grep -q $ID_A ls_out && + test_expect_code 1 grep -q $ID_B ls_out ' test_expect_success "'ipfs pin remote ls' returns error for deleted pin" ' From a080d841dc39ce2c9bcd66d69a7e59832d09cbd0 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 25 Nov 2020 00:25:39 -0500 Subject: [PATCH 102/127] bump version of go-ipfs-cmds --- core/commands/pin/remotepin.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 42e61098500..656bf6c41e9 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -237,7 +237,7 @@ Returns a list of objects that are pinned to a remote pinning service. Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), - cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses (queued,pinning,pinned,failed)").WithDefault("pinned"), + cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses (queued,pinning,pinned,failed)").WithDefault([]string{"pinned"}), cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -368,7 +368,7 @@ collected if needed. cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s)."), - cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses (queued,pinning,pinned,failed).").WithDefault("pinned"), + cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), cmds.BoolOption(pinForceOptionName, "Remove multiple pins without confirmation.").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { diff --git a/go.mod b/go.mod index 322628a8074..a0e6da6a2b7 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/ipfs/go-graphsync v0.1.1 github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 - github.com/ipfs/go-ipfs-cmds v0.4.0 + github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 diff --git a/go.sum b/go.sum index da60abba9b5..af512f4ba81 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= -github.com/ipfs/go-ipfs-cmds v0.4.0 h1:xUavIxA9Ts8U6PAHmQBvDGMlGfUrQ13Rymd+5t8LIF4= -github.com/ipfs/go-ipfs-cmds v0.4.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= +github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa h1:e3O5uBLuOFZI+G7Np0iZJ2y7CdYec48j70zeiaWlDW8= +github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb h1:cK7DEEHIsYIBVwAaxlQsO3ScvTOOy+CSkeJ53yCxgMY= github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From c5a845119620b3736ad1324b84fe540d40806852 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 25 Nov 2020 00:26:49 -0500 Subject: [PATCH 103/127] check option types in pin remote commands more defensively --- core/commands/pin/remotepin.go | 74 +++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 656bf6c41e9..276292694a0 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -93,6 +93,8 @@ func printRemotePinDetails(w io.Writer, out *RemotePinOutput) { // remote pin commands +var pinServiceNameOption = cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use.") + var addRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Pin objects to remote storage.", @@ -104,7 +106,7 @@ var addRemotePinCmd = &cmds.Command{ }, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), - cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + pinServiceNameOption, cmds.BoolOption(pinBackgroundOptionName, "Add to the queue on the remote service and return RequestID immediately.").WithDefault(false), }, Type: RemotePinOutput{}, @@ -113,10 +115,14 @@ var addRemotePinCmd = &cmds.Command{ defer cancel() // Get remote service - service, _ := req.Options[pinServiceNameOptionName].(string) - c, err := getRemotePinServiceOrEnv(env, service) - if err != nil { - return err + var c *pinclient.Client + if service, serviceFound := req.Options[pinServiceNameOptionName]; serviceFound { + serviceStr := service.(string) + var err error + c, err = getRemotePinService(env, serviceStr) + if err != nil { + return err + } } // Prepare value for Pin.cid @@ -134,8 +140,9 @@ var addRemotePinCmd = &cmds.Command{ // Prepare Pin.name opts := []pinclient.AddOption{} - if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { - opts = append(opts, pinclient.PinOpts.WithName(name)) + if name, nameFound := req.Options[pinNameOptionName]; nameFound { + nameStr := name.(string) + opts = append(opts, pinclient.PinOpts.WithName(nameStr)) } // Prepare Pin.origins @@ -238,16 +245,20 @@ Returns a list of objects that are pinned to a remote pinning service. cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), cmds.StringsOption(pinStatusOptionName, "Return only pin objects with the specified statuses (queued,pinning,pinned,failed)").WithDefault([]string{"pinned"}), - cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + pinServiceNameOption, }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(req.Context) defer cancel() - service, _ := req.Options[pinServiceNameOptionName].(string) - c, err := getRemotePinServiceOrEnv(env, service) - if err != nil { - return err + var c *pinclient.Client + if service, serviceFound := req.Options[pinServiceNameOptionName]; serviceFound { + serviceStr := service.(string) + var err error + c, err = getRemotePinService(env, serviceStr) + if err != nil { + return err + } } psCh, errCh, err := lsRemote(ctx, req, c) @@ -312,13 +323,15 @@ func lsRemoteByRequestId(ctx context.Context, req *cmds.Request, c *pinclient.Cl // Executes GET /pins/?query-with-filters func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { opts := []pinclient.LsOption{} - if name, nameFound := req.Options[pinNameOptionName].(string); nameFound { - opts = append(opts, pinclient.PinOpts.FilterName(name)) + if name, nameFound := req.Options[pinNameOptionName]; nameFound { + nameStr := name.(string) + opts = append(opts, pinclient.PinOpts.FilterName(nameStr)) } - if cidsRaw, cidsFound := req.Options[pinCIDsOptionName].([]string); cidsFound { + if cidsRaw, cidsFound := req.Options[pinCIDsOptionName]; cidsFound { + cidsRawArr := cidsRaw.([]string) parsedCIDs := []cid.Cid{} - for _, rawCID := range flattenCommaList(cidsRaw) { + for _, rawCID := range flattenCommaList(cidsRawArr) { parsedCID, err := cid.Decode(rawCID) if err != nil { return nil, nil, fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) @@ -327,9 +340,10 @@ func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Cl } opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) } - if statusRaw, statusFound := req.Options[pinStatusOptionName].([]string); statusFound { + if statusRaw, statusFound := req.Options[pinStatusOptionName]; statusFound { + statusRawArr := statusRaw.([]string) parsedStatuses := []pinclient.Status{} - for _, rawStatus := range flattenCommaList(statusRaw) { + for _, rawStatus := range flattenCommaList(statusRawArr) { s := pinclient.Status(rawStatus) if s.String() == string(pinclient.StatusUnknown) { return nil, nil, fmt.Errorf("status %q is not valid", rawStatus) @@ -365,7 +379,7 @@ collected if needed. cmds.StringArg("request-id", false, true, "Request ID of the pin to be removed."), }, Options: []cmds.Option{ - cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use."), + pinServiceNameOption, cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Remove only pin objects for the specified CID(s)."), cmds.StringsOption(pinStatusOptionName, "Remove only pin objects with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), @@ -375,10 +389,14 @@ collected if needed. ctx, cancel := context.WithCancel(req.Context) defer cancel() - service, _ := req.Options[pinServiceNameOptionName].(string) - c, err := getRemotePinServiceOrEnv(env, service) - if err != nil { - return err + var c *pinclient.Client + if service, serviceFound := req.Options[pinServiceNameOptionName]; serviceFound { + serviceStr := service.(string) + var err error + c, err = getRemotePinService(env, serviceStr) + if err != nil { + return err + } } rmIDs := []string{} @@ -402,7 +420,7 @@ collected if needed. } for _, rmID := range rmIDs { - if err = c.DeleteByID(ctx, rmID); err != nil { + if err := c.DeleteByID(ctx, rmID); err != nil { return fmt.Errorf("removing pin identified by requestid=%q failed: %v", rmID, err) } } @@ -543,7 +561,7 @@ var lsRemotePinServiceCmd = &cmds.Command{ // if --pin-count is passed, we try to fetch pin numbers from remote service if req.Options[pinServicePinCountOptionName].(bool) { lsRemotePinCount := func(ctx context.Context, env cmds.Environment, svcName string) (*PinCount, error) { - c, err := getRemotePinServiceOrEnv(env, svcName) + c, err := getRemotePinService(env, svcName) if err != nil { return nil, err } @@ -640,18 +658,18 @@ func (l PinServicesList) Less(i, j int) bool { return s[i].Service < s[j].Service } -func getRemotePinServiceOrEnv(env cmds.Environment, name string) (*pinclient.Client, error) { +func getRemotePinService(env cmds.Environment, name string) (*pinclient.Client, error) { if name == "" { return nil, fmt.Errorf("remote pinning service name not specified") } - url, key, err := getRemotePinService(env, name) + url, key, err := getRemotePinServiceInfo(env, name) if err != nil { return nil, err } return pinclient.NewClient(url, key), nil } -func getRemotePinService(env cmds.Environment, name string) (url, key string, err error) { +func getRemotePinServiceInfo(env cmds.Environment, name string) (url, key string, err error) { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { return "", "", err From ca3b866019451ebe0b4a08933335645959068b28 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 26 Nov 2020 15:32:03 +0100 Subject: [PATCH 104/127] fix: remote remote services from config outputs This ensures contents of Pinning.RemoteServices get the same protection as Identity.PrivKey. The goal here is it ensure that even if API access is compromised, the remote service ApiKey won't get leaked. --- core/commands/config.go | 47 ++++++++++++++++++++++++++------ go.mod | 3 +- go.sum | 5 ++-- test/sharness/t0700-remotepin.sh | 44 ++++++++++++++++++++++++++---- 4 files changed, 82 insertions(+), 17 deletions(-) diff --git a/core/commands/config.go b/core/commands/config.go index 47c21308edb..d5e495e6a65 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -17,6 +17,7 @@ import ( "github.com/elgris/jsondiff" cmds "github.com/ipfs/go-ipfs-cmds" config "github.com/ipfs/go-ipfs-config" + "github.com/mitchellh/mapstructure" ) // ConfigUpdateOutput is config profile apply command's output @@ -36,6 +37,8 @@ const ( configDryRunOptionName = "dry-run" ) +var tryRemoteServiceApiErr = errors.New("cannot show or change pinning services through this API (try: ipfs pin remote service --help)") + var ConfigCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get and set ipfs config values.", @@ -83,13 +86,13 @@ Set the value of the 'Datastore.Path' key: switch strings.ToLower(key) { case "identity", "identity.privkey": return errors.New("cannot show or change private key through API") - case "pinning", "pinning.remoteservices": - return errors.New("cannot show or change pinning services through API, use 'ipfs pin remote service' instead") default: } - if blocked := inBlockedScope(key, "Pinning.RemoteServices"); blocked { - return errors.New("cannot show or change pinning services through API, use 'ipfs pin remote service' instead") + // Temporary fix until we move ApiKey secrets out of the config file + // (remote services are a map, so more advanced blocking is required) + if blocked := inBlockedScope(key, config.RemoteServicesSelector); blocked { + return tryRemoteServiceApiErr } cfgRoot, err := cmdenv.GetConfigRoot(env) @@ -155,7 +158,9 @@ func inBlockedScope(testKey string, blockedScope string) bool { for _, name := range roots { scope := append(scope, name) impactedKey := strings.Join(scope, ".") - if strings.HasPrefix(impactedKey, blockedScope) { + // blockedScope=foo.bar.BLOCKED should return true + // for parent and child impactedKeys: foo.bar and foo.bar.BLOCKED.subkey + if strings.HasPrefix(impactedKey, blockedScope) || strings.HasPrefix(blockedScope, impactedKey) { return true } } @@ -197,8 +202,7 @@ NOTE: For security reasons, this command will omit your private key and remote s return err } - // TODO: needs tests similar to PrivKey ones in test/sharness/t0021-config.sh - err = scrubOptionalValue(cfg, []string{"Pinning", "RemoteServices"}) + err = scrubOptionalValue(cfg, []string{config.PinningTag, config.RemoteServicesTag}) if err != nil { return err } @@ -218,12 +222,16 @@ NOTE: For security reasons, this command will omit your private key and remote s }, } +// Scrubs value and returns error if missing func scrubValue(m map[string]interface{}, key []string) error { return scrub(m, key, false) } + +// Scrubs value and returns no error if missing func scrubOptionalValue(m map[string]interface{}, key []string) error { return scrub(m, key, true) } + func scrub(m map[string]interface{}, key []string, okIfMissing bool) error { find := func(m map[string]interface{}, k string) (string, interface{}, bool) { lckey := strings.ToLower(k) @@ -500,6 +508,9 @@ func replaceConfig(r repo.Repo, file io.Reader) error { if err := json.NewDecoder(file).Decode(&cfg); err != nil { return errors.New("failed to decode file as config") } + + // Handle Identity.PrivKey (secret) + if len(cfg.Identity.PrivKey) != 0 { return errors.New("setting private key with API is not supported") } @@ -516,7 +527,27 @@ func replaceConfig(r repo.Repo, file io.Reader) error { cfg.Identity.PrivKey = pkstr - // TODO: preserve Pinning.RemoteServices (if present in original config) + // Handle Pinning.RemoteServices (ApiKey of each service is secret) + // Note: these settings are opt-in and may be missing + + if len(cfg.Pinning.RemoteServices) != 0 { + return tryRemoteServiceApiErr + } + + // detect if existing config has any remote services defined.. + if remoteServicesTag, err := getConfig(r, config.RemoteServicesSelector); err == nil { + // seems that golang cannot type assert map[string]interface{} to map[string]config.RemotePinningService + // so we have to manually copy the data :-| + if val, ok := remoteServicesTag.Value.(map[string]interface{}); ok { + var services map[string]config.RemotePinningService + err := mapstructure.Decode(val, &services) + if err != nil { + return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err) + } + // .. if so, apply them on top of the new config + cfg.Pinning.RemoteServices = services + } + } return r.SetConfig(&cfg) } diff --git a/go.mod b/go.mod index a0e6da6a2b7..b51be42ab9c 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa - github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb + github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 @@ -88,6 +88,7 @@ require ( github.com/lucas-clemente/quic-go v0.18.1 github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/go-homedir v1.1.0 + github.com/mitchellh/mapstructure v1.1.2 github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 diff --git a/go.sum b/go.sum index af512f4ba81..721976fbb80 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa h1:e3O5uBLuOFZI+G7Np0iZJ2y7CdYec48j70zeiaWlDW8= github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb h1:cK7DEEHIsYIBVwAaxlQsO3ScvTOOy+CSkeJ53yCxgMY= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201116161859-f69c8d6d47cb/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64 h1:36d77+JS1TqRF3+WaJvMqB+0Rq62jyGq1iRGypssYGY= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -941,6 +941,7 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 1abf07de1e7..f842b17006a 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -37,25 +37,55 @@ test_expect_success "test 'ipfs pin remote service ls'" ' grep -q test_invalid_url_dns_svc ls_out ' -# SECURITY: prevent exposing the ApiKey on the network +# SECURITY of access tokens in ApiKey fields: +# Pinning.RemoteServices includes ApiKey, and we give it the same treatment +# as Identity.PrivKey to prevent exposing it on the network + test_expect_success "'ipfs config Pinning' fails" ' test_expect_code 1 ipfs config Pinning 2> config_out ' - -test_expect_success "output includes meaningful error" " - echo \"Error: cannot show or change pinning services through API, use 'ipfs pin remote service' instead\" > config_exp && +test_expect_success "output includes meaningful error" ' + echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > config_exp && test_cmp config_exp config_out -" +' test_expect_success "'ipfs config Pinning.RemoteServices' fails" ' test_expect_code 1 ipfs config Pinning.RemoteServices 2> config_out ' +test_expect_success "output includes meaningful error" ' + test_cmp config_exp config_out +' +test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc' fails" ' + test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc 2> config_out +' test_expect_success "output includes meaningful error" ' test_cmp config_exp config_out ' -# TODO: test ipfs config replace +test_expect_success "'ipfs config show' doesn't include RemoteServices" ' + ipfs config show > show_config && + test_expect_code 1 grep RemoteServices show_config +' + +test_expect_success "'ipfs config replace' injects remote services back" ' + test_expect_code 1 grep -q -E "test_.+_svc" show_config && + ipfs config replace show_config && + test_expect_code 0 grep -q test_pin_svc "$IPFS_PATH/config" && + test_expect_code 0 grep -q test_invalid_key_svc "$IPFS_PATH/config" && + test_expect_code 0 grep -q test_invalid_url_path_svc "$IPFS_PATH/config" && + test_expect_code 0 grep -q test_invalid_url_dns_svc "$IPFS_PATH/config" +' + +# note: we remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices +test_expect_success "'ipfs config replace' with remote services errors out" ' + jq -M "del(.Identity.PrivKey)" "$IPFS_PATH/config" | jq ".Pinning += { RemoteServices: {\"foo\": {} }}" > new_config && + test_expect_code 1 ipfs config replace - < new_config 2> replace_out +' +test_expect_success "output includes meaningful error" ' + echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > replace_expected + test_cmp replace_out replace_expected +' # /SECURITY @@ -218,3 +248,5 @@ test_remote_pins "" test_kill_ipfs_daemon test_done + +# vim: ts=2 sw=2 sts=2 et: From 9ff07fa1d6f838aa8680e43312871705829f3fb8 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 27 Nov 2020 00:30:57 +0100 Subject: [PATCH 105/127] refactor: parallel remote service ls --pin-count --- core/commands/pin/remotepin.go | 46 +++++++++++++++++++++++----------- go.mod | 1 + 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 276292694a0..6fd8d673f51 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -11,6 +11,8 @@ import ( neturl "net/url" + "golang.org/x/sync/errgroup" + cid "github.com/ipfs/go-cid" cmds "github.com/ipfs/go-ipfs-cmds" config "github.com/ipfs/go-ipfs-config" @@ -569,25 +571,41 @@ var lsRemotePinServiceCmd = &cmds.Command{ batch := pinclient.PinOpts.Limit(1) fs := pinclient.PinOpts.FilterStatus - // TODO run in parallel? - _, queued, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusQueued)) - if err != nil { - return nil, err - } - _, pinning, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusPinning)) - if err != nil { - return nil, err + statuses := []pinclient.Status{ + pinclient.StatusQueued, + pinclient.StatusPinning, + pinclient.StatusPinned, + pinclient.StatusFailed, } - _, pinned, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusPinned)) - if err != nil { - return nil, err + + g, ctx := errgroup.WithContext(ctx) + pc := &PinCount{} + + for _, s := range statuses { + status := s // lol https://golang.org/doc/faq#closures_and_goroutines + g.Go(func() error { + _, n, err := c.LsBatchSync(ctx, batch, fs(status)) + if err != nil { + return err + } + switch status { + case pinclient.StatusQueued: + pc.Queued = n + case pinclient.StatusPinning: + pc.Pinning = n + case pinclient.StatusPinned: + pc.Pinned = n + case pinclient.StatusFailed: + pc.Failed = n + } + return nil + }) } - _, failed, err := c.LsBatchSync(ctx, batch, fs(pinclient.StatusFailed)) - if err != nil { + if err := g.Wait(); err != nil { return nil, err } - return &PinCount{queued, pinning, pinned, failed}, nil + return pc, nil } pinCount, err := lsRemotePinCount(ctx, env, svcName) diff --git a/go.mod b/go.mod index b51be42ab9c..9c983e12640 100644 --- a/go.mod +++ b/go.mod @@ -106,6 +106,7 @@ require ( go.uber.org/zap v1.15.0 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect + golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 // indirect ) From 8f89163f6cee371dd46f6b087fa973fc8d71524c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 27 Nov 2020 02:01:43 +0100 Subject: [PATCH 106/127] fix: escape HTML in error response body See https://github.com/ipfs/go-pinning-service-http-client/pull/5 --- go.mod | 2 +- go.sum | 4 ++-- test/sharness/t0700-remotepin.sh | 5 ----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 9c983e12640..ca1882d1df2 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client v0.0.0-20201119193934-34aa0b8a7959 + github.com/ipfs/go-pinning-service-http-client v0.0.0-20201127004627-34a6066675f7 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 721976fbb80..441d547fd40 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,8 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201119193934-34aa0b8a7959 h1:+7p/zpQWfElnHKzbjzks7VRuQtnUg2+8FhX43hyf8c8= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201119193934-34aa0b8a7959/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201127004627-34a6066675f7 h1:2rh10QEOOES5g5W+n+e/NAMQmw2OAml3ZKyjgFeC6B8= +github.com/ipfs/go-pinning-service-http-client v0.0.0-20201127004627-34a6066675f7/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index f842b17006a..53ecaa61472 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -101,17 +101,14 @@ test_expect_success "check connection to test pinning service" ' ipfs pin remote ls --service=test_pin_svc --enc=json ' -# TODO: improve error returned below test_expect_success "unathorized pinning service calls fail" ' test_expect_code 1 ipfs pin remote ls --service=test_invalid_key_svc ' -# TODO: improve error returned below test_expect_success "misconfigured pinning service calls fail (wrong path)" ' test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_path_svc ' -# TODO: improve error returned below (panic when offline mode) test_expect_success "misconfigured pinning service calls fail (dns error)" ' test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_dns_svc ' @@ -182,7 +179,6 @@ test_remote_pins() { test_expect_code 0 grep -q $HASH_B ls_out ' - # TODO: this does not seem to find $HASH_MISSING test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" ' ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | tee ls_out && test_expect_code 0 grep -q $HASH_A ls_out && @@ -205,7 +201,6 @@ test_remote_pins() { test_expect_code 0 grep -q $HASH_A ls_out ' - # TODO: this does not seem to find $HASH_MISSING test_expect_success "'ipfs pin remote ls' for ongoing pins by status" ' ipfs pin remote ls --service=test_pin_svc --status=queued,pinning | tee ls_out && test_expect_code 0 grep -q $HASH_MISSING ls_out From 4f44ac39232f8b004b6bf34d2dbd3bfa89644db9 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 30 Nov 2020 22:42:35 +0100 Subject: [PATCH 107/127] refactor: remove mapstructure built-in json marshal/unmarshal does the same trick --- core/commands/config.go | 4 ++-- go.mod | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/core/commands/config.go b/core/commands/config.go index d5e495e6a65..65c01270a53 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -17,7 +17,6 @@ import ( "github.com/elgris/jsondiff" cmds "github.com/ipfs/go-ipfs-cmds" config "github.com/ipfs/go-ipfs-config" - "github.com/mitchellh/mapstructure" ) // ConfigUpdateOutput is config profile apply command's output @@ -540,10 +539,11 @@ func replaceConfig(r repo.Repo, file io.Reader) error { // so we have to manually copy the data :-| if val, ok := remoteServicesTag.Value.(map[string]interface{}); ok { var services map[string]config.RemotePinningService - err := mapstructure.Decode(val, &services) + jsonString, err := json.Marshal(val) if err != nil { return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err) } + json.Unmarshal(jsonString, &services) // .. if so, apply them on top of the new config cfg.Pinning.RemoteServices = services } diff --git a/go.mod b/go.mod index ca1882d1df2..46f8a495a89 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,6 @@ require ( github.com/lucas-clemente/quic-go v0.18.1 github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/go-homedir v1.1.0 - github.com/mitchellh/mapstructure v1.1.2 github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 From 4eb3757dabb03b969b6b1d783ec342a89178c73b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 1 Dec 2020 00:26:39 +0100 Subject: [PATCH 108/127] chore: go-pinning-service-http-client v0.1.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 46f8a495a89..b053c65d4f7 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 github.com/ipfs/go-path v0.0.8 - github.com/ipfs/go-pinning-service-http-client v0.0.0-20201127004627-34a6066675f7 + github.com/ipfs/go-pinning-service-http-client v0.1.0 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.0 diff --git a/go.sum b/go.sum index 441d547fd40..a01eb857f53 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,8 @@ github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3 github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN1EJOjc= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201127004627-34a6066675f7 h1:2rh10QEOOES5g5W+n+e/NAMQmw2OAml3ZKyjgFeC6B8= -github.com/ipfs/go-pinning-service-http-client v0.0.0-20201127004627-34a6066675f7/go.mod h1:7aOggtq0hGoNLsv5pV7zfvrDDaqqNd1/GxdnS4MkSZA= +github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY= +github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= From ef975b0c95aa40c00e81dca0eb2209a5989e3cec Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 1 Dec 2020 00:34:27 +0100 Subject: [PATCH 109/127] chore: golint --- core/commands/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/commands/config.go b/core/commands/config.go index 65c01270a53..3931bfc25dc 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -543,7 +543,10 @@ func replaceConfig(r repo.Repo, file io.Reader) error { if err != nil { return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err) } - json.Unmarshal(jsonString, &services) + err = json.Unmarshal(jsonString, &services) + if err != nil { + return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err) + } // .. if so, apply them on top of the new config cfg.Pinning.RemoteServices = services } From 5e9e89c2d710c60469202e18cd05545cfd71ed11 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 1 Dec 2020 00:35:43 +0100 Subject: [PATCH 110/127] fix(ci): remove reference to missing commit --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c9beaa1515..ba01a7c6fa4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,6 @@ jobs: git clone --depth 1 https://github.com/ipfs-shipyard/rb-pinning-service-api.git - run: cd rb-pinning-service-api && - git checkout 3a48a62027a19b365e19b49d1f65ee14b87d0754 && docker-compose pull && docker-compose up -d && cd .. From 965b7092cd776e7b44ae6db9128eb155735ec48d Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 1 Dec 2020 19:37:39 -0800 Subject: [PATCH 111/127] use newest config structure --- core/commands/pin/remotepin.go | 10 ++++++---- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 6fd8d673f51..d3f715e53a3 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -480,8 +480,10 @@ var addRemotePinServiceCmd = &cmds.Command{ cfg.Pinning.RemoteServices = map[string]config.RemotePinningService{} } cfg.Pinning.RemoteServices[name] = config.RemotePinningService{ - ApiEndpoint: url, - ApiKey: key, + Api: config.RemotePinningServiceApi{ + ApiEndpoint: url, + ApiKey: key, + }, } return repo.SetConfig(cfg) @@ -558,7 +560,7 @@ var lsRemotePinServiceCmd = &cmds.Command{ services := cfg.Pinning.RemoteServices result := PinServicesList{make([]ServiceDetails, 0, len(services))} for svcName, svcConfig := range services { - svcDetails := ServiceDetails{svcName, svcConfig.ApiEndpoint, nil} + svcDetails := ServiceDetails{svcName, svcConfig.Api.ApiEndpoint, nil} // if --pin-count is passed, we try to fetch pin numbers from remote service if req.Options[pinServicePinCountOptionName].(bool) { @@ -708,5 +710,5 @@ func getRemotePinServiceInfo(env cmds.Environment, name string) (url, key string if !present { return "", "", fmt.Errorf("service not known") } - return service.ApiEndpoint, service.ApiKey, nil + return service.Api.ApiEndpoint, service.Api.ApiKey, nil } diff --git a/go.mod b/go.mod index b053c65d4f7..6452bbd930b 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa - github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64 + github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 diff --git a/go.sum b/go.sum index a01eb857f53..c5ff281c198 100644 --- a/go.sum +++ b/go.sum @@ -360,6 +360,8 @@ github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa h1:e3O5uBLuOFZ github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64 h1:36d77+JS1TqRF3+WaJvMqB+0Rq62jyGq1iRGypssYGY= github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1 h1:5r/LVBdMP3zRlFhonGFBfEXFV9Igh4Fpd1erc+SLwrg= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From 7d63db36979fab2fb43736945834330da315b1cd Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 2 Dec 2020 20:30:01 -0500 Subject: [PATCH 112/127] commands: pin remote add,ls, and rm now require a service flag to be passed --- core/commands/pin/remotepin.go | 49 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index d3f715e53a3..3fd8dc84c0f 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -117,14 +117,9 @@ var addRemotePinCmd = &cmds.Command{ defer cancel() // Get remote service - var c *pinclient.Client - if service, serviceFound := req.Options[pinServiceNameOptionName]; serviceFound { - serviceStr := service.(string) - var err error - c, err = getRemotePinService(env, serviceStr) - if err != nil { - return err - } + c, err := getRemotePinServiceFromRequest(req, env) + if err != nil { + return err } // Prepare value for Pin.cid @@ -253,14 +248,9 @@ Returns a list of objects that are pinned to a remote pinning service. ctx, cancel := context.WithCancel(req.Context) defer cancel() - var c *pinclient.Client - if service, serviceFound := req.Options[pinServiceNameOptionName]; serviceFound { - serviceStr := service.(string) - var err error - c, err = getRemotePinService(env, serviceStr) - if err != nil { - return err - } + c, err := getRemotePinServiceFromRequest(req, env) + if err != nil { + return err } psCh, errCh, err := lsRemote(ctx, req, c) @@ -391,14 +381,9 @@ collected if needed. ctx, cancel := context.WithCancel(req.Context) defer cancel() - var c *pinclient.Client - if service, serviceFound := req.Options[pinServiceNameOptionName]; serviceFound { - serviceStr := service.(string) - var err error - c, err = getRemotePinService(env, serviceStr) - if err != nil { - return err - } + c, err := getRemotePinServiceFromRequest(req, env) + if err != nil { + return err } rmIDs := []string{} @@ -678,6 +663,22 @@ func (l PinServicesList) Less(i, j int) bool { return s[i].Service < s[j].Service } +func getRemotePinServiceFromRequest(req *cmds.Request, env cmds.Environment) (*pinclient.Client, error) { + service, serviceFound := req.Options[pinServiceNameOptionName] + if !serviceFound { + return nil, fmt.Errorf("a service name must be passed") + } + + serviceStr := service.(string) + var err error + c, err := getRemotePinService(env, serviceStr) + if err != nil { + return nil, err + } + + return c, nil +} + func getRemotePinService(env cmds.Environment, name string) (*pinclient.Client, error) { if name == "" { return nil, fmt.Errorf("remote pinning service name not specified") From 8564996cd9905f30a8c0dc07769d01a4088d2971 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 3 Dec 2020 07:26:54 -0800 Subject: [PATCH 113/127] add pin policy flag --- core/commands/pin/remotepin.go | 20 ++++++++++++++++++-- test/sharness/t0700-remotepin.sh | 3 ++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 3fd8dc84c0f..29abea074d7 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -62,6 +62,7 @@ const pinServiceKeyOptionName = "key" const pinServicePinCountOptionName = "pin-count" const pinBackgroundOptionName = "background" const pinForceOptionName = "force" +const pinPolicyOptionName = "policy" type RemotePinOutput struct { RequestID string @@ -427,8 +428,10 @@ var addRemotePinServiceCmd = &cmds.Command{ cmds.StringArg(pinServiceURLOptionName, true, false, "Service URL."), cmds.StringArg(pinServiceKeyOptionName, true, false, "Service key."), }, - Options: []cmds.Option{}, - Type: nil, + Options: []cmds.Option{ + cmds.StringsOption(pinPolicyOptionName, "list of pin policies from: mfs; optional, comma separated."), + }, + Type: nil, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) if err != nil { @@ -464,11 +467,24 @@ var addRemotePinServiceCmd = &cmds.Command{ } else { cfg.Pinning.RemoteServices = map[string]config.RemotePinningService{} } + + var pinMFS bool + if policyRaw, policyFound := req.Options[pinCIDsOptionName]; policyFound { + for _, p := range policyRaw.([]string) { + if p == "mfs" { + pinMFS = true + } + } + } + cfg.Pinning.RemoteServices[name] = config.RemotePinningService{ Api: config.RemotePinningServiceApi{ ApiEndpoint: url, ApiKey: key, }, + Policies: config.RemotePinningServicePolicies{ + PinMFS: &pinMFS, + }, } return repo.SetConfig(cfg) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 53ecaa61472..006b22f9bf2 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -26,7 +26,8 @@ test_expect_success "creating test user on remote pinning service" ' ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} && ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key && ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key && - ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key + ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key && + ipfs pin remote service add --policy=mfs test_pin_mfs_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} ' test_expect_success "test 'ipfs pin remote service ls'" ' From 5dfbe2f42a7deee6aa9291b9241072d8cef6d8d8 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Thu, 3 Dec 2020 15:15:31 -0800 Subject: [PATCH 114/127] update config dep --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6452bbd930b..f8c30060689 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa - github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1 + github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 diff --git a/go.sum b/go.sum index c5ff281c198..856f8779e7e 100644 --- a/go.sum +++ b/go.sum @@ -362,6 +362,8 @@ github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64 h1:36d77+JS1 github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1 h1:5r/LVBdMP3zRlFhonGFBfEXFV9Igh4Fpd1erc+SLwrg= github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38 h1:YTUQ/PfhhuG9MEfoqdf+VHW2leUUxD5/ZFyyXVurr9w= +github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From 1706647faf7021987592f1524e69bb4b98b1a300 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 4 Dec 2020 12:47:46 +0100 Subject: [PATCH 115/127] fix: avoid redundant /pins/pins Small UX improvement, if someone passes URL with operation path: https://api.service.example.com/pins/ --- core/commands/pin/remotepin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 29abea074d7..4d4e2cd1380 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -448,7 +448,7 @@ var addRemotePinServiceCmd = &cmds.Command{ } name := req.Arguments[0] - url := req.Arguments[1] + url := strings.TrimSuffix(req.Arguments[1], "/pins") // fix /pins/pins :-) key := req.Arguments[2] u, err := neturl.ParseRequestURI(url) From e4e2b046c3ea7c5e242b2fdcf3c0c2de6efc34db Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 4 Dec 2020 06:59:35 -0800 Subject: [PATCH 116/127] simplify inBlockScope(); now Pinning.MFSRepinInterval is settable with ipfs config --- core/commands/config.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/core/commands/config.go b/core/commands/config.go index 3931bfc25dc..426fe607696 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -148,22 +148,9 @@ Set the value of the 'Datastore.Path' key: Type: ConfigField{}, } -// Returns bool to indicate if tested key is in the blocked scope. -// (scope includes parent, direct, and child match) +// Returns bool to indicate if tested key is inside blockedScope. func inBlockedScope(testKey string, blockedScope string) bool { - blockedScope = strings.ToLower(blockedScope) - roots := strings.Split(strings.ToLower(testKey), ".") - var scope []string - for _, name := range roots { - scope := append(scope, name) - impactedKey := strings.Join(scope, ".") - // blockedScope=foo.bar.BLOCKED should return true - // for parent and child impactedKeys: foo.bar and foo.bar.BLOCKED.subkey - if strings.HasPrefix(impactedKey, blockedScope) || strings.HasPrefix(blockedScope, impactedKey) { - return true - } - } - return false + return strings.HasPrefix(strings.ToLower(testKey), strings.ToLower(blockedScope)) } var configShowCmd = &cmds.Command{ From a1ea3d65f527f4cf1ea496fc988a50bfa8823082 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 4 Dec 2020 22:08:20 +0100 Subject: [PATCH 117/127] refactor: remove requestid and delegates Rationale: https://github.com/ipfs/go-ipfs/pull/7661#issuecomment-739024909 Note: one config test fails due to unrelated changes, this will be addressed separatelly. --- core/commands/pin/remotepin.go | 75 +++++------------------ test/sharness/t0700-remotepin.sh | 100 ++++++++++++++++--------------- 2 files changed, 68 insertions(+), 107 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 4d4e2cd1380..7206e8945f3 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -65,20 +65,16 @@ const pinForceOptionName = "force" const pinPolicyOptionName = "policy" type RemotePinOutput struct { - RequestID string - Status string - Cid string - Name string - Delegates []string // multiaddr + Status string + Cid string + Name string } func toRemotePinOutput(ps pinclient.PinStatusGetter) RemotePinOutput { return RemotePinOutput{ - RequestID: ps.GetRequestId(), - Name: ps.GetPin().GetName(), - Delegates: multiaddrsToStrings(ps.GetDelegates()), - Status: ps.GetStatus().String(), - Cid: ps.GetPin().GetCid().String(), + Name: ps.GetPin().GetName(), + Status: ps.GetStatus().String(), + Cid: ps.GetPin().GetCid().String(), } } @@ -88,7 +84,6 @@ func printRemotePinDetails(w io.Writer, out *RemotePinOutput) { fw := func(k string, v string) { fmt.Fprintf(tw, "%s:\t%s\n", k, v) } - fw("RequestID", out.RequestID) fw("CID", out.Cid) fw("Name", out.Name) fw("Status", out.Status) @@ -100,8 +95,8 @@ var pinServiceNameOption = cmds.StringOption(pinServiceNameOptionName, "Name of var addRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ - Tagline: "Pin objects to remote storage.", - ShortDescription: "Stores an IPFS object(s) from a given path to a remote pinning service.", + Tagline: "Pin object to remote pinning service.", + ShortDescription: "Stores an IPFS object from a given path to a remote pinning service.", }, Arguments: []cmds.Argument{ @@ -110,7 +105,7 @@ var addRemotePinCmd = &cmds.Command{ Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "An optional name for the pin."), pinServiceNameOption, - cmds.BoolOption(pinBackgroundOptionName, "Add to the queue on the remote service and return RequestID immediately.").WithDefault(false), + cmds.BoolOption(pinBackgroundOptionName, "Add to the queue on the remote service and return immediately (does not wait for pinned status).").WithDefault(false), }, Type: RemotePinOutput{}, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -202,7 +197,7 @@ var addRemotePinCmd = &cmds.Command{ select { case <-tmr.C: case <-ctx.Done(): - return fmt.Errorf("waiting for pin interrupted, requestid=%q remains on remote service: check its status with 'ls' or abort with 'rm'", requestId) + return fmt.Errorf("waiting for pin interrupted, requestid=%q remains on remote service", requestId) } } } @@ -236,9 +231,7 @@ Returns a list of objects that are pinned to a remote pinning service. `, }, - Arguments: []cmds.Argument{ - cmds.StringArg("request-id", false, true, "Request ID of the pin to be listed."), - }, + Arguments: []cmds.Argument{}, Options: []cmds.Option{ cmds.StringOption(pinNameOptionName, "Return pins objects with names that contain provided value (case-sensitive, exact match)."), cmds.StringsOption(pinCIDsOptionName, "Return only pin objects for the specified CID(s); optional, comma separated."), @@ -271,50 +264,14 @@ Returns a list of objects that are pinned to a remote pinning service. Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { // pin remote ls produces a flat output similar to legacy pin ls - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", out.RequestID, out.Cid, out.Status, out.Name) + fmt.Fprintf(w, "%s\t%s\t%s\n", out.Cid, out.Status, out.Name) return nil }), }, } -// Returns pin objects with passed RequestIDs or matching filters. -func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { - // If request-ids are provided, do direct lookup for specific objects - if len(req.Arguments) > 0 { - return lsRemoteByRequestId(ctx, req, c) - } - // else, apply filters to find matching pin objects - return lsRemoteWithFilters(ctx, req, c) -} - -// Executes GET /pins/{requestid} for each requestid passed as argument. -// Status checks are executed one by one and operation is aborted on first error. -func lsRemoteByRequestId(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { - lsIDs := req.Arguments - res := make(chan pinclient.PinStatusGetter, 1) - errs := make(chan error, 1) - go func() { - defer close(res) - defer close(errs) - for _, requestId := range lsIDs { - ps, err := c.GetStatusByID(ctx, requestId) - if err != nil { - errs <- err - return - } - select { - case res <- ps: - case <-ctx.Done(): - errs <- ctx.Err() - return - } - } - }() - return res, errs, nil -} - // Executes GET /pins/?query-with-filters -func lsRemoteWithFilters(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { +func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName]; nameFound { nameStr := name.(string) @@ -368,9 +325,7 @@ collected if needed. `, }, - Arguments: []cmds.Argument{ - cmds.StringArg("request-id", false, true, "Request ID of the pin to be removed."), - }, + Arguments: []cmds.Argument{}, Options: []cmds.Option{ pinServiceNameOption, cmds.StringOption(pinNameOptionName, "Remove pin objects with names that contain provided value (case-sensitive, exact match)."), @@ -404,7 +359,7 @@ collected if needed. return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") } } else { - rmIDs = append(rmIDs, req.Arguments[0]) + return fmt.Errorf("unexpected argument %q", req.Arguments[0]) } for _, rmID := range rmIDs { diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 006b22f9bf2..10966a2ee75 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -42,18 +42,19 @@ test_expect_success "test 'ipfs pin remote service ls'" ' # Pinning.RemoteServices includes ApiKey, and we give it the same treatment # as Identity.PrivKey to prevent exposing it on the network -test_expect_success "'ipfs config Pinning' fails" ' - test_expect_code 1 ipfs config Pinning 2> config_out +test_expect_success "'ipfs config Pinning' works" ' + test_expect_code 0 ipfs config Pinning > config_out ' -test_expect_success "output includes meaningful error" ' - echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > config_exp && - test_cmp config_exp config_out +test_expect_success "output does not include ApiKey" ' + test_expect_code 1 grep -q ApiKey config_out ' -test_expect_success "'ipfs config Pinning.RemoteServices' fails" ' - test_expect_code 1 ipfs config Pinning.RemoteServices 2> config_out +test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.Api.ApiKey' fails" ' + test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.Api.ApiKey 2> config_out ' + test_expect_success "output includes meaningful error" ' + echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > config_exp && test_cmp config_exp config_out ' @@ -84,7 +85,7 @@ test_expect_success "'ipfs config replace' with remote services errors out" ' test_expect_code 1 ipfs config replace - < new_config 2> replace_out ' test_expect_success "output includes meaningful error" ' - echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > replace_expected + echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > replace_expected && test_cmp replace_out replace_expected ' @@ -102,7 +103,7 @@ test_expect_success "check connection to test pinning service" ' ipfs pin remote ls --service=test_pin_svc --enc=json ' -test_expect_success "unathorized pinning service calls fail" ' +test_expect_success "unauthorized pinning service calls fail" ' test_expect_code 1 ipfs pin remote ls --service=test_invalid_key_svc ' @@ -123,7 +124,10 @@ test_expect_success "remove pinning service" ' ' test_expect_success "verify pinning service removal works" ' - ipfs pin remote service ls | grep -L test_invalid_key_svc + ipfs pin remote service ls | tee ls_out && + test_expect_code 1 grep test_invalid_key_svc ls_out && + test_expect_code 1 grep test_invalid_url_path_svc ls_out && + test_expect_code 1 grep test_invalid_url_dns_svc ls_out ' # pin remote add @@ -145,38 +149,44 @@ test_remote_pins() { # note: HAS_MISSING is not inlined nor imported to IPFS on purpose, to reliably test 'queued' state test_expect_success "create some hashes using base $BASE" ' - export HASH_A=$(echo "A @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && - export HASH_B=$(echo "B @ $(date)" | ipfs add $BASE_ARGS -q --inline --inline-limit 100 --pin=false) && - export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date)" | ipfs add $BASE_ARGS -q --only-hash) && + export HASH_A=$(echo -n "A @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && + export HASH_B=$(echo -n "B @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && + export HASH_C=$(echo -n "C @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) && + export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --only-hash) && echo "A: $HASH_A" && echo "B: $HASH_B" && + echo "C: $HASH_C" && echo "M: $HASH_MISSING" ' test_expect_success "'ipfs pin remote add --background=true'" ' - export ID_A=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A | jq --raw-output .RequestID) + ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A ' - test_expect_success "'ipfs pin remote add --background=true' with CID that is not available" ' - export ID_M=$(ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING | jq --raw-output .RequestID) + test_expect_success "verify background add worked (instantly pinned variant)" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out && + test_expect_code 0 grep -q name_a ls_out && + test_expect_code 0 grep -q $HASH_A ls_out ' - test_expect_success "'ipfs pin remote ls' includes queued CID that is not available" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_M + test_expect_success "'ipfs pin remote add --background=true' with CID that is not available" ' + test_expect_code 0 ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING ' - test_expect_success "verify background add worked" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A $ID_M | tee ls_out && - test_expect_code 0 grep -q $HASH_A ls_out && + test_expect_success "verify background add worked (queued variant)" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_m --status=queued,pinning | tee ls_out && + test_expect_code 0 grep -q name_m ls_out && test_expect_code 0 grep -q $HASH_MISSING ls_out ' test_expect_success "'ipfs pin remote add --background=false'" ' - export ID_B=$(ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) + test_expect_code 0 ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B ' test_expect_success "verify foreground add worked" ' ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out && + test_expect_code 0 grep -q name_b ls_out && + test_expect_code 0 grep -q pinned ls_out && test_expect_code 0 grep -q $HASH_B ls_out ' @@ -187,11 +197,6 @@ test_remote_pins() { test_expect_code 0 grep -q $HASH_MISSING ls_out ' - test_expect_success "'ipfs pin remote ls' for existing pins by RequestID" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_A | tee ls_out && - test_expect_code 0 grep -q $HASH_A ls_out - ' - test_expect_success "'ipfs pin remote ls' for existing pins by CID" ' ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_B | tee ls_out && test_expect_code 0 grep -q $HASH_B ls_out @@ -207,35 +212,36 @@ test_remote_pins() { test_expect_code 0 grep -q $HASH_MISSING ls_out ' - test_expect_success "'ipfs pin remote rm' an existing pin by RequestID" ' - test_expect_code 0 ipfs pin remote ls --service=test_pin_svc $ID_A && - ipfs pin remote rm --service=test_pin_svc $ID_A && - test_expect_code 1 ipfs pin remote ls --service=test_pin_svc $ID_A - ' - # --force is required only when more than a single match is found, - # so we add second pin with the same name to simulate that scenario + # so we add second pin with the same name (but different CID) to simulate that scenario test_expect_success "'ipfs pin remote rm --name' fails without --force when matching multiple pins" ' - export ID_B2=$(ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B | jq --raw-output .RequestID) && - test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --enc=json --name=name_b + test_expect_code 0 ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_C && + test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --name=name_b 2> rm_out && + echo "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal" > rm_expected && + test_cmp rm_out rm_expected ' test_expect_success "'ipfs pin remote rm --name' without --force did not remove matching pins" ' - ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .RequestID | tee ls_out && - test_expect_code 0 grep -q $ID_B ls_out && - test_expect_code 0 grep -q $ID_B2 ls_out + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out && + test_expect_code 0 grep -q $HASH_B ls_out && + test_expect_code 0 grep -q $HASH_C ls_out ' - test_expect_success "'ipfs pin remote rm --force' removes all pinned items" ' - ipfs pin remote rm --service=test_pin_svc --enc=json --force && - ipfs pin remote ls --service=test_pin_svc --enc=json --status=pinned | tee ls_out && - test_expect_code 1 grep -q $ID_A ls_out && - test_expect_code 1 grep -q $ID_B ls_out + test_expect_success "'ipfs pin remote rm --name' with --force removes all matching pins" ' + test_expect_code 0 ipfs pin remote rm --service=test_pin_svc --name=name_b --force && + ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out && + test_expect_code 1 grep -q $HASH_B ls_out && + test_expect_code 1 grep -q $HASH_C ls_out ' - test_expect_success "'ipfs pin remote ls' returns error for deleted pin" ' - ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out && - test_expect_code 1 grep -q $HASH_B ls_out + test_expect_success "'ipfs pin remote rm --force' removes all pinned items" ' + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out && + test_expect_code 0 grep -q $HASH_A ls_out && + test_expect_code 0 grep -q $HASH_MISSING ls_out && + ipfs pin remote rm --service=test_pin_svc --status=queued,pinning,pinned,failed --force && + ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out && + test_expect_code 1 grep -q $HASH_A ls_out && + test_expect_code 1 grep -q $HASH_MISSING ls_out ' } From fa9f36f034444ebdcd2699e620bd5bb842be013a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 4 Dec 2020 22:34:55 +0100 Subject: [PATCH 118/127] refactor: sanitize entire Pinning block for now We will replace this with smarter logic in separate PR with MFSRepinInterval --- core/commands/config.go | 17 +++++++++++++++-- core/commands/pin/remotepin.go | 9 --------- test/sharness/t0700-remotepin.sh | 4 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/core/commands/config.go b/core/commands/config.go index 426fe607696..3931bfc25dc 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -148,9 +148,22 @@ Set the value of the 'Datastore.Path' key: Type: ConfigField{}, } -// Returns bool to indicate if tested key is inside blockedScope. +// Returns bool to indicate if tested key is in the blocked scope. +// (scope includes parent, direct, and child match) func inBlockedScope(testKey string, blockedScope string) bool { - return strings.HasPrefix(strings.ToLower(testKey), strings.ToLower(blockedScope)) + blockedScope = strings.ToLower(blockedScope) + roots := strings.Split(strings.ToLower(testKey), ".") + var scope []string + for _, name := range roots { + scope := append(scope, name) + impactedKey := strings.Join(scope, ".") + // blockedScope=foo.bar.BLOCKED should return true + // for parent and child impactedKeys: foo.bar and foo.bar.BLOCKED.subkey + if strings.HasPrefix(impactedKey, blockedScope) || strings.HasPrefix(blockedScope, impactedKey) { + return true + } + } + return false } var configShowCmd = &cmds.Command{ diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 7206e8945f3..a811157149f 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -23,7 +23,6 @@ import ( path "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-core/host" peer "github.com/libp2p/go-libp2p-core/peer" - "github.com/multiformats/go-multiaddr" ) var log = logging.Logger("core/commands/cmdenv") @@ -212,14 +211,6 @@ var addRemotePinCmd = &cmds.Command{ }, } -func multiaddrsToStrings(m []multiaddr.Multiaddr) []string { - r := make([]string, len(m)) - for i := range m { - r[i] = m[i].String() - } - return r -} - var listRemotePinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects pinned to remote pinning service.", diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 10966a2ee75..970b41684ef 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -42,8 +42,8 @@ test_expect_success "test 'ipfs pin remote service ls'" ' # Pinning.RemoteServices includes ApiKey, and we give it the same treatment # as Identity.PrivKey to prevent exposing it on the network -test_expect_success "'ipfs config Pinning' works" ' - test_expect_code 0 ipfs config Pinning > config_out +test_expect_success "'ipfs config Pinning' fails" ' + test_expect_code 1 ipfs config Pinning 2>&1 > config_out ' test_expect_success "output does not include ApiKey" ' test_expect_code 1 grep -q ApiKey config_out From 244b186d7d7f817fb42d0752a61a4d351df630a7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 5 Dec 2020 00:16:38 +0100 Subject: [PATCH 119/127] refactor: rename --pin-count to --stat Applying suggestions from https://github.com/ipfs/go-ipfs/pull/7661#issuecomment-739049969 --- core/commands/pin/remotepin.go | 33 ++++++++++++++++++++------------ test/sharness/t0700-remotepin.sh | 28 ++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index a811157149f..43fca49b1b3 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -58,7 +58,7 @@ const pinStatusOptionName = "status" const pinServiceNameOptionName = "service" const pinServiceURLOptionName = "url" const pinServiceKeyOptionName = "key" -const pinServicePinCountOptionName = "pin-count" +const pinServiceStatOptionName = "stat" const pinBackgroundOptionName = "background" const pinForceOptionName = "force" const pinPolicyOptionName = "policy" @@ -481,7 +481,7 @@ var lsRemotePinServiceCmd = &cmds.Command{ }, Arguments: []cmds.Argument{}, Options: []cmds.Option{ - cmds.BoolOption(pinServicePinCountOptionName, "Try to fetch and display current pin count on remote service (queued/pinning/pinned/failed).").WithDefault(false), + cmds.BoolOption(pinServiceStatOptionName, "Try to fetch and display current pin count on remote service (queued/pinning/pinned/failed).").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { ctx, cancel := context.WithCancel(req.Context) @@ -507,10 +507,10 @@ var lsRemotePinServiceCmd = &cmds.Command{ services := cfg.Pinning.RemoteServices result := PinServicesList{make([]ServiceDetails, 0, len(services))} for svcName, svcConfig := range services { - svcDetails := ServiceDetails{svcName, svcConfig.Api.ApiEndpoint, nil} + svcDetails := ServiceDetails{svcName, svcConfig.Api.ApiEndpoint, Stat{Status: "unknown", PinCount: nil}} // if --pin-count is passed, we try to fetch pin numbers from remote service - if req.Options[pinServicePinCountOptionName].(bool) { + if req.Options[pinServiceStatOptionName].(bool) { lsRemotePinCount := func(ctx context.Context, env cmds.Environment, svcName string) (*PinCount, error) { c, err := getRemotePinService(env, svcName) if err != nil { @@ -563,7 +563,10 @@ var lsRemotePinServiceCmd = &cmds.Command{ // We don't want to break listing of services so this is best-effort. // (verbose err is returned by 'pin remote ls', if needed) if err == nil { - svcDetails.PinCount = pinCount + svcDetails.Stat.Status = "valid" + svcDetails.Stat.PinCount = pinCount + } else { + svcDetails.Stat.Status = "invalid" } } result.RemoteServices = append(result.RemoteServices, svcDetails) @@ -575,14 +578,15 @@ var lsRemotePinServiceCmd = &cmds.Command{ Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *PinServicesList) error { tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) - withPinCount := req.Options[pinServicePinCountOptionName].(bool) + withStat := req.Options[pinServiceStatOptionName].(bool) for _, s := range list.RemoteServices { - if withPinCount { - pinCount := "offline" - if s.PinCount != nil { - pinCount = fmt.Sprintf("%d/%d/%d/%d", s.PinCount.Queued, s.PinCount.Pinning, s.PinCount.Pinned, s.PinCount.Failed) + if withStat { + stat := s.Stat.Status + pc := s.Stat.PinCount + if s.Stat.PinCount != nil { + stat = fmt.Sprintf("%d/%d/%d/%d", pc.Queued, pc.Pinning, pc.Pinned, pc.Failed) } - fmt.Fprintf(tw, "%s\t%s\t%s\n", s.Service, s.ApiEndpoint, pinCount) + fmt.Fprintf(tw, "%s\t%s\t%s\n", s.Service, s.ApiEndpoint, stat) } else { fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.ApiEndpoint) } @@ -596,7 +600,12 @@ var lsRemotePinServiceCmd = &cmds.Command{ type ServiceDetails struct { Service string ApiEndpoint string - PinCount *PinCount `json:",omitempty"` // missing when --pin-service is passed means service is offline + Stat Stat +} + +type Stat struct { + Status string + PinCount *PinCount `json:",omitempty"` // missing when --stat is passed means service is offline } type PinCount struct { diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 970b41684ef..c0931c75d0c 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -91,15 +91,33 @@ test_expect_success "output includes meaningful error" ' # /SECURITY -test_expect_success "pin remote service ls --pin-count' returns numbers for a valid service" ' - ipfs pin remote service ls --pin-count | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$" +test_expect_success "pin remote service ls --stat' returns numbers for a valid service" ' + ipfs pin remote service ls --stat | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$" ' -test_expect_success "pin remote service ls --pin-count' returns 'offline' for invalid service" ' - ipfs pin remote service ls --pin-count | grep -E "^test_invalid_url_path_svc.+offline$" +test_expect_success "pin remote service ls --enc=json --stat' returns valid status" " + ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_pin_svc\") | .Stat.Status' | tee stat_out && + echo valid > stat_expected && + test_cmp stat_out stat_expected +" + +test_expect_success "pin remote service ls --stat' returns invalid status for invalid service" ' + ipfs pin remote service ls --stat | grep -E "^test_invalid_url_path_svc.+invalid$" ' -test_expect_success "check connection to test pinning service" ' +test_expect_success "pin remote service ls --enc=json --stat' returns invalid status" " + ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat.Status' | tee stat_out && + echo invalid > stat_expected && + test_cmp stat_out stat_expected +" + +test_expect_success "pin remote service ls --enc=json' (without --stat) returns unknown status" " + ipfs pin remote service ls --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat.Status' | tee stat_out && + echo unknown > stat_expected && + test_cmp stat_out stat_expected +" + +test_expect_success "check connection to the test pinning service" ' ipfs pin remote ls --service=test_pin_svc --enc=json ' From 808e74ea3ce3b320b09b1961667276895c0becb0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 5 Dec 2020 01:39:26 +0100 Subject: [PATCH 120/127] fix(ci): deterministic rb-pinning-service-api --- .circleci/config.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba01a7c6fa4..664cf1e9e07 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,12 +133,16 @@ jobs: - checkout - run: - git clone --depth 1 https://github.com/ipfs-shipyard/rb-pinning-service-api.git + mkdir rb-pinning-service-api && + cd rb-pinning-service-api && + git init && + git remote add origin https://github.com/ipfs-shipyard/rb-pinning-service-api.git && + git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 && + git checkout FETCH_HEAD - run: cd rb-pinning-service-api && docker-compose pull && - docker-compose up -d && - cd .. + docker-compose up -d - *make_out_dirs - *restore_gomod From 32f89502e6a0f4b15b1fad0e3384ad6ac49d91d1 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 5 Dec 2020 02:57:52 +0100 Subject: [PATCH 121/127] refactor: no Stat without --stat https://github.com/ipfs/go-ipfs/pull/7661#issuecomment-739098557 --- core/commands/pin/remotepin.go | 7 ++++--- test/sharness/t0700-remotepin.sh | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 43fca49b1b3..f0a231b596f 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -507,7 +507,7 @@ var lsRemotePinServiceCmd = &cmds.Command{ services := cfg.Pinning.RemoteServices result := PinServicesList{make([]ServiceDetails, 0, len(services))} for svcName, svcConfig := range services { - svcDetails := ServiceDetails{svcName, svcConfig.Api.ApiEndpoint, Stat{Status: "unknown", PinCount: nil}} + svcDetails := ServiceDetails{svcName, svcConfig.Api.ApiEndpoint, nil} // if --pin-count is passed, we try to fetch pin numbers from remote service if req.Options[pinServiceStatOptionName].(bool) { @@ -562,6 +562,7 @@ var lsRemotePinServiceCmd = &cmds.Command{ // PinCount is present only if we were able to fetch counts. // We don't want to break listing of services so this is best-effort. // (verbose err is returned by 'pin remote ls', if needed) + svcDetails.Stat = &Stat{} if err == nil { svcDetails.Stat.Status = "valid" svcDetails.Stat.PinCount = pinCount @@ -600,12 +601,12 @@ var lsRemotePinServiceCmd = &cmds.Command{ type ServiceDetails struct { Service string ApiEndpoint string - Stat Stat + Stat *Stat `json:",omitempty"` // present only when --stat not passed } type Stat struct { Status string - PinCount *PinCount `json:",omitempty"` // missing when --stat is passed means service is offline + PinCount *PinCount `json:",omitempty"` // missing when --stat is passed but the service is offline } type PinCount struct { diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index c0931c75d0c..909b76cf28e 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -111,9 +111,9 @@ test_expect_success "pin remote service ls --enc=json --stat' returns invalid st test_cmp stat_out stat_expected " -test_expect_success "pin remote service ls --enc=json' (without --stat) returns unknown status" " - ipfs pin remote service ls --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat.Status' | tee stat_out && - echo unknown > stat_expected && +test_expect_success "pin remote service ls --enc=json' (without --stat) returns no Stat object" " + ipfs pin remote service ls --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat' | tee stat_out && + echo null > stat_expected && test_cmp stat_out stat_expected " From eda4ea435e3af00ff8c32a3eb7a3c326c6468b2f Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Sat, 5 Dec 2020 21:34:16 -0800 Subject: [PATCH 122/127] remove references to pinmfs --- core/commands/pin/remotepin.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index f0a231b596f..8d1b1b3a8d0 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -61,7 +61,6 @@ const pinServiceKeyOptionName = "key" const pinServiceStatOptionName = "stat" const pinBackgroundOptionName = "background" const pinForceOptionName = "force" -const pinPolicyOptionName = "policy" type RemotePinOutput struct { Status string @@ -374,9 +373,6 @@ var addRemotePinServiceCmd = &cmds.Command{ cmds.StringArg(pinServiceURLOptionName, true, false, "Service URL."), cmds.StringArg(pinServiceKeyOptionName, true, false, "Service key."), }, - Options: []cmds.Option{ - cmds.StringsOption(pinPolicyOptionName, "list of pin policies from: mfs; optional, comma separated."), - }, Type: nil, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cfgRoot, err := cmdenv.GetConfigRoot(env) From 876abdacdf33b21205572e76cfc7a826c6b769bd Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Sat, 5 Dec 2020 21:36:37 -0800 Subject: [PATCH 123/127] fix compile; fix config dependency --- core/commands/pin/remotepin.go | 20 ++++---------------- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 8d1b1b3a8d0..12a2639c998 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -410,22 +410,10 @@ var addRemotePinServiceCmd = &cmds.Command{ cfg.Pinning.RemoteServices = map[string]config.RemotePinningService{} } - var pinMFS bool - if policyRaw, policyFound := req.Options[pinCIDsOptionName]; policyFound { - for _, p := range policyRaw.([]string) { - if p == "mfs" { - pinMFS = true - } - } - } - cfg.Pinning.RemoteServices[name] = config.RemotePinningService{ Api: config.RemotePinningServiceApi{ - ApiEndpoint: url, - ApiKey: key, - }, - Policies: config.RemotePinningServicePolicies{ - PinMFS: &pinMFS, + Endpoint: url, + Key: key, }, } @@ -503,7 +491,7 @@ var lsRemotePinServiceCmd = &cmds.Command{ services := cfg.Pinning.RemoteServices result := PinServicesList{make([]ServiceDetails, 0, len(services))} for svcName, svcConfig := range services { - svcDetails := ServiceDetails{svcName, svcConfig.Api.ApiEndpoint, nil} + svcDetails := ServiceDetails{svcName, svcConfig.Api.Endpoint, nil} // if --pin-count is passed, we try to fetch pin numbers from remote service if req.Options[pinServiceStatOptionName].(bool) { @@ -679,5 +667,5 @@ func getRemotePinServiceInfo(env cmds.Environment, name string) (url, key string if !present { return "", "", fmt.Errorf("service not known") } - return service.Api.ApiEndpoint, service.Api.ApiKey, nil + return service.Api.Endpoint, service.Api.Key, nil } diff --git a/go.mod b/go.mod index f8c30060689..96b6ffff2db 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa - github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38 + github.com/ipfs/go-ipfs-config v0.10.1-0.20201205222155-84cb43b0a8c9 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 diff --git a/go.sum b/go.sum index 856f8779e7e..17c30a36625 100644 --- a/go.sum +++ b/go.sum @@ -364,6 +364,8 @@ github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1 h1:5r/LVBdMP github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38 h1:YTUQ/PfhhuG9MEfoqdf+VHW2leUUxD5/ZFyyXVurr9w= github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= +github.com/ipfs/go-ipfs-config v0.10.1-0.20201205222155-84cb43b0a8c9 h1:fvVDfvbM3szTh5BzoswyTG4EAMAKkQ0auzYgljGHcFI= +github.com/ipfs/go-ipfs-config v0.10.1-0.20201205222155-84cb43b0a8c9/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From da85c7445d9b188d44d4c6f6fbd0dd72b81648b2 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Sat, 5 Dec 2020 21:38:47 -0800 Subject: [PATCH 124/127] remove mfs and policy from test --- test/sharness/t0700-remotepin.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index 909b76cf28e..f3c7826921b 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -27,7 +27,6 @@ test_expect_success "creating test user on remote pinning service" ' ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key && ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key && ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key && - ipfs pin remote service add --policy=mfs test_pin_mfs_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} ' test_expect_success "test 'ipfs pin remote service ls'" ' From 121ae86f442bcadf619559c96830f1d447b198d4 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 7 Dec 2020 15:30:43 +0100 Subject: [PATCH 125/127] =?UTF-8?q?fix(test):=20Api.ApiKey=20=E2=86=92=20A?= =?UTF-8?q?pi.Key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/sharness/t0700-remotepin.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/sharness/t0700-remotepin.sh b/test/sharness/t0700-remotepin.sh index f3c7826921b..a033f87457f 100755 --- a/test/sharness/t0700-remotepin.sh +++ b/test/sharness/t0700-remotepin.sh @@ -26,7 +26,7 @@ test_expect_success "creating test user on remote pinning service" ' ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} && ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key && ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key && - ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key && + ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key ' test_expect_success "test 'ipfs pin remote service ls'" ' @@ -37,19 +37,19 @@ test_expect_success "test 'ipfs pin remote service ls'" ' grep -q test_invalid_url_dns_svc ls_out ' -# SECURITY of access tokens in ApiKey fields: -# Pinning.RemoteServices includes ApiKey, and we give it the same treatment +# SECURITY of access tokens in Api.Key fields: +# Pinning.RemoteServices includes Api.Key, and we give it the same treatment # as Identity.PrivKey to prevent exposing it on the network test_expect_success "'ipfs config Pinning' fails" ' test_expect_code 1 ipfs config Pinning 2>&1 > config_out ' -test_expect_success "output does not include ApiKey" ' - test_expect_code 1 grep -q ApiKey config_out +test_expect_success "output does not include Api.Key" ' + test_expect_code 1 grep -q Key config_out ' -test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.Api.ApiKey' fails" ' - test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.Api.ApiKey 2> config_out +test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.Api.Key' fails" ' + test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.Api.Key 2> config_out ' test_expect_success "output includes meaningful error" ' From 0cc482df57d379442ac6e83fa124ba2cf86f23c9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 8 Dec 2020 14:56:29 -0500 Subject: [PATCH 126/127] update deps --- go.mod | 4 ++-- go.sum | 52 ++++------------------------------------------------ 2 files changed, 6 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index 96b6ffff2db..1884ad22536 100644 --- a/go.mod +++ b/go.mod @@ -30,8 +30,8 @@ require ( github.com/ipfs/go-graphsync v0.1.1 github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 - github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa - github.com/ipfs/go-ipfs-config v0.10.1-0.20201205222155-84cb43b0a8c9 + github.com/ipfs/go-ipfs-cmds v0.5.0 + github.com/ipfs/go-ipfs-config v0.11.0 github.com/ipfs/go-ipfs-ds-help v0.1.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 diff --git a/go.sum b/go.sum index 17c30a36625..7cd0a3dbc70 100644 --- a/go.sum +++ b/go.sum @@ -35,7 +35,6 @@ github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMG github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -138,7 +137,6 @@ github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= @@ -244,7 +242,6 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -258,7 +255,6 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -267,7 +263,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitswap v0.0.3/go.mod h1:jadAZYsP/tcRMl47ZhFxhaNuDQoXawT8iHMg+iFoQbg= github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= @@ -278,7 +273,6 @@ github.com/ipfs/go-bitswap v0.2.20/go.mod h1:C7TwBgHnu89Q8sHsTJP7IhUqF9XYLe71P4t github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRMtYNGrlxZ8KuI= github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= @@ -356,16 +350,10 @@ github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= -github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa h1:e3O5uBLuOFZI+G7Np0iZJ2y7CdYec48j70zeiaWlDW8= -github.com/ipfs/go-ipfs-cmds v0.4.1-0.20201125035704-4373bad335aa/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64 h1:36d77+JS1TqRF3+WaJvMqB+0Rq62jyGq1iRGypssYGY= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201125175347-280c4dbe8b64/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1 h1:5r/LVBdMP3zRlFhonGFBfEXFV9Igh4Fpd1erc+SLwrg= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201130210916-af9d35a8dce1/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38 h1:YTUQ/PfhhuG9MEfoqdf+VHW2leUUxD5/ZFyyXVurr9w= -github.com/ipfs/go-ipfs-config v0.9.1-0.20201203231406-d4056861ff38/go.mod h1:GQUxqb0NfkZmEU92PxqqqLVVFTLpoGGUlBaTyDaAqrE= -github.com/ipfs/go-ipfs-config v0.10.1-0.20201205222155-84cb43b0a8c9 h1:fvVDfvbM3szTh5BzoswyTG4EAMAKkQ0auzYgljGHcFI= -github.com/ipfs/go-ipfs-config v0.10.1-0.20201205222155-84cb43b0a8c9/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= +github.com/ipfs/go-ipfs-cmds v0.5.0 h1:T1ZT6Qu3IUCp6FgU2IzVtvGLaexEWo9q13+S5ic+Q5Y= +github.com/ipfs/go-ipfs-cmds v0.5.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= +github.com/ipfs/go-ipfs-config v0.11.0 h1:w4t2pz415Gtg6MTUKAq06C7ezC59/Us+k3+n1Tje+wg= +github.com/ipfs/go-ipfs-config v0.11.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -377,11 +365,9 @@ github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6r github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= -github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= -github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= github.com/ipfs/go-ipfs-pinner v0.0.4 h1:EmxhS3vDsCK/rZrsgxX0Le9m2drBcGlUd7ah/VyFYVE= github.com/ipfs/go-ipfs-pinner v0.0.4/go.mod h1:s4kFZWLWGDudN8Jyd/GTpt222A12C2snA2+OTdy/7p8= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= @@ -398,7 +384,6 @@ github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-cbor v0.0.1/go.mod h1:RXHr8s4k0NE0TKhnrxqZC9M888QfsBN9rhS5NjfKzY8= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= @@ -433,7 +418,6 @@ github.com/ipfs/go-log/v2 v2.0.5 h1:fL4YI+1g5V/b1Yxr1qAiXTMg1H8z9vx/VmJxBuQMHvU= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.1 h1:G4TtqN+V9y9HY9TA6BwbCVyyBZ2B9MbCjR2MtGx8FR0= github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= -github.com/ipfs/go-merkledag v0.0.3/go.mod h1:Oc5kIXLHokkE1hWGMBHw+oxehkAaTOqtEb7Zbh6BhLA= github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= @@ -450,7 +434,6 @@ github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZa github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks= github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg= github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0= -github.com/ipfs/go-path v0.0.3/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo= github.com/ipfs/go-path v0.0.7 h1:H06hKMquQ0aYtHiHryOMLpQC1qC3QwXwkahcEVD51Ho= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= github.com/ipfs/go-path v0.0.8 h1:R0k6t9x/pa+g8qzl5apQIPurJFozXhopks3iw3MX+jU= @@ -462,14 +445,11 @@ github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-pinning-service-http-client v0.1.0 h1:Au0P4NglL5JfzhNSZHlZ1qra+IcJyO3RWMd9EYCwqSY= github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= -github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s= github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipfs/interface-go-ipfs-core v0.3.0 h1:oZdLLfh256gPGcYPURjivj/lv296GIcr8mUqZUnXOEI= -github.com/ipfs/interface-go-ipfs-core v0.3.0/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= github.com/ipfs/interface-go-ipfs-core v0.4.0 h1:+mUiamyHIwedqP8ZgbCIwpy40oX7QcXUbo4CZOeJVJg= github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= github.com/ipld/go-car v0.1.1-0.20200429200904-c222d793c339 h1:YEYaf6mrrjoTfGpi7MajslcGvhP23Sh0b3ubcGYRMw0= @@ -478,7 +458,6 @@ github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e h1:ZISbJlM0ur github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1 h1:K1Ysr7kgIlo7YQkPqdkA6H7BVdIugvuAz7OQUTJxLdE= github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs= -github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -543,7 +522,6 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= -github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= @@ -559,7 +537,6 @@ github.com/libp2p/go-flow-metrics v0.0.2 h1:U5TvqfoyR6GVRM+bC15Ux1ltar1kbj6Zw6xO github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= -github.com/libp2p/go-libp2p v0.0.2/go.mod h1:Qu8bWqFXiocPloabFGUcVG4kk94fLvfC8mWTDdFC9wE= github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= @@ -578,7 +555,6 @@ github.com/libp2p/go-libp2p v0.11.0 h1:jb5mqdqYEBAybTEhD8io43Cz5LzVKuWxOK7znSN69 github.com/libp2p/go-libp2p v0.11.0/go.mod h1:3/ogJDXsbbepEfqtZKBR/DedzxJXCeK17t2Z9RE9bEE= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 h1:BM7aaOF7RpmNn9+9g6uTjGJ0cTzWr5j9i9IKeun2M8U= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= -github.com/libp2p/go-libp2p-autonat v0.0.2/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -600,7 +576,6 @@ github.com/libp2p/go-libp2p-blankhost v0.1.6 h1:CkPp1/zaCrCnBo0AdsQA0O1VkUYoUOty github.com/libp2p/go-libp2p-blankhost v0.1.6/go.mod h1:jONCAJqEP+Z8T6EQviGL4JsQcLx1LgTGtVqFNY8EMfQ= github.com/libp2p/go-libp2p-blankhost v0.2.0 h1:3EsGAi0CBGcZ33GwRuXEYJLLPoVWyXJ1bcJzAJjINkk= github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= -github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE= github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8= @@ -650,7 +625,6 @@ github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI= github.com/libp2p/go-libp2p-discovery v0.0.5/go.mod h1:YtF20GUxjgoKZ4zmXj8j3Nb2TUSBHFlOCetzYdbZL5I= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= @@ -690,7 +664,6 @@ github.com/libp2p/go-libp2p-mplex v0.2.3 h1:2zijwaJvpdesST2MXpI5w9wWFRgYtMcpRX7r github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= github.com/libp2p/go-libp2p-mplex v0.2.4 h1:XFFXaN4jhqnIuJVjYOR3k6bnRj0mFfJOlIuDVww+4Zo= github.com/libp2p/go-libp2p-mplex v0.2.4/go.mod h1:mI7iOezdWFOisvUwaYd3IDrJ4oVmgoXK8H331ui39CE= -github.com/libp2p/go-libp2p-nat v0.0.2/go.mod h1:QrjXQSD5Dj4IJOdEcjHRkWTSomyxRo6HnUkf/TfQpLQ= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= github.com/libp2p/go-libp2p-nat v0.0.5 h1:/mH8pXFVKleflDL1YwqMg27W9GD8kjEx7NY0P6eGc98= @@ -748,7 +721,6 @@ github.com/libp2p/go-libp2p-record v0.1.3/go.mod h1:yNUff/adKIfPnYQXgp6FQmNu3gLJ github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= github.com/libp2p/go-libp2p-routing-helpers v0.2.3 h1:xY61alxJ6PurSi+MXbywZpelvuU4U4p/gPTxjqCqTzY= github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= -github.com/libp2p/go-libp2p-secio v0.0.1/go.mod h1:IdG6iQybdcYmbTzxp4J5dwtUEDTOvZrT0opIDVNPrJs= github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= @@ -757,7 +729,6 @@ github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+Ildkg github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= github.com/libp2p/go-libp2p-secio v0.2.2 h1:rLLPvShPQAcY6eNurKNZq3eZjPWfU9kXF2eI9jIYdrg= github.com/libp2p/go-libp2p-secio v0.2.2/go.mod h1:wP3bS+m5AUnFA+OFO7Er03uO1mncHG0uVwGrwvjYlNY= -github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs= github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= @@ -781,9 +752,7 @@ github.com/libp2p/go-libp2p-testing v0.2.0/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexI github.com/libp2p/go-libp2p-tls v0.1.3 h1:twKMhMu44jQO+HgQK9X8NHO5HkeJu2QbhLzLJpa8oNM= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= -github.com/libp2p/go-libp2p-transport v0.0.4/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= github.com/libp2p/go-libp2p-transport v0.0.5/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= -github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2kn/m1w6YXxcIAYJYeI90h6BGgUc= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= @@ -809,7 +778,6 @@ github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9 github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= -github.com/libp2p/go-mplex v0.0.1/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.4/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= @@ -818,7 +786,6 @@ github.com/libp2p/go-mplex v0.1.1 h1:huPH/GGRJzmsHR9IZJJsrSwIM5YE2gL4ssgl1YWb/ps github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.1.2 h1:qOg1s+WdGLlpkrczDqmhYzyk3vCfsQ8+RxRTQjOZWwI= github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= -github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= @@ -849,7 +816,6 @@ github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FW github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= -github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3 h1:zzOeXnTooCkRvoH+bSXEfXhn76+LAiwoneM0gnXjF2M= @@ -867,7 +833,6 @@ github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROm github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= github.com/libp2p/go-stream-muxer-multistream v0.3.0 h1:TqnSHPJEIqDEO7h1wZZ0p3DXdvDSiLHQidKKUGZtiOY= github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA= -github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6mK0en8rlpA6BBTs= github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19K427vCzQ+xHKH/o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= github.com/libp2p/go-tcp-transport v0.1.1 h1:yGlqURmqgNA2fvzjSgZNlHcsd/IulAnKM8Ncu+vlqnw= @@ -878,7 +843,6 @@ github.com/libp2p/go-tcp-transport v0.2.1 h1:ExZiVQV+h+qL16fzCWtd1HSzPsqWottJ8KX github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M= github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= -github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= github.com/libp2p/go-ws-transport v0.0.5/go.mod h1:Qbl4BxPfXXhhd/o0wcrgoaItHqA9tnZjoFZnxykuaXU= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-ws-transport v0.2.0 h1:MJCw2OrPA9+76YNRvdo1wMnSOxb9Bivj6sVFY1Xrj6w= @@ -930,7 +894,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28 h1:gQhy5bsJa8zTlVI8lywCTZp1lguor+xevFoYlzeCTQY= @@ -1011,7 +974,6 @@ github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= -github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= @@ -1202,10 +1164,6 @@ github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46 github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= -github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= -github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= -github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= -github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1 h1:ctS9Anw/KozviCCtK6VWMz5kPL9nbQzbQY4yfqlIV4M= github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= @@ -1219,7 +1177,6 @@ github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0d github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1325,7 +1282,6 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 6beca407f1636a079187c0ebcb618fc7f02968c9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 8 Dec 2020 19:51:40 -0500 Subject: [PATCH 127/127] add pinning service config docs --- docs/config.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/config.md b/docs/config.md index 938477b719d..2ea82bca39a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -176,6 +176,11 @@ does (e.g, `"1d2h4m40.01s"`). - [`Mounts.IPFS`](#mountsipfs) - [`Mounts.IPNS`](#mountsipns) - [`Mounts.FuseAllowOther`](#mountsfuseallowother) +- [`Pinning`](#pinning) + - [`Pinning.RemoteServices`](#pinningremoteservices) + - [`Pinning.RemoteServices.API`](#pinningremoteservices-api) + - [`Pinning.RemoteServices.API.Endpoint`](#pinningremoteservices-apiendpoint) + - [`Pinning.RemoteServices.API.Key`](#pinningremoteservices-apikey) - [`Pubsub`](#pubsub) - [`Pubsub.Router`](#pubsubrouter) - [`Pubsub.DisableSigning`](#pubsubdisablesigning) @@ -813,6 +818,55 @@ Type: `string` (filesystem path) Sets the FUSE allow other option on the mountpoint. +## `Pinning` + +Pinning configures the options available for pinning content +(i.e. keeping content longer term instead of as temporarily cached storage). + +### `Pinning.RemoteServices` + +`RemoteServices` maps a name for a remote pinning service to its configuration. + +A remote pinning service is a remote service that exposes an API for managing +that service's interest in longer term data storage. + +The exposed API conforms to the specification defined at +https://ipfs.github.io/pinning-services-api-spec/ + +#### `Pinning.RemoteServices: API` + +Contains information relevant to utilizing the remote pinning service + +Example: +```json +{ + "Pinning": { + "RemoteServices": { + "myPinningService": { + "API" : { + "Endpoint" : "https://pinningservice.tld:1234/my/api/path", + "Key" : "someOpaqueKey" + } + } + } + } +} +``` + +##### `Pinning.RemoteServices: API.Endpoint` + +The HTTP(S) endpoint through which to access the pinning service + +Example: "https://pinningservice.tld:1234/my/api/path" + +Type: `string` + +##### `Pinning.RemoteServices: API.Key` + +The key through which access to the pinning service is granted + +Type: `string` + ## `Pubsub` Pubsub configures the `ipfs pubsub` subsystem. To use, it must be enabled by