From 202081523fdc8bc54b16e5e4447b221586febef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:12:10 +0100 Subject: [PATCH 001/159] Setup repo This commit was moved from ipfs/go-ipfs-http-client@4e7edce41d91825f0ea21dec426de2e128db5b38 --- client/httpapi/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 client/httpapi/README.md diff --git a/client/httpapi/README.md b/client/httpapi/README.md new file mode 100644 index 000000000000..ab37a4cf8aa8 --- /dev/null +++ b/client/httpapi/README.md @@ -0,0 +1,27 @@ +# go-ipfs-http-api + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) +[![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) + +> IPFS CoreAPI implementation using HTTP API + +## Documentation + +https://godoc.org/github.com/ipfs/go-ipfs-http-api + +## Contribute + +Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +### Want to hack on IPFS? + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) + +## License + +MIT From 6d85aff407d3fb3eb6d2549fbac94615fd147b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:13:15 +0100 Subject: [PATCH 002/159] Initial structure, path stuff This commit was moved from ipfs/go-ipfs-http-client@93943f7f5671948b0d2aee19c30cf689307faf13 --- client/httpapi/api.go | 127 +++++++++++++++++++++++++++++ client/httpapi/name.go | 35 ++++++++ client/httpapi/path.go | 48 +++++++++++ client/httpapi/request.go | 34 ++++++++ client/httpapi/requestbuilder.go | 100 +++++++++++++++++++++++ client/httpapi/response.go | 132 +++++++++++++++++++++++++++++++ 6 files changed, 476 insertions(+) create mode 100644 client/httpapi/api.go create mode 100644 client/httpapi/name.go create mode 100644 client/httpapi/path.go create mode 100644 client/httpapi/request.go create mode 100644 client/httpapi/requestbuilder.go create mode 100644 client/httpapi/response.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go new file mode 100644 index 000000000000..82e030350f72 --- /dev/null +++ b/client/httpapi/api.go @@ -0,0 +1,127 @@ +package httpapi + +import ( + "github.com/pkg/errors" + "io/ioutil" + gohttp "net/http" + "os" + "path" + "strings" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + homedir "github.com/mitchellh/go-homedir" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" + ) + +const ( + DefaultPathName = ".ipfs" + DefaultPathRoot = "~/" + DefaultPathName + DefaultApiFile = "api" + EnvDir = "IPFS_PATH" +) + +var ErrNotImplemented = errors.New("not implemented") + +type HttpApi struct { + url string + httpcli *gohttp.Client +} + +func NewLocalApi() iface.CoreAPI { + baseDir := os.Getenv(EnvDir) + if baseDir == "" { + baseDir = DefaultPathRoot + } + + baseDir, err := homedir.Expand(baseDir) + if err != nil { + return nil + } + + apiFile := path.Join(baseDir, DefaultApiFile) + + if _, err := os.Stat(apiFile); err != nil { + return nil + } + + api, err := ioutil.ReadFile(apiFile) + if err != nil { + return nil + } + + return NewApi(strings.TrimSpace(string(api))) +} + +func NewApi(url string) *HttpApi { + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + } + + return NewApiWithClient(url, c) +} + +func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { + if a, err := ma.NewMultiaddr(url); err == nil { + _, host, err := manet.DialArgs(a) + if err == nil { + url = host + } + } + + return &HttpApi{ + url: url, + httpcli: c, + } +} + +func (api *HttpApi) request(command string, args ...string) *RequestBuilder { + return &RequestBuilder{ + command: command, + args: args, + shell: api, + } +} + +func (api *HttpApi) Unixfs() iface.UnixfsAPI { + return nil +} + +func (api *HttpApi) Block() iface.BlockAPI { + return nil +} + +func (api *HttpApi) Dag() iface.DagAPI { + return nil +} + +func (api *HttpApi) Name() iface.NameAPI { + return (*NameAPI)(api) +} + +func (api *HttpApi) Key() iface.KeyAPI { + return nil +} + +func (api *HttpApi) Pin() iface.PinAPI { + return nil +} + +func (api *HttpApi) Object() iface.ObjectAPI { + return nil +} + +func (api *HttpApi) Dht() iface.DhtAPI { + return nil +} + +func (api *HttpApi) Swarm() iface.SwarmAPI { + return nil +} + +func (api *HttpApi) PubSub() iface.PubSubAPI { + return nil +} diff --git a/client/httpapi/name.go b/client/httpapi/name.go new file mode 100644 index 000000000000..41426ef57c9c --- /dev/null +++ b/client/httpapi/name.go @@ -0,0 +1,35 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type NameAPI HttpApi + +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { + // TODO: options! + + req := api.core().request("name/resolve") + req.Arguments(name) + + var out struct{ Path string } + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + + return iface.ParsePath(out.Path) +} + +func (api *NameAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/path.go b/client/httpapi/path.go new file mode 100644 index 000000000000..28656fbd4c03 --- /dev/null +++ b/client/httpapi/path.go @@ -0,0 +1,48 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { + var out struct { + Cid cid.Cid + RemPath string + } + + //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help + + var err error + if path.Namespace() == "ipns" { + if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + return nil, err + } + } + + if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + // TODO: + ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + if err != nil { + return nil, err + } + + root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + if err != nil { + return nil, err + } + + return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil +} + +func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { + return nil, ErrNotImplemented +} diff --git a/client/httpapi/request.go b/client/httpapi/request.go new file mode 100644 index 000000000000..58c61ac67805 --- /dev/null +++ b/client/httpapi/request.go @@ -0,0 +1,34 @@ +package httpapi + +import ( + "context" + "io" + "strings" +) + +type Request struct { + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string +} + +func NewRequest(ctx context.Context, url, command string, args ...string) *Request { + if !strings.HasPrefix(url, "http") { + url = "http://" + url + } + + opts := map[string]string{ + "encoding": "json", + "stream-channels": "true", + } + return &Request{ + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), + } +} diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go new file mode 100644 index 000000000000..9ccc8cf9720e --- /dev/null +++ b/client/httpapi/requestbuilder.go @@ -0,0 +1,100 @@ +package httpapi + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "strings" +) + +// RequestBuilder is an IPFS commands request builder. +type RequestBuilder struct { + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader + + shell *HttpApi +} + +// Arguments adds the arguments to the args. +func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { + r.args = append(r.args, args...) + return r +} + +// BodyString sets the request body to the given string. +func (r *RequestBuilder) BodyString(body string) *RequestBuilder { + return r.Body(strings.NewReader(body)) +} + +// BodyBytes sets the request body to the given buffer. +func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { + return r.Body(bytes.NewReader(body)) +} + +// Body sets the request body to the given reader. +func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { + r.body = body + return r +} + +// Option sets the given option. +func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { + var s string + switch v := value.(type) { + case bool: + s = strconv.FormatBool(v) + case string: + s = v + case []byte: + s = string(v) + default: + // slow case. + s = fmt.Sprint(value) + } + if r.opts == nil { + r.opts = make(map[string]string, 1) + } + r.opts[key] = s + return r +} + +// Header sets the given header. +func (r *RequestBuilder) Header(name, value string) *RequestBuilder { + if r.headers == nil { + r.headers = make(map[string]string, 1) + } + r.headers[name] = value + return r +} + +// Send sends the request and return the response. +func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + req := NewRequest(ctx, r.shell.url, r.command, r.args...) + req.Opts = r.opts + req.Headers = r.headers + req.Body = r.body + return req.Send(r.shell.httpcli) +} + +// Exec sends the request a request and decodes the response. +func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { + httpRes, err := r.Send(ctx) + if err != nil { + return err + } + + if res == nil { + httpRes.Close() + if httpRes.Error != nil { + return httpRes.Error + } + return nil + } + + return httpRes.Decode(res) +} diff --git a/client/httpapi/response.go b/client/httpapi/response.go new file mode 100644 index 000000000000..27709769b1f6 --- /dev/null +++ b/client/httpapi/response.go @@ -0,0 +1,132 @@ +package httpapi + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + files "github.com/ipfs/go-ipfs-files" +) + +type Response struct { + Output io.ReadCloser + Error *Error +} + +func (r *Response) Close() error { + if r.Output != nil { + // always drain output (response body) + ioutil.ReadAll(r.Output) + return r.Output.Close() + } + return nil +} + +func (r *Response) Decode(dec interface{}) error { + defer r.Close() + if r.Error != nil { + return r.Error + } + + return json.NewDecoder(r.Output).Decode(dec) +} + +type Error struct { + Command string + Message string + Code int +} + +func (e *Error) Error() string { + var out string + if e.Command != "" { + out = e.Command + ": " + } + if e.Code != 0 { + out = fmt.Sprintf("%s%d: ", out, e.Code) + } + return out + e.Message +} + +func (r *Request) Send(c *http.Client) (*Response, error) { + url := r.getURL() + req, err := http.NewRequest("POST", url, r.Body) + if err != nil { + return nil, err + } + + // Add any headers that were supplied via the RequestBuilder. + for k, v := range r.Headers { + req.Header.Add(k, v) + } + + if fr, ok := r.Body.(*files.MultiFileReader); ok { + req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) + req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + } + + resp, err := c.Do(req) + if err != nil { + return nil, err + } + + contentType := resp.Header.Get("Content-Type") + parts := strings.Split(contentType, ";") + contentType = parts[0] + + nresp := new(Response) + + nresp.Output = resp.Body + if resp.StatusCode >= http.StatusBadRequest { + e := &Error{ + Command: r.Command, + } + switch { + case resp.StatusCode == http.StatusNotFound: + e.Message = "command not found" + case contentType == "text/plain": + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = string(out) + case contentType == "application/json": + if err = json.NewDecoder(resp.Body).Decode(e); err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) + } + default: + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out) + } + nresp.Error = e + nresp.Output = nil + + // drain body and close + ioutil.ReadAll(resp.Body) + resp.Body.Close() + } + + return nresp, nil +} + +func (r *Request) getURL() string { + + values := make(url.Values) + for _, arg := range r.Args { + values.Add("arg", arg) + } + for k, v := range r.Opts { + values.Add(k, v) + } + + return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode()) +} From 35c271e3d88725a9ab25be40c1837435de110a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:40:02 +0100 Subject: [PATCH 003/159] wip notice in readme This commit was moved from ipfs/go-ipfs-http-client@e823507553ca82115e303e1ba26f76b83f5bd12c --- client/httpapi/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index ab37a4cf8aa8..dd09142e32bc 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,6 +8,8 @@ > IPFS CoreAPI implementation using HTTP API +This project is WIP, use https://github.com/ipfs/go-ipfs-api for now + ## Documentation https://godoc.org/github.com/ipfs/go-ipfs-http-api From 6c927fd9624807e124f0376445c41daaa975fbd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:38:11 +0100 Subject: [PATCH 004/159] Partial ipld node impl This commit was moved from ipfs/go-ipfs-http-client@df916c7849d6cfd7f479c4fe80c0f75d6705a0fc --- client/httpapi/api.go | 6 +-- client/httpapi/block.go | 40 ++++++++++++++ client/httpapi/ipldnode.go | 107 +++++++++++++++++++++++++++++++++++++ client/httpapi/path.go | 2 +- 4 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/block.go create mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 82e030350f72..cd3fb9fd021e 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,7 @@ package httpapi import ( - "github.com/pkg/errors" + "errors" "io/ioutil" gohttp "net/http" "os" @@ -12,7 +12,7 @@ import ( homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" - ) +) const ( DefaultPathName = ".ipfs" @@ -91,7 +91,7 @@ func (api *HttpApi) Unixfs() iface.UnixfsAPI { } func (api *HttpApi) Block() iface.BlockAPI { - return nil + return (*BlockAPI)(api) } func (api *HttpApi) Dag() iface.DagAPI { diff --git a/client/httpapi/block.go b/client/httpapi/block.go new file mode 100644 index 000000000000..8bdb4c502ebc --- /dev/null +++ b/client/httpapi/block.go @@ -0,0 +1,40 @@ +package httpapi + +import ( + "context" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type BlockAPI HttpApi + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + if err != nil { + return nil, err + } + + //TODO: is close on the reader enough? + //defer resp.Close() + + //TODO: make blockApi return ReadCloser + return resp.Output, resp.Error +} + +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { + return ErrNotImplemented +} + +func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go new file mode 100644 index 000000000000..a3dd6204b2b6 --- /dev/null +++ b/client/httpapi/ipldnode.go @@ -0,0 +1,107 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + "io/ioutil" + "strconv" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +type ipldNode struct { + ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? + path iface.ResolvedPath + api *HttpApi +} + +func (n *ipldNode) RawData() []byte { + r, err := n.api.Block().Get(n.ctx, n.path) + if err != nil { + panic(err) // TODO: eww, should we add errors too / better ideas? + } + + b, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + + return b +} + +func (n *ipldNode) Cid() cid.Cid { + return n.path.Cid() +} + +func (n *ipldNode) String() string { + return fmt.Sprintf("[Block %s]", n.Cid()) +} + +func (n *ipldNode) Loggable() map[string]interface{} { + return nil //TODO: we can't really do better here, can we? +} + +// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. +// We can also give people a choice +func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { + p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) + + var out interface{} + n.api.request("dag/get", p).Exec(n.ctx, &out) + + // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack + for len(path) > 0 { + switch o := out.(type) { + case map[string]interface{}: + v, ok := o[path[0]] + if !ok { + // TODO: ipld links + return nil, nil, errors.New("no element under this path") + } + out = v + case []interface{}: + n, err := strconv.ParseUint(path[0], 10, 32) + if err != nil { + return nil, nil, err + } + if len(o) < int(n) { + return nil, nil, errors.New("no element under this path") + } + out = o[n] + } + path = path[1:] + } + + return out, path, nil +} + +func (n *ipldNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (n *ipldNode) Copy() ipld.Node { + panic("implement me") +} + +func (n *ipldNode) Links() []*ipld.Link { + panic("implement me") +} + +func (n *ipldNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (n *ipldNode) Size() (uint64, error) { + panic("implement me") +} + +var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 28656fbd4c03..6b6e4b027bc7 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -30,7 +30,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } // TODO: - ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } From 1cd2ec05b7524e921f58750a4b9c8f48ad1a2e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Dec 2018 16:04:19 +0100 Subject: [PATCH 005/159] Init gx This commit was moved from ipfs/go-ipfs-http-client@e06cddbedd360b1982e414571dd08ab25df38546 --- client/httpapi/ipldnode.go | 8 ++++---- client/httpapi/name.go | 1 + client/httpapi/path.go | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index a3dd6204b2b6..b8e6fba01806 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -2,16 +2,16 @@ package httpapi import ( "context" + "errors" "fmt" - "github.com/pkg/errors" - "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" "io/ioutil" "strconv" "github.com/ipfs/go-ipfs/core/coreapi/interface" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) type ipldNode struct { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 41426ef57c9c..7315ac2c3918 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 6b6e4b027bc7..5701326fc560 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -5,9 +5,9 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { From 16e97bf5e8f80b681c5a8d4b96f406b4b83a8e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 21 Dec 2018 18:44:34 +0100 Subject: [PATCH 006/159] It builds This commit was moved from ipfs/go-ipfs-http-client@a23d794e5fceadb9115d8d0c8466c10e9c5d0891 --- client/httpapi/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cd3fb9fd021e..dc9acb9f6067 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -78,6 +79,10 @@ func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { } } +func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { + return nil, ErrNotImplemented +} + func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, From fe4c9fd8033c94c5ea707b09e76479a6d018635d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 30 Dec 2018 04:24:09 +0100 Subject: [PATCH 007/159] Skeleton for tests This commit was moved from ipfs/go-ipfs-http-client@dfbe0026ad9438d4e795bbb43debeffb6bb9ca1a --- client/httpapi/api.go | 24 ++++++++--- client/httpapi/api_test.go | 84 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 client/httpapi/api_test.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dc9acb9f6067..63ea3aad7d33 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -35,7 +35,11 @@ func NewLocalApi() iface.CoreAPI { baseDir = DefaultPathRoot } - baseDir, err := homedir.Expand(baseDir) + return NewPathApi(baseDir) +} + +func NewPathApi(p string) iface.CoreAPI { + baseDir, err := homedir.Expand(p) if err != nil { return nil } @@ -51,10 +55,15 @@ func NewLocalApi() iface.CoreAPI { return nil } - return NewApi(strings.TrimSpace(string(api))) + maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) + if err != nil { + return nil + } + + return NewApi(maddr) } -func NewApi(url string) *HttpApi { +func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -62,10 +71,15 @@ func NewApi(url string) *HttpApi { }, } - return NewApiWithClient(url, c) + return NewApiWithClient(a, c) } -func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { + _, url, err := manet.DialArgs(a) + if err != nil { + return nil // TODO: return that error + } + if a, err := ma.NewMultiaddr(url); err == nil { _, host, err := manet.DialArgs(a) if err == nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go new file mode 100644 index 000000000000..b925b5c348f2 --- /dev/null +++ b/client/httpapi/api_test.go @@ -0,0 +1,84 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/iptb/testbed/interfaces" + "io/ioutil" + "os" + "path" + "strconv" + "testing" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" + + local "github.com/ipfs/iptb-plugins/local" + "github.com/ipfs/iptb/cli" + "github.com/ipfs/iptb/testbed" +) + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ + From: "", + NewNode: local.NewNode, + GetAttrList: local.GetAttrList, + GetAttrDesc: local.GetAttrDesc, + PluginName: local.PluginName, + BuiltIn: true, + }, false) + if err != nil { + return nil, err + } + + dir, err := ioutil.TempDir("", "httpapi-tb-") + if err != nil { + return nil, err + } + + c := cli.NewCli() + if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + + defer os.Remove(dir) + + defer func() { + _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + }() + }() + + apis := make([]iface.CoreAPI, n) + + for i := range apis { + tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) + + node, err := tb.Node(i) + if err != nil { + return nil, err + } + + attrNode, ok := node.(testbedi.Attribute) + if !ok { + return nil, fmt.Errorf("node does not implement attributes") + } + + pth, err := attrNode.Attr("path") + if err != nil { + return nil, err + } + + apis[i] = NewPathApi(pth) + } + + return apis, nil +} + +func TestHttpApi(t *testing.T) { + tests.TestApi(&NodeProvider{})(t) +} From e8da6e2cf9a8fd40c5fd960a60ce1b1b27ada713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 7 Jan 2019 23:44:43 +0100 Subject: [PATCH 008/159] Block API This commit was moved from ipfs/go-ipfs-http-client@d0c98b870e55bed78fa777793e520f08157deb4f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 11 +++- client/httpapi/block.go | 109 ++++++++++++++++++++++++++++--- client/httpapi/requestbuilder.go | 12 ++++ 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 63ea3aad7d33..1768e8477497 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return (*NameAPI)(api) + return nil } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b925b5c348f2..e32c456e5a36 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/iptb/testbed/interfaces" "io/ioutil" "os" "path" @@ -16,6 +15,7 @@ import ( local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" + "github.com/ipfs/iptb/testbed/interfaces" ) type NodeProvider struct{} @@ -39,7 +39,14 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := cli.NewCli() - if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + + initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} + if err := c.Run(initArgs); err != nil { + return nil, err + } + + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 8bdb4c502ebc..185aa0a42be0 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -1,17 +1,69 @@ package httpapi import ( + "bytes" "context" + "errors" + "fmt" "io" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + mh "github.com/multiformats/go-multihash" ) type BlockAPI HttpApi -func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { - return nil, ErrNotImplemented +type blockStat struct { + Key string + BSize int `json:"Size"` +} + +func (s *blockStat) Size() int { + return s.BSize +} + +func (s *blockStat) valid() (iface.ResolvedPath, error) { + c, err := cid.Parse(s.Key) + if err != nil { + return nil, err + } + + return iface.IpldPath(c), nil +} + +func (s *blockStat) Path() iface.ResolvedPath { + p, _ := s.valid() + return p +} + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { + options, _, err := caopts.BlockPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("block/put"). + Option("mhtype", mht). + Option("mhlen", options.MhLength). + Option("format", options.Codec). + FileBody(r) + + var out blockStat + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { @@ -19,20 +71,57 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } - //TODO: is close on the reader enough? - //defer resp.Close() + //TODO: make get return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } - //TODO: make blockApi return ReadCloser - return resp.Output, resp.Error + return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { - return ErrNotImplemented +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { + options, err := caopts.BlockRmOptions(opts...) + if err != nil { + return err + } + + removedBlock := struct { + Hash string `json:",omitempty"` + Error string `json:",omitempty"` + }{} + + req := api.core().request("block/rm"). + Option("force", options.Force). + Arguments(p.String()) + + if err := req.Exec(ctx, &removedBlock); err != nil { + return err + } + + if removedBlock.Error != "" { + return errors.New(removedBlock.Error) + } + + return nil } func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { - return nil, ErrNotImplemented + var out blockStat + err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) core() *HttpApi { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 9ccc8cf9720e..6e5a89ebdb13 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,8 +5,11 @@ import ( "context" "fmt" "io" + "io/ioutil" "strconv" "strings" + + "github.com/ipfs/go-ipfs-files" ) // RequestBuilder is an IPFS commands request builder. @@ -42,6 +45,15 @@ func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { return r } +// FileBody sets the request body to the given reader wrapped into multipartreader. +func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { + pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + d := files.NewMapDirectory(map[string]files.Node{"": pr}) + r.body = files.NewMultiFileReader(d, false) + + return r +} + // Option sets the given option. func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { var s string From ab89e0abf99a0bc42ee2d8fb55a9b3de79573635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 00:30:22 +0100 Subject: [PATCH 009/159] Partial Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@44696b84f59f6f707858787c9a0750f224596635 --- client/httpapi/api.go | 4 +- client/httpapi/unixfs.go | 95 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/unixfs.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 1768e8477497..099b45123113 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -106,7 +106,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { } func (api *HttpApi) Unixfs() iface.UnixfsAPI { - return nil + return (*UnixfsAPI)(api) } func (api *HttpApi) Block() iface.BlockAPI { @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return nil + return (*NameAPI)(api) } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go new file mode 100644 index 000000000000..567f60284d44 --- /dev/null +++ b/client/httpapi/unixfs.go @@ -0,0 +1,95 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type addEvent struct { + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + +type UnixfsAPI HttpApi + +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { + options, _, err := caopts.UnixfsAddOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("add"). + Option("hash", mht). + Option("chunker", options.Chunker). + Option("cid-version", options.CidVersion). + //Option("", options.Events). + Option("fscache", options.FsCache). + Option("hidden", options.Hidden). + Option("inline", options.Inline). + Option("inline-limit", options.InlineLimit). + Option("nocopy", options.NoCopy). + Option("only-hash", options.OnlyHash). + Option("pin", options.Pin). + //Option("", options.Progress). + Option("silent", options.Silent). + Option("stdin-name", options.StdinName). + Option("wrap-with-directory", options.Wrap). + Option("quieter", true) // TODO: rm after event impl + + if options.RawLeavesSet { + req.Option("raw-leaves", options.RawLeaves) + } + + switch options.Layout { + case caopts.BalancedLayout: + // noop, default + case caopts.TrickleLayout: + req.Option("trickle", true) + } + + switch c := f.(type) { + case files.Directory: + req.Body(files.NewMultiFileReader(c, false)) + case files.File: + req.Body(c) + } + + var out addEvent + if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil +} + +func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { + panic("implement me") +} + +func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *UnixfsAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cf74d391603457ad626b7f6b27ff0655bc946ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:07:21 +0100 Subject: [PATCH 010/159] Partial Key API, ApiAddr funcion This commit was moved from ipfs/go-ipfs-http-client@0ffdef159261e19cfd76edabb394179aa9295003 --- client/httpapi/api.go | 13 ++++++- client/httpapi/key.go | 86 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/key.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 099b45123113..d104d98dffb9 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,6 +29,7 @@ type HttpApi struct { httpcli *gohttp.Client } +//TODO: Return errors here func NewLocalApi() iface.CoreAPI { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -39,6 +40,14 @@ func NewLocalApi() iface.CoreAPI { } func NewPathApi(p string) iface.CoreAPI { + a := ApiAddr(p) + if a == nil { + return nil + } + return NewApi(a) +} + +func ApiAddr(p string) ma.Multiaddr { baseDir, err := homedir.Expand(p) if err != nil { return nil @@ -60,7 +69,7 @@ func NewPathApi(p string) iface.CoreAPI { return nil } - return NewApi(maddr) + return maddr } func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? @@ -122,7 +131,7 @@ func (api *HttpApi) Name() iface.NameAPI { } func (api *HttpApi) Key() iface.KeyAPI { - return nil + return (*KeyAPI)(api) } func (api *HttpApi) Pin() iface.PinAPI { diff --git a/client/httpapi/key.go b/client/httpapi/key.go new file mode 100644 index 000000000000..87b573f98a56 --- /dev/null +++ b/client/httpapi/key.go @@ -0,0 +1,86 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type KeyAPI HttpApi + +type keyOutput struct { + JName string `json:"Name"` + Id string +} + +func (k *keyOutput) Name() string { + return k.JName +} + +func (k *keyOutput) Path() iface.Path { + p, _ := iface.ParsePath("/ipns/" + k.Id) + return p +} + +func (k *keyOutput) ID() peer.ID { + p, _ := peer.IDB58Decode(k.Id) + return p +} + +func (k *keyOutput) valid() error { + _, err := peer.IDB58Decode(k.Id) + return err +} + + +func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { + options, err := caopts.KeyGenerateOptions(opts...) + if err != nil { + return nil, err + } + + var out keyOutput + err = api.core().request("key/gen", name). + Option("type", options.Algorithm). + Option("size", options.Size). + Exec(ctx, &out) + if err != nil { + return nil, err + } + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { + panic("implement me") +} + +func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { + var id struct{ID string} + if err := api.core().request("id").Exec(ctx, &id); err != nil { + return nil, err + } + + out := keyOutput{JName: "self", Id: id.ID} + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c236393733bf3491c45002d67189ba78303332a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:08:18 +0100 Subject: [PATCH 011/159] Connect test swarms, don't compress api calls This commit was moved from ipfs/go-ipfs-http-client@c6472d9b8286c932492db4d6bf03f890bf97759c --- client/httpapi/api_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index e32c456e5a36..02c7830bb650 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io/ioutil" + gohttp "net/http" "os" "path" "strconv" @@ -50,6 +51,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + if n > 1 { + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + if err := c.Run(connectArgs); err != nil { + return nil, err + } + } + go func() { <-ctx.Done() @@ -80,7 +88,18 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - apis[i] = NewPathApi(pth) + a := ApiAddr(pth) + if a == nil { + return nil, fmt.Errorf("nil addr for node") + } + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(a, c) } return apis, nil From 7861315f49398e20e7e1189671e42d0a16adbef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:00 +0100 Subject: [PATCH 012/159] Wrap single files in Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@16f77b24a1b150e7ec624d5675ded478ac6cb853 --- client/httpapi/unixfs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 567f60284d44..21f75f85ae90 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -66,7 +66,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix case files.Directory: req.Body(files.NewMultiFileReader(c, false)) case files.File: - req.Body(c) + d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) } var out addEvent From f638bae3a960a74d284503c30086f0bbffbe73c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:17 +0100 Subject: [PATCH 013/159] implement .Name This commit was moved from ipfs/go-ipfs-http-client@e19e5f54e48d0909796add6d57e3e74e84eba85e --- client/httpapi/name.go | 70 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 7315ac2c3918..fbc440b96664 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,26 +2,80 @@ package httpapi import ( "context" + "fmt" + "github.com/ipfs/go-ipfs/namesys/opts" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) type NameAPI HttpApi -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { - return nil, ErrNotImplemented +type ipnsEntry struct { + JName string `json:"Name"` + JValue string `json:"Value"` +} + +func (e *ipnsEntry) valid() (iface.Path, error) { + return iface.ParsePath(e.JValue) +} + +func (e *ipnsEntry) Name() string { + return e.JName +} + +func (e *ipnsEntry) Value() iface.Path { + p, _ := e.valid() + return p } -func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { + options, err := caopts.NamePublishOptions(opts...) + if err != nil { + return nil, err + } + + req := api.core().request("name/publish", p.String()). + Option("key", options.Key). + Option("allow-offline", options.AllowOffline). + Option("lifetime", options.ValidTime.String()). + Option("resolve", false) + + if options.TTL != nil { + req.Option("ttl", options.TTL.String()) + } + + var out ipnsEntry + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { return nil, ErrNotImplemented } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { - // TODO: options! +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } + + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } - req := api.core().request("name/resolve") - req.Arguments(name) + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout.String()) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { From 1acf4163902ae3b9e0e182761deafee5e67415ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:53:24 +0100 Subject: [PATCH 014/159] Implement .Unixfs.Ls() This commit was moved from ipfs/go-ipfs-http-client@eb1944fae32f4ccc43969ecd632681c2f2cbfa00 --- client/httpapi/response.go | 14 ++++++++++++- client/httpapi/unixfs.go | 43 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 27709769b1f6..5749ca29eea1 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -33,7 +33,19 @@ func (r *Response) Decode(dec interface{}) error { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + n := 0 + var err error + for { + err = json.NewDecoder(r.Output).Decode(dec) + if err != nil { + break + } + n++ + } + if n > 0 && err == io.EOF { + err = nil + } + return err } type Error struct { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 21f75f85ae90..75e565a4a910 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -11,6 +12,7 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { @@ -87,8 +89,45 @@ func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { panic("implement me") } -func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +type lsLink struct { + Name, Hash string + Size uint64 + Type unixfspb.Data_DataType +} + +type lsObject struct { + Hash string + Links []lsLink +} + +type lsOutput struct { + Objects []lsObject +} + +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out lsOutput + err := api.core().request("ls", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + + if len(out.Objects) != 1 { + return nil, errors.New("unexpected objects len") + } + + links := make([]*format.Link, len(out.Objects[0].Links)) + for i, l := range out.Objects[0].Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + links[i] = &format.Link{ + Name: l.Name, + Size: l.Size, + Cid: c, + } + } + return links, nil } func (api *UnixfsAPI) core() *HttpApi { From a6636aac599a1a8a4802ccadb885909f0ab88629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:18:21 +0100 Subject: [PATCH 015/159] Imprement partian Pin API This commit was moved from ipfs/go-ipfs-http-client@dbf90eac67d95722e88056ca16fa769a32dcd48f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 16 +++++++- client/httpapi/pin.go | 78 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pin.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d104d98dffb9..d5741da089d4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -135,7 +135,7 @@ func (api *HttpApi) Key() iface.KeyAPI { } func (api *HttpApi) Pin() iface.PinAPI { - return nil + return (*PinAPI)(api) } func (api *HttpApi) Object() iface.ObjectAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 02c7830bb650..5fdf1835241f 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -39,7 +40,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - c := cli.NewCli() + c := cli.NewCli() //TODO: is there a better way? initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} if err := c.Run(initArgs); err != nil { @@ -100,6 +101,19 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) }, } apis[i] = NewApiWithClient(a, c) + + // node cleanup + // TODO: pass --empty-repo somehow (how?) + pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + for _, pin := range pins { //TODO: parallel + if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { + return nil, err + } + } + } return apis, nil diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go new file mode 100644 index 000000000000..02dcc4222473 --- /dev/null +++ b/client/httpapi/pin.go @@ -0,0 +1,78 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type PinAPI HttpApi + +type pinRefKeyObject struct { + Type string +} + +type pinRefKeyList struct { + Keys map[string]pinRefKeyObject +} + +type pin struct { + path iface.ResolvedPath + typ string +} + +func (p *pin) Path() iface.ResolvedPath { + return p.path +} + +func (p *pin) Type() string { + return p.typ +} + + +func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { + panic("implement me") +} + +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { + options, err := caopts.PinLsOptions(opts...) + if err != nil { + return nil, err + } + + var out pinRefKeyList + err = api.core().request("pin/ls"). + Option("type", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + pins := make([]iface.Pin, 0, len(out.Keys)) + for hash, p := range out.Keys { + c, err := cid.Parse(hash) + if err != nil { + return nil, err + } + pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + } + + return pins, nil +} + +func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { + return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +} + +func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { + panic("implement me") +} + +func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { + panic("implement me") +} + +func (api *PinAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 0f7c83956b2d769c801c270fe01d4819cb070881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:20:04 +0100 Subject: [PATCH 016/159] Import missing unixfs dep This commit was moved from ipfs/go-ipfs-http-client@6169321d1df6495cbd1c06cca9f2e0777a00f638 --- client/httpapi/unixfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 75e565a4a910..5a1495c24b45 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -11,8 +11,8 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" - unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { From af197cb6d90da1b51525e68f97a30ebfc322e134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 14:46:52 +0100 Subject: [PATCH 017/159] api.WithOption This commit was moved from ipfs/go-ipfs-http-client@634b00bf1a42dc7f2a2dc75f85c77e5bb5ef727c --- client/httpapi/api.go | 21 ++++++++++++++++++--- client/httpapi/api_test.go | 4 ++-- client/httpapi/requestbuilder.go | 2 ++ client/httpapi/response.go | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d5741da089d4..3df45f38c242 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -27,6 +27,8 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string httpcli *gohttp.Client + + applyGlobal func(*RequestBuilder) } //TODO: Return errors here @@ -99,11 +101,24 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return &HttpApi{ url: url, httpcli: c, + applyGlobal: func(*RequestBuilder) {}, } } -func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { - return nil, ErrNotImplemented +func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { + options, err := caopts.ApiOptions(opts...) + if err != nil { + return nil, err + } + + subApi := *api + subApi.applyGlobal = func(req *RequestBuilder) { + if options.Offline { + req.Option("offline", options.Offline) + } + } + + return &subApi, nil } func (api *HttpApi) request(command string, args ...string) *RequestBuilder { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5fdf1835241f..67a2dbdce533 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -104,7 +104,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) // node cleanup // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 6e5a89ebdb13..831e6d71c0ba 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -86,6 +86,8 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + r.shell.applyGlobal(r) + req := NewRequest(ctx, r.shell.url, r.command, r.args...) req.Opts = r.opts req.Headers = r.headers diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 5749ca29eea1..f6e7f3ab7099 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -21,7 +21,7 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { // always drain output (response body) - ioutil.ReadAll(r.Output) + //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data return r.Output.Close() } return nil From 6d9dea62825012678ab0f117ec3cfe28c96fdaf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 17:47:22 +0100 Subject: [PATCH 018/159] Implement Unixfs.Get() This commit was moved from ipfs/go-ipfs-http-client@b31bee083d1fc5002779d755b6a624b517d2b10c --- client/httpapi/apifile.go | 231 ++++++++++++++++++++++++++++++++++++++ client/httpapi/unixfs.go | 4 - 2 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/apifile.go diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go new file mode 100644 index 000000000000..541189b609e6 --- /dev/null +++ b/client/httpapi/apifile.go @@ -0,0 +1,231 @@ +package httpapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + "github.com/ipfs/go-ipfs-files" + unixfspb "github.com/ipfs/go-unixfs/pb" +) + +func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { + if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS + var err error + p, err = api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + } + + var stat struct{ + Hash string + Type string + Size int64 // unixfs size + } + err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + if err != nil { + return nil, err + } + + switch stat.Type { + case "file": + return api.getFile(ctx, p, stat.Size) + case "directory": + return api.getDir(ctx, p, stat.Size) + default: + return nil, fmt.Errorf("unsupported file type '%s'", stat.Type) + } +} + +type apiFile struct { + ctx context.Context + core *HttpApi + size int64 + path iface.Path + + r io.ReadCloser + at int64 +} + +func (f *apiFile) reset() error { + if f.r != nil { + f.r.Close() + } + req := f.core.request("cat", f.path.String()) + if f.at != 0 { + req.Option("offset", f.at) + } + resp, err := req.Send(f.ctx) + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + f.r = resp.Output + return nil +} + +func (f *apiFile) Read(p []byte) (int, error) { + n, err := f.r.Read(p) + if n > 0 { + f.at += int64(n) + } + return n, err +} + +func (f *apiFile) Seek(offset int64, whence int) (int64, error) { + panic("implement me") //TODO +} + +func (f *apiFile) Close() error { + if f.r != nil { + return f.r.Close() + } + return nil +} + +func (f *apiFile) Size() (int64, error) { + return f.size, nil +} + +func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + f := &apiFile{ + ctx: ctx, + core: api.core(), + size: size, + path: p, + } + + return f, f.reset() +} + +type apiIter struct { + ctx context.Context + core *UnixfsAPI + + err error + + dec *json.Decoder + curFile files.Node + cur lsLink +} + +func (it *apiIter) Err() error { + return it.err +} + +func (it *apiIter) Name() string { + return it.cur.Name +} + +func (it *apiIter) Next() bool { + var out lsOutput + if err := it.dec.Decode(&out); err != nil { + if err != io.EOF { + it.err = err + } + return false + } + + if len(out.Objects) != 1 { + it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + return false + } + + if len(out.Objects[0].Links) != 1 { + it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + return false + } + + it.cur = out.Objects[0].Links[0] + c, err := cid.Parse(it.cur.Hash) + if err != nil { + it.err = err + return false + } + + switch it.cur.Type { + case unixfspb.Data_HAMTShard: + fallthrough + case unixfspb.Data_Metadata: + fallthrough + case unixfspb.Data_Directory: + it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + case unixfspb.Data_File: + it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + default: + it.err = fmt.Errorf("file type %d not supported", it.cur.Type) + return false + } + return true +} + +func (it *apiIter) Node() files.Node { + return it.curFile +} + +type apiDir struct { + ctx context.Context + core *UnixfsAPI + size int64 + path iface.Path + + dec *json.Decoder +} + +func (d *apiDir) Close() error { + return nil +} + +func (d *apiDir) Size() (int64, error) { + return d.size, nil +} + +func (d *apiDir) Entries() files.DirIterator { + return &apiIter{ + ctx: d.ctx, + core: d.core, + dec: d.dec, + } +} + +func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + resp, err := api.core().request("ls", p.String()). + Option("resolve-size", true). + Option("stream", true).Send(ctx) + + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + d := &apiDir{ + ctx: ctx, + core: api, + size: size, + path: p, + + dec: json.NewDecoder(resp.Output), + } + + return d, nil +} + +var _ files.File = &apiFile{} +var _ files.Directory = &apiDir{} diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5a1495c24b45..77daf7b9b7e1 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -85,10 +85,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return iface.IpfsPath(c), nil } -func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { - panic("implement me") -} - type lsLink struct { Name, Hash string Size uint64 From 00597e6dff81846a2dbf742dd1ab9bf546b856e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:04:22 +0100 Subject: [PATCH 019/159] Dag.Put This commit was moved from ipfs/go-ipfs-http-client@3217104469020f0c66d8457842f8221b64289d74 --- client/httpapi/api.go | 2 +- client/httpapi/dag.go | 69 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dag.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3df45f38c242..c2ec1ad6719e 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { } func (api *HttpApi) Dag() iface.DagAPI { - return nil + return (*DagAPI)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go new file mode 100644 index 000000000000..c3b29e7ee0bc --- /dev/null +++ b/client/httpapi/dag.go @@ -0,0 +1,69 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type DagAPI HttpApi + +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { + options, err := caopts.DagPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + codec, ok := cid.CodecToStr[options.Codec] + if !ok { + return nil, fmt.Errorf("unknowm codec %d", options.MhType) + } + + if options.MhLength != -1 { + return nil, fmt.Errorf("setting hash len is not supported yet") + } + + var out struct{ + Cid cid.Cid + } + err = api.core().request("dht/put"). + Option("hash", mht). + Option("format", codec). + Option("input-enc", options.InputEnc). + FileBody(src). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + return iface.IpldPath(out.Cid), nil +} + +func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { + panic("implement me") +} + +func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { + panic("implement me") +} + +func (api *DagAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 101910f04ed0575be19efaacb44b1bce73717033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:22:10 +0100 Subject: [PATCH 020/159] Object.New This commit was moved from ipfs/go-ipfs-http-client@60321ed42fd8f42e964e37ab5e96839fb8fd77b6 --- client/httpapi/api.go | 2 +- client/httpapi/ipldnode.go | 8 ++++ client/httpapi/object.go | 82 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/object.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c2ec1ad6719e..cf9a633bc8b4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -154,7 +154,7 @@ func (api *HttpApi) Pin() iface.PinAPI { } func (api *HttpApi) Object() iface.ObjectAPI { - return nil + return (*ObjectAPI)(api) } func (api *HttpApi) Dht() iface.DhtAPI { diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index b8e6fba01806..1f584ac0700a 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -20,6 +20,14 @@ type ipldNode struct { api *HttpApi } +func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { + return &ipldNode{ + ctx: ctx, + path: p, + api: a, + } +} + func (n *ipldNode) RawData() []byte { r, err := n.api.Block().Get(n.ctx, n.path) if err != nil { diff --git a/client/httpapi/object.go b/client/httpapi/object.go new file mode 100644 index 000000000000..9c1063af9938 --- /dev/null +++ b/client/httpapi/object.go @@ -0,0 +1,82 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" +) + +type ObjectAPI HttpApi + +type objectOut struct { + Hash string +} + +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { + options, err := caopts.ObjectNewOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/new", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil +} + +func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { + panic("implement me") +} + +func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { + panic("implement me") +} + +func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { + panic("implement me") +} + +func (api *ObjectAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 31a4c3754b87e68f60525533ba9c6bb9c0b1021e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:47:32 +0100 Subject: [PATCH 021/159] Fix Dag.Put This commit was moved from ipfs/go-ipfs-http-client@5b2c99abdedaf464f061511bc7b24b1ade66546f --- client/httpapi/dag.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index c3b29e7ee0bc..152d2c6b0a74 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -3,30 +3,25 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-cid" "io" + "math" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" ) type DagAPI HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { options, err := caopts.DagPutOptions(opts...) if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) - } - codec, ok := cid.CodecToStr[options.Codec] if !ok { return nil, fmt.Errorf("unknowm codec %d", options.MhType) @@ -39,12 +34,19 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPu var out struct{ Cid cid.Cid } - err = api.core().request("dht/put"). - Option("hash", mht). + req := api.core().request("dag/put"). Option("format", codec). - Option("input-enc", options.InputEnc). - FileBody(src). - Exec(ctx, &out) + Option("input-enc", options.InputEnc) + + if options.MhType != math.MaxUint64 { + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + req.Option("hash", mht) + } + + err = req.FileBody(src).Exec(ctx, &out) if err != nil { return nil, err } @@ -56,7 +58,7 @@ func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error panic("implement me") } -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { panic("implement me") } From c213e2654235ffbdb6ce2f3a7df3c027ba53e02f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 21:55:45 +0100 Subject: [PATCH 022/159] Unixfs.Add progress events This commit was moved from ipfs/go-ipfs-http-client@2f3a77b686ee8c4194ee6c21cbdecd57f2081519 --- client/httpapi/api.go | 4 +-- client/httpapi/api_test.go | 6 ++--- client/httpapi/apifile.go | 24 +++++++++--------- client/httpapi/block.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/ipldnode.go | 4 +-- client/httpapi/key.go | 5 ++-- client/httpapi/object.go | 2 +- client/httpapi/pin.go | 3 +-- client/httpapi/unixfs.go | 51 ++++++++++++++++++++++++++++++++++---- 10 files changed, 71 insertions(+), 32 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cf9a633bc8b4..c686a8d0dac4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -99,8 +99,8 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } return &HttpApi{ - url: url, - httpcli: c, + url: url, + httpcli: c, applyGlobal: func(*RequestBuilder) {}, } } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 67a2dbdce533..b822378f73a2 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -53,7 +53,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} if err := c.Run(connectArgs); err != nil { return nil, err } @@ -95,8 +95,8 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := &gohttp.Client{ Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, DisableCompression: true, }, } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 541189b609e6..d9da2397598f 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -22,12 +22,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } } - var stat struct{ + var stat struct { Hash string Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -43,12 +43,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } type apiFile struct { - ctx context.Context + ctx context.Context core *HttpApi size int64 path iface.Path - r io.ReadCloser + r io.ReadCloser at int64 } @@ -96,7 +96,7 @@ func (f *apiFile) Size() (int64, error) { func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { f := &apiFile{ - ctx: ctx, + ctx: ctx, core: api.core(), size: size, path: p, @@ -106,14 +106,14 @@ func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (fi } type apiIter struct { - ctx context.Context + ctx context.Context core *UnixfsAPI err error - dec *json.Decoder + dec *json.Decoder curFile files.Node - cur lsLink + cur lsLink } func (it *apiIter) Err() error { @@ -179,7 +179,7 @@ func (it *apiIter) Node() files.Node { } type apiDir struct { - ctx context.Context + ctx context.Context core *UnixfsAPI size int64 path iface.Path @@ -197,9 +197,9 @@ func (d *apiDir) Size() (int64, error) { func (d *apiDir) Entries() files.DirIterator { return &apiIter{ - ctx: d.ctx, + ctx: d.ctx, core: d.core, - dec: d.dec, + dec: d.dec, } } @@ -216,7 +216,7 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (fil } d := &apiDir{ - ctx: ctx, + ctx: ctx, core: api, size: size, path: p, diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 185aa0a42be0..f933d3c4afaf 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -16,7 +16,7 @@ import ( type BlockAPI HttpApi type blockStat struct { - Key string + Key string BSize int `json:"Size"` } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 152d2c6b0a74..20c233196d83 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -31,7 +31,7 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPut return nil, fmt.Errorf("setting hash len is not supported yet") } - var out struct{ + var out struct { Cid cid.Cid } req := api.core().request("dag/put"). diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 1f584ac0700a..8e028e1a7ed7 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -22,9 +22,9 @@ type ipldNode struct { func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ - ctx: ctx, + ctx: ctx, path: p, - api: a, + api: a, } } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 87b573f98a56..3653526ebef6 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -13,7 +13,7 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` - Id string + Id string } func (k *keyOutput) Name() string { @@ -35,7 +35,6 @@ func (k *keyOutput) valid() error { return err } - func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { options, err := caopts.KeyGenerateOptions(opts...) if err != nil { @@ -65,7 +64,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { - var id struct{ID string} + var id struct{ ID string } if err := api.core().request("id").Exec(ctx, &id); err != nil { return nil, err } diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 9c1063af9938..6259473c7ce9 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,7 +14,7 @@ import ( type ObjectAPI HttpApi type objectOut struct { - Hash string + Hash string } func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 02dcc4222473..ea34dc5409d6 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -20,7 +20,7 @@ type pinRefKeyList struct { type pin struct { path iface.ResolvedPath - typ string + typ string } func (p *pin) Path() iface.ResolvedPath { @@ -31,7 +31,6 @@ func (p *pin) Type() string { return p.typ } - func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { panic("implement me") } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 77daf7b9b7e1..7e63d241f878 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -2,9 +2,11 @@ package httpapi import ( "context" + "encoding/json" "fmt" "github.com/ipfs/go-cid" "github.com/pkg/errors" + "io" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -39,7 +41,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). - //Option("", options.Events). Option("fscache", options.FsCache). Option("hidden", options.Hidden). Option("inline", options.Inline). @@ -47,11 +48,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). - //Option("", options.Progress). Option("silent", options.Silent). Option("stdin-name", options.StdinName). Option("wrap-with-directory", options.Wrap). - Option("quieter", true) // TODO: rm after event impl + Option("progress", options.Progress) if options.RawLeavesSet { req.Option("raw-leaves", options.RawLeaves) @@ -73,9 +73,50 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix } var out addEvent - if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + resp, err := req.Send(ctx) + if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Output.Close() + dec := json.NewDecoder(resp.Output) +loop: + for { + var evt addEvent + switch err := dec.Decode(&evt); err { + case nil: + case io.EOF: + break loop + default: + return nil, err + } + out = evt + + if options.Events != nil { + ifevt := &iface.AddEvent{ + Name: out.Name, + Size: out.Size, + Bytes: out.Bytes, + } + + if out.Hash != "" { + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + ifevt.Path = iface.IpfsPath(c) + } + + select { + case options.Events <- ifevt: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } c, err := cid.Parse(out.Hash) if err != nil { @@ -120,7 +161,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, err links[i] = &format.Link{ Name: l.Name, Size: l.Size, - Cid: c, + Cid: c, } } return links, nil From 266c2f92c50df9b50bbc5aa093c793bb5d1a2265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 14:33:03 +0100 Subject: [PATCH 023/159] Implement Key API This commit was moved from ipfs/go-ipfs-http-client@281b2bf65b19b17e4f0c047ac8f9901306bf171f --- client/httpapi/key.go | 57 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 3653526ebef6..417415ddbf4f 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -49,18 +50,47 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { - panic("implement me") + options, err := caopts.KeyRenameOptions(opts...) + if err != nil { + return nil, false, err + } + + var out struct{ + Was string + Now string + Id string + Overwrite bool + } + err = api.core().request("key/rename", oldName, newName). + Option("force", options.Force). + Exec(ctx, &out) + if err != nil { + return nil, false, err + } + + id := &keyOutput{JName: out.Now, Id: out.Id} + return id, out.Overwrite, id.valid() } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { - panic("implement me") + var out struct{ Keys []*keyOutput } + if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]iface.Key, len(out.Keys)) + for i, k := range out.Keys { + if err := k.valid(); err != nil { + return nil, err + } + res[i] = k + } + + return res, nil } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { @@ -70,14 +100,19 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { } out := keyOutput{JName: "self", Id: id.ID} - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { - panic("implement me") + var out struct{ Keys []keyOutput } + if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + return nil, err + } + if len(out.Keys) != 1 { + return nil, errors.New("got unexpected number of keys back") + } + + return &out.Keys[0], out.Keys[0].valid() } func (api *KeyAPI) core() *HttpApi { From 9d647d011efa551bae28c50cafb402a35d33b04a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 15:02:15 +0100 Subject: [PATCH 024/159] Implement Pin API This commit was moved from ipfs/go-ipfs-http-client@5bb7a581323aad4dd765a0916f6552cd5902d1e2 --- client/httpapi/pin.go | 92 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ea34dc5409d6..34cafad71b67 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -31,8 +33,14 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { - panic("implement me") +func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { + options, err := caopts.PinAddOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/add", p.String()). + Option("recursive", options.Recursive).Exec(ctx, nil) } func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { @@ -65,11 +73,85 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { - panic("implement me") + options, err := caopts.PinUpdateOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/update"). + Option("unpin", options.Unpin).Exec(ctx, nil) +} + +type pinVerifyRes struct { + Cid string + JOk bool `json:"Ok"` + JBadNodes []*badNode `json:"BadNodes,omitempty"` +} + +func (r *pinVerifyRes) Ok() bool { + return r.JOk +} + +func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { + out := make([]iface.BadPinNode, len(r.JBadNodes)) + for i, n := range r.JBadNodes { + out[i] = n + } + return out +} + +type badNode struct { + Cid string + JErr string `json:"Err"` } -func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { - panic("implement me") +func (n *badNode) Path() iface.ResolvedPath { + c, err := cid.Parse(n.Cid) + if err != nil { + return nil // todo: handle this better + } + return iface.IpldPath(c) +} + +func (n *badNode) Err() error { + if n.JErr != "" { + return errors.New(n.JErr) + } + if _, err := cid.Parse(n.Cid); err != nil { + return err + } + return nil +} + +func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { + resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan iface.PinStatus) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + for { + var out pinVerifyRes + if err := dec.Decode(&out); err != nil { + return // todo: handle non io.EOF somehow + } + + select { + case res <- &out: + case <-ctx.Done(): + return + } + } + }() + + return res, nil } func (api *PinAPI) core() *HttpApi { From eaa19388d4d86eebf09d93a3a637e0690df722bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 16:31:07 +0100 Subject: [PATCH 025/159] Implement Object API This commit was moved from ipfs/go-ipfs-http-client@38149e46c8c3a1544dc3e3108ef72ffbc885e410 --- client/httpapi/block.go | 2 +- client/httpapi/ipldnode.go | 7 +- client/httpapi/key.go | 8 +- client/httpapi/object.go | 216 +++++++++++++++++++++++++++++++++---- client/httpapi/pin.go | 6 +- 5 files changed, 208 insertions(+), 31 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f933d3c4afaf..f8334a10980d 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -67,7 +67,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + resp, err := api.core().request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 8e028e1a7ed7..2294a95b42bd 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "errors" - "fmt" "io/ioutil" "strconv" @@ -20,11 +19,11 @@ type ipldNode struct { api *HttpApi } -func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { +func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ ctx: ctx, path: p, - api: a, + api: api, } } @@ -47,7 +46,7 @@ func (n *ipldNode) Cid() cid.Cid { } func (n *ipldNode) String() string { - return fmt.Sprintf("[Block %s]", n.Cid()) + return n.Cid().String() } func (n *ipldNode) Loggable() map[string]interface{} { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 417415ddbf4f..5087b6d99893 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -59,10 +59,10 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o return nil, false, err } - var out struct{ - Was string - Now string - Id string + var out struct { + Was string + Now string + Id string Overwrite bool } err = api.core().request("key/rename", oldName, newName). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6259473c7ce9..4bc78799166e 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -1,14 +1,17 @@ package httpapi import ( + "bytes" "context" - "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" ) type ObjectAPI HttpApi @@ -37,44 +40,219 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil } -func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + options, err := caopts.ObjectPutOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/put"). + Option("inputenc", options.InputEnc). + Option("datafieldenc", options.DataType). + Option("pin", options.Pin). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { - panic("implement me") +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { + r, err := api.core().Block().Get(ctx, p) + if err != nil { + return nil, err + } + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { - panic("implement me") +func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("object/data", p.String()).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + //TODO: make Data return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } + + return b, nil } -func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out struct { + Links []struct { + Name string + Hash string + Size uint64 + } + } + if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]*format.Link, len(out.Links)) + for i, l := range out.Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + + res[i] = &format.Link{ + Cid: c, + Name: l.Name, + Size: l.Size, + } + } + + return res, nil } -func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { - panic("implement me") +func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { + var out struct { + Hash string + NumLinks int + BlockSize int + LinksSize int + DataSize int + CumulativeSize int + } + if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return &iface.ObjectStat{ + Cid: c, + NumLinks: out.NumLinks, + BlockSize: out.BlockSize, + LinksSize: out.LinksSize, + DataSize: out.DataSize, + CumulativeSize: out.CumulativeSize, + }, nil } func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { - panic("implement me") + options, err := caopts.ObjectAddLinkOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + Option("create", options.Create). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { - panic("implement me") + var out objectOut + err := api.core().request("object/patch/rm-link", base.String(), link). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil +} + +func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/append-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/set-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +type change struct { + Type iface.ChangeType + Path string + Before cid.Cid + After cid.Cid } -func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { - panic("implement me") +func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { + var out struct { + Changes []change + } + if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]iface.ObjectChange, len(out.Changes)) + for i, ch := range out.Changes { + res[i] = iface.ObjectChange{ + Type: ch.Type, + Path: ch.Path, + } + if ch.Before != cid.Undef { + res[i].Before = iface.IpfsPath(ch.Before) + } + if ch.After != cid.Undef { + res[i].After = iface.IpfsPath(ch.After) + } + } + return res, nil } func (api *ObjectAPI) core() *HttpApi { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 34cafad71b67..1624b7867580 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -83,8 +83,8 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` + Cid string + JOk bool `json:"Ok"` JBadNodes []*badNode `json:"BadNodes,omitempty"` } @@ -101,7 +101,7 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { } type badNode struct { - Cid string + Cid string JErr string `json:"Err"` } From ed9f2dd091693e914e9de072b2c23c2f991210a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:01:33 +0100 Subject: [PATCH 026/159] Implement DHT Api This commit was moved from ipfs/go-ipfs-http-client@c77355067a22300a8abe6b93b354de0ae0a3b548 --- client/httpapi/api.go | 2 +- client/httpapi/dht.go | 98 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dht.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c686a8d0dac4..208b958e9312 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -158,7 +158,7 @@ func (api *HttpApi) Object() iface.ObjectAPI { } func (api *HttpApi) Dht() iface.DhtAPI { - return nil + return (*DhtAPI)(api) } func (api *HttpApi) Swarm() iface.SwarmAPI { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go new file mode 100644 index 000000000000..17d3b0500c2b --- /dev/null +++ b/client/httpapi/dht.go @@ -0,0 +1,98 @@ +package httpapi + +import ( + "context" + "encoding/json" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + notif "github.com/libp2p/go-libp2p-routing/notifications" +) + +type DhtAPI HttpApi + +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + if err != nil { + return peerstore.PeerInfo{}, err + } + if resp.Error != nil { + return peerstore.PeerInfo{}, resp.Error + } + defer resp.Close() + dec := json.NewDecoder(resp.Output) + for { + if err := dec.Decode(&out); err != nil { + return peerstore.PeerInfo{}, err + } + if out.Type == notif.FinalPeer { + return out.Responses[0], nil + } + } +} + +func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { + options, err := caopts.DhtFindProvidersOptions(opts...) + if err != nil { + return nil, err + } + resp, err := api.core().request("dht/findprovs", p.String()). + Option("num-providers", options.NumProviders). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan peerstore.PeerInfo) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + + for { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + + if err := dec.Decode(&out); err != nil { + return // todo: handle this somehow + } + if out.Type == notif.Provider { + for _, pi := range out.Responses { + select { + case res <- pi: + case <-ctx.Done(): + return + } + } + } + } + }() + + return res, nil +} + +func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { + options, err := caopts.DhtProvideOptions(opts...) + if err != nil { + return err + } + + return api.core().request("dht/provide", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) +} + +func (api *DhtAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cae0ff2379f29a205426f7a8199c398885f33897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:36:35 +0100 Subject: [PATCH 027/159] Implement Swarm Api This commit was moved from ipfs/go-ipfs-http-client@01105690d2da6e3f3581fa364c6284248a251a35 --- client/httpapi/api.go | 2 +- client/httpapi/swarm.go | 176 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/swarm.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 208b958e9312..a5b56eff6347 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -162,7 +162,7 @@ func (api *HttpApi) Dht() iface.DhtAPI { } func (api *HttpApi) Swarm() iface.SwarmAPI { - return nil + return (*SwarmAPI)(api) } func (api *HttpApi) PubSub() iface.PubSubAPI { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go new file mode 100644 index 000000000000..cf0ee2a9b369 --- /dev/null +++ b/client/httpapi/swarm.go @@ -0,0 +1,176 @@ +package httpapi + +import ( + "context" + "github.com/libp2p/go-libp2p-protocol" + "time" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + inet "github.com/libp2p/go-libp2p-net" + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + "github.com/multiformats/go-multiaddr" +) + +type SwarmAPI HttpApi + +func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + saddrs := make([]string, len(pi.Addrs)) + for i, addr := range pi.Addrs { + saddrs[i] = addr.String() + } + + return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) +} + +func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { + return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) +} + +type streamInfo struct { + Protocol string +} + +type connInfo struct { + Addr string + Peer string + JLatency time.Duration `json:"Latency"` + Muxer string + JDirection inet.Direction `json:"Direction"` + JStreams []streamInfo `json:"Streams"` +} + +func (c *connInfo) valid() error { + _, err := multiaddr.NewMultiaddr(c.Addr) + if err != nil { + return err + } + + _, err = peer.IDB58Decode(c.Peer) + return err +} + +func (c *connInfo) ID() peer.ID { + id, _ := peer.IDB58Decode(c.Peer) + return id +} + +func (c *connInfo) Address() multiaddr.Multiaddr { + a, _ := multiaddr.NewMultiaddr(c.Addr) + return a +} + +func (c *connInfo) Direction() inet.Direction { + return c.JDirection +} + +func (c *connInfo) Latency() (time.Duration, error) { + return c.JLatency, nil +} + +func (c *connInfo) Streams() ([]protocol.ID, error) { + res := make([]protocol.ID, len(c.JStreams)) + for i, stream := range c.JStreams { + res[i] = protocol.ID(stream.Protocol) + } + return res, nil +} + +func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { + var out struct { + Peers []*connInfo + } + + err := api.core().request("swarm/peers"). + Option("streams", true). + Option("latency", true). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + res := make([]iface.ConnectionInfo, len(out.Peers)) + for i, conn := range out.Peers { + if err := conn.valid(); err != nil { + return nil, err + } + res[i] = conn + } + + return res, nil +} + +func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Multiaddr, error) { + var out struct { + Addrs map[string][]string + } + if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + return nil, err + } + res := map[peer.ID][]multiaddr.Multiaddr{} + for spid, saddrs := range out.Addrs { + addrs := make([]multiaddr.Multiaddr, len(saddrs)) + + for i, addr := range saddrs { + a, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + addrs[i] = a + } + + pid, err := peer.IDB58Decode(spid) + if err != nil { + return nil, err + } + + res[pid] = addrs + } + + return res, nil +} + +func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c2e0872f6dfba6f6d3f74a4b9eb755053a3f483e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:19:50 +0100 Subject: [PATCH 028/159] Implement PubSub Api This commit was moved from ipfs/go-ipfs-http-client@7abddda1d33595dffb0e0da782b52660147f6855 --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 2 +- client/httpapi/pubsub.go | 129 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pubsub.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index a5b56eff6347..3a8e0e0031fe 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -166,5 +166,5 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { } func (api *HttpApi) PubSub() iface.PubSubAPI { - return nil + return (*PubsubAPI)(api) } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b822378f73a2..1acece88c3c6 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,7 +47,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go new file mode 100644 index 000000000000..3c5e3c50d606 --- /dev/null +++ b/client/httpapi/pubsub.go @@ -0,0 +1,129 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type PubsubAPI HttpApi + +func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + return nil, err + } + + return out.Strings, nil +} + +func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { + options, err := caopts.PubSubPeersOptions(opts...) + if err != nil { + return nil, err + } + + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]peer.ID, len(out.Strings)) + for i, sid := range out.Strings { + id, err := peer.IDB58Decode(sid) + if err != nil { + return nil, err + } + res[i] = id + } + return res, nil +} + +func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { + return api.core().request("pubsub/pub", topic). + FileBody(bytes.NewReader(message)). + Exec(ctx, nil) +} + +type pubsubSub struct { + io.Closer + dec *json.Decoder +} + +type pubsubMessage struct { + JFrom []byte `json:"from,omitempty"` + JData []byte `json:"data,omitempty"` + JSeqno []byte `json:"seqno,omitempty"` + JTopicIDs []string `json:"topicIDs,omitempty"` +} + +func (msg *pubsubMessage) valid() error { + _, err := peer.IDFromBytes(msg.JFrom) + return err +} + +func (msg *pubsubMessage) From() peer.ID { + id, _ := peer.IDFromBytes(msg.JFrom) + return id +} + +func (msg *pubsubMessage) Data() []byte { + return msg.JData +} + +func (msg *pubsubMessage) Seq() []byte { + return msg.JSeqno +} + +func (msg *pubsubMessage) Topics() []string { + return msg.JTopicIDs +} + +func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { + // TODO: handle ctx + + var msg pubsubMessage + if err := s.dec.Decode(&msg); err != nil { + return nil, err + } + return &msg, msg.valid() +} + +func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + options, err := caopts.PubSubSubscribeOptions(opts...) + if err != nil { + return nil, err + } + + resp, err := api.core().request("pubsub/sub", topic). + Option("discover", options.Discover). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + return &pubsubSub{ + Closer: resp, + dec: json.NewDecoder(resp.Output), + }, nil +} + +func (api *PubsubAPI) core() *HttpApi { + return (*HttpApi)(api) +} + From 9b24cf0aafa94a7b4cb4401b6fbb9c959bde09b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:49:11 +0100 Subject: [PATCH 029/159] Use cids in DHT calls This commit was moved from ipfs/go-ipfs-http-client@163b25f8b88eb98ddae50978440e3940070fabc0 --- client/httpapi/dht.go | 20 ++++++++++++++++++-- client/httpapi/pubsub.go | 3 +-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 17d3b0500c2b..8fedabbf83de 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -42,7 +42,13 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err != nil { return nil, err } - resp, err := api.core().request("dht/findprovs", p.String()). + + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + resp, err := api.core().request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -60,6 +66,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop for { var out struct { + Extra string Type notif.QueryEventType Responses []peerstore.PeerInfo } @@ -67,6 +74,10 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } + if out.Type == notif.QueryError { + return // usually a 'not found' error + // todo: handle other errors + } if out.Type == notif.Provider { for _, pi := range out.Responses { select { @@ -88,7 +99,12 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", p.String()). + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return err + } + + return api.core().request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 3c5e3c50d606..fb9bb74601c5 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -119,11 +119,10 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return &pubsubSub{ Closer: resp, - dec: json.NewDecoder(resp.Output), + dec: json.NewDecoder(resp.Output), }, nil } func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } - From b7e258cc1015e518198c50b02fcc6ebdd8767375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Jan 2019 21:31:09 +0100 Subject: [PATCH 030/159] Reimplement DAG as DAGService This commit was moved from ipfs/go-ipfs-http-client@f34a5f6d2569bb21f022d3231e811b3db6262ca8 --- client/httpapi/api.go | 6 ++- client/httpapi/apifile.go | 5 ++ client/httpapi/dag.go | 103 +++++++++++++++++++++++++------------- 3 files changed, 77 insertions(+), 37 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3a8e0e0031fe..7d4fbd0e0811 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -10,6 +10,8 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -137,8 +139,8 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() iface.DagAPI { - return (*DagAPI)(api) +func (api *HttpApi) Dag() format.DAGService { + return (*HttpDagServ)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index d9da2397598f..99ae72059c8d 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -125,6 +125,11 @@ func (it *apiIter) Name() string { } func (it *apiIter) Next() bool { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + var out lsOutput if err := it.dec.Decode(&out); err != nil { if err != io.EOF { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 20c233196d83..de2934c2f115 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -1,71 +1,104 @@ package httpapi import ( + "bytes" "context" "fmt" - "io" - "math" + "io/ioutil" + "sync" "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - mh "github.com/multiformats/go-multihash" ) -type DagAPI HttpApi +type HttpDagServ HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { - options, err := caopts.DagPutOptions(opts...) +func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { + r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) if err != nil { return nil, err } - codec, ok := cid.CodecToStr[options.Codec] - if !ok { - return nil, fmt.Errorf("unknowm codec %d", options.MhType) + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err } - if options.MhLength != -1 { - return nil, fmt.Errorf("setting hash len is not supported yet") + blk, err := blocks.NewBlockWithCid(data, c) + if err != nil { + return nil, err } - var out struct { - Cid cid.Cid + return format.DefaultBlockDecoder.Decode(blk) +} + +func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { + out := make(chan *format.NodeOption) + wg := sync.WaitGroup{} + wg.Add(len(cids)) + + for _, c := range cids { + // TODO: Consider limiting concurrency of this somehow + go func() { + defer wg.Done() + n, err := api.Get(ctx, c) + + select { + case out <- &format.NodeOption{Node: n, Err: err}: + case <-ctx.Done(): + } + }() } - req := api.core().request("dag/put"). - Option("format", codec). - Option("input-enc", options.InputEnc) - - if options.MhType != math.MaxUint64 { - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) - } - req.Option("hash", mht) + return out +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + c := nd.Cid() + prefix := c.Prefix() + format := cid.CodecToStr[prefix.Codec] + if prefix.Version == 0 { + format = "v0" } - err = req.FileBody(src).Exec(ctx, &out) + stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), + options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) if err != nil { - return nil, err + return err } - - return iface.IpldPath(out.Cid), nil + if !stat.Path().Cid().Equals(c) { + return fmt.Errorf("cids didn't match - local %s, remote %s", c.String(), stat.Path().Cid().String()) + } + return nil } -func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { - panic("implement me") +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + for _, nd := range nds { + // TODO: optimize + if err := api.Add(ctx, nd); err != nil { + return err + } + } + return nil } -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { - panic("implement me") +func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { + return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? } -func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { - panic("implement me") +func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { + for _, c := range cids { + // TODO: optimize + if err := api.Remove(ctx, c); err != nil { + return err + } + } + return nil } -func (api *DagAPI) core() *HttpApi { +func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From c3f2970f8a7302344154e6d7fe74485f0dd272c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:42:32 +0100 Subject: [PATCH 031/159] block: Pin option This commit was moved from ipfs/go-ipfs-http-client@def66919dfbee162595852334468719f57158121 --- client/httpapi/block.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f8334a10980d..24df9629ed81 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -53,6 +53,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). + Option("pin", options.Pin). FileBody(r) var out blockStat From af2edd12eb44f477ef2e0dc22ae31a75df645b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:00 +0100 Subject: [PATCH 032/159] tests: enable filestore This commit was moved from ipfs/go-ipfs-http-client@904e8eeeb1b677d458c819122d91264040120d3b --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 1acece88c3c6..bf1d6b9b04b9 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,6 +47,11 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} + if err := c.Run(filestoreArgs); err != nil { + return nil, err + } + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err From 75cf2be1026a7f206e61d164342dae4807d36bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:26 +0100 Subject: [PATCH 033/159] apifile: Implement Seek This commit was moved from ipfs/go-ipfs-http-client@69cc3e8106552605ecfdfa5c6f352996dc6f9497 --- client/httpapi/apifile.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 99ae72059c8d..ec916d8e2990 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,13 +6,15 @@ import ( "fmt" "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs-files" - unixfspb "github.com/ipfs/go-unixfs/pb" ) +const forwardSeekLimit = 1 << 14 //16k + func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error @@ -80,7 +82,24 @@ func (f *apiFile) Read(p []byte) (int, error) { } func (f *apiFile) Seek(offset int64, whence int) (int64, error) { - panic("implement me") //TODO + switch whence { + case io.SeekEnd: + offset = f.size + offset + case io.SeekCurrent: + offset = f.at + offset + } + if f.at == offset { //noop + return offset, nil + } + + if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + + f.at += r + return f.at, err + } + f.at = offset + return f.at, f.reset() } func (f *apiFile) Close() error { @@ -156,17 +175,17 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case unixfspb.Data_HAMTShard: + case iface.THAMTShard: fallthrough - case unixfspb.Data_Metadata: + case iface.TMetadata: fallthrough - case unixfspb.Data_Directory: + case iface.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case unixfspb.Data_File: + case iface.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err From bb83ccb15feb4854c7bdb4821fbcf90b89f36163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:44:48 +0100 Subject: [PATCH 034/159] response: read trailing error headers This commit was moved from ipfs/go-ipfs-http-client@83dfd84ba8bf2341478bb98d437ae1f14fe29e18 --- client/httpapi/response.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f6e7f3ab7099..745c9a2a99da 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -2,6 +2,7 @@ package httpapi import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -13,6 +14,24 @@ import ( files "github.com/ipfs/go-ipfs-files" ) +type trailerReader struct { + resp *http.Response +} + +func (r *trailerReader) Read(b []byte) (int, error) { + n, err := r.resp.Body.Read(b) + if err != nil { + if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + err = errors.New(e) + } + } + return n, err +} + +func (r *trailerReader) Close() error { + return r.resp.Body.Close() +} + type Response struct { Output io.ReadCloser Error *Error @@ -56,9 +75,6 @@ type Error struct { func (e *Error) Error() string { var out string - if e.Command != "" { - out = e.Command + ": " - } if e.Code != 0 { out = fmt.Sprintf("%s%d: ", out, e.Code) } @@ -93,7 +109,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.Output = resp.Body + nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ Command: r.Command, From 88139ddc50e111e39721990bb86bb1c018c488b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:45:24 +0100 Subject: [PATCH 035/159] dag: Interface updates This commit was moved from ipfs/go-ipfs-http-client@7bea2efb45cba072b4e24c20b46a4372be3a716d --- client/httpapi/api.go | 3 +-- client/httpapi/dag.go | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 7d4fbd0e0811..2c7a97c99265 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -11,7 +11,6 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -139,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() format.DAGService { +func (api *HttpApi) Dag() iface.APIDagService { return (*HttpDagServ)(api) } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index de2934c2f115..eacf631b92cf 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -15,7 +15,9 @@ import ( "github.com/ipfs/go-ipld-format" ) -type HttpDagServ HttpApi +type httpNodeAdder HttpApi +type HttpDagServ httpNodeAdder +type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) @@ -56,7 +58,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for return out } -func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { +func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() format := cid.CodecToStr[prefix.Codec] @@ -65,7 +67,9 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), - options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) + options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.Format(format), + options.Block.Pin(pin)) if err != nil { return err } @@ -75,16 +79,36 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { return nil } -func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { +func (api *httpNodeAdder) addMany(ctx context.Context, nds []format.Node, pin bool) error { for _, nd := range nds { // TODO: optimize - if err := api.Add(ctx, nd); err != nil { + if err := api.add(ctx, nd, pin); err != nil { return err } } return nil } +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, false) +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, false) +} + +func (api *pinningHttpNodeAdder) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, true) +} + +func (api *pinningHttpNodeAdder) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, true) +} + +func (api *HttpDagServ) Pinning() format.NodeAdder { + return (*pinningHttpNodeAdder)(api) +} + func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? } @@ -99,6 +123,10 @@ func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { return nil } +func (api *httpNodeAdder) core() *HttpApi { + return (*HttpApi)(api) +} + func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From 93bfcf91cf979b267758c04aa14c6435a687fc36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:46:00 +0100 Subject: [PATCH 036/159] unixfs: updated ls This commit was moved from ipfs/go-ipfs-http-client@bb8d9d1a60e4161c4cf3a33c468746050f267a12 --- client/httpapi/unixfs.go | 94 ++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 7e63d241f878..67d05309b295 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,7 +13,6 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" - unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" ) @@ -129,7 +128,7 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type unixfspb.Data_DataType + Type iface.FileType } type lsObject struct { @@ -141,30 +140,87 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { - var out lsOutput - err := api.core().request("ls", p.String()).Exec(ctx, &out) +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { + options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err } - if len(out.Objects) != 1 { - return nil, errors.New("unexpected objects len") + resp, err := api.core().request("ls", p.String()). + Option("resolve-type", options.ResolveChildren). + Option("size", options.ResolveChildren). + Option("stream", true). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error } - links := make([]*format.Link, len(out.Objects[0].Links)) - for i, l := range out.Objects[0].Links { - c, err := cid.Parse(l.Hash) - if err != nil { - return nil, err - } - links[i] = &format.Link{ - Name: l.Name, - Size: l.Size, - Cid: c, + dec := json.NewDecoder(resp.Output) + out := make(chan iface.LsLink) + + go func() { + defer resp.Close() + defer close(out) + + for { + var link lsOutput + if err := dec.Decode(&link); err != nil { + if err == io.EOF { + return + } + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects[0].Links) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case <-ctx.Done(): + } + return + } + + l0 := link.Objects[0].Links[0] + + c, err := cid.Decode(l0.Hash) + if err != nil { + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + select { + case out <- iface.LsLink{ + Link: &format.Link{ + Cid: c, + Name: l0.Name, + Size: l0.Size, + }, + Size: l0.Size, + Type: l0.Type, + }: + case <-ctx.Done(): + } } - } - return links, nil + }() + + return out, nil } func (api *UnixfsAPI) core() *HttpApi { From 1bc854bf2169f82d2275232e123adae721b8dc0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:53:02 +0100 Subject: [PATCH 037/159] pin: handle Rm options This commit was moved from ipfs/go-ipfs-http-client@e85e856ea2841cd3e0cf4cb7b87441539af42bbe --- client/httpapi/pin.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 1624b7867580..237b3232976d 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -68,8 +68,15 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { - return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { + options, err := caopts.PinRmOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/rm", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { From c543354b1772323cb2cfbf41c06522297bf27667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Feb 2019 22:36:30 +0100 Subject: [PATCH 038/159] Check for redirects This commit was moved from ipfs/go-ipfs-http-client@19c65db4f0fd1549fc6f224efd868d4f00da997d --- client/httpapi/api.go | 18 +++++++++++++----- client/httpapi/requestbuilder.go | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 2c7a97c99265..da0836d4a6f8 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -2,6 +2,7 @@ package httpapi import ( "errors" + "fmt" "io/ioutil" gohttp "net/http" "os" @@ -27,7 +28,7 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string - httpcli *gohttp.Client + httpcli gohttp.Client applyGlobal func(*RequestBuilder) } @@ -50,8 +51,8 @@ func NewPathApi(p string) iface.CoreAPI { return NewApi(a) } -func ApiAddr(p string) ma.Multiaddr { - baseDir, err := homedir.Expand(p) +func ApiAddr(ipfspath string) ma.Multiaddr { + baseDir, err := homedir.Expand(ipfspath) if err != nil { return nil } @@ -99,11 +100,18 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } } - return &HttpApi{ + api := &HttpApi{ url: url, - httpcli: c, + httpcli: *c, applyGlobal: func(*RequestBuilder) {}, } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 831e6d71c0ba..628ad03cdf39 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -92,7 +92,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { req.Opts = r.opts req.Headers = r.headers req.Body = r.body - return req.Send(r.shell.httpcli) + return req.Send(&r.shell.httpcli) } // Exec sends the request a request and decodes the response. From ef5bf40df1d0d790aeb213147c53412145894bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 11 Feb 2019 18:40:38 +0100 Subject: [PATCH 039/159] Update imports to use extracted interface This commit was moved from ipfs/go-ipfs-http-client@fad467bc437a1282ed3813d828c6f8ec5b7b2d7a --- client/httpapi/api.go | 5 ++--- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 9 ++++----- client/httpapi/block.go | 4 ++-- client/httpapi/dag.go | 5 ++--- client/httpapi/dht.go | 4 ++-- client/httpapi/ipldnode.go | 3 +-- client/httpapi/key.go | 5 ++--- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 5 ++--- client/httpapi/path.go | 3 +-- client/httpapi/pin.go | 6 +++--- client/httpapi/pubsub.go | 5 ++--- client/httpapi/swarm.go | 5 ++--- client/httpapi/unixfs.go | 11 +++++------ 15 files changed, 36 insertions(+), 47 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index da0836d4a6f8..698e36524e7d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,9 +9,8 @@ import ( "path" "strings" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + iface "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index bf1d6b9b04b9..5621ab87f0f3 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,10 +10,9 @@ import ( "strconv" "testing" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec916d8e2990..1e5f61a9a7b3 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -4,13 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/ipfs/go-cid" "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -92,8 +91,8 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { return offset, nil } - if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 24df9629ed81..45c73472c599 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -8,8 +8,8 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index eacf631b92cf..d0059698ab60 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -7,12 +7,11 @@ import ( "io/ioutil" "sync" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/options" ) type httpNodeAdder HttpApi diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 8fedabbf83de..dc7dd6beaf6f 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -3,9 +3,9 @@ package httpapi import ( "context" "encoding/json" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 2294a95b42bd..43fa5d50de94 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -6,11 +6,10 @@ import ( "io/ioutil" "strconv" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) type ipldNode struct { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 5087b6d99893..dc2adf8b7fe1 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,8 @@ import ( "context" "errors" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index fbc440b96664..d760e6188aeb 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -3,10 +3,10 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-ipfs/namesys/opts" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/options/namesys" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 4bc78799166e..3b648d82bc0a 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,12 +6,11 @@ import ( "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 5701326fc560..68dc2633dd5d 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,11 +3,10 @@ package httpapi import ( "context" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 237b3232976d..4c4e5713ce64 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,11 +3,11 @@ package httpapi import ( "context" "encoding/json" + "github.com/ipfs/go-cid" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/pkg/errors" - - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) type PinAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index fb9bb74601c5..3345097806db 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,9 +6,8 @@ import ( "encoding/json" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index cf0ee2a9b369..d179b6540fe2 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -2,14 +2,13 @@ package httpapi import ( "context" - "github.com/libp2p/go-libp2p-protocol" "time" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/interface-go-ipfs-core" inet "github.com/libp2p/go-libp2p-net" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" + "github.com/libp2p/go-libp2p-protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 67d05309b295..1f340b6573a2 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -3,16 +3,15 @@ package httpapi import ( "context" "encoding/json" + "errors" "fmt" - "github.com/ipfs/go-cid" - "github.com/pkg/errors" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -208,7 +207,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf select { case out <- iface.LsLink{ Link: &format.Link{ - Cid: c, + Cid: c, Name: l0.Name, Size: l0.Size, }, From 0813d808b54341414441a56e5cd3a7722a516a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:29:31 +0100 Subject: [PATCH 040/159] Fix govet warning in Dag This commit was moved from ipfs/go-ipfs-http-client@62552b33959401c7a9d9463dffba433e522b2f15 --- client/httpapi/dag.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index d0059698ab60..3f54ced34f02 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -44,7 +44,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for for _, c := range cids { // TODO: Consider limiting concurrency of this somehow - go func() { + go func(c cid.Cid) { defer wg.Done() n, err := api.Get(ctx, c) @@ -52,7 +52,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for case out <- &format.NodeOption{Node: n, Err: err}: case <-ctx.Done(): } - }() + }(c) } return out } From abc30e384fafe157490df0dae80a22ddeb6255a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:48:04 +0100 Subject: [PATCH 041/159] Register iptb plugin once This commit was moved from ipfs/go-ipfs-http-client@d6c8cbd5e643d2131477120a76159074d1a07cd3 --- client/httpapi/api_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5621ab87f0f3..fa38866e1a24 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -19,9 +19,7 @@ import ( "github.com/ipfs/iptb/testbed/interfaces" ) -type NodeProvider struct{} - -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", NewNode: local.NewNode, @@ -31,8 +29,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) BuiltIn: true, }, false) if err != nil { - return nil, err + panic(err) } +} + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { From 9a6ee6f5e153b96afda0f4ca1ee893cf28aa4f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 17:30:04 +0100 Subject: [PATCH 042/159] Improve test node spawning This commit was moved from ipfs/go-ipfs-http-client@6bb2a287a6f5090792e07f3e3a74871c459b395f --- client/httpapi/api_test.go | 198 ++++++++++++++++++++++++------------- 1 file changed, 131 insertions(+), 67 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fa38866e1a24..af2180384d62 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,23 +2,23 @@ package httpapi import ( "context" - "fmt" "io/ioutil" gohttp "net/http" "os" - "path" "strconv" + "sync" "testing" "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" - "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" "github.com/ipfs/iptb/testbed/interfaces" + ma "github.com/multiformats/go-multiaddr" ) +const parallelSpeculativeNodes = 15 // 15 seems to work best + func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", @@ -33,99 +33,163 @@ func init() { } } -type NodeProvider struct{} +type NodeProvider struct { + simple <-chan func(context.Context) ([]iface.CoreAPI, error) +} + +func newNodeProvider(ctx context.Context) *NodeProvider { + simpleNodes := make(chan func(context.Context) ([]iface.CoreAPI, error), parallelSpeculativeNodes) + + np := &NodeProvider{ + simple: simpleNodes, + } + + // start basic nodes speculatively in parallel + for i := 0; i < parallelSpeculativeNodes; i++ { + go func() { + for { + ctx, cancel := context.WithCancel(ctx) + + snd, err := np.makeAPISwarm(ctx, false, 1) + + res := func(ctx context.Context) ([]iface.CoreAPI, error) { + if err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + cancel() + }() + + return snd, nil + } + + select { + case simpleNodes <- res: + case <-ctx.Done(): + return + } + } + }() + } + + return np +} + +func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + if !fullIdentity && n == 1 { + return (<-np.simple)(ctx) + } + return np.makeAPISwarm(ctx, fullIdentity, n) +} -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { return nil, err } - c := cli.NewCli() //TODO: is there a better way? + tb := testbed.NewTestbed(dir) - initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} - if err := c.Run(initArgs); err != nil { + specs, err := testbed.BuildSpecs(tb.Dir(), n, "localipfs", nil) + if err != nil { return nil, err } - filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} - if err := c.Run(filestoreArgs); err != nil { + if err := testbed.WriteNodeSpecs(tb.Dir(), specs); err != nil { return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} - if err := c.Run(startArgs); err != nil { + nodes, err := tb.Nodes() + if err != nil { return nil, err } - if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} - if err := c.Run(connectArgs); err != nil { - return nil, err - } + apis := make([]iface.CoreAPI, n) + + wg := sync.WaitGroup{} + zero := sync.WaitGroup{} + + wg.Add(len(nodes)) + zero.Add(1) + + for i, nd := range nodes { + go func(i int, nd testbedi.Core) { + defer wg.Done() + + if _, err := nd.Init(ctx, "--empty-repo"); err != nil { + panic(err) + } + + if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { + panic(err) + } + + if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { + panic(err) + } + + if i > 0 { + zero.Wait() + if err := nd.Connect(ctx, nodes[0]); err != nil { + panic(err) + } + } else { + zero.Done() + } + + addr, err := nd.APIAddr() + if err != nil { + panic(err) + } + + maddr, err := ma.NewMultiaddr(addr) + if err != nil { + panic(err) + } + + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(maddr, c) + + // empty node is pinned even with --empty-repo, we don't want that + emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + if err != nil { + panic(err) + } + if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { + panic(err) + } + }(i, nd) } + wg.Wait() + go func() { <-ctx.Done() defer os.Remove(dir) defer func() { - _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + for _, nd := range nodes { + _ = nd.Stop(context.Background()) + } }() }() - apis := make([]iface.CoreAPI, n) - - for i := range apis { - tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) - - node, err := tb.Node(i) - if err != nil { - return nil, err - } - - attrNode, ok := node.(testbedi.Attribute) - if !ok { - return nil, fmt.Errorf("node does not implement attributes") - } - - pth, err := attrNode.Attr("path") - if err != nil { - return nil, err - } - - a := ApiAddr(pth) - if a == nil { - return nil, fmt.Errorf("nil addr for node") - } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - DisableCompression: true, - }, - } - apis[i] = NewApiWithClient(a, c) - - // node cleanup - // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) - if err != nil { - return nil, err - } - for _, pin := range pins { //TODO: parallel - if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { - return nil, err - } - } - - } - return apis, nil } func TestHttpApi(t *testing.T) { - tests.TestApi(&NodeProvider{})(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tests.TestApi(newNodeProvider(ctx))(t) } From f7dd0c690960ea79a7aad7f40b11362512d173bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:12:02 +0100 Subject: [PATCH 043/159] Improve apifile error messages This commit was moved from ipfs/go-ipfs-http-client@3393b8379021695126da83afdf3bf54eaff828f4 --- client/httpapi/apifile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1e5f61a9a7b3..8647606844b3 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -157,12 +157,12 @@ func (it *apiIter) Next() bool { } if len(out.Objects) != 1 { - it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + it.err = fmt.Errorf("ls returned more objects than expected (%d)", len(out.Objects)) return false } if len(out.Objects[0].Links) != 1 { - it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + it.err = fmt.Errorf("ls returned more links than expected (%d)", len(out.Objects[0].Links)) return false } From e1b14d78c72edf0a782e362afb0e818e032c31cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:45:19 +0100 Subject: [PATCH 044/159] Implement missing methods This commit was moved from ipfs/go-ipfs-http-client@b76413dfe55fd33d39a680458282d85565cc1c69 --- client/httpapi/api.go | 3 -- client/httpapi/name.go | 63 +++++++++++++++++++++++++++++++++++++++--- client/httpapi/path.go | 9 ++++-- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 698e36524e7d..e106d8d87a63 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,6 @@ package httpapi import ( - "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -23,8 +22,6 @@ const ( EnvDir = "IPFS_PATH" ) -var ErrNotImplemented = errors.New("not implemented") - type HttpApi struct { url string httpcli gohttp.Client diff --git a/client/httpapi/name.go b/client/httpapi/name.go index d760e6188aeb..58fb59249b9a 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "fmt" + "io" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -38,11 +40,11 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na req := api.core().request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). - Option("lifetime", options.ValidTime.String()). + Option("lifetime", options.ValidTime). Option("resolve", false) if options.TTL != nil { - req.Option("ttl", options.TTL.String()) + req.Option("ttl", options.TTL) } var out ipnsEntry @@ -57,7 +59,60 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { - return nil, ErrNotImplemented + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } + + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } + + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout). + Option("stream", true) + resp, err := req.Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + res := make(chan iface.IpnsResult) + + go func() { + defer close(res) + defer resp.Close() + + dec := json.NewDecoder(resp.Output) + + for { + var out struct{ Path string } + err := dec.Decode(&out) + if err == io.EOF { + return + } + var ires iface.IpnsResult + if err == nil { + ires.Path, err = iface.ParsePath(out.Path) + } + + select { + case res <- ires: + case <-ctx.Done(): + } + if err != nil { + return + } + } + }() + + return res, nil } func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { @@ -75,7 +130,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). - Option("dht-timeout", ropts.DhtTimeout.String()) + Option("dht-timeout", ropts.DhtTimeout) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 68dc2633dd5d..8c819121ad16 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -42,6 +42,11 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { - return nil, ErrNotImplemented +func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { + rp, err := api.ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + return api.Dag().Get(ctx, rp.Cid()) } From 42273cab06348b19a813dee77f7cbe5ed7cc78d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:05:17 +0100 Subject: [PATCH 045/159] Don't use valid() pattern This commit was moved from ipfs/go-ipfs-http-client@4d07c48f98b0af0b18c6c13f80f04c966fc43bdc --- client/httpapi/block.go | 20 +++++++------------- client/httpapi/key.go | 28 ++++++++++++++++------------ client/httpapi/name.go | 14 ++++---------- client/httpapi/pubsub.go | 12 +++++------- 4 files changed, 32 insertions(+), 42 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 45c73472c599..fd4d9bab9711 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -18,24 +18,16 @@ type BlockAPI HttpApi type blockStat struct { Key string BSize int `json:"Size"` + + cid cid.Cid } func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) valid() (iface.ResolvedPath, error) { - c, err := cid.Parse(s.Key) - if err != nil { - return nil, err - } - - return iface.IpldPath(c), nil -} - func (s *blockStat) Path() iface.ResolvedPath { - p, _ := s.valid() - return p + return iface.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +52,8 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } @@ -118,7 +111,8 @@ func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, e if err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index dc2adf8b7fe1..a16c30d8ef89 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -14,6 +14,8 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` Id string + + pid peer.ID } func (k *keyOutput) Name() string { @@ -26,13 +28,7 @@ func (k *keyOutput) Path() iface.Path { } func (k *keyOutput) ID() peer.ID { - p, _ := peer.IDB58Decode(k.Id) - return p -} - -func (k *keyOutput) valid() error { - _, err := peer.IDB58Decode(k.Id) - return err + return k.pid } func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { @@ -49,7 +45,8 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { @@ -72,7 +69,8 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - return id, out.Overwrite, id.valid() + id.pid, err = peer.IDB58Decode(id.Id) + return id, out.Overwrite, err } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { @@ -83,7 +81,9 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { - if err := k.valid(); err != nil { + var err error + k.pid, err = peer.IDB58Decode(k.Id) + if err != nil { return nil, err } res[i] = k @@ -98,8 +98,10 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { return nil, err } + var err error out := keyOutput{JName: "self", Id: id.ID} - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { @@ -111,7 +113,9 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { return nil, errors.New("got unexpected number of keys back") } - return &out.Keys[0], out.Keys[0].valid() + var err error + out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + return &out.Keys[0], err } func (api *KeyAPI) core() *HttpApi { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 58fb59249b9a..b848aa819783 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -16,10 +16,8 @@ type NameAPI HttpApi type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` -} -func (e *ipnsEntry) valid() (iface.Path, error) { - return iface.ParsePath(e.JValue) + path iface.Path } func (e *ipnsEntry) Name() string { @@ -27,8 +25,7 @@ func (e *ipnsEntry) Name() string { } func (e *ipnsEntry) Value() iface.Path { - p, _ := e.valid() - return p + return e.path } func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { @@ -51,11 +48,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { - return nil, err - } - - return &out, nil + out.path, err = iface.ParsePath(out.JValue) + return &out, err } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 3345097806db..e27cb565557d 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -66,16 +66,12 @@ type pubsubMessage struct { JData []byte `json:"data,omitempty"` JSeqno []byte `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` -} -func (msg *pubsubMessage) valid() error { - _, err := peer.IDFromBytes(msg.JFrom) - return err + from peer.ID } func (msg *pubsubMessage) From() peer.ID { - id, _ := peer.IDFromBytes(msg.JFrom) - return id + return msg.from } func (msg *pubsubMessage) Data() []byte { @@ -97,7 +93,9 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if err := s.dec.Decode(&msg); err != nil { return nil, err } - return &msg, msg.valid() + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { From 93f684617a3adbd7cba751ee421e32b75cd3dafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:07:45 +0100 Subject: [PATCH 046/159] dag: remove unused waitgroup This commit was moved from ipfs/go-ipfs-http-client@934fc60a7c41a14b679fd89fec3b349650a76c81 --- client/httpapi/dag.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 3f54ced34f02..a613d18dec5b 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -39,13 +39,10 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { out := make(chan *format.NodeOption) - wg := sync.WaitGroup{} - wg.Add(len(cids)) for _, c := range cids { // TODO: Consider limiting concurrency of this somehow go func(c cid.Cid) { - defer wg.Done() n, err := api.Get(ctx, c) select { From cc964b4ab8ea5d059be2c05e64ecdca851dcd8b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:15:21 +0100 Subject: [PATCH 047/159] Simplify Object.New, remove ipldnode.go This commit was moved from ipfs/go-ipfs-http-client@0752a6ee63a2ec42aabb6c9acac6ef6c4a94af38 --- client/httpapi/dag.go | 1 - client/httpapi/ipldnode.go | 113 ------------------------------------- client/httpapi/object.go | 34 +++++------ 3 files changed, 18 insertions(+), 130 deletions(-) delete mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index a613d18dec5b..669b5f893ad8 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io/ioutil" - "sync" "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go deleted file mode 100644 index 43fa5d50de94..000000000000 --- a/client/httpapi/ipldnode.go +++ /dev/null @@ -1,113 +0,0 @@ -package httpapi - -import ( - "context" - "errors" - "io/ioutil" - "strconv" - - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" -) - -type ipldNode struct { - ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? - path iface.ResolvedPath - api *HttpApi -} - -func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { - return &ipldNode{ - ctx: ctx, - path: p, - api: api, - } -} - -func (n *ipldNode) RawData() []byte { - r, err := n.api.Block().Get(n.ctx, n.path) - if err != nil { - panic(err) // TODO: eww, should we add errors too / better ideas? - } - - b, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - - return b -} - -func (n *ipldNode) Cid() cid.Cid { - return n.path.Cid() -} - -func (n *ipldNode) String() string { - return n.Cid().String() -} - -func (n *ipldNode) Loggable() map[string]interface{} { - return nil //TODO: we can't really do better here, can we? -} - -// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. -// We can also give people a choice -func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { - p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) - - var out interface{} - n.api.request("dag/get", p).Exec(n.ctx, &out) - - // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack - for len(path) > 0 { - switch o := out.(type) { - case map[string]interface{}: - v, ok := o[path[0]] - if !ok { - // TODO: ipld links - return nil, nil, errors.New("no element under this path") - } - out = v - case []interface{}: - n, err := strconv.ParseUint(path[0], 10, 32) - if err != nil { - return nil, nil, err - } - if len(o) < int(n) { - return nil, nil, errors.New("no element under this path") - } - out = o[n] - } - path = path[1:] - } - - return out, path, nil -} - -func (n *ipldNode) Tree(path string, depth int) []string { - panic("implement me") -} - -func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { - panic("implement me") -} - -func (n *ipldNode) Copy() ipld.Node { - panic("implement me") -} - -func (n *ipldNode) Links() []*ipld.Link { - panic("implement me") -} - -func (n *ipldNode) Stat() (*ipld.NodeStat, error) { - panic("implement me") -} - -func (n *ipldNode) Size() (uint64, error) { - panic("implement me") -} - -var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 3b648d82bc0a..5a06f74d99aa 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -3,12 +3,15 @@ package httpapi import ( "bytes" "context" + "fmt" "io" "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + dag "github.com/ipfs/go-merkledag" + ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" ) @@ -19,24 +22,23 @@ type objectOut struct { Hash string } -func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (ipld.Node, error) { options, err := caopts.ObjectNewOptions(opts...) if err != nil { return nil, err } - var out objectOut - err = api.core().request("object/new", options.Type).Exec(ctx, &out) - if err != nil { - return nil, err - } - - c, err := cid.Parse(out.Hash) - if err != nil { - return nil, err + var n ipld.Node + switch options.Type { + case "empty": + n = new(dag.ProtoNode) + case "unixfs-dir": + n = ft.EmptyDirNode() + default: + return nil, fmt.Errorf("unknown object type: %s", options.Type) } - return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil + return n, nil } func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { @@ -64,7 +66,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -96,7 +98,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -107,14 +109,14 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } - res := make([]*format.Link, len(out.Links)) + res := make([]*ipld.Link, len(out.Links)) for i, l := range out.Links { c, err := cid.Parse(l.Hash) if err != nil { return nil, err } - res[i] = &format.Link{ + res[i] = &ipld.Link{ Cid: c, Name: l.Name, Size: l.Size, From a23da822de54922faf1f82a86c25660e8cca8bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:35:32 +0100 Subject: [PATCH 048/159] swarm: attach peerid if needed This commit was moved from ipfs/go-ipfs-http-client@aa88b18ac497400d16b36af6489af50a003c298c --- client/httpapi/swarm.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index d179b6540fe2..0bb36aca3faa 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -17,6 +17,13 @@ type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { + if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + addr = addr.Encapsulate(pidma) + } saddrs[i] = addr.String() } From f005b8dc5666916ff2a0ff1eef58b7795670ec73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 16:39:00 +0100 Subject: [PATCH 049/159] pin: verify: parse bad node cids early This commit was moved from ipfs/go-ipfs-http-client@7032dfc9b016f2c584e2102e80fb60c3ecf573ea --- client/httpapi/pin.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 4c4e5713ce64..b11efd35c5db 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -110,23 +110,18 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { type badNode struct { Cid string JErr string `json:"Err"` + + cid cid.Cid } func (n *badNode) Path() iface.ResolvedPath { - c, err := cid.Parse(n.Cid) - if err != nil { - return nil // todo: handle this better - } - return iface.IpldPath(c) + return iface.IpldPath(n.cid) } func (n *badNode) Err() error { if n.JErr != "" { return errors.New(n.JErr) } - if _, err := cid.Parse(n.Cid); err != nil { - return err - } return nil } @@ -150,6 +145,13 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { return // todo: handle non io.EOF somehow } + for i, n := range out.JBadNodes { + out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + if err != nil { + return + } + } + select { case res <- &out: case <-ctx.Done(): From 4a6d36d98b6527e9c13918ac901c4cbc68630b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:02:13 +0100 Subject: [PATCH 050/159] pubsub: handle ctx This commit was moved from ipfs/go-ipfs-http-client@d451a4943c5311d7e32fcf86f56bdc1eda977fc7 --- client/httpapi/pubsub.go | 70 ++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index e27cb565557d..edc1a9709ef7 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -57,8 +57,10 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) } type pubsubSub struct { - io.Closer - dec *json.Decoder + messages chan pubsubMessage + + done chan struct{} + rcloser io.Closer } type pubsubMessage struct { @@ -68,6 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID + err error } func (msg *pubsubMessage) From() peer.ID { @@ -87,15 +90,20 @@ func (msg *pubsubMessage) Topics() []string { } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { - // TODO: handle ctx - - var msg pubsubMessage - if err := s.dec.Decode(&msg); err != nil { - return nil, err + select { + case msg, ok := <-s.messages: + if !ok { + return nil, io.EOF + } + if msg.err != nil { + return nil, msg.err + } + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err + case <-ctx.Done(): + return nil, ctx.Err() } - var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { @@ -114,10 +122,44 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, resp.Error } - return &pubsubSub{ - Closer: resp, - dec: json.NewDecoder(resp.Output), - }, nil + sub := &pubsubSub{ + messages: make(chan pubsubMessage), + done: make(chan struct{}), + } + + dec := json.NewDecoder(resp.Output) + + go func() { + defer close(sub.messages) + + for { + var msg pubsubMessage + if err := dec.Decode(&msg); err != nil { + if err == io.EOF { + return + } + msg.err = err + } + + select { + case sub.messages <- msg: + case <-sub.done: + return + case <-ctx.Done(): + return + } + } + }() + + return sub, nil +} + +func (s*pubsubSub) Close() error { + if s.done != nil { + close(s.done) + s.done = nil + } + return s.rcloser.Close() } func (api *PubsubAPI) core() *HttpApi { From 7229dbbf73fd925da390fc5db5a25caf398b1fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:04:33 +0100 Subject: [PATCH 051/159] don't read all and then throw away the buffer This commit was moved from ipfs/go-ipfs-http-client@5c96c2954a5d45dd907dca9942d2773d76b8a71b --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 745c9a2a99da..b9e83eb3d491 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -139,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - ioutil.ReadAll(resp.Body) + io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } From b6ace8dd401321b842c5f0054d9ed025fd19cd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:12:37 +0100 Subject: [PATCH 052/159] response: handle late errors This commit was moved from ipfs/go-ipfs-http-client@139e9e5ff1b52a92c169d28bf56ff58e36b49ca3 --- client/httpapi/requestbuilder.go | 4 ++-- client/httpapi/response.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 628ad03cdf39..af43ce236baa 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -103,11 +103,11 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { } if res == nil { - httpRes.Close() + lateErr := httpRes.Close() if httpRes.Error != nil { return httpRes.Error } - return nil + return lateErr } return httpRes.Decode(res) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index b9e83eb3d491..f773130e83ce 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -39,9 +39,14 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data - return r.Output.Close() + + // always drain output (response body) //TODO: make optional for things like cat + _, err1 := io.Copy(ioutil.Discard, r.Output) + err2 := r.Output.Close() + if err1 != nil { + return err1 + } + return err2 } return nil } From 7be8d01ee732be71f08f4482e0d5c5b61731ee81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:27:12 +0100 Subject: [PATCH 053/159] response: option to disable output draining This commit was moved from ipfs/go-ipfs-http-client@9f3d9635fa4e977b6ec202c9e4f78ca1c64fc0dd --- client/httpapi/api.go | 1 + client/httpapi/apifile.go | 2 +- client/httpapi/pubsub.go | 4 ++-- client/httpapi/request.go | 2 ++ client/httpapi/requestbuilder.go | 7 +++++++ client/httpapi/response.go | 10 ++++++++-- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e106d8d87a63..f74300216e35 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -131,6 +131,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, + drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 8647606844b3..e3cb85ea460d 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,7 +57,7 @@ func (f *apiFile) reset() error { if f.r != nil { f.r.Close() } - req := f.core.request("cat", f.path.String()) + req := f.core.request("cat", f.path.String()).NoDrain() if f.at != 0 { req.Option("offset", f.at) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index edc1a9709ef7..b3e45ed36c2c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -113,8 +113,8 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover). - Send(ctx) + Option("discover", options.Discover).NoDrain().Send(ctx) + if err != nil { return nil, err } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac67805..18cfb7fd0c5e 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,6 +13,7 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string + DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -30,5 +31,6 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), + DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236baa..8b040522e2f9 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,6 +19,7 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader + drainOut bool shell *HttpApi } @@ -84,6 +85,12 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } +// NoDrain disables output draining in response closer +func (r *RequestBuilder) NoDrain() *RequestBuilder { + r.drainOut = false + return r +} + // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f773130e83ce..cd3cc2b71e7c 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -35,13 +35,18 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error + + drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) //TODO: make optional for things like cat - _, err1 := io.Copy(ioutil.Discard, r.Output) + // always drain output (response body) + var err1 error + if r.drainOutput { + _, err1 = io.Copy(ioutil.Discard, r.Output) + } err2 := r.Output.Close() if err1 != nil { return err1 @@ -114,6 +119,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) + nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ From 3088776f8150fe6967a245cf0360f5219e2dc9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:30:02 +0100 Subject: [PATCH 054/159] swarm: always append peer IDs This commit was moved from ipfs/go-ipfs-http-client@49267901c71f77f098deaf8966aa2dcecf22c2df --- client/httpapi/swarm.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0bb36aca3faa..13a814bc4133 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -15,16 +15,14 @@ import ( type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { - if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { - pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) - if err != nil { - return err - } - addr = addr.Encapsulate(pidma) - } - saddrs[i] = addr.String() + saddrs[i] = addr.Encapsulate(pidma).String() } return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) From 745bf92506bd488bde4f695c4d644e7d3b58f0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:32:40 +0100 Subject: [PATCH 055/159] gofmt This commit was moved from ipfs/go-ipfs-http-client@f3c2c350861470c05617cf74e7b78d06782bb706 --- client/httpapi/api.go | 6 +++--- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f74300216e35..6dbfe3b6a81f 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -128,9 +128,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, drainOut: true, } } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index b3e45ed36c2c..49c58ab88225 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -59,7 +59,7 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) type pubsubSub struct { messages chan pubsubMessage - done chan struct{} + done chan struct{} rcloser io.Closer } @@ -70,7 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID - err error + err error } func (msg *pubsubMessage) From() peer.ID { @@ -124,7 +124,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), - done: make(chan struct{}), + done: make(chan struct{}), } dec := json.NewDecoder(resp.Output) @@ -154,7 +154,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return sub, nil } -func (s*pubsubSub) Close() error { +func (s *pubsubSub) Close() error { if s.done != nil { close(s.done) s.done = nil diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 18cfb7fd0c5e..d65d638350e6 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string DrainOut bool // if set, resp.Close will read all remaining data } @@ -26,11 +26,11 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 8b040522e2f9..c5bc44124f27 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader drainOut bool shell *HttpApi From 27aa13fe44561e8d9469ce537e64baa60d17a705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 21:53:17 +0100 Subject: [PATCH 056/159] return errors from constructor methods This commit was moved from ipfs/go-ipfs-http-client@dbee4e27aaf9ef8e3fa7e25ac1f4f28df8a31e0d --- client/httpapi/api.go | 39 ++++++++++++++++---------------------- client/httpapi/api_test.go | 5 ++++- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6dbfe3b6a81f..e4603bd380de 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,8 +29,7 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } -//TODO: Return errors here -func NewLocalApi() iface.CoreAPI { +func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -39,40 +38,34 @@ func NewLocalApi() iface.CoreAPI { return NewPathApi(baseDir) } -func NewPathApi(p string) iface.CoreAPI { - a := ApiAddr(p) - if a == nil { - return nil +func NewPathApi(p string) (iface.CoreAPI, error) { + a, err := ApiAddr(p) + if err != nil { + if err == os.ErrNotExist { + err = nil + } + return nil, err } return NewApi(a) } -func ApiAddr(ipfspath string) ma.Multiaddr { +func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { - return nil + return nil, err } apiFile := path.Join(baseDir, DefaultApiFile) - if _, err := os.Stat(apiFile); err != nil { - return nil - } - api, err := ioutil.ReadFile(apiFile) if err != nil { - return nil - } - - maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) - if err != nil { - return nil + return nil, err } - return maddr + return ma.NewMultiaddr(strings.TrimSpace(string(api))) } -func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? +func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -83,10 +76,10 @@ func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? return NewApiWithClient(a, c) } -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { - return nil // TODO: return that error + return nil, err } if a, err := ma.NewMultiaddr(url); err == nil { @@ -107,7 +100,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return fmt.Errorf("unexpected redirect") } - return api + return api, nil } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index af2180384d62..2f6d47c58eda 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -157,7 +157,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) DisableCompression: true, }, } - apis[i] = NewApiWithClient(maddr, c) + apis[i], err = NewApiWithClient(maddr, c) + if err != nil { + panic(err) + } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") From fc299e7e849d46e364204234f9ea3d3aa52e42cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:19:56 +0100 Subject: [PATCH 057/159] pin verify: use temporary struct This commit was moved from ipfs/go-ipfs-http-client@e400fa3b380b589c2ae385a66a007d685147e4c7 --- client/httpapi/pin.go | 57 +++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index b11efd35c5db..f667a54875c2 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -90,27 +90,20 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` - JBadNodes []*badNode `json:"BadNodes,omitempty"` + ok bool + badNodes []iface.BadPinNode } func (r *pinVerifyRes) Ok() bool { - return r.JOk + return r.ok } func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { - out := make([]iface.BadPinNode, len(r.JBadNodes)) - for i, n := range r.JBadNodes { - out[i] = n - } - return out + return r.badNodes } type badNode struct { - Cid string - JErr string `json:"Err"` - + err error cid cid.Cid } @@ -119,10 +112,7 @@ func (n *badNode) Path() iface.ResolvedPath { } func (n *badNode) Err() error { - if n.JErr != "" { - return errors.New(n.JErr) - } - return nil + return n.err } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { @@ -140,20 +130,45 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { defer close(res) dec := json.NewDecoder(resp.Output) for { - var out pinVerifyRes + var out struct { + Cid string + Ok bool + + BadNodes []struct{ + Cid string + Err string + } + } if err := dec.Decode(&out); err != nil { return // todo: handle non io.EOF somehow } - for i, n := range out.JBadNodes { - out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + badNodes := make([]iface.BadPinNode, len(out.BadNodes)) + for i, n := range out.BadNodes { + c, err := cid.Decode(n.Cid) if err != nil { - return + badNodes[i] = &badNode{ + cid: c, + err: err, + } + continue + } + + if n.Err != "" { + err = errors.New(n.Err) + } + badNodes[i] = &badNode{ + cid: c, + err: err, } } select { - case res <- &out: + case res <- &pinVerifyRes{ + ok: out.Ok, + + badNodes: badNodes, + }: case <-ctx.Done(): return } From a058e7d31a32d1bca98e41886a310149b872af84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:27:48 +0100 Subject: [PATCH 058/159] response: Document zero-result Decode behaviour This commit was moved from ipfs/go-ipfs-http-client@902bc5ef8c1b9595e97728dca3da0d0da124f142 --- client/httpapi/response.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index cd3cc2b71e7c..26f549c9af7b 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -71,6 +71,9 @@ func (r *Response) Decode(dec interface{}) error { } n++ } + + // Decode expects at least one result. For calls where zero results are valid, + // use Send and construct json Decoder manually. if n > 0 && err == io.EOF { err = nil } From db1af499c9d661a70ac6fcce9cc7347b934385e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:33:57 +0100 Subject: [PATCH 059/159] use mime.ParseMediaType for Content-Type response parsing This commit was moved from ipfs/go-ipfs-http-client@217d1945237321c12f96abef82707217a30813b0 --- client/httpapi/response.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 26f549c9af7b..708d2714fb78 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,14 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" + "mime" "net/http" "net/url" "os" - "strings" - - files "github.com/ipfs/go-ipfs-files" ) type trailerReader struct { @@ -116,9 +115,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType := resp.Header.Get("Content-Type") - parts := strings.Split(contentType, ";") - contentType = parts[0] + + contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return nil, err + } nresp := new(Response) From f34788e6ee61aad0ee1970a4f3d004bdebdbcd27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:11:08 +0100 Subject: [PATCH 060/159] cleanup Swarm.Peers This commit was moved from ipfs/go-ipfs-http-client@fd7858dc57b86761599b0e844f3536f2d4a0d953 --- client/httpapi/api.go | 1 - client/httpapi/apifile.go | 14 +++--- client/httpapi/pubsub.go | 9 ++-- client/httpapi/request.go | 2 - client/httpapi/requestbuilder.go | 7 --- client/httpapi/response.go | 19 +++++--- client/httpapi/swarm.go | 83 +++++++++++++++++--------------- 7 files changed, 69 insertions(+), 66 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e4603bd380de..c5a706d05741 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -124,7 +124,6 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, - drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e3cb85ea460d..a8eb0de1af24 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -49,15 +49,15 @@ type apiFile struct { size int64 path iface.Path - r io.ReadCloser + r *Response at int64 } func (f *apiFile) reset() error { if f.r != nil { - f.r.Close() + f.r.Cancel() } - req := f.core.request("cat", f.path.String()).NoDrain() + req := f.core.request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -68,12 +68,12 @@ func (f *apiFile) reset() error { if resp.Error != nil { return resp.Error } - f.r = resp.Output + f.r = resp return nil } func (f *apiFile) Read(p []byte) (int, error) { - n, err := f.r.Read(p) + n, err := f.r.Output.Read(p) if n > 0 { f.at += int64(n) } @@ -92,7 +92,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) + r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err @@ -103,7 +103,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { func (f *apiFile) Close() error { if f.r != nil { - return f.r.Close() + return f.r.Cancel() } return nil } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 49c58ab88225..2ac04b53ce36 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -60,7 +60,7 @@ type pubsubSub struct { messages chan pubsubMessage done chan struct{} - rcloser io.Closer + rcloser func() error } type pubsubMessage struct { @@ -113,7 +113,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover).NoDrain().Send(ctx) + Option("discover", options.Discover).Send(ctx) if err != nil { return nil, err @@ -125,6 +125,9 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), done: make(chan struct{}), + rcloser: func() error { + return resp.Cancel() + }, } dec := json.NewDecoder(resp.Output) @@ -159,7 +162,7 @@ func (s *pubsubSub) Close() error { close(s.done) s.done = nil } - return s.rcloser.Close() + return s.rcloser() } func (api *PubsubAPI) core() *HttpApi { diff --git a/client/httpapi/request.go b/client/httpapi/request.go index d65d638350e6..f6f7ee486ef0 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,7 +13,6 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string - DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -31,6 +30,5 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), - DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index c5bc44124f27..ba407217f216 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,7 +19,6 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader - drainOut bool shell *HttpApi } @@ -85,12 +84,6 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } -// NoDrain disables output draining in response closer -func (r *RequestBuilder) NoDrain() *RequestBuilder { - r.drainOut = false - return r -} - // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 708d2714fb78..c9f178f9ce8c 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -34,18 +34,13 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error - - drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - var err1 error - if r.drainOutput { - _, err1 = io.Copy(ioutil.Discard, r.Output) - } + // drain output (response body) + _, err1 := io.Copy(ioutil.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -55,6 +50,15 @@ func (r *Response) Close() error { return nil } +// Cancel aborts running request (without draining request body) +func (r *Response) Cancel() error { + if r.Output != nil { + return r.Output.Close() + } + + return nil +} + func (r *Response) Decode(dec interface{}) error { defer r.Close() if r.Error != nil { @@ -123,7 +127,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 13a814bc4133..8827119178a7 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -32,74 +32,81 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) } -type streamInfo struct { - Protocol string -} - type connInfo struct { - Addr string - Peer string - JLatency time.Duration `json:"Latency"` - Muxer string - JDirection inet.Direction `json:"Direction"` - JStreams []streamInfo `json:"Streams"` -} - -func (c *connInfo) valid() error { - _, err := multiaddr.NewMultiaddr(c.Addr) - if err != nil { - return err - } - - _, err = peer.IDB58Decode(c.Peer) - return err + addr multiaddr.Multiaddr + peer peer.ID + latency time.Duration + muxer string + direction inet.Direction + streams []protocol.ID } func (c *connInfo) ID() peer.ID { - id, _ := peer.IDB58Decode(c.Peer) - return id + return c.peer } func (c *connInfo) Address() multiaddr.Multiaddr { - a, _ := multiaddr.NewMultiaddr(c.Addr) - return a + return c.addr } func (c *connInfo) Direction() inet.Direction { - return c.JDirection + return c.direction } func (c *connInfo) Latency() (time.Duration, error) { - return c.JLatency, nil + return c.latency, nil } func (c *connInfo) Streams() ([]protocol.ID, error) { - res := make([]protocol.ID, len(c.JStreams)) - for i, stream := range c.JStreams { - res[i] = protocol.ID(stream.Protocol) - } - return res, nil + return c.streams, nil } func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { - var out struct { - Peers []*connInfo + var resp struct { + Peers []struct{ + Addr string + Peer string + Latency time.Duration + Muxer string + Direction inet.Direction + Streams []struct { + Protocol string + } + } } err := api.core().request("swarm/peers"). Option("streams", true). Option("latency", true). - Exec(ctx, &out) + Exec(ctx, &resp) if err != nil { return nil, err } - res := make([]iface.ConnectionInfo, len(out.Peers)) - for i, conn := range out.Peers { - if err := conn.valid(); err != nil { + res := make([]iface.ConnectionInfo, len(resp.Peers)) + for i, conn := range resp.Peers { + out := &connInfo{ + latency: conn.Latency, + muxer: conn.Muxer, + direction: conn.Direction, + } + + out.peer, err = peer.IDB58Decode(conn.Peer) + if err != nil { return nil, err } - res[i] = conn + + out.addr, err = multiaddr.NewMultiaddr(conn.Addr) + if err != nil { + return nil, err + } + + out.streams = make([]protocol.ID, len(conn.Streams)) + for i, p := range conn.Streams { + out.streams[i] = protocol.ID(p.Protocol) + } + + res[i] = out } return res, nil From aa4d6e16f13bd753b77d0a8fc2174e41af89310e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:26:34 +0100 Subject: [PATCH 061/159] Add some docs to constructors This commit was moved from ipfs/go-ipfs-http-client@e34cd600e690bfa9bd20d6089b9335ac6090d4dd --- client/httpapi/api.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c5a706d05741..9e8d55188178 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -22,6 +22,11 @@ const ( EnvDir = "IPFS_PATH" ) +// HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using +// IPFS HTTP API. +// +// For interface docs see +// https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string httpcli gohttp.Client @@ -29,6 +34,11 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } +// NewLocalApi tries to construct new HttpApi instance communicating with local +// IPFS daemon +// +// Daemon api address is pulled from the $IPFS_PATH/api file. +// If $IPFS_PATH env var is not present, it defaults to ~/.ipfs func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -38,8 +48,10 @@ func NewLocalApi() (iface.CoreAPI, error) { return NewPathApi(baseDir) } -func NewPathApi(p string) (iface.CoreAPI, error) { - a, err := ApiAddr(p) +// NewPathApi constructs new HttpApi by pulling api address from specified +// ipfspath. Api file should be located at $ipfspath/api +func NewPathApi(ipfspath string) (iface.CoreAPI, error) { + a, err := ApiAddr(ipfspath) if err != nil { if err == os.ErrNotExist { err = nil @@ -49,6 +61,7 @@ func NewPathApi(p string) (iface.CoreAPI, error) { return NewApi(a) } +// ApiAddr reads api file in specified ipfs path func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { @@ -65,6 +78,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return ma.NewMultiaddr(strings.TrimSpace(string(api))) } +// NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ @@ -76,6 +90,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { return NewApiWithClient(a, c) } +// NewApiWithClient constructs HttpApi with specified endpoint and custom http client func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { From f6a3a4fb2d715ffc0394408287b23eae0d322631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:28:39 +0100 Subject: [PATCH 062/159] gofmt This commit was moved from ipfs/go-ipfs-http-client@65cd935e13a2ea05befbc69d72d64a89056952ea --- client/httpapi/api.go | 6 +++--- client/httpapi/pin.go | 4 ++-- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- client/httpapi/response.go | 1 - client/httpapi/swarm.go | 18 +++++++++--------- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 9e8d55188178..b67fb0b483ac 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -136,9 +136,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, } } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f667a54875c2..0111d626a091 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -132,9 +132,9 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { for { var out struct { Cid string - Ok bool + Ok bool - BadNodes []struct{ + BadNodes []struct { Cid string Err string } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index f6f7ee486ef0..58c61ac67805 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -25,10 +25,10 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index ba407217f216..af43ce236baa 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader shell *HttpApi } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index c9f178f9ce8c..84f28214dcfa 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -119,7 +119,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil { return nil, err diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 8827119178a7..0814debee3c4 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -33,10 +33,10 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e } type connInfo struct { - addr multiaddr.Multiaddr - peer peer.ID + addr multiaddr.Multiaddr + peer peer.ID latency time.Duration - muxer string + muxer string direction inet.Direction streams []protocol.ID } @@ -63,11 +63,11 @@ func (c *connInfo) Streams() ([]protocol.ID, error) { func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { var resp struct { - Peers []struct{ - Addr string - Peer string + Peers []struct { + Addr string + Peer string Latency time.Duration - Muxer string + Muxer string Direction inet.Direction Streams []struct { Protocol string @@ -86,8 +86,8 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { out := &connInfo{ - latency: conn.Latency, - muxer: conn.Muxer, + latency: conn.Latency, + muxer: conn.Muxer, direction: conn.Direction, } From ad844e3d0b0221333c04896ed80f11afed301495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 16:50:38 +0100 Subject: [PATCH 063/159] response: simplify Decode This commit was moved from ipfs/go-ipfs-http-client@adbfda4c1c6ee037df3f91ca73f60de1f4620d67 --- client/httpapi/api.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 20 +++----------------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b67fb0b483ac..17a289cc5973 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -53,7 +53,7 @@ func NewLocalApi() (iface.CoreAPI, error) { func NewPathApi(ipfspath string) (iface.CoreAPI, error) { a, err := ApiAddr(ipfspath) if err != nil { - if err == os.ErrNotExist { + if os.IsNotExist(err) { err = nil } return nil, err diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236baa..2ffed7a0ae7d 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -110,5 +110,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return lateErr } - return httpRes.Decode(res) + return httpRes.decode(res) } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 84f28214dcfa..794cd14704fa 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -59,28 +59,14 @@ func (r *Response) Cancel() error { return nil } -func (r *Response) Decode(dec interface{}) error { +// Decode reads request body and decodes it as json +func (r *Response) decode(dec interface{}) error { defer r.Close() if r.Error != nil { return r.Error } - n := 0 - var err error - for { - err = json.NewDecoder(r.Output).Decode(dec) - if err != nil { - break - } - n++ - } - - // Decode expects at least one result. For calls where zero results are valid, - // use Send and construct json Decoder manually. - if n > 0 && err == io.EOF { - err = nil - } - return err + return json.NewDecoder(r.Output).Decode(dec) } type Error struct { From 19d91fbb259220e91c8887ee8a09218865418720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 22:08:02 +0100 Subject: [PATCH 064/159] response: pass close error in decode This commit was moved from ipfs/go-ipfs-http-client@4a87232ecaa96246f038d15eb7ee6094216a201e --- client/httpapi/response.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 794cd14704fa..624a488d7d79 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -61,12 +61,17 @@ func (r *Response) Cancel() error { // Decode reads request body and decodes it as json func (r *Response) decode(dec interface{}) error { - defer r.Close() if r.Error != nil { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + err := json.NewDecoder(r.Output).Decode(dec) + err2 := r.Close() + if err != nil { + return err + } + + return err2 } type Error struct { From cc9968d5dda71c21b3c90eb376458cacadb59e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:39:12 +0100 Subject: [PATCH 065/159] request: fix Content-Disposition header in Send This commit was moved from ipfs/go-ipfs-http-client@b7db17c63b69dc4fbb726dc720e9c594fa9ed495 --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 624a488d7d79..339c73658619 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -102,7 +102,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { if fr, ok := r.Body.(*files.MultiFileReader); ok { req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) - req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + req.Header.Set("Content-Disposition", "form-data; name=\"files\"") } resp, err := c.Do(req) From 47b820150bddb9868274ed2c6faf79c6b8cbe369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:46:18 +0100 Subject: [PATCH 066/159] test: don't panic on errors in async node construction This commit was moved from ipfs/go-ipfs-http-client@95ce0f3949da47b2db92508b8c7b0a00f502682b --- client/httpapi/api_test.go | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2f6d47c58eda..df45c15af508 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -114,27 +114,32 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) wg.Add(len(nodes)) zero.Add(1) + errs := make(chan error, len(nodes)) for i, nd := range nodes { go func(i int, nd testbedi.Core) { defer wg.Done() if _, err := nd.Init(ctx, "--empty-repo"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { - panic(err) + errs <- err + return } if i > 0 { zero.Wait() if err := nd.Connect(ctx, nodes[0]); err != nil { - panic(err) + errs <- err + return } } else { zero.Done() @@ -142,12 +147,14 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) addr, err := nd.APIAddr() if err != nil { - panic(err) + errs <- err + return } maddr, err := ma.NewMultiaddr(addr) if err != nil { - panic(err) + errs <- err + return } c := &gohttp.Client{ @@ -159,16 +166,19 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } apis[i], err = NewApiWithClient(maddr, c) if err != nil { - panic(err) + errs <- err + return } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err != nil { - panic(err) + errs <- err + return } if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { - panic(err) + errs <- err + return } }(i, nd) } @@ -187,7 +197,12 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) }() }() - return apis, nil + select { + case err = <-errs: + default: + } + + return apis, err } func TestHttpApi(t *testing.T) { From 0e800c3d8431fdcdc22ee2a2ac3337cb009553c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 18:15:37 +0000 Subject: [PATCH 067/159] update interface-go-ipfs-core and handle breaking changes * No more Hidden, StdinName, and Wrap options. * LsLink -> DirEntry with file types that don't expose internals. This commit was moved from ipfs/go-ipfs-http-client@5a7161eeab7127da4348dda6a6dc16f07b051f36 --- client/httpapi/apifile.go | 13 ++++----- client/httpapi/unixfs.go | 58 ++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index a8eb0de1af24..e0c0c9e1b97b 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -8,8 +8,9 @@ import ( "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + iface "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -174,17 +175,13 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case iface.THAMTShard: - fallthrough - case iface.TMetadata: - fallthrough - case iface.TDirectory: + case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case iface.TFile: + case unixfs.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 1f340b6573a2..4f717b1291bc 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,9 +8,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + unixfs_pb "github.com/ipfs/go-unixfs/pb" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -40,15 +41,12 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). Option("fscache", options.FsCache). - Option("hidden", options.Hidden). Option("inline", options.Inline). Option("inline-limit", options.InlineLimit). Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). Option("silent", options.Silent). - Option("stdin-name", options.StdinName). - Option("wrap-with-directory", options.Wrap). Option("progress", options.Progress) if options.RawLeavesSet { @@ -62,13 +60,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix req.Option("trickle", true) } - switch c := f.(type) { - case files.Directory: - req.Body(files.NewMultiFileReader(c, false)) - case files.File: - d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side - req.Body(files.NewMultiFileReader(d, false)) - } + d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) var out addEvent resp, err := req.Send(ctx) @@ -127,7 +120,8 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type iface.FileType + Type unixfs_pb.Data_DataType + Target string } type lsObject struct { @@ -139,7 +133,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err @@ -158,7 +152,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf } dec := json.NewDecoder(resp.Output) - out := make(chan iface.LsLink) + out := make(chan iface.DirEntry) go func() { defer resp.Close() @@ -171,7 +165,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return } select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return @@ -179,7 +173,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Objects len")}: case <-ctx.Done(): } return @@ -187,7 +181,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects[0].Links) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Links len")}: case <-ctx.Done(): } return @@ -198,21 +192,29 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf c, err := cid.Decode(l0.Hash) if err != nil { select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return } + var ftype iface.FileType + switch l0.Type { + case unixfs.TRaw, unixfs.TFile: + ftype = iface.TFile + case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: + ftype = iface.TDirectory + case unixfs.TSymlink: + ftype = iface.TSymlink + } + select { - case out <- iface.LsLink{ - Link: &format.Link{ - Cid: c, - Name: l0.Name, - Size: l0.Size, - }, - Size: l0.Size, - Type: l0.Type, + case out <- iface.DirEntry{ + Name: l0.Name, + Cid: c, + Size: l0.Size, + Type: ftype, + Target: l0.Target, }: case <-ctx.Done(): } From e76ecd30384799dfe388671ebd153e88903d9388 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 16:54:05 +0000 Subject: [PATCH 068/159] nit: return concrete types This is generally considered the "better" way to do things in go. It also allows us to expose functions like `request` at some point without changing the API interface itself. This commit was moved from ipfs/go-ipfs-http-client@524e5d82e6bcd2c4956151a4811addad656d7bf0 --- client/httpapi/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 17a289cc5973..ef6eff3e8234 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -39,7 +39,7 @@ type HttpApi struct { // // Daemon api address is pulled from the $IPFS_PATH/api file. // If $IPFS_PATH env var is not present, it defaults to ~/.ipfs -func NewLocalApi() (iface.CoreAPI, error) { +func NewLocalApi() (*HttpApi, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -50,7 +50,7 @@ func NewLocalApi() (iface.CoreAPI, error) { // NewPathApi constructs new HttpApi by pulling api address from specified // ipfspath. Api file should be located at $ipfspath/api -func NewPathApi(ipfspath string) (iface.CoreAPI, error) { +func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { From c6f704b453cd72734ca03793cd370021f4a39db9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:02:04 +0000 Subject: [PATCH 069/159] return an error when the API file isn't found fixes #7 This commit was moved from ipfs/go-ipfs-http-client@897d1b1bcdbf9f6616991366e66ac55e18a99a9c --- client/httpapi/api.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ef6eff3e8234..6c41ef3ab8f1 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,6 +1,7 @@ package httpapi import ( + "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -22,6 +23,9 @@ const ( EnvDir = "IPFS_PATH" ) +// ErrApiNotFound if we fail to find a running daemon. +var ErrApiNotFound = errors.New("ipfs api address could not be found") + // HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using // IPFS HTTP API. // @@ -54,7 +58,7 @@ func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { - err = nil + err = ErrApiNotFound } return nil, err } From 40f40b5a6593af189b81f43e393316b381bd541d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:04:33 +0000 Subject: [PATCH 070/159] fix golang-ci lint nits This commit was moved from ipfs/go-ipfs-http-client@ccf20b0ef5f440aca754f94325fca7202275ba8b --- client/httpapi/apifile.go | 2 +- client/httpapi/response.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e0c0c9e1b97b..1fd751ede7b2 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -56,7 +56,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { - f.r.Cancel() + _ = f.r.Cancel() } req := f.core.request("cat", f.path.String()) if f.at != 0 { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c73658619..ed3033886070 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -147,8 +147,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() + _, _ = io.Copy(ioutil.Discard, resp.Body) + _ = resp.Body.Close() } return nresp, nil From c791d73355e33139e2face5fc96884c5444f96c7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:05:02 +0000 Subject: [PATCH 071/159] avoid canceling multiple times This commit was moved from ipfs/go-ipfs-http-client@fe16408225d4c9b688528f67ffabe08198b53ab5 --- client/httpapi/apifile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1fd751ede7b2..9e6df95c24b5 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,6 +57,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { _ = f.r.Cancel() + f.r = nil } req := f.core.request("cat", f.path.String()) if f.at != 0 { From 1c07bed90465210c27dba3cda850913cf397b5c5 Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Mon, 1 Apr 2019 17:13:39 -0700 Subject: [PATCH 072/159] Export (*HttpApi).request to enable building custom requests This commit was moved from ipfs/go-ipfs-http-client@8c9ed7dbc284b4d112f6805914a43e63f234df78 --- client/httpapi/api.go | 2 +- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 8 ++++---- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 10 +++++----- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 18 +++++++++--------- client/httpapi/path.go | 2 +- client/httpapi/pin.go | 10 +++++----- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/swarm.go | 12 ++++++------ client/httpapi/unixfs.go | 4 ++-- 12 files changed, 46 insertions(+), 46 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6c41ef3ab8f1..dad205fd09d4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, args: args, diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 9e6df95c24b5..6e3433a51bf3 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -29,7 +29,7 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) + err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -59,7 +59,7 @@ func (f *apiFile) reset() error { _ = f.r.Cancel() f.r = nil } - req := f.core.request("cat", f.path.String()) + req := f.core.Request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -225,7 +225,7 @@ func (d *apiDir) Entries() files.DirIterator { } func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index fd4d9bab9711..3389f4c317ed 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -41,7 +41,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("block/put"). + req := api.core().Request("block/put"). Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). @@ -61,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(ctx) + resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR Error string `json:",omitempty"` }{} - req := api.core().request("block/rm"). + req := api.core().Request("block/rm"). Option("force", options.Force). Arguments(p.String()) @@ -107,7 +107,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { var out blockStat - err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { return nil, err } diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index dc7dd6beaf6f..f7417cb7fc3a 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -18,7 +18,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, Type notif.QueryEventType Responses []peerstore.PeerInfo } - resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { return peerstore.PeerInfo{}, err } @@ -48,7 +48,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return nil, err } - resp, err := api.core().request("dht/findprovs", rp.Cid().String()). + resp, err := api.core().Request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -104,7 +104,7 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", rp.Cid().String()). + return api.core().Request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index a16c30d8ef89..1571a3350bf8 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -38,7 +38,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key } var out keyOutput - err = api.core().request("key/gen", name). + err = api.core().Request("key/gen", name). Option("type", options.Algorithm). Option("size", options.Size). Exec(ctx, &out) @@ -61,7 +61,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o Id string Overwrite bool } - err = api.core().request("key/rename", oldName, newName). + err = api.core().Request("key/rename", oldName, newName). Option("force", options.Force). Exec(ctx, &out) if err != nil { @@ -75,7 +75,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { var out struct{ Keys []*keyOutput } - if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + if err := api.core().Request("key/list").Exec(ctx, &out); err != nil { return nil, err } @@ -94,7 +94,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var id struct{ ID string } - if err := api.core().request("id").Exec(ctx, &id); err != nil { + if err := api.core().Request("id").Exec(ctx, &id); err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { var out struct{ Keys []keyOutput } - if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + if err := api.core().Request("key/rm", name).Exec(ctx, &out); err != nil { return nil, err } if len(out.Keys) != 1 { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b848aa819783..0641d1a22d60 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -34,7 +34,7 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na return nil, err } - req := api.core().request("name/publish", p.String()). + req := api.core().Request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). Option("lifetime", options.ValidTime). @@ -63,7 +63,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). @@ -120,7 +120,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 5a06f74d99aa..a1ee486b32c4 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -48,7 +48,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec } var out objectOut - err = api.core().request("object/put"). + err = api.core().Request("object/put"). Option("inputenc", options.InputEnc). Option("datafieldenc", options.DataType). Option("pin", options.Pin). @@ -80,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) } func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("object/data", p.String()).Send(ctx) + resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er Size uint64 } } - if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]*ipld.Link, len(out.Links)) @@ -135,7 +135,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat DataSize int CumulativeSize int } - if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/stat", p.String()).Exec(ctx, &out); err != nil { return nil, err } @@ -161,7 +161,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, } var out objectOut - err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + err = api.core().Request("object/patch/add-link", base.String(), name, child.String()). Option("create", options.Create). Exec(ctx, &out) if err != nil { @@ -178,7 +178,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/rm-link", base.String(), link). + err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) if err != nil { return nil, err @@ -194,7 +194,7 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/append-data", p.String()). + err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -211,7 +211,7 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/set-data", p.String()). + err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -237,7 +237,7 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i var out struct { Changes []change } - if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]iface.ObjectChange, len(out.Changes)) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 8c819121ad16..639ca2f8e20a 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -24,7 +24,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } } - if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 0111d626a091..f779640f1740 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -39,7 +39,7 @@ func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddO return err } - return api.core().request("pin/add", p.String()). + return api.core().Request("pin/add", p.String()). Option("recursive", options.Recursive).Exec(ctx, nil) } @@ -50,7 +50,7 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. } var out pinRefKeyList - err = api.core().request("pin/ls"). + err = api.core().Request("pin/ls"). Option("type", options.Type).Exec(ctx, &out) if err != nil { return nil, err @@ -74,7 +74,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt return err } - return api.core().request("pin/rm", p.String()). + return api.core().Request("pin/rm", p.String()). Option("recursive", options.Recursive). Exec(ctx, nil) } @@ -85,7 +85,7 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o return err } - return api.core().request("pin/update"). + return api.core().Request("pin/update"). Option("unpin", options.Unpin).Exec(ctx, nil) } @@ -116,7 +116,7 @@ func (n *badNode) Err() error { } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { - resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + resp, err := api.core().Request("pin/verify").Option("verbose", true).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 2ac04b53ce36..380b933dc95b 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -18,7 +18,7 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { Strings []string } - if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } @@ -35,7 +35,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +51,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", topic). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -112,7 +112,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, err } - resp, err := api.core().request("pubsub/sub", topic). + resp, err := api.core().Request("pubsub/sub", topic). Option("discover", options.Discover).Send(ctx) if err != nil { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0814debee3c4..0e47df2ab36c 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -25,11 +25,11 @@ func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs[i] = addr.Encapsulate(pidma).String() } - return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) + return api.core().Request("swarm/connect", saddrs...).Exec(ctx, nil) } func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { - return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) + return api.core().Request("swarm/disconnect", addr.String()).Exec(ctx, nil) } type connInfo struct { @@ -75,7 +75,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) } } - err := api.core().request("swarm/peers"). + err := api.core().Request("swarm/peers"). Option("streams", true). Option("latency", true). Exec(ctx, &resp) @@ -116,7 +116,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu var out struct { Addrs map[string][]string } - if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs").Exec(ctx, &out); err != nil { return nil, err } res := map[peer.ID][]multiaddr.Multiaddr{} @@ -147,7 +147,7 @@ func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, err Strings []string } - if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/local").Exec(ctx, &out); err != nil { return nil, err } @@ -167,7 +167,7 @@ func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, er Strings []string } - if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/listen").Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 4f717b1291bc..84609194bff9 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -36,7 +36,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("add"). + req := api.core().Request("add"). Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). @@ -139,7 +139,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return nil, err } - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-type", options.ResolveChildren). Option("size", options.ResolveChildren). Option("stream", true). From 70158edd5950671a26631788d15204350d174f52 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 4 Apr 2019 00:36:32 -0700 Subject: [PATCH 073/159] feed through context see https://github.com/ipfs/go-ipfs-api/pull/173 all credit to @optman This commit was moved from ipfs/go-ipfs-http-client@7b75a8c577988e4728474a02144bf5bf83260527 --- client/httpapi/request.go | 2 ++ client/httpapi/response.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac67805..dd5293b7a6ed 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,6 +7,7 @@ import ( ) type Request struct { + Ctx context.Context ApiBase string Command string Args []string @@ -25,6 +26,7 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ + Ctx: ctx, ApiBase: url + "/api/v0", Command: command, Args: args, diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c73658619..29ae38c69ee0 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -95,6 +95,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } + req = req.WithContext(r.Ctx) + // Add any headers that were supplied via the RequestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) From 3b02ea46462d1f87a64e3fe23f73ee0f186b40d5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 9 Apr 2019 19:53:07 -0700 Subject: [PATCH 074/159] api: add authenticated transport, and direct connection This commit was moved from ipfs/go-ipfs-http-client@d04afa02b059d2c6a631f23a6923eb57ecb21015 --- client/httpapi/api.go | 36 ++++++++++++++++++++++++++++++++++++ client/httpapi/api_test.go | 25 +++++++++++++++++++++++-- client/httpapi/transport.go | 21 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dad205fd09d4..c0eacd7cbfd2 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -122,6 +122,42 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } +// NewDirectAPIClient is used to instantiate a HttpApi client +// that connects to an endpoint which leverages additional http paths. +// +// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 +// you should use NewDirectAPIClient. +func NewDirectAPIClient(url string) (*HttpApi, error) { + api := &HttpApi{ + url: url, + httpcli: gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }, + applyGlobal: func(*RequestBuilder) {}, + } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api, nil +} + +// WithAuthorization is used to wrap an instance of HttpApi +// with an authenticated transport, such as JWT +func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { + return &HttpApi{ + url: api.url, + httpcli: gohttp.Client{ + Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), + }, + } +} + func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af508..f1b1bf269426 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -9,11 +9,11 @@ import ( "sync" "testing" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" - "github.com/ipfs/iptb/testbed/interfaces" + testbedi "github.com/ipfs/iptb/testbed/interfaces" ma "github.com/multiformats/go-multiaddr" ) @@ -211,3 +211,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } + +func TestDirectAPI(t *testing.T) { + type args struct { + url, header, value string + } + tests := []struct { + name string + args args + }{ + {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + api, err := NewDirectAPIClient(tt.args.url) + if err != nil { + t.Fatal(err) + } + api = api.WithAuthorization(tt.args.header, tt.args.value) + }) + } +} diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go new file mode 100644 index 000000000000..3fb0e6e00a2a --- /dev/null +++ b/client/httpapi/transport.go @@ -0,0 +1,21 @@ +package httpapi + +import "net/http" + +type transport struct { + header, value string + httptr http.RoundTripper +} + +func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { + return &transport{ + header: header, + value: value, + httptr: tr, + } +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set(t.header, t.value) + return t.httptr.RoundTrip(req) +} From 3b67615201cd38aceb6de61e87da1552be7689fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 26 Apr 2019 20:30:55 +0200 Subject: [PATCH 075/159] gomod: update iface with path refactor This commit was moved from ipfs/go-ipfs-http-client@0bd65a67ee2994966538fae6d034ddb9fac466e6 --- client/httpapi/api_test.go | 7 ++----- client/httpapi/apifile.go | 16 ++++++++-------- client/httpapi/block.go | 11 ++++++----- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 17 +++++++++-------- client/httpapi/object.go | 35 ++++++++++++++++++----------------- client/httpapi/path.go | 18 +++++++++--------- client/httpapi/pin.go | 17 +++++++++-------- client/httpapi/unixfs.go | 9 +++++---- 11 files changed, 75 insertions(+), 73 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af508..b1bae5565e9e 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" @@ -171,11 +172,7 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } // empty node is pinned even with --empty-repo, we don't want that - emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - errs <- err - return - } + emptyNode := path.New("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { errs <- err return diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 6e3433a51bf3..afd9934e61fa 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -10,12 +10,12 @@ import ( "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" unixfs "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k -func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { +func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error p, err = api.core().ResolvePath(ctx, p) @@ -48,7 +48,7 @@ type apiFile struct { ctx context.Context core *HttpApi size int64 - path iface.Path + path path.Path r *Response at int64 @@ -114,7 +114,7 @@ func (f *apiFile) Size() (int64, error) { return f.size, nil } -func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) { f := &apiFile{ ctx: ctx, core: api.core(), @@ -177,13 +177,13 @@ func (it *apiIter) Next() bool { switch it.cur.Type { case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: - it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getDir(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } case unixfs.TFile: - it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getFile(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false @@ -203,7 +203,7 @@ type apiDir struct { ctx context.Context core *UnixfsAPI size int64 - path iface.Path + path path.Path dec *json.Decoder } @@ -224,7 +224,7 @@ func (d *apiDir) Entries() files.DirIterator { } } -func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) { resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 3389f4c317ed..640f186f5044 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -26,8 +27,8 @@ func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) Path() iface.ResolvedPath { - return iface.IpldPath(s.cid) +func (s *blockStat) Path() path.Resolved { + return path.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return &out, nil } -func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { +func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRmOption) error { options, err := caopts.BlockRmOptions(opts...) if err != nil { return err @@ -105,7 +106,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR return nil } -func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { +func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 669b5f893ad8..15d9a9c62ed2 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -9,8 +9,8 @@ import ( "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type httpNodeAdder HttpApi @@ -18,7 +18,7 @@ type HttpDagServ httpNodeAdder type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { - r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) + r, err := api.core().Block().Get(ctx, path.IpldPath(c)) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (api *HttpDagServ) Pinning() format.NodeAdder { } func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { - return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? + return api.core().Block().Rm(ctx, path.IpldPath(c)) //TODO: should we force rm? } func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index f7417cb7fc3a..e12caa1c5341 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" @@ -37,7 +37,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, } } -func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -93,7 +93,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return res, nil } -func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { +func (api *DhtAPI) Provide(ctx context.Context, p path.Path, opts ...caopts.DhtProvideOption) error { options, err := caopts.DhtProvideOptions(opts...) if err != nil { return err diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 1571a3350bf8..9a47e2ed010f 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" ) @@ -22,9 +23,8 @@ func (k *keyOutput) Name() string { return k.JName } -func (k *keyOutput) Path() iface.Path { - p, _ := iface.ParsePath("/ipns/" + k.Id) - return p +func (k *keyOutput) Path() path.Path { + return path.New("/ipns/" + k.Id) } func (k *keyOutput) ID() peer.ID { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 0641d1a22d60..47227a2ab893 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/options/namesys" + "github.com/ipfs/interface-go-ipfs-core/path" ) type NameAPI HttpApi @@ -17,18 +18,18 @@ type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` - path iface.Path + path path.Path } func (e *ipnsEntry) Name() string { return e.JName } -func (e *ipnsEntry) Value() iface.Path { +func (e *ipnsEntry) Value() path.Path { return e.path } -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { +func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { options, err := caopts.NamePublishOptions(opts...) if err != nil { return nil, err @@ -48,8 +49,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - out.path, err = iface.ParsePath(out.JValue) - return &out, err + out.path = path.New(out.JValue) + return &out, out.path.IsValid() } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { @@ -93,7 +94,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name } var ires iface.IpnsResult if err == nil { - ires.Path, err = iface.ParsePath(out.Path) + ires.Path = path.New(out.Path) } select { @@ -109,7 +110,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return res, nil } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (path.Path, error) { options, err := caopts.NameResolveOptions(opts...) if err != nil { return nil, err @@ -131,7 +132,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, err } - return iface.ParsePath(out.Path) + return path.New(out.Path), nil } func (api *NameAPI) core() *HttpApi { diff --git a/client/httpapi/object.go b/client/httpapi/object.go index a1ee486b32c4..280b2490f93d 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,6 +14,7 @@ import ( ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi @@ -41,7 +42,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return n, nil } -func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (path.Resolved, error) { options, err := caopts.ObjectPutOptions(opts...) if err != nil { return nil, err @@ -63,10 +64,10 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *ObjectAPI) Data(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err @@ -98,7 +99,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p path.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -126,7 +127,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er return res, nil } -func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { +func (api *ObjectAPI) Stat(ctx context.Context, p path.Path) (*iface.ObjectStat, error) { var out struct { Hash string NumLinks int @@ -154,7 +155,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat }, nil } -func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...caopts.ObjectAddLinkOption) (path.Resolved, error) { options, err := caopts.ObjectAddLinkOptions(opts...) if err != nil { return nil, err @@ -173,10 +174,10 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { +func (api *ObjectAPI) RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) @@ -189,10 +190,10 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AppendData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). @@ -206,10 +207,10 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) SetData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). @@ -223,7 +224,7 @@ func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (i return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type change struct { @@ -233,7 +234,7 @@ type change struct { After cid.Cid } -func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { +func (api *ObjectAPI) Diff(ctx context.Context, a path.Path, b path.Path) ([]iface.ObjectChange, error) { var out struct { Changes []change } @@ -247,10 +248,10 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i Path: ch.Path, } if ch.Before != cid.Undef { - res[i].Before = iface.IpfsPath(ch.Before) + res[i].Before = path.IpfsPath(ch.Before) } if ch.After != cid.Undef { - res[i].After = iface.IpfsPath(ch.After) + res[i].After = path.IpfsPath(ch.After) } } return res, nil diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 639ca2f8e20a..619dd0e0d2cb 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -6,10 +6,10 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) -func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { +func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { var out struct { Cid cid.Cid RemPath string @@ -18,31 +18,31 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help var err error - if path.Namespace() == "ipns" { - if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + if p.Namespace() == "ipns" { + if p, err = api.Name().Resolve(ctx, p.String()); err != nil { return nil, err } } - if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", p.String()).Exec(ctx, &out); err != nil { return nil, err } // TODO: - ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+p.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } - root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + root, err := cid.Parse(ipfspath.Path(p.String()).Segments()[1]) if err != nil { return nil, err } - return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil + return path.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *HttpApi) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) { rp, err := api.ResolvePath(ctx, p) if err != nil { return nil, err diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f779640f1740..ec0fc91a8c2d 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) @@ -21,11 +22,11 @@ type pinRefKeyList struct { } type pin struct { - path iface.ResolvedPath + path path.Resolved typ string } -func (p *pin) Path() iface.ResolvedPath { +func (p *pin) Path() path.Resolved { return p.path } @@ -33,7 +34,7 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { +func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error { options, err := caopts.PinAddOptions(opts...) if err != nil { return err @@ -62,13 +63,13 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. if err != nil { return nil, err } - pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) } return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { +func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { options, err := caopts.PinRmOptions(opts...) if err != nil { return err @@ -79,7 +80,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt Exec(ctx, nil) } -func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { +func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error { options, err := caopts.PinUpdateOptions(opts...) if err != nil { return err @@ -107,8 +108,8 @@ type badNode struct { cid cid.Cid } -func (n *badNode) Path() iface.ResolvedPath { - return iface.IpldPath(n.cid) +func (n *badNode) Path() path.Resolved { + return path.IpldPath(n.cid) } func (n *badNode) Err() error { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 84609194bff9..5e27fb0367f1 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,6 +13,7 @@ import ( unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -25,7 +26,7 @@ type addEvent struct { type UnixfsAPI HttpApi -func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (path.Resolved, error) { options, _, err := caopts.UnixfsAddOptions(opts...) if err != nil { return nil, err @@ -98,7 +99,7 @@ loop: return nil, err } - ifevt.Path = iface.IpfsPath(c) + ifevt.Path = path.IpfsPath(c) } select { @@ -114,7 +115,7 @@ loop: return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type lsLink struct { @@ -133,7 +134,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err From 11a990c7b12db59d9d03cbd9b28c2b8c75abfa26 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:13:59 -0700 Subject: [PATCH 076/159] api: expose headers via HttpApi, copy headers during request build This commit was moved from ipfs/go-ipfs-http-client@033befdef3f06a4a80fb9cf7c3afecc1a383700c --- client/httpapi/api.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c0eacd7cbfd2..712801d53233 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" gohttp "net/http" "os" "path" @@ -32,9 +33,9 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // For interface docs see // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { - url string - httpcli gohttp.Client - + url string + httpcli gohttp.Client + Headers http.Header applyGlobal func(*RequestBuilder) } @@ -175,10 +176,17 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { + var headers map[string]string + if api.Headers != nil { + for k := range api.Headers { + headers[k] = api.Headers.Get(k) + } + } return &RequestBuilder{ command: command, args: args, shell: api, + headers: headers, } } From d8c286bb15e628dbe7ddd2370c77ef02a5dc16b2 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:15 -0700 Subject: [PATCH 077/159] api: remove WithAuthorization, DirectAPI client, instantiate header map This commit was moved from ipfs/go-ipfs-http-client@9c7d495b03db4d130a8fd87217093beb33e6fdb7 --- client/httpapi/api.go | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 712801d53233..ffcc85f2b517 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -112,6 +112,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -123,20 +124,11 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } -// NewDirectAPIClient is used to instantiate a HttpApi client -// that connects to an endpoint which leverages additional http paths. -// -// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 -// you should use NewDirectAPIClient. -func NewDirectAPIClient(url string) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ - url: url, - httpcli: gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - }, - }, + url: url, + httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -144,21 +136,9 @@ func NewDirectAPIClient(url string) (*HttpApi, error) { api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { return fmt.Errorf("unexpected redirect") } - return api, nil } -// WithAuthorization is used to wrap an instance of HttpApi -// with an authenticated transport, such as JWT -func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { - return &HttpApi{ - url: api.url, - httpcli: gohttp.Client{ - Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), - }, - } -} - func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { @@ -176,7 +156,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { - var headers map[string]string + headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) From 89600e62ce160cf5f3aa2a752b3806a273d1d142 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:37 -0700 Subject: [PATCH 078/159] api: add wip api test This commit was moved from ipfs/go-ipfs-http-client@2b48dd36ad170149bd4c62a4194587bf7c704f7e --- client/httpapi/api_test.go | 62 ++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index f1b1bf269426..7b519ce7dd66 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,11 +3,13 @@ package httpapi import ( "context" "io/ioutil" + "net/http" gohttp "net/http" "os" "strconv" "sync" "testing" + "time" iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +214,51 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func TestDirectAPI(t *testing.T) { - type args struct { - url, header, value string +func Test_NewURLApiWithClient(t *testing.T) { + t.Skip() + var ( + url = "127.0.0.1:65501" + headerToTest = "Test-Header" + expectedHeaderValue = "thisisaheadertest" + ) + server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) + if err != nil { + t.Fatal(err) } - tests := []struct { - name string - args args - }{ - {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + defer server.Close() + go func() { + server.ListenAndServe() + }() + time.Sleep(time.Second * 2) + api, err := NewURLApiWithClient(url, &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }) + if err != nil { + t.Fatal(err) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - api, err := NewDirectAPIClient(tt.args.url) - if err != nil { - t.Fatal(err) - } - api = api.WithAuthorization(tt.args.header, tt.args.value) - }) + api.Headers.Set(headerToTest, expectedHeaderValue) + if _, err := api.Pin().Ls(context.Background()); err != nil { + t.Fatal(err) } } + +/// testHTTPServer spins up a test go http server +// used to check headers +func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { + r := http.NewServeMux() + r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }) + return &http.Server{ + Handler: r, + Addr: url, + }, nil +} From efe3b6054e8f62191e74e6664adec06c306c06aa Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:00:48 -0700 Subject: [PATCH 079/159] api: call NewURLApiWithClient from NewApiWithClient This commit was moved from ipfs/go-ipfs-http-client@aea6890f36001f15d24a4b373979bb1b9bf81483 --- client/httpapi/api.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ffcc85f2b517..eb99230eb1e4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -109,19 +109,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { } } - api := &HttpApi{ - url: url, - httpcli: *c, - Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, - } - - // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { - return fmt.Errorf("unexpected redirect") - } - - return api, nil + return NewURLApiWithClient(url, c) } func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { From fd5010bd2a852ea8fa3a2b08e199312fb3b71097 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:13:47 -0700 Subject: [PATCH 080/159] api: cleanup header and NewURLApiWithClient test This commit was moved from ipfs/go-ipfs-http-client@b8b55cb89a5dd830915f83e43e8afa7a01597905 --- client/httpapi/api_test.go | 46 +++++++++++++------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 52fa67d0e2ca..fadd9663e9dd 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -5,13 +5,14 @@ import ( "io/ioutil" "net/http" gohttp "net/http" + "net/http/httptest" "os" "strconv" "sync" "testing" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +213,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func Test_NewURLApiWithClient(t *testing.T) { - t.Skip() +func Test_NewURLApiWithClient_With_Headers(t *testing.T) { var ( - url = "127.0.0.1:65501" headerToTest = "Test-Header" expectedHeaderValue = "thisisaheadertest" ) - server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) - if err != nil { - t.Fatal(err) - } - defer server.Close() - go func() { - server.ListenAndServe() - }() + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }), + ) + defer ts.Close() time.Sleep(time.Second * 2) - api, err := NewURLApiWithClient(url, &http.Client{ + api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, @@ -242,21 +244,3 @@ func Test_NewURLApiWithClient(t *testing.T) { t.Fatal(err) } } - -/// testHTTPServer spins up a test go http server -// used to check headers -func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { - r := http.NewServeMux() - r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { - val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { - w.WriteHeader(400) - return - } - w.WriteHeader(200) - }) - return &http.Server{ - Handler: r, - Addr: url, - }, nil -} From daf1b72809a3d1af819d7ff44aabcc6bc6471fe8 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 17:19:35 -0700 Subject: [PATCH 081/159] api: fix failing test This commit was moved from ipfs/go-ipfs-http-client@ea507216d821455c351440ba33775f564ea18ea8 --- client/httpapi/api_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fadd9663e9dd..5f012b476721 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "strconv" + "strings" "sync" "testing" "time" @@ -221,15 +222,14 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { ts := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { + if val != expectedHeaderValue { w.WriteHeader(400) return } - w.WriteHeader(200) + http.ServeContent(w, r, "", time.Now(), strings.NewReader("test")) }), ) defer ts.Close() - time.Sleep(time.Second * 2) api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -240,7 +240,7 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } api.Headers.Set(headerToTest, expectedHeaderValue) - if _, err := api.Pin().Ls(context.Background()); err != nil { + if err := api.Pin().Rm(context.Background(), path.New("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv")); err != nil { t.Fatal(err) } } From 8b9e189169b51c46956a18e08f9dc7bc668401b4 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 19:14:15 -0700 Subject: [PATCH 082/159] remove unused transport.go file This commit was moved from ipfs/go-ipfs-http-client@320130421f4727d72d68a3c6a5c0e633e1dabc38 --- client/httpapi/transport.go | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go deleted file mode 100644 index 3fb0e6e00a2a..000000000000 --- a/client/httpapi/transport.go +++ /dev/null @@ -1,21 +0,0 @@ -package httpapi - -import "net/http" - -type transport struct { - header, value string - httptr http.RoundTripper -} - -func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { - return &transport{ - header: header, - value: value, - httptr: tr, - } -} - -func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header.Set(t.header, t.value) - return t.httptr.RoundTrip(req) -} From 8d953a7359c488cde5299ca704594bd8e1349ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 10 May 2019 15:38:20 +0200 Subject: [PATCH 083/159] Use interface for RequestBuilder This commit was moved from ipfs/go-ipfs-http-client@0534b7ca8319329459c505a546f3025862a735f5 --- client/httpapi/api.go | 10 ++++----- client/httpapi/requestbuilder.go | 36 ++++++++++++++++++++++---------- client/httpapi/response.go | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index eb99230eb1e4..5c2eb76d6d06 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -36,7 +36,7 @@ type HttpApi struct { url string httpcli gohttp.Client Headers http.Header - applyGlobal func(*RequestBuilder) + applyGlobal func(*requestBuilder) } // NewLocalApi tries to construct new HttpApi instance communicating with local @@ -117,7 +117,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { url: url, httpcli: *c, Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, + applyGlobal: func(*requestBuilder) {}, } // We don't support redirects. @@ -134,7 +134,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } subApi := *api - subApi.applyGlobal = func(req *RequestBuilder) { + subApi.applyGlobal = func(req *requestBuilder) { if options.Offline { req.Option("offline", options.Offline) } @@ -143,14 +143,14 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) RequestBuilder { headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) } } - return &RequestBuilder{ + return &requestBuilder{ command: command, args: args, shell: api, diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 2ffed7a0ae7d..7012a8935bfa 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -12,8 +12,20 @@ import ( "github.com/ipfs/go-ipfs-files" ) -// RequestBuilder is an IPFS commands request builder. -type RequestBuilder struct { +type RequestBuilder interface { + Arguments(args ...string) RequestBuilder + BodyString(body string) RequestBuilder + BodyBytes(body []byte) RequestBuilder + Body(body io.Reader) RequestBuilder + FileBody(body io.Reader) RequestBuilder + Option(key string, value interface{}) RequestBuilder + Header(name, value string) RequestBuilder + Send(ctx context.Context) (*Response, error) + Exec(ctx context.Context, res interface{}) error +} + +// requestBuilder is an IPFS commands request builder. +type requestBuilder struct { command string args []string opts map[string]string @@ -24,29 +36,29 @@ type RequestBuilder struct { } // Arguments adds the arguments to the args. -func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { +func (r *requestBuilder) Arguments(args ...string) RequestBuilder { r.args = append(r.args, args...) return r } // BodyString sets the request body to the given string. -func (r *RequestBuilder) BodyString(body string) *RequestBuilder { +func (r *requestBuilder) BodyString(body string) RequestBuilder { return r.Body(strings.NewReader(body)) } // BodyBytes sets the request body to the given buffer. -func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { +func (r *requestBuilder) BodyBytes(body []byte) RequestBuilder { return r.Body(bytes.NewReader(body)) } // Body sets the request body to the given reader. -func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { +func (r *requestBuilder) Body(body io.Reader) RequestBuilder { r.body = body return r } // FileBody sets the request body to the given reader wrapped into multipartreader. -func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { +func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) @@ -55,7 +67,7 @@ func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { } // Option sets the given option. -func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { +func (r *requestBuilder) Option(key string, value interface{}) RequestBuilder { var s string switch v := value.(type) { case bool: @@ -76,7 +88,7 @@ func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { } // Header sets the given header. -func (r *RequestBuilder) Header(name, value string) *RequestBuilder { +func (r *requestBuilder) Header(name, value string) RequestBuilder { if r.headers == nil { r.headers = make(map[string]string, 1) } @@ -85,7 +97,7 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { } // Send sends the request and return the response. -func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { +func (r *requestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) req := NewRequest(ctx, r.shell.url, r.command, r.args...) @@ -96,7 +108,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { } // Exec sends the request a request and decodes the response. -func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { +func (r *requestBuilder) Exec(ctx context.Context, res interface{}) error { httpRes, err := r.Send(ctx) if err != nil { return err @@ -112,3 +124,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return httpRes.decode(res) } + +var _ RequestBuilder = &requestBuilder{} diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 2d6af49d2ca6..db22a66faf4f 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -97,7 +97,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { req = req.WithContext(r.Ctx) - // Add any headers that were supplied via the RequestBuilder. + // Add any headers that were supplied via the requestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) } From 1b81f2ef10c40b5ba5a50ae545a62cc050c11813 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 11 Jun 2019 17:41:49 -0700 Subject: [PATCH 084/159] add extended error handling fixes #19 This commit was moved from ipfs/go-ipfs-http-client@8e3552ac1e6ea18ffd5f49918eef578de2b224fb --- client/httpapi/response.go | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index db22a66faf4f..056b2d68d8bb 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -13,6 +13,48 @@ import ( "os" ) +// Error codes adapted from go-ipfs-cmds. We should find a better solution. + +// ErrorType signfies a category of errors +type ErrorType uint + +// ErrorTypes convey what category of error ocurred +const ( + // ErrNormal is a normal error. The command failed for some reason that's not a bug. + ErrNormal ErrorType = iota + // ErrClient means the client made an invalid request. + ErrClient + // ErrImplementation means there's a bug in the implementation. + ErrImplementation + // ErrRateLimited is returned when the operation has been rate-limited. + ErrRateLimited + // ErrForbidden is returned when the client doesn't have permission to + // perform the requested operation. + ErrForbidden +) + +func (e ErrorType) Error() string { + return e.String() +} + +func (e ErrorType) String() string { + switch e { + case ErrNormal: + return "command failed" + case ErrClient: + return "invalid argument" + case ErrImplementation: + return "internal error" + case ErrRateLimited: + return "rate limited" + case ErrForbidden: + return "request forbidden" + default: + return "unknown error code" + } + +} + type trailerReader struct { resp *http.Response } @@ -77,7 +119,13 @@ func (r *Response) decode(dec interface{}) error { type Error struct { Command string Message string - Code int + Code ErrorType +} + +// Unwrap returns the base error (an ErrorType). Works with go 1.14 error +// helpers. +func (e *Error) Unwrap() error { + return e.Code } func (e *Error) Error() string { @@ -133,11 +181,23 @@ func (r *Request) Send(c *http.Client) (*Response, error) { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } e.Message = string(out) + + // set special status codes. + switch resp.StatusCode { + case http.StatusNotFound, http.StatusBadRequest: + e.Code = ErrClient + case http.StatusTooManyRequests: + e.Code = ErrRateLimited + case http.StatusForbidden: + e.Code = ErrForbidden + } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) } default: + // This is a server-side bug (probably). + e.Code = ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 2612353d85385a595e889dfa247363a8a4ab02c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 13 Jun 2019 12:00:58 -0700 Subject: [PATCH 085/159] use error from go-ipfs-cmds note: this drops the Command field This commit was moved from ipfs/go-ipfs-http-client@523a26f0a853a2d6779a21d7bad239bf2012833b --- client/httpapi/response.go | 79 +++++--------------------------------- 1 file changed, 10 insertions(+), 69 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 056b2d68d8bb..95cbf13ec1bf 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,56 +4,19 @@ import ( "encoding/json" "errors" "fmt" - "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" "mime" "net/http" "net/url" "os" -) -// Error codes adapted from go-ipfs-cmds. We should find a better solution. - -// ErrorType signfies a category of errors -type ErrorType uint - -// ErrorTypes convey what category of error ocurred -const ( - // ErrNormal is a normal error. The command failed for some reason that's not a bug. - ErrNormal ErrorType = iota - // ErrClient means the client made an invalid request. - ErrClient - // ErrImplementation means there's a bug in the implementation. - ErrImplementation - // ErrRateLimited is returned when the operation has been rate-limited. - ErrRateLimited - // ErrForbidden is returned when the client doesn't have permission to - // perform the requested operation. - ErrForbidden + cmds "github.com/ipfs/go-ipfs-cmds" + cmdhttp "github.com/ipfs/go-ipfs-cmds/http" + files "github.com/ipfs/go-ipfs-files" ) -func (e ErrorType) Error() string { - return e.String() -} - -func (e ErrorType) String() string { - switch e { - case ErrNormal: - return "command failed" - case ErrClient: - return "invalid argument" - case ErrImplementation: - return "internal error" - case ErrRateLimited: - return "rate limited" - case ErrForbidden: - return "request forbidden" - default: - return "unknown error code" - } - -} +type Error = cmds.Error type trailerReader struct { resp *http.Response @@ -62,7 +25,7 @@ type trailerReader struct { func (r *trailerReader) Read(b []byte) (int, error) { n, err := r.resp.Body.Read(b) if err != nil { - if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + if e := r.resp.Trailer.Get(cmdhttp.StreamErrHeader); e != "" { err = errors.New(e) } } @@ -116,26 +79,6 @@ func (r *Response) decode(dec interface{}) error { return err2 } -type Error struct { - Command string - Message string - Code ErrorType -} - -// Unwrap returns the base error (an ErrorType). Works with go 1.14 error -// helpers. -func (e *Error) Unwrap() error { - return e.Code -} - -func (e *Error) Error() string { - var out string - if e.Code != 0 { - out = fmt.Sprintf("%s%d: ", out, e.Code) - } - return out + e.Message -} - func (r *Request) Send(c *http.Client) (*Response, error) { url := r.getURL() req, err := http.NewRequest("POST", url, r.Body) @@ -169,9 +112,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { - e := &Error{ - Command: r.Command, - } + e := new(Error) switch { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" @@ -185,11 +126,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // set special status codes. switch resp.StatusCode { case http.StatusNotFound, http.StatusBadRequest: - e.Code = ErrClient + e.Code = cmds.ErrClient case http.StatusTooManyRequests: - e.Code = ErrRateLimited + e.Code = cmds.ErrRateLimited case http.StatusForbidden: - e.Code = ErrForbidden + e.Code = cmds.ErrForbidden } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { @@ -197,7 +138,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { } default: // This is a server-side bug (probably). - e.Code = ErrImplementation + e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 9345cfa7b83d3fb2fb470fa1e77e7af89087452b Mon Sep 17 00:00:00 2001 From: godcong Date: Sun, 21 Jul 2019 12:33:35 +0800 Subject: [PATCH 086/159] update to consolidated libp2p interface package (#21) and fix parsing of connection latencies This commit was moved from ipfs/go-ipfs-http-client@fd5cce4cbc9d95617ded33849c437a492f5e9d96 --- client/httpapi/dht.go | 31 +++++++++++++++---------------- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/swarm.go | 20 ++++++++++---------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index e12caa1c5341..0a78514f9739 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,38 +6,37 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - notif "github.com/libp2p/go-libp2p-routing/notifications" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/routing" ) type DhtAPI HttpApi -func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) { var out struct { - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } if resp.Error != nil { - return peerstore.PeerInfo{}, resp.Error + return peer.AddrInfo{}, resp.Error } defer resp.Close() dec := json.NewDecoder(resp.Output) for { if err := dec.Decode(&out); err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } - if out.Type == notif.FinalPeer { + if out.Type == routing.FinalPeer { return out.Responses[0], nil } } } -func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peer.AddrInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -57,7 +56,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt if resp.Error != nil { return nil, resp.Error } - res := make(chan peerstore.PeerInfo) + res := make(chan peer.AddrInfo) go func() { defer resp.Close() @@ -67,18 +66,18 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt for { var out struct { Extra string - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } - if out.Type == notif.QueryError { + if out.Type == routing.QueryError { return // usually a 'not found' error // todo: handle other errors } - if out.Type == notif.Provider { + if out.Type == routing.Provider { for _, pi := range out.Responses { select { case res <- pi: diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 9a47e2ed010f..cf22c61089c8 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 380b933dc95b..81ddb5211c3c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type PubsubAPI HttpApi diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0e47df2ab36c..10280c39f080 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,16 +5,15 @@ import ( "time" "github.com/ipfs/interface-go-ipfs-core" - inet "github.com/libp2p/go-libp2p-net" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - "github.com/libp2p/go-libp2p-protocol" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/multiformats/go-multiaddr" ) type SwarmAPI HttpApi -func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { +func (api *SwarmAPI) Connect(ctx context.Context, pi peer.AddrInfo) error { pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) if err != nil { return err @@ -37,7 +36,7 @@ type connInfo struct { peer peer.ID latency time.Duration muxer string - direction inet.Direction + direction network.Direction streams []protocol.ID } @@ -49,7 +48,7 @@ func (c *connInfo) Address() multiaddr.Multiaddr { return c.addr } -func (c *connInfo) Direction() inet.Direction { +func (c *connInfo) Direction() network.Direction { return c.direction } @@ -66,9 +65,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) Peers []struct { Addr string Peer string - Latency time.Duration + Latency string Muxer string - Direction inet.Direction + Direction network.Direction Streams []struct { Protocol string } @@ -85,8 +84,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { + latency, _ := time.ParseDuration(conn.Latency) out := &connInfo{ - latency: conn.Latency, + latency: latency, muxer: conn.Muxer, direction: conn.Direction, } From be8f80237d4ba8ed8d2ef278180b2b3d446c60f0 Mon Sep 17 00:00:00 2001 From: godcong Date: Tue, 13 Aug 2019 07:16:47 +0800 Subject: [PATCH 087/159] fix path miss in windows/add a new api create function (#23) * fix path was not always correct in windows * add new api create function for if the port was different with api file. * Update api.go remove no need commit This commit was moved from ipfs/go-ipfs-http-client@8c87debb1233f2af7a0cd102a934b630d8a12147 --- client/httpapi/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 5c2eb76d6d06..ec16c4b2da83 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -7,12 +7,12 @@ import ( "net/http" gohttp "net/http" "os" - "path" + "path/filepath" "strings" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - homedir "github.com/mitchellh/go-homedir" + "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) @@ -73,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return nil, err } - apiFile := path.Join(baseDir, DefaultApiFile) + apiFile := filepath.Join(baseDir, DefaultApiFile) api, err := ioutil.ReadFile(apiFile) if err != nil { From b6b2ae355bf86c4859f234240b44b35656685ece Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 30 Aug 2019 18:44:24 -0700 Subject: [PATCH 088/159] doc: mark this package experimental instead of WIP (#30) We're at a point where this package works pretty well and users should consider using it. This commit was moved from ipfs/go-ipfs-http-client@3e8506bbecd9b77ae30d04c7b997d9d612654beb --- client/httpapi/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index dd09142e32bc..e5ab1c612273 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,7 +8,10 @@ > IPFS CoreAPI implementation using HTTP API -This project is WIP, use https://github.com/ipfs/go-ipfs-api for now +This package is experimental and subject to change. If you need to depend on +something less likely to change, please use +[go-ipfs-api](https://github.com/ipfs/go-ipfs-api). If you'd like the latest and +greatest features, please use _this_ package. ## Documentation From d48f8fce64d6496a18b442c74021829554151823 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 10 Sep 2019 18:07:48 -0700 Subject: [PATCH 089/159] file: implement ReadAt This commit was moved from ipfs/go-ipfs-http-client@88a9b615205816ea20ea38fbee170b3256e51b94 --- client/httpapi/apifile.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index afd9934e61fa..ec2f7588d591 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -82,6 +82,25 @@ func (f *apiFile) Read(p []byte) (int, error) { return n, err } +func (f *apiFile) ReadAt(p []byte, off int64) (int, error) { + // Always make a new request. This method should be parallel-safe. + resp, err := f.core.Request("cat", f.path.String()). + Option("offset", off).Option("length", len(p)).Send(f.ctx) + if err != nil { + return 0, err + } + if resp.Error != nil { + return 0, resp.Error + } + defer resp.Output.Close() + + n, err := io.ReadFull(resp.Output, p) + if err == io.ErrUnexpectedEOF { + err = io.EOF + } + return n, err +} + func (f *apiFile) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekEnd: From 8f9306304efdc60147eccbb49380b83e3c7cb4ea Mon Sep 17 00:00:00 2001 From: Joel Gustafson Date: Sat, 7 Dec 2019 01:08:48 -0500 Subject: [PATCH 090/159] Update pin.go This commit was moved from ipfs/go-ipfs-http-client@31b4e60d961ada25cb7b40317b18011b284436a7 --- client/httpapi/pin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ec0fc91a8c2d..6026c71b38ab 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -86,7 +86,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt return err } - return api.core().Request("pin/update"). + return api.core().Request("pin/update", from.String(), to.String()). Option("unpin", options.Unpin).Exec(ctx, nil) } From d9004a9061070263dcc01976334836acb9493530 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 3 May 2020 12:04:04 -0700 Subject: [PATCH 091/159] IDB58Decode -> Decode Allow Non-RSA keys to be decoded by the HTTP client This commit was moved from ipfs/go-ipfs-http-client@b0549d86231d69a933b0e4b771c350cf1bf5c27b --- client/httpapi/key.go | 12 ++++++------ client/httpapi/pubsub.go | 4 ++-- client/httpapi/swarm.go | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index cf22c61089c8..78b67517c011 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-core/peer" @@ -45,7 +45,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -69,7 +69,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - id.pid, err = peer.IDB58Decode(id.Id) + id.pid, err = peer.Decode(id.Id) return id, out.Overwrite, err } @@ -82,7 +82,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { var err error - k.pid, err = peer.IDB58Decode(k.Id) + k.pid, err = peer.Decode(k.Id) if err != nil { return nil, err } @@ -100,7 +100,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var err error out := keyOutput{JName: "self", Id: id.ID} - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -114,7 +114,7 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { } var err error - out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + out.Keys[0].pid, err = peer.Decode(out.Keys[0].Id) return &out.Keys[0], err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 81ddb5211c3c..da7c59ef1d2f 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,7 +6,7 @@ import ( "encoding/json" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" ) @@ -41,7 +41,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio res := make([]peer.ID, len(out.Strings)) for i, sid := range out.Strings { - id, err := peer.IDB58Decode(sid) + id, err := peer.Decode(sid) if err != nil { return nil, err } diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 10280c39f080..1cb0d91df7c6 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" @@ -91,7 +91,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) direction: conn.Direction, } - out.peer, err = peer.IDB58Decode(conn.Peer) + out.peer, err = peer.Decode(conn.Peer) if err != nil { return nil, err } @@ -131,7 +131,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu addrs[i] = a } - pid, err := peer.IDB58Decode(spid) + pid, err := peer.Decode(spid) if err != nil { return nil, err } From 7ebef96ba12bfdfc550412748ee16eadcaffd25d Mon Sep 17 00:00:00 2001 From: Bryan Stenson Date: Tue, 12 May 2020 20:09:16 +0000 Subject: [PATCH 092/159] update contributing link This commit was moved from ipfs/go-ipfs-http-client@ef2a556e19e8d9ebafcb24b3574724c6d7718bc8 --- client/httpapi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index e5ab1c612273..376e6590e85d 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -25,7 +25,7 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ### Want to hack on IPFS? -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) ## License From 247cb05fbc4fa3863542ecd9e0e5109c927790c6 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 31 Jul 2020 10:20:59 -0700 Subject: [PATCH 093/159] Bump interface-go-ipfs-core version (#117) * fix issues #26 Implement an IsPinned function * fix issues #113 Implement interface v0.3.0 * ci: update to use go-ipfs v0.6.0 * ci: bump go version Co-authored-by: godcong Co-authored-by: Adin Schmahmann This commit was moved from ipfs/go-ipfs-http-client@4e8a6c8b981f808062caa4ef7481babc0a39f298 --- client/httpapi/pin.go | 54 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 6026c71b38ab..bd3e5423bc36 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,9 +3,10 @@ package httpapi import ( "context" "encoding/json" + "strings" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" @@ -24,6 +25,11 @@ type pinRefKeyList struct { type pin struct { path path.Resolved typ string + err error +} + +func (p *pin) Err() error { + return p.err } func (p *pin) Path() path.Resolved { @@ -44,7 +50,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp Option("recursive", options.Recursive).Exec(ctx, nil) } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan iface.Pin, error) { options, err := caopts.PinLsOptions(opts...) if err != nil { return nil, err @@ -57,16 +63,46 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return nil, err } - pins := make([]iface.Pin, 0, len(out.Keys)) - for hash, p := range out.Keys { - c, err := cid.Parse(hash) - if err != nil { - return nil, err + pins := make(chan iface.Pin) + go func(ch chan<- iface.Pin) { + defer close(ch) + for hash, p := range out.Keys { + c, e := cid.Parse(hash) + if e != nil { + ch <- &pin{typ: p.Type, err: e} + return + } + ch <- &pin{typ: p.Type, path: path.IpldPath(c), err: e} } - pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) + }(pins) + return pins, nil +} + +// IsPinned returns whether or not the given cid is pinned +// and an explanation of why its pinned +func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { + options, err := caopts.PinIsPinnedOptions(opts...) + if err != nil { + return "", false, err + } + var out pinRefKeyList + err = api.core().Request("pin/ls"). + Option("type", options.WithType). + Option("arg", p.String()). + Exec(ctx, &out) + if err != nil { + // TODO: This error-type discrimination based on sub-string matching is brittle. + // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 + if strings.Index(err.Error(), "is not pinned") != -1 { + return "", false, nil + } + return "", false, err } - return pins, nil + for _, obj := range out.Keys { + return obj.Type, true, nil + } + return "", false, errors.New("http api returned no error and no results") } func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { From ea2a35686db523b544245cbaea5592021d0cd163 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 15 May 2021 18:21:24 -0700 Subject: [PATCH 094/159] fix staticcheck This commit was moved from ipfs/go-ipfs-http-client@4461a0bb952ce0a05a5bab9728cc64b658375068 --- client/httpapi/api.go | 17 ++++++++--------- client/httpapi/api_test.go | 7 +++---- client/httpapi/object.go | 3 +-- client/httpapi/pin.go | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ec16c4b2da83..f589e4267a47 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "net/http" - gohttp "net/http" "os" "path/filepath" "strings" @@ -14,7 +13,7 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) const ( @@ -34,7 +33,7 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string - httpcli gohttp.Client + httpcli http.Client Headers http.Header applyGlobal func(*requestBuilder) } @@ -85,9 +84,9 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { // NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, }, } @@ -96,7 +95,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { } // NewApiWithClient constructs HttpApi with specified endpoint and custom http client -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { +func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { return nil, err @@ -112,7 +111,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return NewURLApiWithClient(url, c) } -func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, @@ -121,7 +120,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { } // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("unexpected redirect") } return api, nil diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5f012b476721..35a57d2c089a 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,7 +4,6 @@ import ( "context" "io/ioutil" "net/http" - gohttp "net/http" "net/http/httptest" "os" "strconv" @@ -163,9 +162,9 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) return } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, DisableCompression: true, }, diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 280b2490f93d..6ec7f5503cd0 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -10,7 +10,6 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" - dag "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -32,7 +31,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( var n ipld.Node switch options.Type { case "empty": - n = new(dag.ProtoNode) + n = new(merkledag.ProtoNode) case "unixfs-dir": n = ft.EmptyDirNode() default: diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index bd3e5423bc36..13de2d3893de 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -93,7 +93,7 @@ func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.Pin if err != nil { // TODO: This error-type discrimination based on sub-string matching is brittle. // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 - if strings.Index(err.Error(), "is not pinned") != -1 { + if strings.Contains(err.Error(), "is not pinned") { return "", false, nil } return "", false, err From 5322cf89e45197afb74cc36104ee581ca179c1d6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 29 Nov 2021 22:24:27 +0100 Subject: [PATCH 095/159] feat: pubsub http rpc with multibase (#151) * feat: pubsub http rpc with multibase This updates HTTP RPC wire format to one from https://github.com/ipfs/go-ipfs/pull/8183 * chore: use updated go-ipfs * chore: switch ci to go-ipfs master This commit was moved from ipfs/go-ipfs-http-client@c832fc0ce1d8235c830fecd0dd44dda04a9fba37 --- client/httpapi/README.md | 6 ++-- client/httpapi/pubsub.go | 78 +++++++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index 376e6590e85d..d3e14ac3f1be 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -1,8 +1,8 @@ # go-ipfs-http-api -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](https://protocol.ai) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.io/) +[![](https://img.shields.io/badge/matrix-%23ipfs-blue.svg?style=flat-square)](https://app.element.io/#/room/#ipfs:matrix.org) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index da7c59ef1d2f..72f5923761f2 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -9,6 +9,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" + mbase "github.com/multiformats/go-multibase" ) type PubsubAPI HttpApi @@ -21,8 +22,15 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } - - return out.Strings, nil + topics := make([]string, len(out.Strings)) + for n, mb := range out.Strings { + _, topic, err := mbase.Decode(mb) + if err != nil { + return nil, err + } + topics[n] = string(topic) + } + return topics, nil } func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { @@ -35,7 +43,11 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + var optionalTopic string + if len(options.Topic) > 0 { + optionalTopic = toMultibase([]byte(options.Topic)) + } + if err := api.core().Request("pubsub/peers", optionalTopic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +63,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().Request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", toMultibase([]byte(topic))). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -64,13 +76,18 @@ type pubsubSub struct { } type pubsubMessage struct { - JFrom []byte `json:"from,omitempty"` - JData []byte `json:"data,omitempty"` - JSeqno []byte `json:"seqno,omitempty"` + JFrom string `json:"from,omitempty"` + JData string `json:"data,omitempty"` + JSeqno string `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` - from peer.ID - err error + // real values after unpacking from text/multibase envelopes + from peer.ID + data []byte + seqno []byte + topics []string + + err error } func (msg *pubsubMessage) From() peer.ID { @@ -78,15 +95,17 @@ func (msg *pubsubMessage) From() peer.ID { } func (msg *pubsubMessage) Data() []byte { - return msg.JData + return msg.data } func (msg *pubsubMessage) Seq() []byte { - return msg.JSeqno + return msg.seqno } +// TODO: do we want to keep this interface as []string, +// or change to more correct [][]byte? func (msg *pubsubMessage) Topics() []string { - return msg.JTopicIDs + return msg.topics } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { @@ -98,22 +117,41 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if msg.err != nil { return nil, msg.err } + // unpack values from text/multibase envelopes var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err + msg.from, err = peer.Decode(msg.JFrom) + if err != nil { + return nil, err + } + _, msg.data, err = mbase.Decode(msg.JData) + if err != nil { + return nil, err + } + _, msg.seqno, err = mbase.Decode(msg.JSeqno) + if err != nil { + return nil, err + } + for _, mbt := range msg.JTopicIDs { + _, topic, err := mbase.Decode(mbt) + if err != nil { + return nil, err + } + msg.topics = append(msg.topics, string(topic)) + } + return &msg, nil case <-ctx.Done(): return nil, ctx.Err() } } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + /* right now we have no options (discover got deprecated) options, err := caopts.PubSubSubscribeOptions(opts...) if err != nil { return nil, err } - - resp, err := api.core().Request("pubsub/sub", topic). - Option("discover", options.Discover).Send(ctx) + */ + resp, err := api.core().Request("pubsub/sub", toMultibase([]byte(topic))).Send(ctx) if err != nil { return nil, err @@ -168,3 +206,9 @@ func (s *pubsubSub) Close() error { func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } + +// Encodes bytes into URL-safe multibase that can be sent over HTTP RPC (URL or body) +func toMultibase(data []byte) string { + mb, _ := mbase.Encode(mbase.Base64url, data) + return mb +} From b52c80ca4da5730f47674946025c9490fcd50e1f Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 14 Jan 2022 16:16:26 +0100 Subject: [PATCH 096/159] skip TestHttpApi on Windows This commit was moved from ipfs/go-ipfs-http-client@a41c7956c0da2a4c51e15794b22dea09395cc7ef --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 35a57d2c089a..d5f27f0ec9d8 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "os" + "runtime" "strconv" "strings" "sync" @@ -207,6 +208,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } func TestHttpApi(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping due to #142") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() From b85aa9639f861d659b49e0396b409398462226a2 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 27 Mar 2022 14:11:47 +0200 Subject: [PATCH 097/159] fix: make Block().* return correct ABI based ipld.ErrNotFound errors This commit was moved from ipfs/go-ipfs-http-client@4f5f8e9b144d1f38e0fc37c5080845611dcbcf1e --- client/httpapi/abyfy_errors.go | 104 ++++++++++++++++++++++++++++ client/httpapi/abyfy_errors_test.go | 60 ++++++++++++++++ client/httpapi/block.go | 11 +-- 3 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 client/httpapi/abyfy_errors.go create mode 100644 client/httpapi/abyfy_errors_test.go diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/abyfy_errors.go new file mode 100644 index 000000000000..e4f4afb7cb3d --- /dev/null +++ b/client/httpapi/abyfy_errors.go @@ -0,0 +1,104 @@ +package httpapi + +import ( + "errors" + "strings" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +type prePostWrappedNotFoundError struct { + pre string + post string + + wrapped ipld.ErrNotFound +} + +func (e prePostWrappedNotFoundError) String() string { + return e.Error() +} + +func (e prePostWrappedNotFoundError) Error() string { + return e.pre + e.wrapped.Error() + e.post +} + +func (e prePostWrappedNotFoundError) Unwrap() error { + return e.wrapped +} + +func abyfyIpldNotFoundFallbackToMSG(msg string) error { + err, handled := abyfyIpldNotFound(msg) + if handled { + return err + } + + return errors.New(msg) +} + +func abyfyIpldNotFoundFallbackToError(msg error) error { + err, handled := abyfyIpldNotFound(msg.Error()) + if handled { + return err + } + + return msg +} + +// This file handle parsing and returning the correct ABI based errors from error messages +//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type +func abyfyIpldNotFound(msg string) (error, bool) { + if msg == "" { + return nil, true // Fast path + } + + // The patern we search for is: + // node not found (fallback) + // or + // CID not found (here we parse the CID) + notFoundIndex := strings.LastIndex(msg, " not found") + + if notFoundIndex == -1 { + // Unknown, ot found not found + return nil, false + } + + preNotFound := msg[:notFoundIndex] + + var c cid.Cid + var preIndex int + if strings.HasSuffix(preNotFound, "node") { + // Fallback case + c = cid.Undef + preIndex = notFoundIndex - len("node") + } else { + // Assume that CIDs does not include whitespace to pull out the CID + preIndex = strings.LastIndexByte(preNotFound, ' ') + // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space + preIndex++ + var err error + c, err = cid.Decode(preNotFound[preIndex:]) + if err != nil { + // Unknown + return nil, false + } + } + + postIndex := notFoundIndex + len(" not found") + + err := ipld.ErrNotFound{Cid: c} + + pre := msg[:preIndex] + post := msg[postIndex:] + + if len(pre) > 0 || len(post) > 0 { + // We have some text to wrap arround the ErrNotFound one + return prePostWrappedNotFoundError{ + pre: pre, + post: post, + wrapped: err, + }, true + } + + return err, true +} diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/abyfy_errors_test.go new file mode 100644 index 000000000000..fe1cf0707bf7 --- /dev/null +++ b/client/httpapi/abyfy_errors_test.go @@ -0,0 +1,60 @@ +package httpapi + +import ( + "errors" + "fmt" + "testing" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} + +func doAbyfyIpldNotFoundTest(t *testing.T, original error) { + originalMsg := original.Error() + + rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + + rebuiltMsg := rebuilt.Error() + + if originalMsg != rebuiltMsg { + t.Errorf("expected message to be %q; got %q", originalMsg, rebuiltMsg) + } + + originalNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(original) + if originalNotFound != rebuiltNotFound { + t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + } +} + +func TestAbyfyIpldNotFound(t *testing.T) { + if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { + t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + for _, err := range [...]error{ + errors.New("file not found"), + errors.New(" not found"), + errors.New("Bad_CID not found"), + errors.New("network connection timeout"), + ipld.ErrNotFound{Cid: cid.Undef}, + ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, + ipld.ErrNotFound{Cid: cid.NewCidV1(cid.Raw, randomSha256MH)}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doAbyfyIpldNotFoundTest(t, err) + } + } +} diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 640f186f5044..e78ba2281759 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -3,7 +3,6 @@ package httpapi import ( "bytes" "context" - "errors" "fmt" "io" @@ -67,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, resp.Error + return nil, abyfyIpldNotFoundFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -99,18 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - if removedBlock.Error != "" { - return errors.New(removedBlock.Error) - } - - return nil + return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, err + return nil, abyfyIpldNotFoundFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { From 5a8e2b77c0e4361a0dd6e57a7f4129adba2f4d8a Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 30 Mar 2022 04:32:12 +0200 Subject: [PATCH 098/159] chore: rename abyfyIpldErrNotFound to parseIPLDErrNotFound This commit was moved from ipfs/go-ipfs-http-client@7aa002992970058e37bb82f9837a442227562d33 --- client/httpapi/block.go | 6 +++--- client/httpapi/{abyfy_errors.go => errors.go} | 10 +++++----- .../httpapi/{abyfy_errors_test.go => errors_test.go} | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) rename client/httpapi/{abyfy_errors.go => errors.go} (88%) rename client/httpapi/{abyfy_errors_test.go => errors_test.go} (84%) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index e78ba2281759..0ee838e8326a 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, abyfyIpldNotFoundFallbackToError(resp.Error) + return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) + return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, abyfyIpldNotFoundFallbackToError(err) + return nil, parseIPLDNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/errors.go similarity index 88% rename from client/httpapi/abyfy_errors.go rename to client/httpapi/errors.go index e4f4afb7cb3d..a527e545232d 100644 --- a/client/httpapi/abyfy_errors.go +++ b/client/httpapi/errors.go @@ -27,8 +27,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func abyfyIpldNotFoundFallbackToMSG(msg string) error { - err, handled := abyfyIpldNotFound(msg) +func parseIPLDNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseIPLDNotFound(msg) if handled { return err } @@ -36,8 +36,8 @@ func abyfyIpldNotFoundFallbackToMSG(msg string) error { return errors.New(msg) } -func abyfyIpldNotFoundFallbackToError(msg error) error { - err, handled := abyfyIpldNotFound(msg.Error()) +func parseIPLDNotFoundWithFallbackToError(msg error) error { + err, handled := parseIPLDNotFound(msg.Error()) if handled { return err } @@ -47,7 +47,7 @@ func abyfyIpldNotFoundFallbackToError(msg error) error { // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func abyfyIpldNotFound(msg string) (error, bool) { +func parseIPLDNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/errors_test.go similarity index 84% rename from client/httpapi/abyfy_errors_test.go rename to client/httpapi/errors_test.go index fe1cf0707bf7..08def204dc8c 100644 --- a/client/httpapi/abyfy_errors_test.go +++ b/client/httpapi/errors_test.go @@ -12,10 +12,10 @@ import ( var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} -func doAbyfyIpldNotFoundTest(t *testing.T, original error) { +func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -30,8 +30,8 @@ func doAbyfyIpldNotFoundTest(t *testing.T, original error) { } } -func TestAbyfyIpldNotFound(t *testing.T) { - if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { +func TestParseIPLDNotFound(t *testing.T) { + if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -54,7 +54,7 @@ func TestAbyfyIpldNotFound(t *testing.T) { err = fmt.Errorf(wrap, err) } - doAbyfyIpldNotFoundTest(t, err) + doParseIpldNotFoundTest(t, err) } } } From a2a60768ea7e5d5dad1ad1f0dcf8811cca24d3eb Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:36:11 +0200 Subject: [PATCH 099/159] fix: parseIPLDErrNotFound test This commit was moved from ipfs/go-ipfs-http-client@2e09c4b3ab580f71c652c82926ba7b1580a2b495 --- client/httpapi/errors_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 08def204dc8c..502c10966dfe 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -24,9 +24,9 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } originalNotFound := ipld.IsNotFound(original) - rebuiltNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(rebuilt) if originalNotFound != rebuiltNotFound { - t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + t.Errorf("for %q expected Ipld.IsNotFound to be %t; got %t", originalMsg, originalNotFound, rebuiltNotFound) } } From ddd36645b25c97651deb81c092c7669efabd959e Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:43:27 +0200 Subject: [PATCH 100/159] feat: update the error parsing for go-ipld-format to v0.4.0 This commit was moved from ipfs/go-ipfs-http-client@296534fd1647a1ec9cae08314daf41ec96d55763 --- client/httpapi/errors.go | 54 +++++++++++++++++++++-------------- client/httpapi/errors_test.go | 14 +++++++-- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index a527e545232d..f42d9257523c 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -45,6 +45,18 @@ func parseIPLDNotFoundWithFallbackToError(msg error) error { return msg } +// Use a string to move it into RODATA +// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) +const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + +func notAsciiLetterOrDigits(r rune) bool { + if r > 'z' { + return true + } + + return notAsciiLetterOrDigitsLUT[r] > 0 +} + // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseIPLDNotFound(msg string) (error, bool) { @@ -53,46 +65,46 @@ func parseIPLDNotFound(msg string) (error, bool) { } // The patern we search for is: - // node not found (fallback) - // or - // CID not found (here we parse the CID) - notFoundIndex := strings.LastIndex(msg, " not found") + const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ + // We try to parse the CID, if it's invalid we give up and return a simple text error. + // We also accept "node" in place of the CID because that means it's an Undefined CID. - if notFoundIndex == -1 { - // Unknown, ot found not found + keyIndex := strings.Index(msg, ipldErrNotFoundKey) + + if keyIndex < 0 { // Unknown error return nil, false } - preNotFound := msg[:notFoundIndex] + cidStart := keyIndex + len(ipldErrNotFoundKey) + msgPostKey := msg[cidStart:] var c cid.Cid - var preIndex int - if strings.HasSuffix(preNotFound, "node") { + var postIndex int + if strings.HasPrefix(msgPostKey, "node") { // Fallback case c = cid.Undef - preIndex = notFoundIndex - len("node") + postIndex = len("node") } else { - // Assume that CIDs does not include whitespace to pull out the CID - preIndex = strings.LastIndexByte(preNotFound, ' ') - // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space - preIndex++ + // Assume that CIDs only contain a-zA-Z0-9 characters. + // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. + postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + if postIndex < 0 { + postIndex = len(msgPostKey) + } + var err error - c, err = cid.Decode(preNotFound[preIndex:]) + c, err = cid.Decode(msgPostKey[:postIndex]) if err != nil { // Unknown return nil, false } } - postIndex := notFoundIndex + len(" not found") - err := ipld.ErrNotFound{Cid: c} - - pre := msg[:preIndex] - post := msg[postIndex:] + pre := msg[:keyIndex] + post := msgPostKey[postIndex:] if len(pre) > 0 || len(post) > 0 { - // We have some text to wrap arround the ErrNotFound one return prePostWrappedNotFoundError{ pre: pre, post: post, diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 502c10966dfe..1b7de8798268 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,6 +3,7 @@ package httpapi import ( "errors" "fmt" + "strings" "testing" "github.com/ipfs/go-cid" @@ -42,9 +43,8 @@ func TestParseIPLDNotFound(t *testing.T) { "%w is wrong", } { for _, err := range [...]error{ - errors.New("file not found"), - errors.New(" not found"), - errors.New("Bad_CID not found"), + errors.New("ipld: could not find "), + errors.New("ipld: could not find Bad_CID"), errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, @@ -58,3 +58,11 @@ func TestParseIPLDNotFound(t *testing.T) { } } } + +func TestNotAsciiLetterOrDigits(t *testing.T) { + for i := rune(0); i <= 256; i++ { + if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { + t.Errorf("%q is incorrectly identified", i) + } + } +} From 775bcb7f09de0d13a64b013ccaefa253e44c3403 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Fri, 1 Apr 2022 00:23:55 +0200 Subject: [PATCH 101/159] feat: add blockstore: block not found matching too This commit was moved from ipfs/go-ipfs-http-client@a3354f062c97bcb6412e74ecfb24e84208edc479 --- client/httpapi/block.go | 6 ++-- client/httpapi/errors.go | 56 +++++++++++++++++++++++++++++++---- client/httpapi/errors_test.go | 25 ++++++++++++++-- 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 0ee838e8326a..c074f79407e2 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) + return nil, parseErrNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) + return parseErrNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, parseIPLDNotFoundWithFallbackToError(err) + return nil, parseErrNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f42d9257523c..b2a6f86de274 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -8,6 +8,8 @@ import ( ipld "github.com/ipfs/go-ipld-format" ) +// This file handle parsing and returning the correct ABI based errors from error messages + type prePostWrappedNotFoundError struct { pre string post string @@ -27,8 +29,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func parseIPLDNotFoundWithFallbackToMSG(msg string) error { - err, handled := parseIPLDNotFound(msg) +func parseErrNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseErrNotFound(msg) if handled { return err } @@ -36,8 +38,8 @@ func parseIPLDNotFoundWithFallbackToMSG(msg string) error { return errors.New(msg) } -func parseIPLDNotFoundWithFallbackToError(msg error) error { - err, handled := parseIPLDNotFound(msg.Error()) +func parseErrNotFoundWithFallbackToError(msg error) error { + err, handled := parseErrNotFound(msg.Error()) if handled { return err } @@ -57,13 +59,25 @@ func notAsciiLetterOrDigits(r rune) bool { return notAsciiLetterOrDigitsLUT[r] > 0 } -// This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func parseIPLDNotFound(msg string) (error, bool) { +func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } + if err, handled := parseIPLDErrNotFound(msg); handled { + return err, true + } + + if err, handled := parseBlockstoreNotFound(msg); handled { + return err, true + } + + return nil, false +} + +//lint:ignore ST1008 using error as values +func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ // We try to parse the CID, if it's invalid we give up and return a simple text error. @@ -114,3 +128,33 @@ func parseIPLDNotFound(msg string) (error, bool) { return err, true } + +// This is a simple error type that just return msg as Error(). +// But that also match ipld.ErrNotFound when called with Is(err). +// That is needed to keep compatiblity with code that use string.Contains(err.Error(), "blockstore: block not found") +// and code using ipld.ErrNotFound +type blockstoreNotFoundMatchingIPLDErrNotFound struct { + msg string +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) String() string { + return e.Error() +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Error() string { + return e.msg +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { + _, ok := err.(ipld.ErrNotFound) + return ok +} + +//lint:ignore ST1008 using error as values +func parseBlockstoreNotFound(msg string) (error, bool) { + if !strings.Contains(msg, "blockstore: block not found") { + return nil, false + } + + return blockstoreNotFoundMatchingIPLDErrNotFound{msg: msg}, true +} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 1b7de8798268..09437bfbfcce 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -16,7 +16,7 @@ var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1 func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) + rebuilt := parseErrNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -32,7 +32,7 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } func TestParseIPLDNotFound(t *testing.T) { - if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { + if err := parseErrNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -59,6 +59,27 @@ func TestParseIPLDNotFound(t *testing.T) { } } +func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { + if !ipld.IsNotFound(blockstoreNotFoundMatchingIPLDErrNotFound{}) { + t.Fatalf("expected blockstoreNotFoundMatchingIPLDErrNotFound to match ipld.IsNotFound; got false") + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doParseIpldNotFoundTest(t, err) + } +} + func TestNotAsciiLetterOrDigits(t *testing.T) { for i := rune(0); i <= 256; i++ { if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { From 54f6e90870c61a686f57b0bd0f1ad4923615897b Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 03:25:39 +0200 Subject: [PATCH 102/159] fix: change CID breaking logic when parsing ipld.ErrNotFound This commit was moved from ipfs/go-ipfs-http-client@a3b49352bfd1b885567018092a06f303de18b7aa --- client/httpapi/errors.go | 24 +++++++++--------------- client/httpapi/errors_test.go | 18 +++++++----------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index b2a6f86de274..f1a57a049600 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -47,18 +47,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -// Use a string to move it into RODATA -// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) -const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - -func notAsciiLetterOrDigits(r rune) bool { - if r > 'z' { - return true - } - - return notAsciiLetterOrDigitsLUT[r] > 0 -} - //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { @@ -76,6 +64,12 @@ func parseErrNotFound(msg string) (error, bool) { return nil, false } +// Assume CIDs break on: +// - Whitespaces: " \t\n\r\v\f" +// - Semicolon: ";" this is to parse ipld.ErrNotFound wrapped in multierr +// - Double Quotes: "\"" this is for parsing %q and %#v formating +const cidBreakSet = " \t\n\r\v\f;\"" + //lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: @@ -99,9 +93,9 @@ func parseIPLDErrNotFound(msg string) (error, bool) { c = cid.Undef postIndex = len("node") } else { - // Assume that CIDs only contain a-zA-Z0-9 characters. - // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. - postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + postIndex = strings.IndexFunc(msgPostKey, func(r rune) bool { + return strings.ContainsAny(string(r), cidBreakSet) + }) if postIndex < 0 { postIndex = len(msgPostKey) } diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 09437bfbfcce..86f91cdf5f7b 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "strings" "testing" "github.com/ipfs/go-cid" @@ -36,12 +35,17 @@ func TestParseIPLDNotFound(t *testing.T) { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } - for _, wrap := range [...]string{ + cidBreaks := make([]string, len(cidBreakSet)) + for i, v := range cidBreakSet { + cidBreaks[i] = "%w" + string(v) + } + + for _, wrap := range append(cidBreaks, "", "merkledag: %w", "testing: %w the test", "%w is wrong", - } { + ) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), @@ -79,11 +83,3 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { doParseIpldNotFoundTest(t, err) } } - -func TestNotAsciiLetterOrDigits(t *testing.T) { - for i := rune(0); i <= 256; i++ { - if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { - t.Errorf("%q is incorrectly identified", i) - } - } -} From f2f2109bc1e8f5b9c636597eb7478e0a4ddca427 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 04:24:12 +0200 Subject: [PATCH 103/159] test: add a false case test to blockstore parsing This commit was moved from ipfs/go-ipfs-http-client@75f597aa16c512ec02f549836e998f1fc29a3846 --- client/httpapi/errors_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 86f91cdf5f7b..7709fc9c53a2 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -74,12 +74,15 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { "testing: %w the test", "%w is wrong", } { - var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + for _, err := range [...]error{ + errors.New("network connection timeout"), + blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } - if wrap != "" { - err = fmt.Errorf(wrap, err) + doParseIpldNotFoundTest(t, err) } - - doParseIpldNotFoundTest(t, err) } } From 60a548b621ff229a648a268c4d06f6e206ad6099 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 5 Apr 2022 20:11:06 +0200 Subject: [PATCH 104/159] correctness: only match CIDs matching go-cid.Cid.String output This commit was moved from ipfs/go-ipfs-http-client@34cc489461985cf4139819e589fdd09bdc034c24 --- client/httpapi/errors.go | 19 +++++++++++++++++-- client/httpapi/errors_test.go | 7 +++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f1a57a049600..1ccf6182c7b4 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -3,9 +3,11 @@ package httpapi import ( "errors" "strings" + "unicode/utf8" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" ) // This file handle parsing and returning the correct ABI based errors from error messages @@ -97,15 +99,28 @@ func parseIPLDErrNotFound(msg string) (error, bool) { return strings.ContainsAny(string(r), cidBreakSet) }) if postIndex < 0 { + // no breakage meaning the string look like this something + "ipld: could not find bafy" postIndex = len(msgPostKey) } + cidStr := msgPostKey[:postIndex] + var err error - c, err = cid.Decode(msgPostKey[:postIndex]) + c, err = cid.Decode(cidStr) if err != nil { - // Unknown + // failed to decode CID give up return nil, false } + + // check that the CID is either a CIDv0 or a base32 multibase + // because that what ipld.ErrNotFound.Error() -> cid.Cid.String() do currently + if c.Version() != 0 { + baseRune, _ := utf8.DecodeRuneInString(cidStr) + if baseRune == utf8.RuneError || baseRune != mbase.Base32 { + // not a multibase we expect, give up + return nil, false + } + } } err := ipld.ErrNotFound{Cid: c} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 7709fc9c53a2..c8b98d08eb24 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" mh "github.com/multiformats/go-multihash" ) @@ -40,6 +41,11 @@ func TestParseIPLDNotFound(t *testing.T) { cidBreaks[i] = "%w" + string(v) } + base58BTCEncoder, err := mbase.NewEncoder(mbase.Base58BTC) + if err != nil { + t.Fatalf("expected to find Base58BTC encoder; got error %q", err.Error()) + } + for _, wrap := range append(cidBreaks, "", "merkledag: %w", @@ -49,6 +55,7 @@ func TestParseIPLDNotFound(t *testing.T) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), + errors.New("ipld: could not find " + cid.NewCidV1(cid.Raw, randomSha256MH).Encode(base58BTCEncoder)), // Test that we only accept CIDv0 and base32 CIDs errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, From fd209019f27c7a16f94bfc41ebc7b815daee9903 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 21 Apr 2022 22:18:19 +0200 Subject: [PATCH 105/159] fix: interop with 'block put' from go-ipfs 0.13 (#158) * chore: interop with go-ipfs 0.13 Applies necessary changes to ensure 'block/put' works and is backward-compatible. Context: https://github.com/ipfs/go-ipfs/pull/8568 * chore: 0.3.1 bumping as patch because we bumped to 0.3.0 recently, as part of other (unreleased) go-ipfs 0.13 work This commit was moved from ipfs/go-ipfs-http-client@ecf364c9898d78034e3232574aae70d1bf85357c --- client/httpapi/block.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index c074f79407e2..7b4cf0edab56 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -7,9 +7,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) @@ -31,20 +32,33 @@ func (s *blockStat) Path() path.Resolved { } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { - options, _, err := caopts.BlockPutOptions(opts...) + options, err := caopts.BlockPutOptions(opts...) + px := options.CidPrefix if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] + mht, ok := mh.Codes[px.MhType] if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + return nil, fmt.Errorf("unknowm mhType %d", px.MhType) + } + + var cidOptKey, cidOptVal string + switch { + case px.Version == 0 && px.Codec == cid.DagProtobuf: + // ensure legacy --format=v0 passes as BlockPutOption still works + cidOptKey = "format" + cidOptVal = "v0" + default: + // pass codec as string + cidOptKey = "cid-codec" + cidOptVal = mc.Code(px.Codec).String() } req := api.core().Request("block/put"). Option("mhtype", mht). - Option("mhlen", options.MhLength). - Option("format", options.Codec). + Option("mhlen", px.MhLength). + Option(cidOptKey, cidOptVal). Option("pin", options.Pin). FileBody(r) From 1cdb9adf898a8a2b9d2cd0d3fe8ba6467a822166 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 23 Jun 2022 22:39:01 +0200 Subject: [PATCH 106/159] fix: interop with go-ipfs 0.13 (#160) This ensures cid-codec introduced in https://github.com/ipfs/go-ipfs/pull/8568 gets correctly passed (+ we maintain backward-compatibility with CIDv0) This commit was moved from ipfs/go-ipfs-http-client@9c9f43fd9ca36c7ebdc664bb6936a6fe9881255b --- client/httpapi/dag.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 15d9a9c62ed2..879c1e499327 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,12 @@ import ( "fmt" "io/ioutil" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + multicodec "github.com/multiformats/go-multicodec" ) type httpNodeAdder HttpApi @@ -56,13 +57,21 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() - format := cid.CodecToStr[prefix.Codec] + + // preserve 'cid-codec' when sent over HTTP + cidCodec := multicodec.Code(prefix.Codec).String() + + // 'format' got replaced by 'cid-codec' in https://github.com/ipfs/interface-go-ipfs-core/pull/80 + // but we still support it here for backward-compatibility with use of CIDv0 + format := "" if prefix.Version == 0 { + cidCodec = "" format = "v0" } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.CidCodec(cidCodec), options.Block.Format(format), options.Block.Pin(pin)) if err != nil { From 0fff1d5d8e4cb2debaff1ee4e85b1266907101af Mon Sep 17 00:00:00 2001 From: web3-bot Date: Fri, 23 Sep 2022 07:43:33 +0000 Subject: [PATCH 107/159] stop using the deprecated io/ioutil package This commit was moved from ipfs/go-ipfs-http-client@026ba730a1fe9a4a25a22449bec8d69c2262ccfc --- client/httpapi/api.go | 3 +-- client/httpapi/api_test.go | 3 +-- client/httpapi/apifile.go | 3 +-- client/httpapi/dag.go | 4 ++-- client/httpapi/name.go | 4 ++-- client/httpapi/object.go | 5 ++--- client/httpapi/requestbuilder.go | 5 ++--- client/httpapi/response.go | 9 ++++----- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f589e4267a47..97440a72436d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "io/ioutil" "net/http" "os" "path/filepath" @@ -74,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { apiFile := filepath.Join(baseDir, DefaultApiFile) - api, err := ioutil.ReadFile(apiFile) + api, err := os.ReadFile(apiFile) if err != nil { return nil, err } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index d5f27f0ec9d8..5960ea2c0270 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,7 +2,6 @@ package httpapi import ( "context" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -92,7 +91,7 @@ func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n i func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { - dir, err := ioutil.TempDir("", "httpapi-tb-") + dir, err := os.MkdirTemp("", "httpapi-tb-") if err != nil { return nil, err } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec2f7588d591..c4884b924f08 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" @@ -113,7 +112,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) + r, err := io.CopyN(io.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 879c1e499327..f32c67c42efa 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "io" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -24,7 +24,7 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) return nil, err } - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 47227a2ab893..b6a7836033ab 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/options/namesys" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6ec7f5503cd0..894369223fc1 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -5,13 +5,12 @@ import ( "context" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -71,7 +70,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { if err != nil { return nil, err } - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 7012a8935bfa..039bca036410 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,11 +5,10 @@ import ( "context" "fmt" "io" - "io/ioutil" "strconv" "strings" - "github.com/ipfs/go-ipfs-files" + files "github.com/ipfs/go-ipfs-files" ) type RequestBuilder interface { @@ -59,7 +58,7 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder { // FileBody sets the request body to the given reader wrapped into multipartreader. func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { - pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 95cbf13ec1bf..8a491ab73169 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime" "net/http" "net/url" @@ -45,7 +44,7 @@ func (r *Response) Close() error { if r.Output != nil { // drain output (response body) - _, err1 := io.Copy(ioutil.Discard, r.Output) + _, err1 := io.Copy(io.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -117,7 +116,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" case contentType == "text/plain": - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } @@ -140,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // This is a server-side bug (probably). e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) } @@ -150,7 +149,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() } From 1193b2978f7859ed8e1df4be50e980ff1bde7015 Mon Sep 17 00:00:00 2001 From: galargh Date: Sat, 1 Oct 2022 17:37:23 +0200 Subject: [PATCH 108/159] chore: remove unused linter directives This commit was moved from ipfs/go-ipfs-http-client@c4fc1a7740bfaad2abcba2563d569422e525324c --- client/httpapi/errors.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index 1ccf6182c7b4..59e4ad705432 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -49,7 +49,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path @@ -72,7 +71,6 @@ func parseErrNotFound(msg string) (error, bool) { // - Double Quotes: "\"" this is for parsing %q and %#v formating const cidBreakSet = " \t\n\r\v\f;\"" -//lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ @@ -159,7 +157,6 @@ func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { return ok } -//lint:ignore ST1008 using error as values func parseBlockstoreNotFound(msg string) (error, bool) { if !strings.Contains(msg, "blockstore: block not found") { return nil, false From 75b6f45f765237d611168a5e1c09915f380e7bd7 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 6 Feb 2023 15:32:57 +0100 Subject: [PATCH 109/159] chore: bumps for Kubo 0.18 and Routing API with stub for Put This commit was moved from ipfs/go-ipfs-http-client@c076c3cb71d08fb9448ae9bf6916dab137322722 --- client/httpapi/api.go | 4 +++ client/httpapi/apifile.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/dht.go | 4 +-- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/routing.go | 55 ++++++++++++++++++++++++++++++++ client/httpapi/swarm.go | 6 ++-- client/httpapi/unixfs.go | 2 +- 11 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 client/httpapi/routing.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 97440a72436d..10df94603f75 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -195,3 +195,7 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { func (api *HttpApi) PubSub() iface.PubSubAPI { return (*PubsubAPI)(api) } + +func (api *HttpApi) Routing() iface.RoutingAPI { + return (*RoutingAPI)(api) +} diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index c4884b924f08..80fd13cd4fe9 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -7,7 +7,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index f32c67c42efa..2fbfea1ff29c 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 0a78514f9739..ebecfcaeb3ae 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,8 +6,8 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" ) type DhtAPI HttpApi diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 78b67517c011..b785bbf453dc 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 72f5923761f2..ec45d1a51bef 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 039bca036410..bc6f4c8a94f2 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 8a491ab73169..99932ca0da7e 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -12,7 +12,7 @@ import ( cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/routing.go b/client/httpapi/routing.go new file mode 100644 index 000000000000..18f5332766fb --- /dev/null +++ b/client/httpapi/routing.go @@ -0,0 +1,55 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + + "github.com/libp2p/go-libp2p/core/routing" +) + +type RoutingAPI HttpApi + +func (api *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) { + resp, err := api.core().Request("routing/get", key).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Close() + + var out routing.QueryEvent + + dec := json.NewDecoder(resp.Output) + if err := dec.Decode(&out); err != nil { + return nil, err + } + + res, err := base64.StdEncoding.DecodeString(out.Extra) + if err != nil { + return nil, err + } + + return res, nil +} + +func (api *RoutingAPI) Put(ctx context.Context, key string, value []byte) error { + resp, err := api.core().Request("routing/put", key). + FileBody(bytes.NewReader(value)). + Send(ctx) + + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + return nil +} + +func (api *RoutingAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 1cb0d91df7c6..c5dfbe564523 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,9 +5,9 @@ import ( "time" iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5e27fb0367f1..38f23d5b9b4c 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,7 +8,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" From 5c21cf0ce95d7c2710d5bb5dd58e2871c0fbc7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= <92101826+PedrobyJoao@users.noreply.github.com> Date: Tue, 28 Mar 2023 07:38:24 -0300 Subject: [PATCH 110/159] docs: adding example of connection and pinning to README (#173) Co-authored-by: JPexplorer This commit was moved from ipfs/go-ipfs-http-client@b0de2b028e6ef03d4dba3ed105641ad299d4f7be --- client/httpapi/README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index d3e14ac3f1be..da25f24feb00 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -17,6 +17,41 @@ greatest features, please use _this_ package. https://godoc.org/github.com/ipfs/go-ipfs-http-api +### Example + +Pin file on your local IPFS node based on its CID: + +```go +package main + +import ( + "context" + "fmt" + + ipfsClient "github.com/ipfs/go-ipfs-http-client" + path "github.com/ipfs/interface-go-ipfs-core/path" +) + +func main() { + // "Connect" to local node + node, err := ipfsClient.NewLocalApi() + if err != nil { + fmt.Printf(err) + return + } + // Pin a given file by its CID + ctx := context.Background() + cid := "bafkreidtuosuw37f5xmn65b3ksdiikajy7pwjjslzj2lxxz2vc4wdy3zku" + p := path.New(cid) + err = node.Pin().Add(ctx, p) + if err != nil { + fmt.Printf(err) + return + } + return +} +``` + ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! From 16d8d972b669ce6658c30719396e38905852a9fb Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 7 Apr 2023 07:12:08 +0000 Subject: [PATCH 111/159] chore: update version --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 992f4a9885b0..850332654b5c 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,7 @@ import ( var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.20.0-dev" +const CurrentVersionNumber = "0.20.0-rc1" const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint From 8bebfc00b7192de63b79082bd2666d8d1bf76126 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 16 Apr 2023 12:31:09 +0200 Subject: [PATCH 112/159] chore: migrate from go-libipfs to boxo This commit was moved from ipfs/go-ipfs-http-client@ae996cbe5a91708d245100267c45094ab52c42ad --- client/httpapi/api.go | 4 ++-- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 6 +++--- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 4 ++-- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 8 ++++---- client/httpapi/object.go | 10 +++++----- client/httpapi/path.go | 4 ++-- client/httpapi/pin.go | 6 +++--- client/httpapi/pubsub.go | 4 ++-- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/swarm.go | 2 +- client/httpapi/unixfs.go | 12 ++++++------ 16 files changed, 44 insertions(+), 45 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 10df94603f75..b7cfb2b520d4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strings" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5960ea2c0270..2832d722d374 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -12,10 +12,9 @@ import ( "testing" "time" - iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/ipfs/interface-go-ipfs-core/path" - - "github.com/ipfs/interface-go-ipfs-core/tests" + iface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/coreiface/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" testbedi "github.com/ipfs/iptb/testbed/interfaces" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 80fd13cd4fe9..25fd7c3b3718 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 7b4cf0edab56..2a794c26f529 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 2fbfea1ff29c..795e1d78d9e3 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,11 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" - "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" ) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index ebecfcaeb3ae..a2910fef6d37 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index b785bbf453dc..434e98fe5214 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,9 @@ import ( "context" "errors" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b6a7836033ab..f82f69f3ac42 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + nsopts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/coreiface/path" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 894369223fc1..4e3b9ef6be18 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,13 +6,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipld/merkledag" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - ft "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 619dd0e0d2cb..d69d425ab664 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,10 +3,10 @@ package httpapi import ( "context" + "github.com/ipfs/boxo/coreiface/path" + ipfspath "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core/path" ) func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 13de2d3893de..30a3d7b7a902 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -5,10 +5,10 @@ import ( "encoding/json" "strings" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index ec45d1a51bef..28f1ef8e609c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,8 +6,8 @@ import ( "encoding/json" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index bc6f4c8a94f2..476aed7866d1 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/boxo/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 99932ca0da7e..189b43671614 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -10,9 +10,9 @@ import ( "net/url" "os" + "github.com/ipfs/boxo/files" cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index c5dfbe564523..9b073078da0a 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - iface "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/boxo/coreiface" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 38f23d5b9b4c..b9c34c59ffcf 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -7,13 +7,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" + unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - unixfs_pb "github.com/ipfs/go-unixfs/pb" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) From acb622fb1162bb53f43e9c82b636aa9cb02d5fec Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 25 Apr 2023 11:44:53 +0000 Subject: [PATCH 113/159] chore: update version --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 850332654b5c..7c1681bf0eb6 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,7 @@ import ( var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.20.0-rc1" +const CurrentVersionNumber = "0.20.0-rc2" const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint From e4fa9cb7ac7f8cc71144d21be476d5f9830578ef Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 12 Apr 2023 01:51:02 +0200 Subject: [PATCH 114/159] feat: boxo tracing and traceparent support (#9811) https://www.w3.org/TR/trace-context/ https://github.com/ipfs/bifrost-gateway/issues/68 --- docs/environment-variables.md | 71 +-------------------- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 12 ++-- go.sum | 4 +- tracing/file_exporter.go | 45 ------------- tracing/tracing.go | 88 +------------------------- 7 files changed, 14 insertions(+), 212 deletions(-) delete mode 100644 tracing/file_exporter.go diff --git a/docs/environment-variables.md b/docs/environment-variables.md index b3d03d1bdc0a..4113be09b823 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -164,74 +164,5 @@ and outputs it to `rcmgr.json.gz` Default: disabled (not set) # Tracing -For advanced configuration (e.g. ratio-based sampling), see also: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md -## `OTEL_TRACES_EXPORTER` -Specifies the exporters to use as a comma-separated string. Each exporter has a set of additional environment variables used to configure it. The following values are supported: - -- `otlp` -- `jaeger` -- `zipkin` -- `file` -- appends traces to a JSON file on the filesystem - -Setting this enables OpenTelemetry tracing. - -**NOTE** Tracing support is experimental: releases may contain tracing-related breaking changes. - -Default: "" (no exporters) - -## `OTLP Exporter` -Unless specified in this section, the OTLP exporter uses the environment variables documented here: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md - -### `OTEL_EXPORTER_OTLP_PROTOCOL` -Specifies the OTLP protocol to use, which is one of: - -- `grpc` -- `http/protobuf` - -Default: "grpc" - -## `Jaeger Exporter` - -See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md#jaeger-exporter - -## `Zipkin Exporter` -See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md#zipkin-exporter - -## `File Exporter` -### `OTEL_EXPORTER_FILE_PATH` -Specifies the filesystem path for the JSON file. - -Default: "$PWD/traces.json" - -### How to use Jaeger UI - -One can use the `jaegertracing/all-in-one` Docker image to run a full Jaeger -stack and configure Kubo to publish traces to it (here, in an ephemeral -container): - -```console -$ docker run --rm -it --name jaeger \ - -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \ - -p 5775:5775/udp \ - -p 6831:6831/udp \ - -p 6832:6832/udp \ - -p 5778:5778 \ - -p 16686:16686 \ - -p 14268:14268 \ - -p 14269:14269 \ - -p 14250:14250 \ - -p 9411:9411 \ - jaegertracing/all-in-one -``` - -Then, in other terminal, start Kubo with Jaeger tracing enabled: -``` -$ OTEL_TRACES_EXPORTER=jaeger ipfs daemon -``` - -Finally, the [Jaeger UI](https://github.com/jaegertracing/jaeger-ui#readme) is available at http://localhost:16686 - -## `OTEL_PROPAGATORS` - -See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md#general-sdk-configuration +For tracing configuration, please check: https://github.com/ipfs/boxo/blob/main/docs/tracing.md diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 1c2af1154537..345f97f3fdd9 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.0 + github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.0 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 802f84be6a19..755f9aceb054 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= -github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e h1:8wmBhjwJk2drWZjNwoN7uc+IkG+N93laIhjY69rjMqw= +github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 58d943ac576f..115bd5749aa7 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.0 + github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 @@ -75,11 +75,6 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/exporters/jaeger v1.14.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 - go.opentelemetry.io/otel/exporters/zipkin v1.14.0 go.opentelemetry.io/otel/sdk v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/dig v1.16.1 @@ -208,8 +203,13 @@ require ( go.opentelemetry.io/contrib/propagators/b3 v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.15.0 // indirect + go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect go.opentelemetry.io/otel/metric v0.37.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect diff --git a/go.sum b/go.sum index c623a9d4d127..90cd6eb79db1 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= -github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e h1:8wmBhjwJk2drWZjNwoN7uc+IkG+N93laIhjY69rjMqw= +github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/tracing/file_exporter.go b/tracing/file_exporter.go deleted file mode 100644 index 32ca20ee2740..000000000000 --- a/tracing/file_exporter.go +++ /dev/null @@ -1,45 +0,0 @@ -package tracing - -import ( - "context" - "fmt" - "os" - - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/sdk/trace" -) - -// fileExporter wraps a file-writing exporter and closes the file when the exporter is shutdown. -type fileExporter struct { - file *os.File - writerExporter *stdouttrace.Exporter -} - -func newFileExporter(file string) (*fileExporter, error) { - f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return nil, fmt.Errorf("opening '%s' for OpenTelemetry file exporter: %w", file, err) - } - stdoutExporter, err := stdouttrace.New(stdouttrace.WithWriter(f)) - if err != nil { - return nil, err - } - return &fileExporter{ - writerExporter: stdoutExporter, - file: f, - }, nil -} - -func (e *fileExporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error { - return e.writerExporter.ExportSpans(ctx, spans) -} - -func (e *fileExporter) Shutdown(ctx context.Context) error { - if err := e.writerExporter.Shutdown(ctx); err != nil { - return err - } - if err := e.file.Close(); err != nil { - return fmt.Errorf("closing trace file: %w", err) - } - return nil -} diff --git a/tracing/tracing.go b/tracing/tracing.go index b22930a6f268..8fd898d7b057 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -3,16 +3,10 @@ package tracing import ( "context" "fmt" - "os" - "path" - "strings" + "github.com/ipfs/boxo/tracing" version "github.com/ipfs/kubo" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/jaeger" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/exporters/zipkin" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" @@ -33,87 +27,9 @@ type noopShutdownTracerProvider struct{ traceapi.TracerProvider } func (n *noopShutdownTracerProvider) Shutdown(ctx context.Context) error { return nil } -func buildExporters(ctx context.Context) ([]trace.SpanExporter, error) { - // These env vars are standardized but not yet supported by opentelemetry-go. - // Once supported, we can remove most of this code. - // - // Specs: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md#exporter-selection - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md - var exporters []trace.SpanExporter - for _, exporterStr := range strings.Split(os.Getenv("OTEL_TRACES_EXPORTER"), ",") { - switch exporterStr { - case "otlp": - protocol := "http/protobuf" - if v := os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL"); v != "" { - protocol = v - } - if v := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"); v != "" { - protocol = v - } - - switch protocol { - case "http/protobuf": - exporter, err := otlptracehttp.New(ctx) - if err != nil { - return nil, fmt.Errorf("building OTLP HTTP exporter: %w", err) - } - exporters = append(exporters, exporter) - case "grpc": - exporter, err := otlptracegrpc.New(ctx) - if err != nil { - return nil, fmt.Errorf("building OTLP gRPC exporter: %w", err) - } - exporters = append(exporters, exporter) - default: - return nil, fmt.Errorf("unknown or unsupported OTLP exporter '%s'", exporterStr) - } - case "jaeger": - exporter, err := jaeger.New(jaeger.WithCollectorEndpoint()) - if err != nil { - return nil, fmt.Errorf("building Jaeger exporter: %w", err) - } - exporters = append(exporters, exporter) - case "zipkin": - exporter, err := zipkin.New("") - if err != nil { - return nil, fmt.Errorf("building Zipkin exporter: %w", err) - } - exporters = append(exporters, exporter) - case "file": - // This is not part of the spec, but provided for convenience - // so that you don't have to setup a collector, - // and because we don't support the stdout exporter. - filePath := os.Getenv("OTEL_EXPORTER_FILE_PATH") - if filePath == "" { - cwd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("finding working directory for the OpenTelemetry file exporter: %w", err) - } - filePath = path.Join(cwd, "traces.json") - } - exporter, err := newFileExporter(filePath) - if err != nil { - return nil, err - } - exporters = append(exporters, exporter) - case "none": - continue - case "": - continue - case "stdout": - // stdout is already used for certain kinds of logging, so we don't support this - fallthrough - default: - return nil, fmt.Errorf("unknown or unsupported exporter '%s'", exporterStr) - } - } - return exporters, nil -} - // NewTracerProvider creates and configures a TracerProvider. func NewTracerProvider(ctx context.Context) (shutdownTracerProvider, error) { - exporters, err := buildExporters(ctx) + exporters, err := tracing.NewSpanExporters(ctx) if err != nil { return nil, err } From b706d644875108786ab44d76833a902daab3b5e2 Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Thu, 13 Apr 2023 06:59:08 -0700 Subject: [PATCH 115/159] docs: preparing 0.20 changelog for release (#9799) Co-authored-by: Gus Eggert Co-authored-by: Marcin Rataj Co-authored-by: Henrique Dias --- docs/changelogs/v0.20.md | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/docs/changelogs/v0.20.md b/docs/changelogs/v0.20.md index c1c3e9639d0b..9809d4703b21 100644 --- a/docs/changelogs/v0.20.md +++ b/docs/changelogs/v0.20.md @@ -6,6 +6,13 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Boxo under the covers](#boxo-under-the-covers) + - [HTTP Gateway](#http-gateway) + - [Switch to `boxo/gateway` library](#switch-to-boxogateway-library) + - [Improved testing](#improved-testing) + - [Trace Context support](#trace-context-support) + - [Removed legacy features](#removed-legacy-features) + - [`--empty-repo` is now the default](#--empty-repo-is-now-the-default) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -13,6 +20,97 @@ ### ๐Ÿ”ฆ Highlights +#### Boxo under the covers +We have consolidated many IPFS repos into [Boxo](https://github.com/ipfs/boxo), and this release switches Kubo over to use Boxo instead of those repos, resulting in the removal of 27 dependencies from Kubo: + +- github.com/ipfs/go-bitswap +- github.com/ipfs/go-ipfs-files +- github.com/ipfs/tar-utils +- gihtub.com/ipfs/go-block-format +- github.com/ipfs/interface-go-ipfs-core +- github.com/ipfs/go-unixfs +- github.com/ipfs/go-pinning-service-http-client +- github.com/ipfs/go-path +- github.com/ipfs/go-namesys +- github.com/ipfs/go-mfs +- github.com/ipfs/go-ipfs-provider +- github.com/ipfs/go-ipfs-pinner +- github.com/ipfs/go-ipfs-keystore +- github.com/ipfs/go-filestore +- github.com/ipfs/go-ipns +- github.com/ipfs/go-blockservice +- github.com/ipfs/go-ipfs-chunker +- github.com/ipfs/go-fetcher +- github.com/ipfs/go-ipfs-blockstore +- github.com/ipfs/go-ipfs-posinfo +- github.com/ipfs/go-ipfs-util +- github.com/ipfs/go-ipfs-ds-help +- github.com/ipfs/go-verifcid +- github.com/ipfs/go-ipfs-exchange-offline +- github.com/ipfs/go-ipfs-routing +- github.com/ipfs/go-ipfs-exchange-interface +- github.com/ipfs/go-libipfs + +Note: if you consume these in your own code, we recommend migrating to Boxo. To ease this process, there's a [tool which will help migrate your code to Boxo](https://github.com/ipfs/boxo#migrating-to-box). + +You can learn more about the [Boxo 0.8 release](https://github.com/ipfs/boxo/releases/tag/v0.8.0) that Kubo now depends and the general effort to get Boxo to be a stable foundation [here](https://github.com/ipfs/boxo/issues/196). + +#### HTTP Gateway + +##### Switch to `boxo/gateway` library + +Gateway code was extracted and refactored into a standalone library that now +lives in [boxo/gateway](https://github.com/ipfs/boxo/tree/main/gateway). This +enabled us to clean up some legacy code and remove dependency on Kubo +internals. + +The GO API is still being refined, but now operates on higher level abstraction +defined by `gateway.IPFSBackend` interface. It is now possible to embed +gateway functionality without the rest of Kubo. + +See the [car](https://github.com/ipfs/boxo/tree/main/examples/gateway/car) +and [proxy](https://github.com/ipfs/boxo/tree/main/examples/gateway/proxy) +examples, or more advanced +[bifrost-gateway](https://github.com/ipfs/bifrost-gateway). + +##### Improved testing + +We are also in the progress of moving away from gateway testing being based on +Kubo sharness tests, and are working on +[ipfs/gateway-conformance](https://github.com/ipfs/gateway-conformance) test +suite that is vendor agnostic and can be run against arbitrary HTTP endpoint to +test specific subset of [HTTP Gateways specifications](https://specs.ipfs.tech/http-gateways/). + +##### Trace Context support + +We've introduced initial support for `traceparent` header from [W3C's Trace +Context spec](https://w3c.github.io/trace-context/). + +If `traceparent` header is +present in the gateway request, one can use its `trace-id` part to inspect +trace spans via selected exporter such as Jaeger UI +([docs](https://github.com/ipfs/boxo/blob/main/docs/tracing.md#using-jaeger-ui), +[demo](https://user-images.githubusercontent.com/157609/231312374-bafc2035-1fc6-4d6b-901b-9e4af039807c.png)). + +To learn more, see [tracing docs](https://github.com/ipfs/boxo/blob/main/docs/tracing.md). + +##### Removed legacy features + +- Some Kubo-specific prometheus metrics are no longer available. + - An up-to-date list of gateway metrics can be found in [boxo/gateway/metrics.go](https://github.com/ipfs/boxo/blob/main/gateway/metrics.go). +- The legacy opt-in `Gateway.Writable` is no longer available as of Kubo 0.20. + - We are working on developing a modern replacement. + To support our efforts, please leave a comment describing your use case in + [ipfs/specs#375](https://github.com/ipfs/specs/issues/375). + +#### `--empty-repo` is now the default + +When creating a repository with `ipfs init`, `--empty-repo=true` is now the default. This means +that your repository will be empty by default instead of containing the introduction files. +You can read more about the rationale behind this decision on the [tracking issue](https://github.com/ipfs/kubo/issues/9757). + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors + + From 4f438e6d4efb06f8b956b9d32736cdf0d4f70e0c Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 14 Apr 2023 02:09:51 -0700 Subject: [PATCH 116/159] deps: bump go-libp2p to v0.27.1 (#9816) Co-authored-by: Jorropo --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 345f97f3fdd9..e61559247cc5 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.27.0 + github.com/libp2p/go-libp2p v0.27.1 github.com/multiformats/go-multiaddr v0.9.0 ) diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 755f9aceb054..643c6ace007b 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -489,8 +489,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.0 h1:QbhrTuB0ln9j9op6yAOR0o+cx/qa9NyNZ5ov0Tql8ZU= -github.com/libp2p/go-libp2p v0.27.0/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.1 h1:k1u6RHsX3hqKnslDjsSgLNURxJ3O1atIZCY4gpMbbus= +github.com/libp2p/go-libp2p v0.27.1/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= diff --git a/go.mod b/go.mod index 115bd5749aa7..a7737edef2fa 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.27.0 + github.com/libp2p/go-libp2p v0.27.1 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-kbucket v0.5.0 diff --git a/go.sum b/go.sum index 90cd6eb79db1..e060b4e25691 100644 --- a/go.sum +++ b/go.sum @@ -540,8 +540,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.0 h1:QbhrTuB0ln9j9op6yAOR0o+cx/qa9NyNZ5ov0Tql8ZU= -github.com/libp2p/go-libp2p v0.27.0/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.1 h1:k1u6RHsX3hqKnslDjsSgLNURxJ3O1atIZCY4gpMbbus= +github.com/libp2p/go-libp2p v0.27.1/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= From 656867b81c7ecdc47e835714d5073937ecfb6fd6 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Mon, 24 Apr 2023 09:44:27 +0200 Subject: [PATCH 117/159] ci: update apt before installing deps (#9831) --- .github/workflows/build.yml | 1 - .github/workflows/sharness.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50889613bea2..1f16d9bf3995 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -148,7 +148,6 @@ jobs: with: repository: ipfs/go-ipfs-http-client path: go-ipfs-http-client - ref: bump-for-rcmgr-last-push - uses: protocol/cache-go-action@v1 with: name: ${{ github.job }} diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index 012a6a9826ed..2d9dfb41da22 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -32,7 +32,7 @@ jobs: with: path: kubo - name: Install missing tools - run: sudo apt install -y socat net-tools fish libxml2-utils + run: sudo apt update && sudo apt install -y socat net-tools fish libxml2-utils - name: Restore Go Cache uses: protocol/cache-go-action@v1 with: From 7ea0b0703816473460614d1705b9c8cc9f12e052 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 25 Apr 2023 09:55:40 +0200 Subject: [PATCH 118/159] docs: add `ipfs pubsub` deprecation reminder to changelog (#9827) --- docs/changelogs/v0.20.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelogs/v0.20.md b/docs/changelogs/v0.20.md index 9809d4703b21..590ef5453a5a 100644 --- a/docs/changelogs/v0.20.md +++ b/docs/changelogs/v0.20.md @@ -13,6 +13,7 @@ - [Trace Context support](#trace-context-support) - [Removed legacy features](#removed-legacy-features) - [`--empty-repo` is now the default](#--empty-repo-is-now-the-default) + - [Reminder: `ipfs pubsub` commands and matching HTTP endpoints are deprecated and will be removed](#reminder-ipfs-pubsub-commands-and-matching-http-endpoints-are-deprecated-and-will-be-removed) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -109,6 +110,10 @@ When creating a repository with `ipfs init`, `--empty-repo=true` is now the defa that your repository will be empty by default instead of containing the introduction files. You can read more about the rationale behind this decision on the [tracking issue](https://github.com/ipfs/kubo/issues/9757). +#### Reminder: `ipfs pubsub` commands and matching HTTP endpoints are deprecated and will be removed + +`ipfs pubsub` commands and all `/api/v0/pubsub/` RPC endpoints and will be removed in the next release. For more information and rational see [#9717](https://github.com/ipfs/kubo/issues/9717). + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors From 8418b083d8fa0db0f8a2ad0fb99a45eed9f38406 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 25 Apr 2023 10:47:28 +0200 Subject: [PATCH 119/159] chore: bump to boxo 0.8.1 (#9836) --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index e61559247cc5..6eb1bcb5eb51 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e + github.com/ipfs/boxo v0.8.1 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.1 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 643c6ace007b..2302548a9701 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e h1:8wmBhjwJk2drWZjNwoN7uc+IkG+N93laIhjY69rjMqw= -github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= +github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU= +github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index a7737edef2fa..b83143f9bb18 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e + github.com/ipfs/boxo v0.8.1 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index e060b4e25691..b00e5aed63f7 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e h1:8wmBhjwJk2drWZjNwoN7uc+IkG+N93laIhjY69rjMqw= -github.com/ipfs/boxo v0.8.1-0.20230411232920-5d6c73c8e35e/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= +github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU= +github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= From 196b5c16c00faac96438368f0a0d9f9ef0d7169f Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 8 May 2023 14:20:51 +0200 Subject: [PATCH 120/159] docs(config): remove mentions of relay v1 (#9860) Context: https://github.com/ipfs/interop/pull/592 --- docs/config.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index bb2d8b7022fb..91866a42323c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1622,8 +1622,8 @@ Type: `flag` #### `Swarm.RelayClient.StaticRelays` -Your node will use these statically configured relay servers (V1 or V2) -instead of discovering public relays V2 from the network. +Your node will use these statically configured relay servers +instead of discovering public relays ([Circuit Relay v2](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md)) from the network. Default: `[]` From c178c51835f89f7b8d9e46962fe5e5d3dd3b2a69 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 8 May 2023 16:11:03 +0200 Subject: [PATCH 121/159] fix: deadlock on retrieving WebTransport addresses (#9857) Co-authored-by: Marco Polo --- core/mock/mock.go | 11 +++++++- core/node/groups.go | 2 +- core/node/libp2p/addrs.go | 38 ++++---------------------- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +-- go.mod | 2 +- go.sum | 4 +-- test/cli/harness/peering.go | 4 +-- test/cli/transports_test.go | 23 ++++++++++++++++ 9 files changed, 48 insertions(+), 42 deletions(-) diff --git a/core/mock/mock.go b/core/mock/mock.go index 7b718c43efef..f9e0876f8d1c 100644 --- a/core/mock/mock.go +++ b/core/mock/mock.go @@ -34,7 +34,16 @@ func NewMockNode() (*core.IpfsNode, error) { } func MockHostOption(mn mocknet.Mocknet) libp2p2.HostOption { - return func(id peer.ID, ps pstore.Peerstore, _ ...libp2p.Option) (host.Host, error) { + return func(id peer.ID, ps pstore.Peerstore, opts ...libp2p.Option) (host.Host, error) { + var cfg libp2p.Config + if err := cfg.Apply(opts...); err != nil { + return nil, err + } + + // The mocknet does not use the provided libp2p.Option. This options include + // the listening addresses we want our peer listening on. Therefore, we have + // to manually parse the configuration and add them here. + ps.AddAddrs(id, cfg.ListenAddrs, pstore.PermanentAddrTTL) return mn.AddPeerWithPeerstore(id, ps) } } diff --git a/core/node/groups.go b/core/node/groups.go index ae8ce85399a9..5486788358cc 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -155,7 +155,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part fx.Provide(libp2p.RelayTransport(enableRelayTransport)), fx.Provide(libp2p.RelayService(enableRelayService, cfg.Swarm.RelayService)), fx.Provide(libp2p.Transports(cfg.Swarm.Transports)), - fx.Invoke(libp2p.StartListening(cfg.Addresses.Swarm)), + fx.Provide(libp2p.ListenOn(cfg.Addresses.Swarm)), fx.Invoke(libp2p.SetupDiscovery(cfg.Discovery.MDNS.Enabled)), fx.Provide(libp2p.ForceReachability(cfg.Internal.Libp2pForceReachability)), fx.Provide(libp2p.HolePunching(cfg.Swarm.EnableHolePunching, enableRelayClient)), diff --git a/core/node/libp2p/addrs.go b/core/node/libp2p/addrs.go index 480b045f8521..b287c20ff3bc 100644 --- a/core/node/libp2p/addrs.go +++ b/core/node/libp2p/addrs.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p/core/host" p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" mamask "github.com/whyrusleeping/multiaddr-filter" @@ -99,37 +98,12 @@ func AddrsFactory(announce []string, appendAnnouce []string, noAnnounce []string } } -func listenAddresses(addresses []string) ([]ma.Multiaddr, error) { - listen := make([]ma.Multiaddr, len(addresses)) - for i, addr := range addresses { - maddr, err := ma.NewMultiaddr(addr) - if err != nil { - return nil, fmt.Errorf("failure to parse config.Addresses.Swarm: %s", addresses) - } - listen[i] = maddr - } - - return listen, nil -} - -func StartListening(addresses []string) func(host host.Host) error { - return func(host host.Host) error { - listenAddrs, err := listenAddresses(addresses) - if err != nil { - return err - } - - // Actually start listening: - if err := host.Network().Listen(listenAddrs...); err != nil { - return err - } - - // list out our addresses - addrs, err := host.Network().InterfaceListenAddresses() - if err != nil { - return err +func ListenOn(addresses []string) interface{} { + return func() (opts Libp2pOpts) { + return Libp2pOpts{ + Opts: []libp2p.Option{ + libp2p.ListenAddrStrings(addresses...), + }, } - log.Infof("Swarm listening at: %s", addrs) - return nil } } diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index fd3ad890990d..3c35f0e8434a 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.27.1 + github.com/libp2p/go-libp2p v0.27.2 github.com/multiformats/go-multiaddr v0.9.0 ) diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 15bf75e82943..c3ccc9ae34ad 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -489,8 +489,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.1 h1:k1u6RHsX3hqKnslDjsSgLNURxJ3O1atIZCY4gpMbbus= -github.com/libp2p/go-libp2p v0.27.1/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= +github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= diff --git a/go.mod b/go.mod index 159bbd3e4fd3..6cb0108d8f33 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.27.1 + github.com/libp2p/go-libp2p v0.27.2 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-kbucket v0.5.0 diff --git a/go.sum b/go.sum index 16d312ef5d71..e136c5c00fe3 100644 --- a/go.sum +++ b/go.sum @@ -540,8 +540,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.1 h1:k1u6RHsX3hqKnslDjsSgLNURxJ3O1atIZCY4gpMbbus= -github.com/libp2p/go-libp2p v0.27.1/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= +github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= diff --git a/test/cli/harness/peering.go b/test/cli/harness/peering.go index 3fcd8bb6bdaf..13c3430f0657 100644 --- a/test/cli/harness/peering.go +++ b/test/cli/harness/peering.go @@ -13,7 +13,7 @@ type Peering struct { To int } -func newRandPort() int { +func NewRandPort() int { n := rand.Int() return 3000 + (n % 1000) } @@ -24,7 +24,7 @@ func CreatePeerNodes(t *testing.T, n int, peerings []Peering) (*Harness, Nodes) nodes.ForEachPar(func(node *Node) { node.UpdateConfig(func(cfg *config.Config) { cfg.Routing.Type = config.NewOptionalString("none") - cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", newRandPort())} + cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", NewRandPort())} }) }) diff --git a/test/cli/transports_test.go b/test/cli/transports_test.go index 78f21fd0540a..3a7b5ed933a8 100644 --- a/test/cli/transports_test.go +++ b/test/cli/transports_test.go @@ -1,6 +1,7 @@ package cli import ( + "fmt" "os" "path/filepath" "testing" @@ -124,4 +125,26 @@ func TestTransports(t *testing.T) { runTests(nodes) }) + t.Run("QUIC connects with non-dialable transports", func(t *testing.T) { + // This test targets specific Kubo internals which may change later. This checks + // if we can announce an address we do not listen on, and then are able to connect + // via a different address that is available. + t.Parallel() + nodes := harness.NewT(t).NewNodes(5).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + // We need a specific port to announce so we first generate a random port. + // We can't use 0 here to automatically assign an available port because + // that would only work with Swarm, but not for the announcing. + port := harness.NewRandPort() + quicAddr := fmt.Sprintf("/ip4/127.0.0.1/udp/%d/quic-v1", port) + cfg.Addresses.Swarm = []string{quicAddr} + cfg.Addresses.Announce = []string{quicAddr, quicAddr + "/webtransport"} + }) + }) + disableRouting(nodes) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + } From c39b8052ea39132b230cbffe0b0942956a605b4d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 9 May 2023 10:59:05 +0200 Subject: [PATCH 122/159] chore: update go-libp2p to v0.27.3 (#9862) --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 3c35f0e8434a..6a54fcad394d 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.27.2 + github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 ) diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index c3ccc9ae34ad..1ac6f8afdcbd 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -489,8 +489,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= -github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.3 h1:tkV/zm3KCZ4R5er9Xcs2pt0YNB4JH0iBfGAtHJdLHRs= +github.com/libp2p/go-libp2p v0.27.3/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= diff --git a/go.mod b/go.mod index 6cb0108d8f33..95f326b14b70 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.27.2 + github.com/libp2p/go-libp2p v0.27.3 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-kbucket v0.5.0 diff --git a/go.sum b/go.sum index e136c5c00fe3..6eaf0919e2be 100644 --- a/go.sum +++ b/go.sum @@ -540,8 +540,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= -github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.3 h1:tkV/zm3KCZ4R5er9Xcs2pt0YNB4JH0iBfGAtHJdLHRs= +github.com/libp2p/go-libp2p v0.27.3/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= From 45b31600d841d69e5129a37c1f2f7e5599b508bf Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 9 May 2023 09:12:56 +0000 Subject: [PATCH 123/159] chore: update version --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 7c1681bf0eb6..86ac8dbf4a3f 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,7 @@ import ( var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.20.0-rc2" +const CurrentVersionNumber = "0.20.0" const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint From 9482d7529233ff8942601a875c705fac55376c06 Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Mon, 1 May 2023 15:29:34 -0400 Subject: [PATCH 124/159] fix: use default HTTP routers when FullRT DHT client is used (#9841) --- cmd/ipfs/daemon.go | 14 ++------- core/node/groups.go | 2 +- core/node/libp2p/routing.go | 47 ++++++++++++++++++----------- core/node/libp2p/routingopt.go | 55 ++++++++++++++++++++-------------- 4 files changed, 65 insertions(+), 53 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index ffcb30a34e91..1e03a8264a8e 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -413,19 +413,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment case routingOptionSupernodeKwd: return errors.New("supernode routing was never fully implemented and has been removed") case routingOptionDefaultKwd, routingOptionAutoKwd: - ncfg.Routing = libp2p.ConstructDefaultRouting( - cfg.Identity.PeerID, - cfg.Addresses.Swarm, - cfg.Identity.PrivKey, - libp2p.DHTOption, - ) + ncfg.Routing = libp2p.ConstructDefaultRouting(cfg, libp2p.DHTOption) case routingOptionAutoClientKwd: - ncfg.Routing = libp2p.ConstructDefaultRouting( - cfg.Identity.PeerID, - cfg.Addresses.Swarm, - cfg.Identity.PrivKey, - libp2p.DHTClientOption, - ) + ncfg.Routing = libp2p.ConstructDefaultRouting(cfg, libp2p.DHTClientOption) case routingOptionDHTClientKwd: ncfg.Routing = libp2p.DHTClientOption case routingOptionDHTKwd: diff --git a/core/node/groups.go b/core/node/groups.go index 0b166026596a..ae8ce85399a9 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -165,7 +165,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part fx.Provide(libp2p.Routing), fx.Provide(libp2p.ContentRouting), - fx.Provide(libp2p.BaseRouting(cfg.Experimental.AcceleratedDHTClient)), + fx.Provide(libp2p.BaseRouting(cfg)), maybeProvide(libp2p.PubsubRouter, bcfg.getOpt("ipnsps")), maybeProvide(libp2p.BandwidthCounter, !cfg.Swarm.DisableBandwidthMetrics), diff --git a/core/node/libp2p/routing.go b/core/node/libp2p/routing.go index f96f772e1337..2e356e447f5d 100644 --- a/core/node/libp2p/routing.go +++ b/core/node/libp2p/routing.go @@ -63,35 +63,34 @@ type processInitialRoutingOut struct { type AddrInfoChan chan peer.AddrInfo -func BaseRouting(experimentalDHTClient bool) interface{} { +func BaseRouting(cfg *config.Config) interface{} { return func(lc fx.Lifecycle, in processInitialRoutingIn) (out processInitialRoutingOut, err error) { - var dr *ddht.DHT + var dualDHT *ddht.DHT if dht, ok := in.Router.(*ddht.DHT); ok { - dr = dht + dualDHT = dht lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { - return dr.Close() + return dualDHT.Close() }, }) } - if pr, ok := in.Router.(routinghelpers.ComposableRouter); ok { - for _, r := range pr.Routers() { + if cr, ok := in.Router.(routinghelpers.ComposableRouter); ok { + for _, r := range cr.Routers() { if dht, ok := r.(*ddht.DHT); ok { - dr = dht + dualDHT = dht lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { - return dr.Close() + return dualDHT.Close() }, }) - break } } } - if dr != nil && experimentalDHTClient { + if dualDHT != nil && cfg.Experimental.AcceleratedDHTClient { cfg, err := in.Repo.Config() if err != nil { return out, err @@ -101,7 +100,7 @@ func BaseRouting(experimentalDHTClient bool) interface{} { return out, err } - expClient, err := fullrt.NewFullRT(in.Host, + fullRTClient, err := fullrt.NewFullRT(in.Host, dht.DefaultPrefix, fullrt.DHTOption( dht.Validator(in.Validator), @@ -116,18 +115,30 @@ func BaseRouting(experimentalDHTClient bool) interface{} { lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { - return expClient.Close() + return fullRTClient.Close() }, }) + // we want to also use the default HTTP routers, so wrap the FullRT client + // in a parallel router that calls them in parallel + httpRouters, err := constructDefaultHTTPRouters(cfg) + if err != nil { + return out, err + } + routers := []*routinghelpers.ParallelRouter{ + {Router: fullRTClient}, + } + routers = append(routers, httpRouters...) + router := routinghelpers.NewComposableParallel(routers) + return processInitialRoutingOut{ Router: Router{ - Routing: expClient, Priority: 1000, + Routing: router, }, - DHT: dr, - DHTClient: expClient, - ContentRouter: expClient, + DHT: dualDHT, + DHTClient: fullRTClient, + ContentRouter: fullRTClient, }, nil } @@ -136,8 +147,8 @@ func BaseRouting(experimentalDHTClient bool) interface{} { Priority: 1000, Routing: in.Router, }, - DHT: dr, - DHTClient: dr, + DHT: dualDHT, + DHTClient: dualDHT, ContentRouter: in.Router, }, nil } diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index 1d47ae273088..b363f25b7daf 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -43,8 +43,35 @@ func init() { } } +func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.ParallelRouter, error) { + var routers []*routinghelpers.ParallelRouter + // Append HTTP routers for additional speed + for _, endpoint := range defaultHTTPRouters { + httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, cfg.Addresses.Swarm, cfg.Identity.PrivKey) + if err != nil { + return nil, err + } + + r := &irouting.Composer{ + GetValueRouter: routinghelpers.Null{}, + PutValueRouter: routinghelpers.Null{}, + ProvideRouter: routinghelpers.Null{}, // modify this when indexers supports provide + FindPeersRouter: routinghelpers.Null{}, + FindProvidersRouter: httpRouter, + } + + routers = append(routers, &routinghelpers.ParallelRouter{ + Router: r, + IgnoreError: true, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387 + Timeout: 15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529 + ExecuteAfter: 0, + }) + } + return routers, nil +} + // ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto" -func ConstructDefaultRouting(peerID string, addrs []string, privKey string, routingOpt RoutingOption) RoutingOption { +func ConstructDefaultRouting(cfg *config.Config, routingOpt RoutingOption) RoutingOption { return func(args RoutingOptionArgs) (routing.Routing, error) { // Defined routers will be queried in parallel (optimizing for response speed) // Different trade-offs can be made by setting Routing.Type = "custom" with own Routing.Routers @@ -60,29 +87,13 @@ func ConstructDefaultRouting(peerID string, addrs []string, privKey string, rout ExecuteAfter: 0, }) - // Append HTTP routers for additional speed - for _, endpoint := range defaultHTTPRouters { - httpRouter, err := irouting.ConstructHTTPRouter(endpoint, peerID, addrs, privKey) - if err != nil { - return nil, err - } - - r := &irouting.Composer{ - GetValueRouter: routinghelpers.Null{}, - PutValueRouter: routinghelpers.Null{}, - ProvideRouter: routinghelpers.Null{}, // modify this when indexers supports provide - FindPeersRouter: routinghelpers.Null{}, - FindProvidersRouter: httpRouter, - } - - routers = append(routers, &routinghelpers.ParallelRouter{ - Router: r, - IgnoreError: true, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387 - Timeout: 15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529 - ExecuteAfter: 0, - }) + httpRouters, err := constructDefaultHTTPRouters(cfg) + if err != nil { + return nil, err } + routers = append(routers, httpRouters...) + routing := routinghelpers.NewComposableParallel(routers) return routing, nil } From b4504be54c6f7265aa5473f5432b801b84c5dec4 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 2 May 2023 04:16:52 -0700 Subject: [PATCH 125/159] feat: webui@3.0.0 (#9835) --- core/corehttp/webui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/corehttp/webui.go b/core/corehttp/webui.go index f7b27c93914f..9b7cf776d566 100644 --- a/core/corehttp/webui.go +++ b/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeifeqt7mvxaniphyu2i3qhovjaf3sayooxbh5enfdqtiehxjv2ldte" // v2.22.0 +const WebUIPath = "/ipfs/bafybeic4gops3d3lyrisqku37uio33nvt6fqxvkxihrwlqsuvf76yln4fm" // v3.0.0 // WebUIPaths is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeifeqt7mvxaniphyu2i3qhovjaf3sayooxbh5enfdqtiehxjv2ldte", "/ipfs/bafybeiequgo72mrvuml56j4gk7crewig5bavumrrzhkqbim6b3s2yqi7ty", "/ipfs/bafybeibjbq3tmmy7wuihhhwvbladjsd3gx3kfjepxzkq6wylik6wc3whzy", "/ipfs/bafybeiavrvt53fks6u32n5p2morgblcmck4bh4ymf4rrwu7ah5zsykmqqa", From d2c578878faf63ec2633788c2424e5b24edfbbf4 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 3 May 2023 09:00:41 +0200 Subject: [PATCH 126/159] docs: add changelog for v0.19.2 --- docs/changelogs/v0.19.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/changelogs/v0.19.md b/docs/changelogs/v0.19.md index 1abc2e518b2a..f7e190a7e9c0 100644 --- a/docs/changelogs/v0.19.md +++ b/docs/changelogs/v0.19.md @@ -1,6 +1,33 @@ # Kubo changelog v0.19 +## v0.19.2 + +### Highlights + +#### FullRT DHT HTTP Routers + +The default HTTP routers are now used when the FullRT DHT client is used. This fixes +the issue where cid.contact is not being queried by default when the accelerated +DHT client was enabled. Read more in ([ipfs/kubo#9841](https://github.com/ipfs/kubo/pull/9841)). + +### Changelog + +
Full Changelog + +- github.com/ipfs/kubo: + - fix: use default HTTP routers when FullRT DHT client is used (#9841) ([ipfs/kubo#9841](https://github.com/ipfs/kubo/pull/9841)) + - chore: update version + +
+ +### Contributors + +| Contributor | Commits | Lines ยฑ | Files Changed | +|-------------|---------|---------|---------------| +| Gus Eggert | 1 | +65/-53 | 4 | +| Henrique Dias | 1 | +1/-1 | 1 | + ## v0.19.1 ### ๐Ÿ”ฆ Highlights From d1b0083d604e0a4b46bcdc17f03b950744b9bc55 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 3 May 2023 09:08:04 +0200 Subject: [PATCH 127/159] Merge pull request #9847 from ipfs/release-v0.19.2 Release: v0.19.2 From 5f5c16e2cc7a42ca64f2ad31bd46591a5a345c79 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 3 May 2023 12:57:21 +0200 Subject: [PATCH 128/159] Merge branch 'master' into merge-release-v0.19.2 --- .github/CODEOWNERS | 3 + test/cli/testutils/random_files.go | 116 ++++++++++++ test/cli/transports_test.go | 127 +++++++++++++ test/ipfs-test-lib.sh | 12 +- ...-docker-image.sh => t0002-docker-image.sh} | 16 +- ...ker-migrate.sh => t0003-docker-migrate.sh} | 9 +- test/sharness/t0125-twonode.sh | 178 ------------------ test/sharness/t0130-multinode.sh | 115 ----------- 8 files changed, 261 insertions(+), 315 deletions(-) create mode 100644 test/cli/testutils/random_files.go create mode 100644 test/cli/transports_test.go rename test/sharness/{t0300-docker-image.sh => t0002-docker-image.sh} (87%) rename test/sharness/{t0301-docker-migrate.sh => t0003-docker-migrate.sh} (92%) delete mode 100755 test/sharness/t0125-twonode.sh delete mode 100755 test/sharness/t0130-multinode.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5151ad178ae8..7d17855e03c0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,6 +2,9 @@ # request that modifies code that they own. Code owners are not automatically # requested to review draft pull requests. +# Default +* @ipfs/kubo-maintainers + # HTTP Gateway core/corehttp/ @lidel test/sharness/*gateway*.sh @lidel diff --git a/test/cli/testutils/random_files.go b/test/cli/testutils/random_files.go new file mode 100644 index 000000000000..0e550a125e3f --- /dev/null +++ b/test/cli/testutils/random_files.go @@ -0,0 +1,116 @@ +package testutils + +import ( + "fmt" + "io" + "math/rand" + "os" + "path" + "time" +) + +var AlphabetEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_") +var AlphabetHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ") + +type RandFiles struct { + Rand *rand.Rand + FileSize int // the size per file. + FilenameSize int + Alphabet []rune // for filenames + + FanoutDepth int // how deep the hierarchy goes + FanoutFiles int // how many files per dir + FanoutDirs int // how many dirs per dir + + RandomSize bool // randomize file sizes + RandomFanout bool // randomize fanout numbers +} + +func NewRandFiles() *RandFiles { + return &RandFiles{ + Rand: rand.New(rand.NewSource(time.Now().UnixNano())), + FileSize: 4096, + FilenameSize: 16, + Alphabet: AlphabetEasy, + FanoutDepth: 2, + FanoutDirs: 5, + FanoutFiles: 10, + RandomSize: true, + } +} + +func (r *RandFiles) WriteRandomFiles(root string, depth int) error { + numfiles := r.FanoutFiles + if r.RandomFanout { + numfiles = rand.Intn(r.FanoutFiles) + 1 + } + + for i := 0; i < numfiles; i++ { + if err := r.WriteRandomFile(root); err != nil { + return err + } + } + + if depth+1 <= r.FanoutDepth { + numdirs := r.FanoutDirs + if r.RandomFanout { + numdirs = r.Rand.Intn(numdirs) + 1 + } + + for i := 0; i < numdirs; i++ { + if err := r.WriteRandomDir(root, depth+1); err != nil { + return err + } + } + } + + return nil +} + +func (r *RandFiles) RandomFilename(length int) string { + b := make([]rune, length) + for i := range b { + b[i] = r.Alphabet[r.Rand.Intn(len(r.Alphabet))] + } + return string(b) +} + +func (r *RandFiles) WriteRandomFile(root string) error { + filesize := int64(r.FileSize) + if r.RandomSize { + filesize = r.Rand.Int63n(filesize) + 1 + } + + n := rand.Intn(r.FilenameSize-4) + 4 + name := r.RandomFilename(n) + filepath := path.Join(root, name) + f, err := os.Create(filepath) + if err != nil { + return fmt.Errorf("creating random file: %w", err) + } + + if _, err := io.CopyN(f, r.Rand, filesize); err != nil { + return fmt.Errorf("copying random file: %w", err) + } + + return f.Close() +} + +func (r *RandFiles) WriteRandomDir(root string, depth int) error { + if depth > r.FanoutDepth { + return nil + } + + n := rand.Intn(r.FilenameSize-4) + 4 + name := r.RandomFilename(n) + root = path.Join(root, name) + if err := os.MkdirAll(root, 0755); err != nil { + return fmt.Errorf("creating random dir: %w", err) + } + + err := r.WriteRandomFiles(root, depth) + if err != nil { + return fmt.Errorf("writing random files in random dir: %w", err) + } + return nil +} diff --git a/test/cli/transports_test.go b/test/cli/transports_test.go new file mode 100644 index 000000000000..78f21fd0540a --- /dev/null +++ b/test/cli/transports_test.go @@ -0,0 +1,127 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/test/cli/harness" + "github.com/ipfs/kubo/test/cli/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTransports(t *testing.T) { + disableRouting := func(nodes harness.Nodes) { + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Routing.Type = config.NewOptionalString("none") + cfg.Bootstrap = nil + }) + }) + } + checkSingleFile := func(nodes harness.Nodes) { + s := testutils.RandomStr(100) + hash := nodes[0].IPFSAddStr(s) + nodes.ForEachPar(func(n *harness.Node) { + val := n.IPFS("cat", hash).Stdout.String() + assert.Equal(t, s, val) + }) + } + checkRandomDir := func(nodes harness.Nodes) { + randDir := filepath.Join(nodes[0].Dir, "foobar") + require.NoError(t, os.Mkdir(randDir, 0777)) + rf := testutils.NewRandFiles() + rf.FanoutDirs = 3 + rf.FanoutFiles = 6 + require.NoError(t, rf.WriteRandomFiles(randDir, 4)) + + hash := nodes[1].IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed() + nodes.ForEachPar(func(n *harness.Node) { + res := n.RunIPFS("refs", "-r", hash) + assert.Equal(t, 0, res.ExitCode()) + }) + } + + runTests := func(nodes harness.Nodes) { + checkSingleFile(nodes) + checkRandomDir(nodes) + } + + tcpNodes := func(t *testing.T) harness.Nodes { + nodes := harness.NewT(t).NewNodes(2).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/tcp/0"} + cfg.Swarm.Transports.Network.QUIC = config.False + cfg.Swarm.Transports.Network.Relay = config.False + cfg.Swarm.Transports.Network.WebTransport = config.False + cfg.Swarm.Transports.Network.Websocket = config.False + }) + }) + disableRouting(nodes) + return nodes + } + + t.Run("tcp", func(t *testing.T) { + t.Parallel() + nodes := tcpNodes(t).StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("tcp with mplex", func(t *testing.T) { + t.Parallel() + nodes := tcpNodes(t) + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Swarm.Transports.Multiplexers.Yamux = config.Disabled + }) + }) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("tcp with NOISE", func(t *testing.T) { + t.Parallel() + nodes := tcpNodes(t) + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Swarm.Transports.Security.TLS = config.Disabled + }) + }) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("QUIC", func(t *testing.T) { + t.Parallel() + nodes := harness.NewT(t).NewNodes(5).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1"} + cfg.Swarm.Transports.Network.QUIC = config.True + cfg.Swarm.Transports.Network.TCP = config.False + }) + }) + disableRouting(nodes) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + + t.Run("QUIC", func(t *testing.T) { + t.Parallel() + nodes := harness.NewT(t).NewNodes(5).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1/webtransport"} + cfg.Swarm.Transports.Network.QUIC = config.True + cfg.Swarm.Transports.Network.WebTransport = config.True + }) + }) + disableRouting(nodes) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + +} diff --git a/test/ipfs-test-lib.sh b/test/ipfs-test-lib.sh index eabf17020d22..a3bc32bbbd3c 100644 --- a/test/ipfs-test-lib.sh +++ b/test/ipfs-test-lib.sh @@ -50,19 +50,19 @@ test_path_cmp() { # Docker -# This takes a Dockerfile, and a build context directory +# This takes a Dockerfile, a tag name, and a build context directory docker_build() { - docker build --rm -f "$1" "$2" | ansi_strip + docker build --rm --tag "$1" --file "$2" "$3" | ansi_strip } # This takes an image as argument and writes a docker ID on stdout docker_run() { - docker run -d "$1" + docker run --detach "$1" } # This takes a docker ID and a command as arguments docker_exec() { - docker exec -t "$1" /bin/sh -c "$2" + docker exec --tty "$1" /bin/sh -c "$2" } # This takes a docker ID as argument @@ -72,12 +72,12 @@ docker_stop() { # This takes a docker ID as argument docker_rm() { - docker rm -f -v "$1" > /dev/null + docker rm --force --volumes "$1" > /dev/null } # This takes a docker image name as argument docker_rmi() { - docker rmi -f "$1" > /dev/null + docker rmi --force "$1" > /dev/null } # Test whether all the expected lines are included in a file. The file diff --git a/test/sharness/t0300-docker-image.sh b/test/sharness/t0002-docker-image.sh similarity index 87% rename from test/sharness/t0300-docker-image.sh rename to test/sharness/t0002-docker-image.sh index 3c390a4535f7..11ccf01b74e5 100755 --- a/test/sharness/t0300-docker-image.sh +++ b/test/sharness/t0002-docker-image.sh @@ -27,18 +27,12 @@ TEST_TRASH_DIR=$(pwd) TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR") TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") +IMAGE_TAG=kubo_test test_expect_success "docker image build succeeds" ' - docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" | tee build-actual || + docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" || test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" || - test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" || - test_fsh cat build-actual -' - -test_expect_success "docker image build output looks good" ' - SUCCESS_LINE=$(egrep "^Successfully built" build-actual) && - IMAGE_ID=$(expr "$SUCCESS_LINE" : "^Successfully built \(.*\)") || - test_fsh cat build-actual + test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" ' test_expect_success "write init scripts" ' @@ -52,7 +46,7 @@ test_expect_success "docker image runs" ' -p 127.0.0.1:5001:5001 -p 127.0.0.1:8080:8080 \ -v "$PWD/001.sh":/container-init.d/001.sh \ -v "$PWD/002.sh":/container-init.d/002.sh \ - "$IMAGE_ID") + "$IMAGE_TAG") ' test_expect_success "docker container gateway is up" ' @@ -100,5 +94,5 @@ test_expect_success "stop docker container" ' ' docker_rm "$DOC_ID" -docker_rmi "$IMAGE_ID" +docker_rmi "$IMAGE_TAG" test_done diff --git a/test/sharness/t0301-docker-migrate.sh b/test/sharness/t0003-docker-migrate.sh similarity index 92% rename from test/sharness/t0301-docker-migrate.sh rename to test/sharness/t0003-docker-migrate.sh index 3f7d32f214b1..ac3c7aee2bcd 100755 --- a/test/sharness/t0301-docker-migrate.sh +++ b/test/sharness/t0003-docker-migrate.sh @@ -24,10 +24,10 @@ TEST_TRASH_DIR=$(pwd) TEST_SCRIPTS_DIR=$(dirname "$TEST_TRASH_DIR") TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") +IMAGE_TAG=kubo_migrate test_expect_success "docker image build succeeds" ' - docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >actual && - IMAGE_ID=$(tail -n1 actual | cut -d " " -f 3) + docker_build "$IMAGE_TAG" "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" ' test_init_ipfs @@ -53,7 +53,7 @@ test_expect_success "startup fake dists server" ' ' test_expect_success "docker image runs" ' - DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_ID") + DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_TAG") ' test_expect_success "docker container tries to pull migrations from netcat" ' @@ -78,6 +78,5 @@ test_expect_success "correct version was requested" ' ' docker_rm "$DOC_ID" -docker_rmi "$IMAGE_ID" +docker_rmi "$IMAGE_TAG" test_done - diff --git a/test/sharness/t0125-twonode.sh b/test/sharness/t0125-twonode.sh deleted file mode 100755 index 8ea1c9e5d1bd..000000000000 --- a/test/sharness/t0125-twonode.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017 Jeromy Johnson -# MIT Licensed; see the LICENSE file in this repository. -# - -test_description="Test two ipfs nodes transferring a file" - -. lib/test-lib.sh - -check_file_fetch() { - node=$1 - fhash=$2 - fname=$3 - - test_expect_success "can fetch file" ' - ipfsi $node cat $fhash > fetch_out - ' - - test_expect_success "file looks good" ' - test_cmp $fname fetch_out - ' -} - -check_dir_fetch() { - node=$1 - ref=$2 - - test_expect_success "node can fetch all refs for dir" ' - ipfsi $node refs -r $ref > /dev/null - ' -} - -run_single_file_test() { - test_expect_success "add a file on node1" ' - random 1000000 > filea && - FILEA_HASH=$(ipfsi 1 add -q filea) - ' - - check_file_fetch 0 $FILEA_HASH filea -} - -run_random_dir_test() { - test_expect_success "create a bunch of random files" ' - random-files -depth=3 -dirs=4 -files=5 -seed=5 foobar > /dev/null - ' - - test_expect_success "add those on node 0" ' - DIR_HASH=$(ipfsi 0 add -r -Q foobar) - ' - - check_dir_fetch 1 $DIR_HASH -} - -flaky_advanced_test() { - startup_cluster 2 "$@" - - test_expect_success "clean repo before test" ' - ipfsi 0 repo gc > /dev/null && - ipfsi 1 repo gc > /dev/null - ' - - run_single_file_test - - run_random_dir_test - - test_expect_success "gather bitswap stats" ' - ipfsi 0 bitswap stat -v > stat0 && - ipfsi 1 bitswap stat -v > stat1 - ' - - test_expect_success "shut down nodes" ' - iptb stop && iptb_wait_stop - ' - - # NOTE: data transferred stats checks are flaky - # trying to debug them by printing out the stats hides the flakiness - # my theory is that the extra time cat calls take to print out the stats - # allow for proper cleanup to happen - go-sleep 1s -} - -run_advanced_test() { - # TODO: investigate why flaky_advanced_test is flaky - # Context: https://github.com/ipfs/kubo/pull/9486 - # sometimes, bitswap status returns unexpected block transfers - # and everyone has been re-running circleci until is passes for at least a year. - # this re-runs test until it passes or a timeout hits - - BLOCKS_0=126 - BLOCKS_1=5 - DATA_0=228113 - DATA_1=1000256 - for i in $(test_seq 1 600); do - flaky_advanced_test - (grep -q "$DATA_0" stat0 && grep -q "$DATA_1" stat1) && break - go-sleep 100ms - done - - test_expect_success "node0 data transferred looks correct" ' - test_should_contain "blocks sent: $BLOCKS_0" stat0 && - test_should_contain "blocks received: $BLOCKS_1" stat0 && - test_should_contain "data sent: $DATA_0" stat0 && - test_should_contain "data received: $DATA_1" stat0 - ' - - test_expect_success "node1 data transferred looks correct" ' - test_should_contain "blocks received: $BLOCKS_0" stat1 && - test_should_contain "blocks sent: $BLOCKS_1" stat1 && - test_should_contain "data received: $DATA_0" stat1 && - test_should_contain "data sent: $DATA_1" stat1 - ' - -} - -test_expect_success "set up tcp testbed" ' - iptb testbed create -type localipfs -count 2 -force -init -' - -test_expect_success "disable routing, use direct peering" ' - iptb run -- ipfs config Routing.Type none && - iptb run -- ipfs config --json Bootstrap "[]" -' - -# Test TCP transport -echo "Testing TCP" -addrs='"[\"/ip4/127.0.0.1/tcp/0\"]"' -test_expect_success "use TCP only" ' - iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' && - iptb run -- ipfs config --json Swarm.Transports.Network.QUIC false && - iptb run -- ipfs config --json Swarm.Transports.Network.Relay false && - iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport false && - iptb run -- ipfs config --json Swarm.Transports.Network.Websocket false -' -run_advanced_test - -# test multiplex muxer -echo "Running TCP tests with mplex" -test_expect_success "disable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false -' -run_advanced_test - -test_expect_success "re-enable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux null -' -# test Noise -echo "Running TCP tests with NOISE" -test_expect_success "use noise only" ' - iptb run -- ipfs config --json Swarm.Transports.Security.TLS false -' -run_advanced_test - -test_expect_success "re-enable TLS" ' - iptb run -- ipfs config --json Swarm.Transports.Security.TLS null -' - -# test QUIC -echo "Running advanced tests over QUIC" -addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1\"]"' -test_expect_success "use QUIC only" ' - iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' && - iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true && - iptb run -- ipfs config --json Swarm.Transports.Network.TCP false -' -run_advanced_test - -# test WebTransport -echo "Running advanced tests over WebTransport" -addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1/webtransport\"]"' -test_expect_success "use WebTransport only" ' - iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' && - iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true && - iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport true -' -run_advanced_test - -test_done diff --git a/test/sharness/t0130-multinode.sh b/test/sharness/t0130-multinode.sh deleted file mode 100755 index 04746850bb12..000000000000 --- a/test/sharness/t0130-multinode.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2015 Jeromy Johnson -# MIT Licensed; see the LICENSE file in this repository. -# - -test_description="Test multiple ipfs nodes" - -. lib/test-lib.sh - -check_file_fetch() { - node=$1 - fhash=$2 - fname=$3 - - test_expect_success "can fetch file" ' - ipfsi $node cat $fhash > fetch_out - ' - - test_expect_success "file looks good" ' - test_cmp $fname fetch_out - ' -} - -check_dir_fetch() { - node=$1 - ref=$2 - - test_expect_success "node can fetch all refs for dir" ' - ipfsi $node refs -r $ref > /dev/null - ' -} - -run_single_file_test() { - test_expect_success "add a file on node1" ' - random 1000000 > filea && - FILEA_HASH=$(ipfsi 1 add -q filea) - ' - - check_file_fetch 4 $FILEA_HASH filea - check_file_fetch 3 $FILEA_HASH filea - check_file_fetch 2 $FILEA_HASH filea - check_file_fetch 1 $FILEA_HASH filea - check_file_fetch 0 $FILEA_HASH filea -} - -run_random_dir_test() { - test_expect_success "create a bunch of random files" ' - random-files -depth=4 -dirs=3 -files=6 foobar > /dev/null - ' - - test_expect_success "add those on node 2" ' - DIR_HASH=$(ipfsi 2 add -r -Q foobar) - ' - - check_dir_fetch 0 $DIR_HASH - check_dir_fetch 1 $DIR_HASH - check_dir_fetch 2 $DIR_HASH - check_dir_fetch 3 $DIR_HASH - check_dir_fetch 4 $DIR_HASH -} - - -run_basic_test() { - startup_cluster 5 - - run_single_file_test - - test_expect_success "shut down nodes" ' - iptb stop && iptb_wait_stop - ' -} - -run_advanced_test() { - startup_cluster 5 "$@" - - run_single_file_test - - run_random_dir_test - - test_expect_success "shut down nodes" ' - iptb stop && iptb_wait_stop || - test_fsh tail -n +1 .iptb/testbeds/default/*/daemon.std* - ' -} - -test_expect_success "set up /tcp testbed" ' - iptb testbed create -type localipfs -count 5 -force -init -' - -# test default configuration -run_advanced_test - -# test multiplex muxer -test_expect_success "disable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false -' -run_advanced_test - -test_expect_success "set up /ws testbed" ' - iptb testbed create -type localipfs -count 5 -attr listentype,ws -force -init -' - -# test default configuration -run_advanced_test - -# test multiplex muxer -test_expect_success "disable yamux" ' - iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false -' - -run_advanced_test - - -test_done From d1c94776bbed16c24b6ec435d1384a531878fda6 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 3 May 2023 14:15:14 +0200 Subject: [PATCH 129/159] Merge pull request #9848 from ipfs/merge-release-v0.19.2 Merge Release: v0.19.2 From 548d082db4b76f2170e34d2de34c7568b627b059 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 8 May 2023 14:20:51 +0200 Subject: [PATCH 130/159] docs(config): remove mentions of relay v1 (#9860) Context: https://github.com/ipfs/interop/pull/592 --- docs/config.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index bb2d8b7022fb..91866a42323c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1622,8 +1622,8 @@ Type: `flag` #### `Swarm.RelayClient.StaticRelays` -Your node will use these statically configured relay servers (V1 or V2) -instead of discovering public relays V2 from the network. +Your node will use these statically configured relay servers +instead of discovering public relays ([Circuit Relay v2](https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md)) from the network. Default: `[]` From 8684f05ef376f5971b8df6f67e9aaddccdba5bed Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 8 May 2023 16:11:03 +0200 Subject: [PATCH 131/159] fix: deadlock on retrieving WebTransport addresses (#9857) Co-authored-by: Marco Polo --- core/mock/mock.go | 11 +++++++- core/node/groups.go | 2 +- core/node/libp2p/addrs.go | 38 ++++---------------------- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +-- go.mod | 2 +- go.sum | 4 +-- test/cli/harness/peering.go | 4 +-- test/cli/transports_test.go | 23 ++++++++++++++++ 9 files changed, 48 insertions(+), 42 deletions(-) diff --git a/core/mock/mock.go b/core/mock/mock.go index 7b718c43efef..f9e0876f8d1c 100644 --- a/core/mock/mock.go +++ b/core/mock/mock.go @@ -34,7 +34,16 @@ func NewMockNode() (*core.IpfsNode, error) { } func MockHostOption(mn mocknet.Mocknet) libp2p2.HostOption { - return func(id peer.ID, ps pstore.Peerstore, _ ...libp2p.Option) (host.Host, error) { + return func(id peer.ID, ps pstore.Peerstore, opts ...libp2p.Option) (host.Host, error) { + var cfg libp2p.Config + if err := cfg.Apply(opts...); err != nil { + return nil, err + } + + // The mocknet does not use the provided libp2p.Option. This options include + // the listening addresses we want our peer listening on. Therefore, we have + // to manually parse the configuration and add them here. + ps.AddAddrs(id, cfg.ListenAddrs, pstore.PermanentAddrTTL) return mn.AddPeerWithPeerstore(id, ps) } } diff --git a/core/node/groups.go b/core/node/groups.go index ae8ce85399a9..5486788358cc 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -155,7 +155,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part fx.Provide(libp2p.RelayTransport(enableRelayTransport)), fx.Provide(libp2p.RelayService(enableRelayService, cfg.Swarm.RelayService)), fx.Provide(libp2p.Transports(cfg.Swarm.Transports)), - fx.Invoke(libp2p.StartListening(cfg.Addresses.Swarm)), + fx.Provide(libp2p.ListenOn(cfg.Addresses.Swarm)), fx.Invoke(libp2p.SetupDiscovery(cfg.Discovery.MDNS.Enabled)), fx.Provide(libp2p.ForceReachability(cfg.Internal.Libp2pForceReachability)), fx.Provide(libp2p.HolePunching(cfg.Swarm.EnableHolePunching, enableRelayClient)), diff --git a/core/node/libp2p/addrs.go b/core/node/libp2p/addrs.go index 480b045f8521..b287c20ff3bc 100644 --- a/core/node/libp2p/addrs.go +++ b/core/node/libp2p/addrs.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p/core/host" p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic" ma "github.com/multiformats/go-multiaddr" mamask "github.com/whyrusleeping/multiaddr-filter" @@ -99,37 +98,12 @@ func AddrsFactory(announce []string, appendAnnouce []string, noAnnounce []string } } -func listenAddresses(addresses []string) ([]ma.Multiaddr, error) { - listen := make([]ma.Multiaddr, len(addresses)) - for i, addr := range addresses { - maddr, err := ma.NewMultiaddr(addr) - if err != nil { - return nil, fmt.Errorf("failure to parse config.Addresses.Swarm: %s", addresses) - } - listen[i] = maddr - } - - return listen, nil -} - -func StartListening(addresses []string) func(host host.Host) error { - return func(host host.Host) error { - listenAddrs, err := listenAddresses(addresses) - if err != nil { - return err - } - - // Actually start listening: - if err := host.Network().Listen(listenAddrs...); err != nil { - return err - } - - // list out our addresses - addrs, err := host.Network().InterfaceListenAddresses() - if err != nil { - return err +func ListenOn(addresses []string) interface{} { + return func() (opts Libp2pOpts) { + return Libp2pOpts{ + Opts: []libp2p.Option{ + libp2p.ListenAddrStrings(addresses...), + }, } - log.Infof("Swarm listening at: %s", addrs) - return nil } } diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 6eb1bcb5eb51..ae1e3b796a6f 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.8.1 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.27.1 + github.com/libp2p/go-libp2p v0.27.2 github.com/multiformats/go-multiaddr v0.9.0 ) diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 2302548a9701..9fc6e5a2937d 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -489,8 +489,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.1 h1:k1u6RHsX3hqKnslDjsSgLNURxJ3O1atIZCY4gpMbbus= -github.com/libp2p/go-libp2p v0.27.1/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= +github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= diff --git a/go.mod b/go.mod index b83143f9bb18..d9310c7bec14 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.27.1 + github.com/libp2p/go-libp2p v0.27.2 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-kbucket v0.5.0 diff --git a/go.sum b/go.sum index b00e5aed63f7..8ca1f10b9615 100644 --- a/go.sum +++ b/go.sum @@ -540,8 +540,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.1 h1:k1u6RHsX3hqKnslDjsSgLNURxJ3O1atIZCY4gpMbbus= -github.com/libp2p/go-libp2p v0.27.1/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= +github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= diff --git a/test/cli/harness/peering.go b/test/cli/harness/peering.go index 3fcd8bb6bdaf..13c3430f0657 100644 --- a/test/cli/harness/peering.go +++ b/test/cli/harness/peering.go @@ -13,7 +13,7 @@ type Peering struct { To int } -func newRandPort() int { +func NewRandPort() int { n := rand.Int() return 3000 + (n % 1000) } @@ -24,7 +24,7 @@ func CreatePeerNodes(t *testing.T, n int, peerings []Peering) (*Harness, Nodes) nodes.ForEachPar(func(node *Node) { node.UpdateConfig(func(cfg *config.Config) { cfg.Routing.Type = config.NewOptionalString("none") - cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", newRandPort())} + cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", NewRandPort())} }) }) diff --git a/test/cli/transports_test.go b/test/cli/transports_test.go index 78f21fd0540a..3a7b5ed933a8 100644 --- a/test/cli/transports_test.go +++ b/test/cli/transports_test.go @@ -1,6 +1,7 @@ package cli import ( + "fmt" "os" "path/filepath" "testing" @@ -124,4 +125,26 @@ func TestTransports(t *testing.T) { runTests(nodes) }) + t.Run("QUIC connects with non-dialable transports", func(t *testing.T) { + // This test targets specific Kubo internals which may change later. This checks + // if we can announce an address we do not listen on, and then are able to connect + // via a different address that is available. + t.Parallel() + nodes := harness.NewT(t).NewNodes(5).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + // We need a specific port to announce so we first generate a random port. + // We can't use 0 here to automatically assign an available port because + // that would only work with Swarm, but not for the announcing. + port := harness.NewRandPort() + quicAddr := fmt.Sprintf("/ip4/127.0.0.1/udp/%d/quic-v1", port) + cfg.Addresses.Swarm = []string{quicAddr} + cfg.Addresses.Announce = []string{quicAddr, quicAddr + "/webtransport"} + }) + }) + disableRouting(nodes) + nodes.StartDaemons().Connect() + runTests(nodes) + }) + } From 34653542f6c16a7aac1e77c47ba27ecbc152aebf Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 9 May 2023 10:59:05 +0200 Subject: [PATCH 132/159] chore: update go-libp2p to v0.27.3 (#9862) --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index ae1e3b796a6f..240bd2402e93 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.8.1 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.27.2 + github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 ) diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 9fc6e5a2937d..c1e617c731bd 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -489,8 +489,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= -github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.3 h1:tkV/zm3KCZ4R5er9Xcs2pt0YNB4JH0iBfGAtHJdLHRs= +github.com/libp2p/go-libp2p v0.27.3/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= diff --git a/go.mod b/go.mod index d9310c7bec14..be52c6d65b72 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.27.2 + github.com/libp2p/go-libp2p v0.27.3 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-kbucket v0.5.0 diff --git a/go.sum b/go.sum index 8ca1f10b9615..cad142edc398 100644 --- a/go.sum +++ b/go.sum @@ -540,8 +540,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.27.2 h1:I1fxqxdm/O0TFoAZKje8wSMu9tfLlLdzTQvgT3HA6v0= -github.com/libp2p/go-libp2p v0.27.2/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p v0.27.3 h1:tkV/zm3KCZ4R5er9Xcs2pt0YNB4JH0iBfGAtHJdLHRs= +github.com/libp2p/go-libp2p v0.27.3/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= From 3a6fa1e2290aea790d4810c755b3f9c05e69b686 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 9 May 2023 09:31:11 +0000 Subject: [PATCH 133/159] chore: update changelog for v0.20 --- docs/changelogs/v0.20.md | 472 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 472 insertions(+) diff --git a/docs/changelogs/v0.20.md b/docs/changelogs/v0.20.md index 590ef5453a5a..3a6ce8f64851 100644 --- a/docs/changelogs/v0.20.md +++ b/docs/changelogs/v0.20.md @@ -116,6 +116,478 @@ You can read more about the rationale behind this decision on the [tracking issu ### ๐Ÿ“ Changelog +
Full Changelog + +- github.com/ipfs/kubo: + - fix: deadlock on retrieving WebTransport addresses (#9857) ([ipfs/kubo#9857](https://github.com/ipfs/kubo/pull/9857)) + - docs(config): remove mentions of relay v1 (#9860) ([ipfs/kubo#9860](https://github.com/ipfs/kubo/pull/9860)) + - Merge branch 'master' into merge-release-v0.19.2 + - docs: add changelog for v0.19.2 + - feat: webui@3.0.0 (#9835) ([ipfs/kubo#9835](https://github.com/ipfs/kubo/pull/9835)) + - fix: use default HTTP routers when FullRT DHT client is used (#9841) ([ipfs/kubo#9841](https://github.com/ipfs/kubo/pull/9841)) + - chore: update version + - docs: add `ipfs pubsub` deprecation reminder to changelog (#9827) ([ipfs/kubo#9827](https://github.com/ipfs/kubo/pull/9827)) + - docs: preparing 0.20 changelog for release (#9799) ([ipfs/kubo#9799](https://github.com/ipfs/kubo/pull/9799)) + - feat: boxo tracing and traceparent support (#9811) ([ipfs/kubo#9811](https://github.com/ipfs/kubo/pull/9811)) + - chore: update version + - chore: update version + - update go-libp2p to v0.27.0 + - docs: add optimistic provide feature description + - feat: add experimental optimistic provide + - fix(ci): speed up docker build (#9800) ([ipfs/kubo#9800](https://github.com/ipfs/kubo/pull/9800)) + - feat(tracing): use OTEL_PROPAGATORS as per OTel spec (#9801) ([ipfs/kubo#9801](https://github.com/ipfs/kubo/pull/9801)) + - docs: fix jaeger command (#9797) ([ipfs/kubo#9797](https://github.com/ipfs/kubo/pull/9797)) + - Merge Release: v0.19.1 (#9794) ([ipfs/kubo#9794](https://github.com/ipfs/kubo/pull/9794)) + - chore: upgrade OpenTelemetry dependencies (#9736) ([ipfs/kubo#9736](https://github.com/ipfs/kubo/pull/9736)) + - test: fix flaky content routing over HTTP test (#9772) ([ipfs/kubo#9772](https://github.com/ipfs/kubo/pull/9772)) + - feat: allow injecting custom path resolvers (#9750) ([ipfs/kubo#9750](https://github.com/ipfs/kubo/pull/9750)) + - feat: add changelog entry for router timeouts for v0.19.1 (#9784) ([ipfs/kubo#9784](https://github.com/ipfs/kubo/pull/9784)) + - feat(gw): new metrics and HTTP range support (#9786) ([ipfs/kubo#9786](https://github.com/ipfs/kubo/pull/9786)) + - feat!: make --empty-repo default (#9758) ([ipfs/kubo#9758](https://github.com/ipfs/kubo/pull/9758)) + - fix: remove timeout on default DHT operations (#9783) ([ipfs/kubo#9783](https://github.com/ipfs/kubo/pull/9783)) + - refactor: switch gateway code to new API from go-libipfs (#9681) ([ipfs/kubo#9681](https://github.com/ipfs/kubo/pull/9681)) + - test: port remote pinning tests to Go (#9720) ([ipfs/kubo#9720](https://github.com/ipfs/kubo/pull/9720)) + - feat: add identify option to swarm peers command + - test: port routing DHT tests to Go (#9709) ([ipfs/kubo#9709](https://github.com/ipfs/kubo/pull/9709)) + - test: fix autoclient flakiness (#9769) ([ipfs/kubo#9769](https://github.com/ipfs/kubo/pull/9769)) + - test: skip flaky pubsub test (#9770) ([ipfs/kubo#9770](https://github.com/ipfs/kubo/pull/9770)) + - chore: migrate go-libipfs to boxo + - feat: add tracing to the commands client + - feat: add client-side metrics for routing-v1 client + - test: increase max wait time for peering assertion + - feat: remove writable gateway (#9743) ([ipfs/kubo#9743](https://github.com/ipfs/kubo/pull/9743)) + - Process Improvement: v0.18.0 ([ipfs/kubo#9484](https://github.com/ipfs/kubo/pull/9484)) + - fix: deadlock while racing `ipfs dag import` and `ipfs repo gc` + - feat: improve dag/import (#9721) ([ipfs/kubo#9721](https://github.com/ipfs/kubo/pull/9721)) + - ci: remove circleci config ([ipfs/kubo#9687](https://github.com/ipfs/kubo/pull/9687)) + - docs: use fx.Decorate instead of fx.Replace in examples (#9725) ([ipfs/kubo#9725](https://github.com/ipfs/kubo/pull/9725)) + - Create Changelog: v0.20 ([ipfs/kubo#9742](https://github.com/ipfs/kubo/pull/9742)) + - Merge Release: v0.19.0 ([ipfs/kubo#9741](https://github.com/ipfs/kubo/pull/9741)) + - feat(gateway): invalid CID returns 400 Bad Request (#9726) ([ipfs/kubo#9726](https://github.com/ipfs/kubo/pull/9726)) + - fix: remove outdated changelog part ([ipfs/kubo#9739](https://github.com/ipfs/kubo/pull/9739)) + - docs: 0.19 changelog ([ipfs/kubo#9707](https://github.com/ipfs/kubo/pull/9707)) + - fix: canonicalize user defined headers + - fix: apply API.HTTPHeaders to /webui redirect + - feat: add heap allocs to 'ipfs diag profile' + - fix: future proof with > rcmgr.DefaultLimit for new enum rcmgr values + - test: add test for presarvation of unlimited configs for inbound systems + - fix: preserve Unlimited StreamsInbound in connmgr reconciliation + - test: fix flaky rcmgr test + - chore: deprecate the pubsub api + - Revert "chore: add hamt directory sharding test" + - chore: add hamt directory sharding test + - test: port peering test from sharness to Go + - test: use `T.TempDir` to create temporary test directory + - fix: --verify forgets the verified key + - test: name --verify forgets the verified key + - chore: fix toc in changelog for 0.18 + - feat: add "autoclient" routing type + - test: parallelize more of rcmgr Go tests + - test: port legacy DHT tests to Go + - fix: t0116-gateway-cache.sh ([ipfs/kubo#9696](https://github.com/ipfs/kubo/pull/9696)) + - docs: add bifrost to early testers ([ipfs/kubo#9699](https://github.com/ipfs/kubo/pull/9699)) + - fix: typo in documentation for install path + - docs: fix typos + - Update Version: v0.19 ([ipfs/kubo#9698](https://github.com/ipfs/kubo/pull/9698)) +- github.com/ipfs/go-block-format (v0.1.1 -> v0.1.2): + - chore: release v0.1.2 + - Revert deprecation and go-libipfs/blocks stub types + - docs: deprecation notice [ci skip] +- github.com/ipfs/go-cid (v0.3.2 -> v0.4.1): + - v0.4.1 + - Add unit test for unexpected eof + - Update cid.go + - CidFromReader should not wrap valid EOF return. + - chore: version 0.4.0 + - feat: wrap parsing errors into ErrInvalidCid + - fix: use crypto/rand.Read + - Fix README.md example error (#146) ([ipfs/go-cid#146](https://github.com/ipfs/go-cid/pull/146)) +- github.com/ipfs/go-delegated-routing (v0.7.0 -> v0.8.0): + - chore: release v0.8.0 + - chore: migrate from go-ipns to boxo + - docs: add deprecation notice [ci skip] +- github.com/ipfs/go-graphsync (v0.14.1 -> v0.14.4): + - Update version to cover latest fixes (#419) ([ipfs/go-graphsync#419](https://github.com/ipfs/go-graphsync/pull/419)) + - Bring changes from #412 + - Bring changes from #391 + - fix: calling message queue Shutdown twice causes panic (because close is called twice on done channel) (#414) ([ipfs/go-graphsync#414](https://github.com/ipfs/go-graphsync/pull/414)) + - docs(CHANGELOG): update for v0.14.3 + - fix: wire up proper linksystem to traverser (#411) ([ipfs/go-graphsync#411](https://github.com/ipfs/go-graphsync/pull/411)) + - sync: update CI config files (#378) ([ipfs/go-graphsync#378](https://github.com/ipfs/go-graphsync/pull/378)) + - chore: remove social links (#398) ([ipfs/go-graphsync#398](https://github.com/ipfs/go-graphsync/pull/398)) + - Removes `main` branch callout. + - release v0.14.2 +- github.com/ipfs/go-ipfs-blockstore (v1.2.0 -> v1.3.0): + - chore: release v1.3.0 + - feat: stub and deprecate NewBlockstoreNoPrefix + - Accept options for blockstore: start with WriteThrough and NoPrefix + - Allow using a NewWriteThrough() blockstore. + - sync: update CI config files (#105) ([ipfs/go-ipfs-blockstore#105](https://github.com/ipfs/go-ipfs-blockstore/pull/105)) + - feat: fast-path for PutMany, falling back to Put for single block call (#97) ([ipfs/go-ipfs-blockstore#97](https://github.com/ipfs/go-ipfs-blockstore/pull/97)) +- github.com/ipfs/go-ipfs-cmds (v0.8.2 -> v0.9.0): + - chore: release v0.9.0 + - chore: change go-libipfs to boxo +- github.com/ipfs/go-libipfs (v0.6.2 -> v0.7.0): + - chore: bump to 0.7.0 (#213) ([ipfs/go-libipfs#213](https://github.com/ipfs/go-libipfs/pull/213)) + - feat: return 400 on /ipfs/invalid-cid (#205) ([ipfs/go-libipfs#205](https://github.com/ipfs/go-libipfs/pull/205)) + - docs: add note in README that go-libipfs is not comprehensive (#163) ([ipfs/go-libipfs#163](https://github.com/ipfs/go-libipfs/pull/163)) +- github.com/ipfs/go-merkledag (v0.9.0 -> v0.10.0): + - chore: bump version to 0.10.0 + - fix: switch to crypto/rand.Read + - stop using the deprecated io/ioutil package +- github.com/ipfs/go-unixfs (v0.4.4 -> v0.4.5): + - chore: release v0.4.5 + - chore: remove go-libipfs dependency +- github.com/ipfs/go-unixfsnode (v1.5.2 -> v1.6.0): + - chore: bump v1.6.0 + - feat: add UnixFSPathSelectorBuilder ([ipfs/go-unixfsnode#45](https://github.com/ipfs/go-unixfsnode/pull/45)) + - fix: update state to allow iter continuance on NotFound errors + - chore!: make PBLinkItr private - not intended for public use + - fix: propagate iteration errors +- github.com/ipld/go-car/v2 (v2.5.1 -> v2.9.1-0.20230325062757-fff0e4397a3d): + - chore: unmigrate from go-libipfs + - Create CODEOWNERS + - blockstore: give a direct access to the index for read operations + - blockstore: only close the file on error in OpenReadWrite, not OpenReadWriteFile + - fix: handle (and test) WholeCID vs not; fast Has() path for storage + - ReadWrite: faster Has() by using the in-memory index instead of reading on disk + - fix: let `extract` skip missing unixfs shard links + - fix: error when no files extracted + - fix: make -f optional, read from stdin if omitted + - fix: update cmd/car/README with latest description + - chore: add test cases for extract modes + - feat: extract accepts '-' as an output path for stdout + - feat: extract specific path, accept stdin as streaming input + - fix: if we don't read the full block data, don't error on !EOF + - blockstore: try to close during Finalize(), even in case of previous error + - ReadWrite: add an alternative FinalizeReadOnly+Close flow + - feat: add WithTrustedCar() reader option (#381) ([ipld/go-car#381](https://github.com/ipld/go-car/pull/381)) + - blockstore: fast path for AllKeysChan using the index + - fix: switch to crypto/rand.Read + - stop using the deprecated io/ioutil package + - fix(doc): fix storage package doc formatting + - fix: return errors for unsupported operations + - chore: move insertionindex into store pkg + - chore: add experimental note + - fix: minor lint & windows fd test problems + - feat: docs for StorageCar interfaces + - feat: ReadableWritable; dedupe shared code + - feat: add Writable functionality to StorageCar + - feat: StorageCar as a Readable storage, separate from blockstore + - feat(blockstore): implement a streaming read only storage + - feat(cmd): add index create subcommand to create an external carv2 index ([ipld/go-car#350](https://github.com/ipld/go-car/pull/350)) + - chore: bump version to 0.6.0 + - fix: use goreleaser instead + - Allow using WalkOption in WriteCar function ([ipld/go-car#357](https://github.com/ipld/go-car/pull/357)) + - fix: update go-block-format to the version that includes the stubs + - feat: upgrade from go-block-format to go-libipfs/blocks + - cleanup readme a bit to make the cli more discoverable (#353) ([ipld/go-car#353](https://github.com/ipld/go-car/pull/353)) + - Update install instructions in README.md + - Add a debugging form for car files. (#341) ([ipld/go-car#341](https://github.com/ipld/go-car/pull/341)) + - ([ipld/go-car#340](https://github.com/ipld/go-car/pull/340)) +- github.com/ipld/go-codec-dagpb (v1.5.0 -> v1.6.0): + - Update version.json +- github.com/ipld/go-ipld-prime (v0.19.0 -> v0.20.0): + - Prepare v0.20.0 + - fix(datamodel): add tests to Copy, make it complain on nil + - feat(dagcbor): mode to allow parsing undelimited streamed objects + - Fix mispatched package declaration. + - Add several pieces of docs to schema/dmt. + - Additional access to schema/dmt package; schema concatenation feature ([ipld/go-ipld-prime#483](https://github.com/ipld/go-ipld-prime/pull/483)) + - Fix hash mismatch error on matching link pointer + - feat: support errors.Is for schema errors +- github.com/ipld/go-ipld-prime/storage/bsadapter (v0.0.0-20211210234204-ce2a1c70cd73 -> v0.0.0-20230102063945-1a409dc236dd): + - build(deps): bump github.com/ipfs/go-blockservice + - Fix mispatched package declaration. + - Add several pieces of docs to schema/dmt. + - Additional access to schema/dmt package; schema concatenation feature ([ipld/go-ipld-prime/storage/bsadapter#483](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/483)) + - fix: go mod tidy + - build(deps): bump github.com/frankban/quicktest from 1.14.3 to 1.14.4 + - Fix hash mismatch error on matching link pointer + - build(deps): bump github.com/warpfork/go-testmark from 0.10.0 to 0.11.0 + - feat: support errors.Is for schema errors + - build(deps): bump github.com/multiformats/go-multicodec + - Prepare v0.19.0 + - fix: correct json codec links & bytes handling + - build(deps): bump github.com/google/go-cmp from 0.5.8 to 0.5.9 (#468) ([ipld/go-ipld-prime/storage/bsadapter#468](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/468)) + - build(deps): bump github.com/ipfs/go-cid from 0.3.0 to 0.3.2 (#466) ([ipld/go-ipld-prime/storage/bsadapter#466](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/466)) + - build(deps): bump github.com/ipfs/go-cid in /storage/bsrvadapter (#464) ([ipld/go-ipld-prime/storage/bsadapter#464](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/464)) + - test(basicnode): increase test coverage for int and map types (#454) ([ipld/go-ipld-prime/storage/bsadapter#454](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/454)) + - build(deps): bump github.com/ipfs/go-cid in /storage/bsrvadapter + - build(deps): bump github.com/ipfs/go-cid from 0.2.0 to 0.3.0 + - build(deps): bump github.com/multiformats/go-multicodec + - fix: remove reliance on ioutil + - fix: update sub-package modules + - build(deps): bump github.com/multiformats/go-multihash + - build(deps): bump github.com/ipfs/go-datastore in /storage/dsadapter + - update .github/workflows/go-check.yml + - update .github/workflows/go-test.yml + - run gofmt -s + - bump go.mod to Go 1.18 and run go fix + - bump go.mod to Go 1.18 and run go fix + - bump go.mod to Go 1.18 and run go fix + - bump go.mod to Go 1.18 and run go fix + - feat: add kinded union to gendemo + - fix: go mod 1.17 compat problems + - build(deps): bump github.com/ipfs/go-blockservice + - Prepare v0.18.0 + - fix(deps): update benchmarks go.sum + - build(deps): bump github.com/multiformats/go-multihash + - feat(bindnode): add a BindnodeRegistry utility (#437) ([ipld/go-ipld-prime/storage/bsadapter#437](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/437)) + - feat(bindnode): support full uint64 range + - chore(bindnode): remove typed functions for options + - chore(bindnode): docs and minor tweaks + - feat(bindnode): make Any converters work for List and Map values + - fix(bindnode): shorten converter option names, minor perf improvements + - fix(bindnode): only custom convert AssignNull for Any converter + - feat(bindnode): pass Null on to nullable custom converters + - chore(bindnode): config helper refactor w/ short-circuit + - feat(bindnode): add AddCustomTypeAnyConverter() to handle `Any` fields + - feat(bindnode): add AddCustomTypeXConverter() options for most scalar kinds + - chore(bindnode): back out of reflection for converters + - feat(bindnode): switch to converter functions instead of type + - feat(bindnode): allow custom type conversions with options + - feat: add release checklist (#442) ([ipld/go-ipld-prime/storage/bsadapter#442](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/442)) + - Prepare v0.17.0 + - feat: introduce UIntNode interface, used within DAG-CBOR codec + - add option to not parse beyond end of structure (#435) ([ipld/go-ipld-prime/storage/bsadapter#435](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/435)) + - sync benchmarks go.sum + - build(deps): bump github.com/multiformats/go-multicodec + - patch: first draft. ([ipld/go-ipld-prime/storage/bsadapter#350](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/350)) + - feat(bindnode): infer links and Any from Go types (#432) ([ipld/go-ipld-prime/storage/bsadapter#432](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/432)) + - fix(codecs): error on cid.Undef links in dag{json,cbor} encoding (#433) ([ipld/go-ipld-prime/storage/bsadapter#433](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/433)) + - chore(bindnode): add test for sub-node unwrapping + - fix(bindnode): more helpful error message for enum value footgun + - fix(bindnode): panic early if API has been passed ptr-to-ptr + - fix(deps): mod tidy for dependencies + - build(deps): bump github.com/warpfork/go-testmark from 0.3.0 to 0.10.0 + - build(deps): bump github.com/multiformats/go-multicodec + - build(deps): bump github.com/ipfs/go-cid from 0.0.4 to 0.2.0 + - build(deps): bump github.com/google/go-cmp from 0.5.7 to 0.5.8 + - build(deps): bump github.com/frankban/quicktest from 1.14.2 to 1.14.3 + - build(deps): bump github.com/ipfs/go-cid in /storage/bsrvadapter + - chore(deps): expand dependabot to sub-modules + - chore(deps): add dependabot config + - printer: fix printing of floats + - add version.json file (#411) ([ipld/go-ipld-prime/storage/bsadapter#411](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/411)) + - ci: use GOFLAGS to control test tags + - ci: disable coverpkg using custom workflow insertion + - ci: add initial web3 unified-ci files + - fix: make 32-bit safe and stable & add to CI + - ci: add go-check.yml workflow from unified-ci + - ci: go mod tidy + - fix: staticcheck and govet fixes + - test: make tests work on Windows, add Windows to CI (#405) ([ipld/go-ipld-prime/storage/bsadapter#405](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/405)) + - schema: enable inline types through dsl parser & compiler (#404) ([ipld/go-ipld-prime/storage/bsadapter#404](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/404)) + - node/bindnode: allow nilable types for IPLD optional/nullable + - test(ci): enable macos in GitHub Actions + - test(gen-go): disable parallelism when testing on macos + - storage: update deps + - dsl support for stringjoin struct repr and stringprefix union repr ([ipld/go-ipld-prime/storage/bsadapter#397](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/397)) + - codec/dagcbor: add DecodeOptions.ExperimentalDeterminism + - node/bindnode: add some more docs + - start testing on Go 1.18.x, drop Go 1.16.x + - readme: getting started pointers. + - readme: bindnode definitely needs a mention! + - Readme updates! + - datamodel: document that repr prototypes produce type nodes + - node/bindnode: minor fuzz improvements + - gengo: update readme. + - fix(dagcbor): don't accept trailing bytes + - schema/dmt: reject duplicate or missing union repr members + - node/bindnode: actually check schemadmt.Compile errors when fuzzing + - node/bindnode: avoid OOM when inferring from cyclic IPLD schemas + - schema/dmt: require enum reprs to refer valid members + - skip NaN/Inf errors for dag-json + - node/bindnode: refuse to decode empty union values + - schema/dmt: error in Compile if union reprs refer to unknown members + - node/bindnode: start fuzzing with schema/dmt and codec/dagcbor + - mark v0.16.0 + - node/bindnode: enforce pointer requirement for nullable maps + - Implement WalkTransforming traversal (#376) ([ipld/go-ipld-prime/storage/bsadapter#376](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/376)) + - docs(datamodel): add comment to LargeBytesNode + - Add partial-match traversal of large bytes (#375) ([ipld/go-ipld-prime/storage/bsadapter#375](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/375)) + - Implement option to start traversals at a path ([ipld/go-ipld-prime/storage/bsadapter#358](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/358)) + - add top-level "go value with schema" example + - Support optional `LargeBytesNode` interface (#372) ([ipld/go-ipld-prime/storage/bsadapter#372](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/372)) + - node/bindnode: support pointers to datamodel.Node to bind with Any + - fix(bindnode): tuple struct iterator should handle absent fields properly + - node/bindnode: make AssignNode work at the repr level + - node/bindnode: add support for unsigned integers + - node/bindnode: cover even more edge case panics + - node/bindnode: polish some more AsT panics + - schema/dmt: stop using a fake test to generate code ([ipld/go-ipld-prime/storage/bsadapter#356](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/356)) + - schema: remove one review note; add another. + - fix: minor EncodedLength fixes, add tests to fully exercise + - feat: add dagcbor.EncodedLength(Node) to calculate length without encoding + - chore: rename Garbage() to Generate() + - fix: minor garbage nits + - fix: Garbage() takes rand parameter, tweak algorithms, improve docs + - feat: add Garbage() Node generator + - node/bindnode: introduce an assembler that always errors + - node/bindnode: polish panics on invalid AssignT calls + - datamodel: don't panic when stringifying an empty KindSet + - node/bindnode: start using ipld.LoadSchema APIs + - selectors: fix for edge case around recursion clauses with an immediate edge. ([ipld/go-ipld-prime/storage/bsadapter#334](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/334)) + - node/bindnode: improve support for pointer types + - node/bindnode: subtract all absents in Length at the repr level + - fix(codecs): error when encoding maps whose lengths don't match entry count + - schema: avoid alloc and copy in Struct and Enum methods + - node/bindnode: allow mapping int-repr enums with Go integers + - schema,node/bindnode: add support for Any + - signaling ADLs in selectors (#301) ([ipld/go-ipld-prime/storage/bsadapter#301](https://github.com/ipld/go-ipld-prime/storage/bsadapter/pull/301)) + - node/bindnode: add support for enums + - schema/...: add support for enum int representations + - node/bindnode: allow binding cidlink.Link to links +- github.com/libp2p/go-libp2p (v0.26.4 -> v0.27.3): + - release v0.27.3 + - quic virtual listener: don't panic when quic-go's accept call errors (#2276) ([libp2p/go-libp2p#2276](https://github.com/libp2p/go-libp2p/pull/2276)) + - Release v0.27.2 (#2270) ([libp2p/go-libp2p#2270](https://github.com/libp2p/go-libp2p/pull/2270)) + - release v0.27.1 (#2252) ([libp2p/go-libp2p#2252](https://github.com/libp2p/go-libp2p/pull/2252)) + - Infer public webtransport addrs from quic-v1 addrs. (#2251) ([libp2p/go-libp2p#2251](https://github.com/libp2p/go-libp2p/pull/2251)) + - basichost: don't allocate when deduplicating multiaddrs (#2206) ([libp2p/go-libp2p#2206](https://github.com/libp2p/go-libp2p/pull/2206)) + - identify: fix normalization of interface listen addresses (#2250) ([libp2p/go-libp2p#2250](https://github.com/libp2p/go-libp2p/pull/2250)) + - autonat: fix flaky TestAutoNATDialRefused (#2245) ([libp2p/go-libp2p#2245](https://github.com/libp2p/go-libp2p/pull/2245)) + - basichost: remove stray print statement in test (#2249) ([libp2p/go-libp2p#2249](https://github.com/libp2p/go-libp2p/pull/2249)) + - swarm: fix multiaddr comparison in ListenClose (#2247) ([libp2p/go-libp2p#2247](https://github.com/libp2p/go-libp2p/pull/2247)) + - release v0.27.0 (#2242) ([libp2p/go-libp2p#2242](https://github.com/libp2p/go-libp2p/pull/2242)) + - add a security policy (#2238) ([libp2p/go-libp2p#2238](https://github.com/libp2p/go-libp2p/pull/2238)) + - chore: 0.27.0 changelog entries (#2241) ([libp2p/go-libp2p#2241](https://github.com/libp2p/go-libp2p/pull/2241)) + - correctly handle WebTransport addresses without certhashes (#2239) ([libp2p/go-libp2p#2239](https://github.com/libp2p/go-libp2p/pull/2239)) + - autorelay: add metrics (#2185) ([libp2p/go-libp2p#2185](https://github.com/libp2p/go-libp2p/pull/2185)) + - autonat: don't change status on dial request refused (#2225) ([libp2p/go-libp2p#2225](https://github.com/libp2p/go-libp2p/pull/2225)) + - autonat: fix closing of listeners in dialPolicy tests (#2226) ([libp2p/go-libp2p#2226](https://github.com/libp2p/go-libp2p/pull/2226)) + - discovery (backoff): fix typo in comment (#2214) ([libp2p/go-libp2p#2214](https://github.com/libp2p/go-libp2p/pull/2214)) + - relaysvc: flaky TestReachabilityChangeEvent (#2215) ([libp2p/go-libp2p#2215](https://github.com/libp2p/go-libp2p/pull/2215)) + - Add wss transport to interop tester impl (#2178) ([libp2p/go-libp2p#2178](https://github.com/libp2p/go-libp2p/pull/2178)) + - tests: add a stream read deadline transport test (#2210) ([libp2p/go-libp2p#2210](https://github.com/libp2p/go-libp2p/pull/2210)) + - autorelay: fix busy loop bug and flaky tests in relay finder (#2208) ([libp2p/go-libp2p#2208](https://github.com/libp2p/go-libp2p/pull/2208)) + - tests: test mplex and Yamux, Noise and TLS in transport tests (#2209) ([libp2p/go-libp2p#2209](https://github.com/libp2p/go-libp2p/pull/2209)) + - tests: add some basic transport integration tests (#2207) ([libp2p/go-libp2p#2207](https://github.com/libp2p/go-libp2p/pull/2207)) + - autorelay: remove unused semaphore (#2184) ([libp2p/go-libp2p#2184](https://github.com/libp2p/go-libp2p/pull/2184)) + - basichost: prevent duplicate dials (#2196) ([libp2p/go-libp2p#2196](https://github.com/libp2p/go-libp2p/pull/2196)) + - websocket: don't set a WSS multiaddr for accepted unencrypted conns (#2199) ([libp2p/go-libp2p#2199](https://github.com/libp2p/go-libp2p/pull/2199)) + - websocket: Don't limit message sizes in the websocket reader (#2193) ([libp2p/go-libp2p#2193](https://github.com/libp2p/go-libp2p/pull/2193)) + - identify: fix stale comment (#2179) ([libp2p/go-libp2p#2179](https://github.com/libp2p/go-libp2p/pull/2179)) + - relay service: add metrics (#2154) ([libp2p/go-libp2p#2154](https://github.com/libp2p/go-libp2p/pull/2154)) + - identify: Fix IdentifyWait when Connected events happen out of order (#2173) ([libp2p/go-libp2p#2173](https://github.com/libp2p/go-libp2p/pull/2173)) + - chore: fix ressource manager's README (#2168) ([libp2p/go-libp2p#2168](https://github.com/libp2p/go-libp2p/pull/2168)) + - relay: fix deadlock when closing (#2171) ([libp2p/go-libp2p#2171](https://github.com/libp2p/go-libp2p/pull/2171)) + - core: remove LocalPrivateKey method from network.Conn interface (#2144) ([libp2p/go-libp2p#2144](https://github.com/libp2p/go-libp2p/pull/2144)) + - routed host: return connection error instead of routing error (#2169) ([libp2p/go-libp2p#2169](https://github.com/libp2p/go-libp2p/pull/2169)) + - connmgr: reduce log level for closing connections (#2165) ([libp2p/go-libp2p#2165](https://github.com/libp2p/go-libp2p/pull/2165)) + - circuitv2: cleanup relay service properly (#2164) ([libp2p/go-libp2p#2164](https://github.com/libp2p/go-libp2p/pull/2164)) + - chore: add patch release to changelog (#2151) ([libp2p/go-libp2p#2151](https://github.com/libp2p/go-libp2p/pull/2151)) + - chore: remove superfluous testing section from README (#2150) ([libp2p/go-libp2p#2150](https://github.com/libp2p/go-libp2p/pull/2150)) + - autonat: don't use autonat for address discovery (#2148) ([libp2p/go-libp2p#2148](https://github.com/libp2p/go-libp2p/pull/2148)) + - swarm metrics: fix connection direction (#2147) ([libp2p/go-libp2p#2147](https://github.com/libp2p/go-libp2p/pull/2147)) + - connmgr: Use eventually equal helper in connmgr tests (#2128) ([libp2p/go-libp2p#2128](https://github.com/libp2p/go-libp2p/pull/2128)) + - swarm: emit PeerConnectedness event from swarm instead of from hosts (#1574) ([libp2p/go-libp2p#1574](https://github.com/libp2p/go-libp2p/pull/1574)) + - relay: initialize the ASN util when starting the service (#2143) ([libp2p/go-libp2p#2143](https://github.com/libp2p/go-libp2p/pull/2143)) + - Fix flaky TestMetricsNoAllocNoCover test (#2142) ([libp2p/go-libp2p#2142](https://github.com/libp2p/go-libp2p/pull/2142)) + - identify: Bump timeouts/sleep in tests (#2135) ([libp2p/go-libp2p#2135](https://github.com/libp2p/go-libp2p/pull/2135)) + - Add sleep to fix flaky test (#2129) ([libp2p/go-libp2p#2129](https://github.com/libp2p/go-libp2p/pull/2129)) + - basic_host: Fix flaky tests (#2136) ([libp2p/go-libp2p#2136](https://github.com/libp2p/go-libp2p/pull/2136)) + - swarm: Check context once more before dialing (#2139) ([libp2p/go-libp2p#2139](https://github.com/libp2p/go-libp2p/pull/2139)) +- github.com/libp2p/go-libp2p-asn-util (v0.2.0 -> v0.3.0): + - release v0.3.0 (#26) ([libp2p/go-libp2p-asn-util#26](https://github.com/libp2p/go-libp2p-asn-util/pull/26)) + - initialize the store lazily (#25) ([libp2p/go-libp2p-asn-util#25](https://github.com/libp2p/go-libp2p-asn-util/pull/25)) +- github.com/libp2p/go-libp2p-gostream (v0.5.0 -> v0.6.0): + - Update libp2p ([libp2p/go-libp2p-gostream#80](https://github.com/libp2p/go-libp2p-gostream/pull/80)) + - fix typo in README (#75) ([libp2p/go-libp2p-gostream#75](https://github.com/libp2p/go-libp2p-gostream/pull/75)) +- github.com/libp2p/go-libp2p-http (v0.4.0 -> v0.5.0): + - sync: update CI config files ([libp2p/go-libp2p-http#82](https://github.com/libp2p/go-libp2p-http/pull/82)) +- github.com/libp2p/go-libp2p-kad-dht (v0.21.1 -> v0.23.0): + - Release v0.23.0 + - Specified CODEOWNERS ([libp2p/go-libp2p-kad-dht#828](https://github.com/libp2p/go-libp2p-kad-dht/pull/828)) + - fix: optimistic provide ci checks in tests ([libp2p/go-libp2p-kad-dht#833](https://github.com/libp2p/go-libp2p-kad-dht/pull/833)) + - feat: add experimental optimistic provide (#783) ([libp2p/go-libp2p-kad-dht#783](https://github.com/libp2p/go-libp2p-kad-dht/pull/783)) + - feat: rework tracing a bit + - feat: add basic tracing + - chore: release v0.22.0 + - chore: migrate go-libipfs to boxo + - Fix multiple ProviderAddrTTL definitions #795 ([libp2p/go-libp2p-kad-dht#831](https://github.com/libp2p/go-libp2p-kad-dht/pull/831)) + - Increase provider Multiaddress TTL ([libp2p/go-libp2p-kad-dht#795](https://github.com/libp2p/go-libp2p-kad-dht/pull/795)) + - Make provider manager options configurable in `fullrt` ([libp2p/go-libp2p-kad-dht#829](https://github.com/libp2p/go-libp2p-kad-dht/pull/829)) + - Adjust PeerSet logic in the DHT lookup process ([libp2p/go-libp2p-kad-dht#802](https://github.com/libp2p/go-libp2p-kad-dht/pull/802)) + - added maintainers in the README ([libp2p/go-libp2p-kad-dht#826](https://github.com/libp2p/go-libp2p-kad-dht/pull/826)) + - Allow DHT crawler to be swappable + - Introduce options to parameterize config of the accelerated DHT client ([libp2p/go-libp2p-kad-dht#822](https://github.com/libp2p/go-libp2p-kad-dht/pull/822)) +- github.com/libp2p/go-libp2p-pubsub (v0.9.0 -> v0.9.3): + - Fix Memory Leak In New Timecache Implementations (#528) ([libp2p/go-libp2p-pubsub#528](https://github.com/libp2p/go-libp2p-pubsub/pull/528)) + - Default validator support (#525) ([libp2p/go-libp2p-pubsub#525](https://github.com/libp2p/go-libp2p-pubsub/pull/525)) + - Refactor timecache implementations (#523) ([libp2p/go-libp2p-pubsub#523](https://github.com/libp2p/go-libp2p-pubsub/pull/523)) + - fix(timecache): remove panic in first seen cache on Add (#522) ([libp2p/go-libp2p-pubsub#522](https://github.com/libp2p/go-libp2p-pubsub/pull/522)) + - chore: update go version and dependencies (#516) ([libp2p/go-libp2p-pubsub#516](https://github.com/libp2p/go-libp2p-pubsub/pull/516)) +- github.com/multiformats/go-multiaddr (v0.8.0 -> v0.9.0): + - Release v0.9.0 ([multiformats/go-multiaddr#196](https://github.com/multiformats/go-multiaddr/pull/196)) + - Update webrtc protocols after rename ([multiformats/go-multiaddr#195](https://github.com/multiformats/go-multiaddr/pull/195)) +- github.com/multiformats/go-multibase (v0.1.1 -> v0.2.0): + - chore: bump v0.2.0 + - fix: math/rand -> crypto/rand + - fuzz: add Decoder fuzzing +- github.com/multiformats/go-multicodec (v0.7.0 -> v0.8.1): + - Bump version to release `ipns-record` code + - chore: update submodules and go generate + - deps: upgrade stringer to compatible version + - v0.8.0 + - chore: update submodules and go generate +- github.com/warpfork/go-testmark (v0.10.0 -> v0.11.0): + - Quick changelog to note we have an API update. + - Index fix ([warpfork/go-testmark#13](https://github.com/warpfork/go-testmark/pull/13)) + - Link to python implementation in the readme! + +
+ ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors +| Contributor | Commits | Lines ยฑ | Files Changed | +|-------------|---------|---------|---------------| +| Rod Vagg | 40 | +4214/-1400 | 102 | +| Sukun | 12 | +3541/-267 | 34 | +| Gus Eggert | 22 | +2387/-1160 | 81 | +| galargh | 23 | +1331/-1734 | 34 | +| Henrique Dias | 23 | +681/-1167 | 79 | +| Marco Munizaga | 19 | +1500/-187 | 55 | +| Jorropo | 25 | +897/-597 | 180 | +| Dennis Trautwein | 4 | +990/-60 | 14 | +| Marten Seemann | 18 | +443/-450 | 53 | +| vyzo | 2 | +595/-152 | 11 | +| Michael Murรฉ | 8 | +427/-182 | 18 | +| Will | 2 | +536/-15 | 5 | +| Adin Schmahmann | 3 | +327/-125 | 11 | +| hannahhoward | 2 | +344/-1 | 4 | +| Arthur Gavazza | 1 | +210/-50 | 4 | +| Hector Sanjuan | 6 | +181/-77 | 13 | +| Masih H. Derkani | 5 | +214/-42 | 12 | +| Calvin Behling | 4 | +158/-58 | 11 | +| Eric Myhre | 7 | +113/-27 | 15 | +| Marcin Rataj | 5 | +72/-30 | 5 | +| Steve Loeppky | 2 | +99/-0 | 2 | +| Piotr Galar | 9 | +60/-18 | 9 | +| gammazero | 4 | +69/-0 | 8 | +| Prithvi Shahi | 2 | +55/-14 | 2 | +| Eng Zer Jun | 1 | +15/-54 | 5 | +| Laurent Senta | 3 | +44/-2 | 3 | +| Ian Davis | 1 | +35/-0 | 1 | +| web3-bot | 4 | +19/-13 | 7 | +| guillaumemichel | 2 | +18/-14 | 3 | +| Guillaume Michel - guissou | 4 | +24/-8 | 4 | +| omahs | 1 | +9/-9 | 3 | +| cortze | 3 | +9/-9 | 3 | +| Nishant Das | 1 | +9/-5 | 3 | +| Hlib Kanunnikov | 2 | +11/-3 | 3 | +| Andrew Gillis | 3 | +6/-8 | 3 | +| Johnny | 1 | +0/-10 | 1 | +| Rafaล‚ Leszko | 1 | +4/-4 | 1 | +| Dirk McCormick | 1 | +4/-1 | 1 | +| Antonio Navarro Perez | 1 | +4/-1 | 1 | +| Richฮ›rd | 2 | +2/-2 | 2 | +| Russell Dempsey | 1 | +2/-1 | 1 | +| Winterhuman | 1 | +1/-1 | 1 | +| Will Hawkins | 1 | +1/-1 | 1 | +| Nikhilesh Susarla | 1 | +1/-1 | 1 | +| Kubo Mage | 1 | +1/-1 | 1 | +| Bryan White | 1 | +1/-1 | 1 | + From c9a8d9ff92c7a578209167e38c898fc1e849de67 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Tue, 9 May 2023 12:53:56 +0200 Subject: [PATCH 134/159] ci: simplify self-hosted runner selection (#9813) * ci: simplify self-hosted runner selection * ci: put interop jobs on self-hosted runners --- .github/workflows/build.yml | 34 +++++++++++++++++++++++++--------- .github/workflows/gobuild.yml | 7 ++----- .github/workflows/gotest.yml | 12 ++++++------ .github/workflows/runner.yml | 34 ---------------------------------- .github/workflows/sharness.yml | 20 +++++++++----------- 5 files changed, 42 insertions(+), 65 deletions(-) delete mode 100644 .github/workflows/runner.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f16d9bf3995..e59532597b1c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + jobs: interop-prep: if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' @@ -44,7 +48,7 @@ jobs: path: cmd/ipfs/ipfs interop: needs: [interop-prep] - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 defaults: run: @@ -75,7 +79,7 @@ jobs: npm install ipfs-interop@^10.0.1 working-directory: interop # Run the interop tests while ignoring the js-js interop test cases - - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))' + - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))' --parallel env: LIBP2P_TCP_REUSEPORT: false LIBP2P_ALLOW_WEAK_RSA_KEYS: 1 @@ -156,7 +160,7 @@ jobs: working-directory: go-ipfs-http-client ipfs-webui: needs: [interop-prep] - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 env: NO_SANDBOX: true @@ -191,14 +195,26 @@ jobs: key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }} restore-keys: | ${{ runner.os }}-${{ github.job }}- - - run: | - npm ci --prefer-offline --no-audit --progress=false - npx playwright install + - env: + NPM_CACHE_DIR: ${{ steps.npm-cache-dir.outputs.dir }} + run: | + npm ci --prefer-offline --no-audit --progress=false --cache "$NPM_CACHE_DIR" + npx playwright install --with-deps + working-directory: ipfs-webui + - id: ref + run: echo "ref=$(git rev-parse --short HEAD)" | tee -a $GITHUB_OUTPUT working-directory: ipfs-webui - - name: Run ipfs-webui@main build and smoke-test to confirm the upstream repo is not broken - run: npm test + - id: state + env: + GITHUB_REPOSITORY: ipfs/ipfs-webui + GITHUB_REF: ${{ steps.ref.outputs.ref }} + GITHUB_TOKEN: ${{ github.token }} + run: | + echo "state=$(curl -L -H "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_REF/status" --jq '.state')" | tee -a $GITHUB_OUTPUT + - name: Build ipfs-webui@main (state=${{ steps.state.outputs.state }}) + run: npm run test:build working-directory: ipfs-webui - - name: Test ipfs-webui@main E2E against the locally built Kubo binary + - name: Test ipfs-webui@main (state=${{ steps.state.outputs.state }}) E2E against the locally built Kubo binary run: npm run test:e2e env: IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs diff --git a/.github/workflows/gobuild.yml b/.github/workflows/gobuild.yml index f1e3a60b7767..8591213c4b2c 100644 --- a/.github/workflows/gobuild.yml +++ b/.github/workflows/gobuild.yml @@ -12,12 +12,9 @@ concurrency: cancel-in-progress: true jobs: - go-build-runner: - if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' - uses: ipfs/kubo/.github/workflows/runner.yml@master go-build: - needs: [go-build-runner] - runs-on: ${{ fromJSON(needs.go-build-runner.outputs.config).labels }} + if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' + runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 env: TEST_DOCKER: 0 diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 81c39fd5e51a..b20dca2a38e6 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -12,12 +12,9 @@ concurrency: cancel-in-progress: true jobs: - go-test-runner: - if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' - uses: ipfs/kubo/.github/workflows/runner.yml@master go-test: - needs: [go-test-runner] - runs-on: ${{ fromJSON(needs.go-test-runner.outputs.config).labels }} + if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' + runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 env: TEST_DOCKER: 0 @@ -41,8 +38,11 @@ jobs: with: name: ${{ github.job }} - name: ๐Ÿ‘‰๏ธ If this step failed, go to ยซSummaryยป (top left) โ†’ inspect the ยซFailures/Errorsยป table + env: + # increasing parallelism beyond 2 doesn't speed up the tests much + PARALLEL: 2 run: | - make -j 2 test/unit/gotest.junit.xml && + make -j "$PARALLEL" test/unit/gotest.junit.xml && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - name: Upload coverage to Codecov uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml deleted file mode 100644 index 74a8b41d764d..000000000000 --- a/.github/workflows/runner.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Runner - -on: - workflow_call: - outputs: - config: - description: "The runner's configuration" - value: ${{ jobs.choose.outputs.config }} - -jobs: - choose: - runs-on: ubuntu-latest - timeout-minutes: 1 - outputs: - config: ${{ steps.config.outputs.result }} - steps: - - uses: actions/github-script@v6 - id: config - with: - script: | - if (`${context.repo.owner}/${context.repo.repo}` === 'ipfs/kubo') { - return { - labels: ['self-hosted', 'linux', 'x64', 'kubo'], - parallel: 10, - aws: true - } - } else { - return { - labels: ['ubuntu-latest'], - parallel: 3, - aws: false - } - } - - run: echo ${{ steps.config.outputs.result }} diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index 2d9dfb41da22..a4c2241bee49 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -12,12 +12,9 @@ concurrency: cancel-in-progress: true jobs: - sharness-runner: - if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' - uses: ipfs/kubo/.github/workflows/runner.yml@master sharness-test: - needs: [sharness-runner] - runs-on: ${{ fromJSON(needs.sharness-runner.outputs.config).labels }} + if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch' + runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }} timeout-minutes: 20 defaults: run: @@ -57,7 +54,8 @@ jobs: TEST_EXPENSIVE: 1 IPFS_CHECK_RCMGR_DEFAULTS: 1 CONTINUE_ON_S_FAILURE: 1 - PARALLEL: ${{ fromJSON(needs.sharness-runner.outputs.config).parallel }} + # increasing parallelism beyond 10 doesn't speed up the tests much + PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }} - name: Upload coverage report uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 if: failure() || success() @@ -86,12 +84,12 @@ jobs: - name: Upload one-page HTML report to S3 id: one-page uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main - if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success()) + if: github.repository == 'ipfs/kubo' && (failure() || success()) with: source: kubo/test/sharness/test-results/sharness.html destination: sharness.html - name: Upload one-page HTML report - if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success()) + if: github.repository != 'ipfs/kubo' && (failure() || success()) uses: actions/upload-artifact@v3 with: name: sharness.html @@ -106,18 +104,18 @@ jobs: - name: Upload full HTML report to S3 id: full uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main - if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success()) + if: github.repository == 'ipfs/kubo' && (failure() || success()) with: source: kubo/test/sharness/test-results/sharness-html destination: sharness-html/ - name: Upload full HTML report - if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success()) + if: github.repository != 'ipfs/kubo' && (failure() || success()) uses: actions/upload-artifact@v3 with: name: sharness-html path: kubo/test/sharness/test-results/sharness-html - name: Add S3 links to the summary - if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success()) + if: github.repository == 'ipfs/kubo' && (failure() || success()) run: echo "$MD" >> $GITHUB_STEP_SUMMARY env: MD: | From 851248aedc954ae8b878e657c986af9bcb99dd85 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 9 May 2023 13:05:13 +0000 Subject: [PATCH 135/159] chore: create next changelog --- CHANGELOG.md | 1 + docs/changelogs/v0.21.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 docs/changelogs/v0.21.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2659f2608418..b99ff4f15375 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Kubo Changelogs +- [v0.21](docs/changelogs/v0.21.md) - [v0.20](docs/changelogs/v0.20.md) - [v0.19](docs/changelogs/v0.19.md) - [v0.18](docs/changelogs/v0.18.md) diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md new file mode 100644 index 000000000000..57eb7002bad4 --- /dev/null +++ b/docs/changelogs/v0.21.md @@ -0,0 +1,18 @@ +# Kubo changelog v0.21 + +- [v0.21.0](#v0210) + +## v0.21.0 + +- [Overview](#overview) +- [๐Ÿ”ฆ Highlights](#-highlights) +- [๐Ÿ“ Changelog](#-changelog) +- [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) + +### Overview + +### ๐Ÿ”ฆ Highlights + +### ๐Ÿ“ Changelog + +### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors From 7ec67fdc04d5c048de474b176fc94bb01b681aab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 12:44:57 +0000 Subject: [PATCH 136/159] chore(deps): bump golang.org/x/crypto from 0.7.0 to 0.9.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.7.0 to 0.9.0. - [Commits](https://github.com/golang/crypto/compare/v0.7.0...v0.9.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/examples/kubo-as-a-library/go.mod | 8 ++++---- docs/examples/kubo-as-a-library/go.sum | 16 ++++++++-------- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 6a54fcad394d..12f13645627e 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -179,13 +179,13 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.7.0 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.11.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 1ac6f8afdcbd..cfbcf4474825 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -879,8 +879,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -961,8 +961,8 @@ golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1046,8 +1046,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1061,8 +1061,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/go.mod b/go.mod index 95f326b14b70..54889a77a979 100644 --- a/go.mod +++ b/go.mod @@ -80,10 +80,10 @@ require ( go.uber.org/dig v1.16.1 go.uber.org/fx v1.19.2 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.7.0 + golang.org/x/crypto v0.9.0 golang.org/x/mod v0.10.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.7.0 + golang.org/x/sys v0.8.0 ) require ( @@ -216,10 +216,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.11.0 // indirect diff --git a/go.sum b/go.sum index 6eaf0919e2be..25a1ce4323f5 100644 --- a/go.sum +++ b/go.sum @@ -1009,8 +1009,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1097,8 +1097,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1199,14 +1199,14 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1216,8 +1216,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From eab0baad3b4c5678f21840ee1a1c7fadf9977b73 Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Wed, 10 May 2023 02:59:13 -0400 Subject: [PATCH 137/159] docs: add "Customizing Kubo" doc (#9854) Co-authored-by: Steve Loeppky Co-authored-by: Henrique Dias --- docs/customizing.md | 60 +++++++++++++++++++++++ docs/examples/kubo-as-a-library/README.md | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 docs/customizing.md diff --git a/docs/customizing.md b/docs/customizing.md new file mode 100644 index 000000000000..4db3c3431ada --- /dev/null +++ b/docs/customizing.md @@ -0,0 +1,60 @@ +# Customizing Kubo + +You may want to customize Kubo if you want to reuse most of Kubo's machinery. This document discusses some approaches you may consider for customizing Kubo, and their tradeoffs. + +Some common use cases for customizing Kubo include: + +- Using a custom datastore for storing blocks, pins, or other Kubo metadata +- Adding a custom data transfer protocol into Kubo +- Customizing Kubo internals, such as adding allowlist/blocklist functionality to Bitswap +- Adding new commands, interfaces, functionality, etc. to Kubo while reusing the libp2p swarm +- Building on top of Kubo's configuration and config migration functionality + +## Summary +This table summarizes the tradeoffs between the approaches below: + +| | [Boxo](#boxo-build-your-own-binary) | [Kubo Plugin](#kubo-plugins) | [Bespoke Extension Point](#bespoke-extension-points) | [Go Plugin](#go-plugins) | [Fork](#fork-kubo) | +|:-------------------:|:-----------------------------------:|:----------------------------:|:----------------------------------------------------:|:------------------------:|:------------------:| +| Supported? | โœ… | โœ… | โœ… | โŒ | โŒ | +| Future-proof? | โœ… | โŒ | โœ… | โŒ | โŒ | +| Fully customizable? | โœ… | โœ… | โŒ | โœ… | โœ… | +| Fast to implement? | โŒ | โœ… | โœ… | โœ… | โœ… | +| Dynamic at runtime? | โŒ | โŒ | โœ… | โœ… | โŒ | +| Add new commands? | โŒ | โœ… | โŒ | โœ… | โœ… | + +## Boxo: build your own binary +The best way to reuse Kubo functionality is to pick the functionality you need directly from [Boxo](https://github.com/ipfs/boxo) and compile your own binary. + +Boxo's raison d'etre is to be an IPFS component toolbox to support building custom-made implementations and applications. If your use case is not easy to implement with Boxo, you may want to consider adding whatever functionality is needed to Boxo instead of customizing Kubo, so that the community can benefit. If you are interested in this option, please reach out to Boxo maintainers, who will be happy to help you scope & plan the work. See [Boxo's FAQ](https://github.com/ipfs/boxo#help) for more info. + +## Kubo Plugins +Kubo plugins are a set of interfaces that may be implemented and injected into Kubo. Generally you should recompile the Kubo binary with your plugins added. A popular example of a Kubo plugin is [go-ds-s3](https://github.com/ipfs/go-ds-s3), which can be used to store blocks in Amazon S3. + +Some plugins, such as the `fx` plugin, allow deep customization of Kubo internals. As a result, Kubo maintainers can't guarantee backwards compatibility with these, so you may need to adapt to breaking changes when upgrading to new Kubo versions. + +For more information about the different types of Kubo plugins, see [plugins.md](./plugins.md). + +Kubo plugins can also be injected at runtime using Go plugins (see below), but these are hard to use and not well supported by Go, so we don't recommend them. + +## Bespoke Extension Points +Certain Kubo functionality may have their own extension points. For example: + +* Kubo supports the [Routing v1](https://github.com/ipfs/specs/blob/main/routing/ROUTING_V1_HTTP.md) API for delegating content routing to external processes +* Kubo supports the [Pinning Service API](https://github.com/ipfs/pinning-services-api-spec) for delegating pinning to external processes +* Kubo supports [DNSLink](https://dnslink.dev/) for delegating name->CID mappings to DNS + +(This list is not exhaustive.) + +These can generally be developed and deployed as sidecars (or full external services) without modifying the Kubo binary. + +## Go Plugins +Go provides [dynamic plugins](https://pkg.go.dev/plugin) which can be loaded at runtime into a Go binary. + +Kubo currently works with Go plugins. But using Go plugins requires that you compile the plugin using the exact same version of the Go toolchain with the same configuration (build flags, environment variables, etc.). As a result, you likely need to build Kubo and the plugins together at the same time, and at that point you may as well just compile the functionality directly into Kubo and avoid Go plugins. + +As a result, we don't recommend using Go plugins, and are likely to remove them in a future release of Kubo. + +## Fork Kubo +The "nuclear option" is to fork Kubo into your own repo, make your changes, and periodically sync your repo with the Kubo repo. This can be a good option if your changes are significant and you can commit to keeping your repo in sync with Kubo. + +Kubo maintainers can't make any backwards compatibility guarantees about Kubo internals, so by choosing this option you're accepting the risk that you may need to spend more time adapting to breaking changes. diff --git a/docs/examples/kubo-as-a-library/README.md b/docs/examples/kubo-as-a-library/README.md index c9a320985685..bd41b5ee7d25 100644 --- a/docs/examples/kubo-as-a-library/README.md +++ b/docs/examples/kubo-as-a-library/README.md @@ -1,6 +1,6 @@ # Use Kubo (go-ipfs) as a library to spawn a node and add a file -> This tutorial is the sibling of the [js-ipfs IPFS 101 tutorial](https://github.com/ipfs-examples/js-ipfs-examples/tree/master/examples/ipfs-101#readme). +> Note: if you are trying to customize or extend Kubo, you should read the [Customizing Kubo](../../customizing.md) doc By the end of this tutorial, you will learn how to: From 0e7331c952b40d481efc717b0d749af2bd6bb92f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 10 May 2023 10:14:48 +0200 Subject: [PATCH 138/159] feat: update boxo with routing streaming --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/cli/content_routing_http_test.go | 5 +++-- test/cli/delegated_routing_http_test.go | 1 + 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 12f13645627e..6aa69fdac92b 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 + github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index cfbcf4474825..7c782c536900 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 h1:ThRTXD/EyoLb/jz+YW+ZlOLbjX9FyaxP0dEpgUp3cCE= -github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w= +github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 h1:WddjqbpZs6VHWQOpWoyj5N5DSy6o1oGsO41Wp9iQInE= +github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 54889a77a979..26415d8bb060 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 + github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 25a1ce4323f5..d2d9980cba70 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 h1:ThRTXD/EyoLb/jz+YW+ZlOLbjX9FyaxP0dEpgUp3cCE= -github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w= +github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 h1:WddjqbpZs6VHWQOpWoyj5N5DSy6o1oGsO41Wp9iQInE= +github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/cli/content_routing_http_test.go b/test/cli/content_routing_http_test.go index a8b825619cf7..acdea7029dfc 100644 --- a/test/cli/content_routing_http_test.go +++ b/test/cli/content_routing_http_test.go @@ -11,6 +11,7 @@ import ( "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" + "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/test/cli/harness" "github.com/ipfs/kubo/test/cli/testutils" @@ -23,11 +24,11 @@ type fakeHTTPContentRouter struct { provideCalls int } -func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid) ([]types.ProviderResponse, error) { +func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) { r.m.Lock() defer r.m.Unlock() r.findProvidersCalls++ - return []types.ProviderResponse{}, nil + return iter.FromSlice([]iter.Result[types.ProviderResponse]{}), nil } func (r *fakeHTTPContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) { diff --git a/test/cli/delegated_routing_http_test.go b/test/cli/delegated_routing_http_test.go index 8cf3368fa05c..a94c0b6085e8 100644 --- a/test/cli/delegated_routing_http_test.go +++ b/test/cli/delegated_routing_http_test.go @@ -17,6 +17,7 @@ func TestHTTPDelegatedRouting(t *testing.T) { fakeServer := func(resp string) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") _, err := w.Write([]byte(resp)) if err != nil { panic(err) From 61f8c7301ac878367c80601796885880933d2dac Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Wed, 10 May 2023 13:49:31 +0200 Subject: [PATCH 139/159] refactor: use reusable IPNS ValidateWithPeerID (#9867) Co-authored-by: Henrique Dias --- core/commands/name/name.go | 29 +------------------------- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 7 insertions(+), 34 deletions(-) diff --git a/core/commands/name/name.go b/core/commands/name/name.go index 75512e8f37de..94f05290cfce 100644 --- a/core/commands/name/name.go +++ b/core/commands/name/name.go @@ -17,7 +17,6 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/codec/dagjson" - ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) @@ -216,33 +215,7 @@ Passing --verify will verify signature against provided public key. PublicKey: id, } - pub, err := id.ExtractPublicKey() - if err != nil { - // Make sure it works with all those RSA that cannot be embedded into the - // Peer ID. - if len(entry.PubKey) > 0 { - pub, err = ic.UnmarshalPublicKey(entry.PubKey) - if err != nil { - return err - } - - // Verify the public key matches the name we are verifying. - entryID, err := peer.IDFromPublicKey(pub) - - if err != nil { - return err - } - - if id != entryID { - return fmt.Errorf("record public key does not match the verified name") - } - } - } - if err != nil { - return err - } - - err = ipns.Validate(pub, &entry) + err = ipns.ValidateWithPeerID(id, &entry) if err == nil { result.Validation.Valid = true } else { diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 6aa69fdac92b..52e3c064699c 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 + github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 7c782c536900..3111d683f532 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 h1:WddjqbpZs6VHWQOpWoyj5N5DSy6o1oGsO41Wp9iQInE= -github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w= +github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc= +github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 26415d8bb060..a756f0f32068 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 + github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index d2d9980cba70..af7ec2fc01ae 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82 h1:WddjqbpZs6VHWQOpWoyj5N5DSy6o1oGsO41Wp9iQInE= -github.com/ipfs/boxo v0.8.2-0.20230510071416-d7db1adc7e82/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w= +github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc= +github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= From 4acadd49ebc445d86afb678477abad880fec36cf Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Sun, 14 May 2023 23:22:35 -0400 Subject: [PATCH 140/159] docs: add Thunderdome docs for releases (#9872) --- docs/RELEASE_ISSUE_TEMPLATE.md | 3 ++ docs/releases_thunderdome.md | 60 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 docs/releases_thunderdome.md diff --git a/docs/RELEASE_ISSUE_TEMPLATE.md b/docs/RELEASE_ISSUE_TEMPLATE.md index 3ece6d4149bb..ed181126ea41 100644 --- a/docs/RELEASE_ISSUE_TEMPLATE.md +++ b/docs/RELEASE_ISSUE_TEMPLATE.md @@ -63,6 +63,7 @@ This section covers tasks to be done ahead of the release. - open an access request in the [pldw](https://github.com/protocol/pldw/issues/new/choose) - [example](https://github.com/protocol/pldw/issues/158) - [ ] [kuboreleaser](https://github.com/ipfs/kuboreleaser) checked out on your system (_only if you're using [kuboreleaser](https://github.com/ipfs/kuboreleaser)_) + - [ ] [Thunderdome](https://github.com/ipfs-shipyard/thunderdome) checked out on your system and configured (see the [Thunderdome release docs](./releases_thunderdome.md) for setup) - [ ] [docker](https://docs.docker.com/get-docker/) installed on your system (_only if you're using [kuboreleaser](https://github.com/ipfs/kuboreleaser)_) - [ ] [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) installed on your system (_only if you're **NOT** using [kuboreleaser](https://github.com/ipfs/kuboreleaser)_) - [ ] [zsh](https://github.com/ohmyzsh/ohmyzsh/wiki/Installing-ZSH#install-and-set-up-zsh-as-default) installed on your system @@ -102,6 +103,8 @@ This section covers tasks to be done during each release. - do **NOT** use `Squash and merge` nor `Rebase and merge` because we need to be able to sign the merge commit - do **NOT** delete the `release-vX.Y` branch +- [ ] Run Thunderdome testing, see the [Thunderdome release docs](./releases_thunderdome.md) for details + - [ ] create a PR and merge the experiment config into Thunderdome - [ ] Create the release tag
using `kuboreleaser release --version vX.Y.Z(-rcN) tag` or ... - This is a dangerous operation! Go and Docker publishing are difficult to reverse! Have the release reviewer verify all the commands marked with โš ๏ธ! - [ ] โš ๏ธ ![](https://img.shields.io/badge/only-RC-blue?style=flat-square) tag the HEAD commit using `git tag -s vX.Y.Z(-RCN) -m 'Prerelease X.Y.Z(-RCN)'` diff --git a/docs/releases_thunderdome.md b/docs/releases_thunderdome.md new file mode 100644 index 000000000000..11057e26a30c --- /dev/null +++ b/docs/releases_thunderdome.md @@ -0,0 +1,60 @@ +# Testing Kubo releases with Thunderdome +This document is for running Thunderdome tests by release engineers as part of releasing Kubo. + +We use Thunderdome to replay ipfs.io gateway traffic in a controlled environment against two different versions of Kubo, and we record metrics and compare them to look for logic or performance regressions before releasing a new Kubo version. + +For background information about how Thunderdome works, see: https://github.com/ipfs-shipyard/thunderdome + +## Prerequisites + +* Ensure you have access to the "IPFS Stewards" vault in 1Password, which contains the requisite AWS Console and API credentials +* Ensure you have Docker and the Docker CLI installed +* Checkout the Thunderdome repo locally (or `git pull` to ensure it's up-to-date) +* Install AWS CLI v2: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html +* Configure the AWS CLI + * Configure the credentials as described in the [Thunderdome documentation](https://github.com/ipfs-shipyard/thunderdome/blob/main/cmd/thunderdome/README.md#credentials), using the credentials from 1Password +* Make sure the `thunderdome` binary is up-to-date: `go build ./cmd/thunderdome` + +## Add & run an experiment + +Create a new release configuration JSON in the `experiments/` directory, based on the most recent `kubo-release` configuration, and tweak as necessary. Generally we setup the targets to run a commit against the tag of the last release, such as: + +```json + "targets": [ + { + "name": "kubo190-4283b9", + "description": "kubo 0.19.0-rc1", + "build_from_git": { + "repo": "https://github.com/ipfs/kubo.git", + "commit":"4283b9d98f8438fc8751ccc840d8fc24eeae6f13" + } + }, + { + "name": "kubo181", + "description": "kubo 0.18.", + "build_from_git": { + "repo": "https://github.com/ipfs/kubo.git", + "tag":"v0.18.1" + } + } + ] +``` + +Run the experiment (where `$EXPERIMENT_CONFIG_JSON` is a path to the config JSON created above): + +```shell +AWS_PROFILE=thunderdome ./thunderdome deploy --verbose --duration 120 $EXPERIMENT_CONFIG_JSON +``` + +This will build the Docker images, upload them to ECR, and then launch the experiment in Thunderdome. Once the experiment starts, the CLI will exit and the experiment will continue to run for the duration. + +## Analyze Results + +Add a log entry in https://www.notion.so/pl-strflt/ce2d1bd56f3541028d960d3711465659 and link to it from the release issue, so that experiment results are publicly visible. + +The `deploy` command will output a link to the Grafana dashboard for the experiment. We don't currently have rigorous acceptance criteria, so you should look for anomalies or changes in the metrics and make sure they are tolerable and explainable. Unexplainable anomalies should be noted in the log with a screenshot, and then root caused. + + +## Open a PR to merge the experiment config into Thunderdome + +This is important for both posterity, and so that someone else can sanity-check the test parameters. From f683bf6cf2326987debb43d51ce3d63967dd230b Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Mon, 15 May 2023 14:29:21 +0200 Subject: [PATCH 141/159] feat: create pull request template (#9620) --- .github/PULL_REQUEST_TEMPLATE/pull_request_template.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE/pull_request_template.md diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md new file mode 100644 index 000000000000..6d06659db73b --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -0,0 +1,4 @@ + From 8ca9b712cfb24edda34d2ffc9b4253889ddbf9dd Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Mon, 15 May 2023 14:53:03 +0200 Subject: [PATCH 142/159] fix: the location of the PR template (#9878) --- .github/{PULL_REQUEST_TEMPLATE => }/pull_request_template.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{PULL_REQUEST_TEMPLATE => }/pull_request_template.md (100%) diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/pull_request_template.md similarity index 100% rename from .github/PULL_REQUEST_TEMPLATE/pull_request_template.md rename to .github/pull_request_template.md From a4f54574d021c01d97de2ea8a210ab09afe7df9d Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Mon, 15 May 2023 15:39:00 +0200 Subject: [PATCH 143/159] ci: skip build on PRs modifying MDs only (#9879) --- .github/workflows/build.yml | 2 ++ .github/workflows/codeql-analysis.yml | 2 ++ .github/workflows/docker-build.yml | 2 ++ .github/workflows/gateway-conformance.yml | 2 ++ .github/workflows/gobuild.yml | 2 ++ .github/workflows/golang-analysis.yml | 2 ++ .github/workflows/golint.yml | 2 ++ .github/workflows/gotest.yml | 2 ++ .github/workflows/sharness.yml | 2 ++ 9 files changed, 18 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e59532597b1c..294729c1aa2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,8 @@ name: Interop on: workflow_dispatch: pull_request: + paths-ignore: + - '**/*.md' push: branches: - 'master' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 805b183a748c..49a47476a927 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -8,6 +8,8 @@ on: pull_request: # The branches below must be a subset of the branches above branches: [ master ] + paths-ignore: + - '**/*.md' schedule: - cron: '30 12 * * 2' diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 13334faa6122..42d9c7c9c7b1 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -3,6 +3,8 @@ name: Docker Build on: workflow_dispatch: pull_request: + paths-ignore: + - '**/*.md' push: branches: - 'master' diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 588aec3464f2..ece4ca4630db 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -5,6 +5,8 @@ on: branches: - master pull_request: + paths-ignore: + - '**/*.md' concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} diff --git a/.github/workflows/gobuild.yml b/.github/workflows/gobuild.yml index 8591213c4b2c..ea71d1b50be5 100644 --- a/.github/workflows/gobuild.yml +++ b/.github/workflows/gobuild.yml @@ -3,6 +3,8 @@ name: Go Build on: workflow_dispatch: pull_request: + paths-ignore: + - '**/*.md' push: branches: - 'master' diff --git a/.github/workflows/golang-analysis.yml b/.github/workflows/golang-analysis.yml index 123e22403ffe..684223edd6e7 100644 --- a/.github/workflows/golang-analysis.yml +++ b/.github/workflows/golang-analysis.yml @@ -3,6 +3,8 @@ name: Go Check on: workflow_dispatch: pull_request: + paths-ignore: + - '**/*.md' push: branches: - 'master' diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml index 216573a468e5..0472e0f71875 100644 --- a/.github/workflows/golint.yml +++ b/.github/workflows/golint.yml @@ -3,6 +3,8 @@ name: Go Lint on: workflow_dispatch: pull_request: + paths-ignore: + - '**/*.md' push: branches: - 'master' diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index b20dca2a38e6..a25cc3e4bff7 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -3,6 +3,8 @@ name: Go Test on: workflow_dispatch: pull_request: + paths-ignore: + - '**/*.md' push: branches: - 'master' diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index a4c2241bee49..505c4cd030b2 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -3,6 +3,8 @@ name: Sharness on: workflow_dispatch: pull_request: + paths-ignore: + - '**/*.md' push: branches: - 'master' From 7d8262307cb40c1f21fd79fe52cdfd36a7dcb5cd Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 15 May 2023 17:18:51 +0100 Subject: [PATCH 144/159] fix: use https URI when multiaddress specifies tls (#177) Currently any clients created through `NewApiWithClient` will make a HTTP request to the api, even if the multiaddress specifies TLS or (the deprecated multiaddr option) https. This commit addresses this by having NewApiWithClient iterate the available protocols for the multiaddress, specifying the URL proto as https if it finds TLS or HTTPS is specified. The default continues to be http for those multiaddresses that do not specify these options. Should resolve #176 This commit was moved from ipfs/go-ipfs-http-client@7e1de1f7ccfa8cd1f616e94664886fe04d70806e --- client/httpapi/api.go | 16 +++++++++++++++- client/httpapi/api_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b7cfb2b520d4..5584de85d998 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -107,7 +107,21 @@ func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { } } - return NewURLApiWithClient(url, c) + proto := "http://" + + // By default, DialArgs is going to provide details suitable for connecting + // a socket to, but not really suitable for making an informed choice of http + // protocol. For multiaddresses specifying tls and/or https we want to make + // a https request instead of a http request. + protocols := a.Protocols() + for _, p := range protocols { + if p.Code == ma.P_HTTPS || p.Code == ma.P_TLS { + proto = "https://" + break + } + } + + return NewURLApiWithClient(proto+url, c) } func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2832d722d374..a6d684b0d075 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -246,3 +246,31 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } } + +func Test_NewURLApiWithClient_HTTP_Variant(t *testing.T) { + testcases := []struct { + address string + expected string + }{ + {address: "/ip4/127.0.0.1/tcp/80", expected: "http://127.0.0.1:80"}, + {address: "/ip4/127.0.0.1/tcp/443/tls", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/https", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/tls/http", expected: "https://127.0.0.1:443"}, + } + + for _, tc := range testcases { + address, err := ma.NewMultiaddr(tc.address) + if err != nil { + t.Fatal(err) + } + + api, err := NewApiWithClient(address, &http.Client{}) + if err != nil { + t.Fatal(err) + } + + if api.url != tc.expected { + t.Errorf("Expected = %s; got %s", tc.expected, api.url) + } + } +} From 13c379604fd39250ca788bfbfc20c90595898aca Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 16 May 2023 13:09:39 +0200 Subject: [PATCH 145/159] fix(gateway): redirect /ipns/b58mh to /ipns/cidb36 (#9785) --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/cli/gateway_test.go | 17 ++++++++++++----- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 52e3c064699c..9542831b51aa 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b + github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 3111d683f532..4285b1b47871 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 h1:5EacqVA/Houq8q7jpecu80sWMB0Gvz0rLgRiK2Mfat0= +github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index a756f0f32068..c61903c257c6 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b + github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index af7ec2fc01ae..c22b20251455 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 h1:5EacqVA/Houq8q7jpecu80sWMB0Gvz0rLgRiK2Mfat0= +github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/cli/gateway_test.go b/test/cli/gateway_test.go index 1d6ac45b965f..67f7e136a24a 100644 --- a/test/cli/gateway_test.go +++ b/test/cli/gateway_test.go @@ -13,8 +13,10 @@ import ( "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/test/cli/harness" . "github.com/ipfs/kubo/test/cli/testutils" + "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" + "github.com/multiformats/go-multibase" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,10 +27,13 @@ func TestGateway(t *testing.T) { node := h.NewNode().Init().StartDaemon("--offline") cid := node.IPFSAddStr("Hello Worlds!") + peerID, err := peer.ToCid(node.PeerID()).StringOfBase(multibase.Base36) + assert.Nil(t, err) + client := node.GatewayClient() client.TemplateData = map[string]string{ "CID": cid, - "PeerID": node.PeerID().String(), + "PeerID": peerID, } t.Run("GET IPFS path succeeds", func(t *testing.T) { @@ -182,7 +187,7 @@ func TestGateway(t *testing.T) { t.Run("GET /ipfs/ipns/{peerid} returns redirect to the valid path", func(t *testing.T) { t.Parallel() resp := client.Get("/ipfs/ipns/{{.PeerID}}?query=to-remember") - peerID := node.PeerID().String() + assert.Contains(t, resp.Body, fmt.Sprintf(``, peerID), @@ -474,6 +479,9 @@ func TestGateway(t *testing.T) { cfg.Gateway.NoFetch = true }) + node2PeerID, err := peer.ToCid(node2.PeerID()).StringOfBase(multibase.Base36) + assert.Nil(t, err) + nodes.StartDaemons().Connect() t.Run("not present", func(t *testing.T) { @@ -486,7 +494,7 @@ func TestGateway(t *testing.T) { t.Run("not present IPNS key from node 1", func(t *testing.T) { t.Parallel() - assert.Equal(t, 500, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode) + assert.Equal(t, 500, node1.GatewayClient().Get("/ipns/"+node2PeerID).StatusCode) }) }) @@ -501,8 +509,7 @@ func TestGateway(t *testing.T) { t.Run("present IPNS key from node 1", func(t *testing.T) { t.Parallel() node2.IPFS("name", "publish", "/ipfs/"+cidBar) - assert.Equal(t, 200, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode) - + assert.Equal(t, 200, node1.GatewayClient().Get("/ipns/"+node2PeerID).StatusCode) }) }) }) From 65b8d530b0bb6f0608264b7e9e6d1aaa2378bd0a Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 16 May 2023 14:05:22 +0200 Subject: [PATCH 146/159] test: add streaming delegated routing test (#9874) --- test/cli/delegated_routing_http_test.go | 50 ++++++++++++++++++++----- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/test/cli/delegated_routing_http_test.go b/test/cli/delegated_routing_http_test.go index a94c0b6085e8..b1ebacd14784 100644 --- a/test/cli/delegated_routing_http_test.go +++ b/test/cli/delegated_routing_http_test.go @@ -15,18 +15,20 @@ func TestHTTPDelegatedRouting(t *testing.T) { t.Parallel() node := harness.NewT(t).NewNode().Init().StartDaemon() - fakeServer := func(resp string) *httptest.Server { + fakeServer := func(contentType string, resp ...string) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, err := w.Write([]byte(resp)) - if err != nil { - panic(err) + w.Header().Set("Content-Type", contentType) + for _, r := range resp { + _, err := w.Write([]byte(r)) + if err != nil { + panic(err) + } } })) } findProvsCID := "baeabep4vu3ceru7nerjjbk37sxb7wmftteve4hcosmyolsbsiubw2vr6pqzj6mw7kv6tbn6nqkkldnklbjgm5tzbi4hkpkled4xlcr7xz4bq" - prov := "12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ" + provs := []string{"12D3KooWAobjw92XDcnQ1rRmRJDA3zAQpdPYUpZKrJxH6yccSpje", "12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ"} t.Run("default routing config has no routers defined", func(t *testing.T) { assert.Nil(t, node.ReadConfig().Routing.Routers) @@ -85,11 +87,11 @@ func TestHTTPDelegatedRouting(t *testing.T) { }) t.Run("adding HTTP delegated routing endpoint to Routing.Routers config works", func(t *testing.T) { - server := fakeServer(ToJSONStr(JSONObj{ + server := fakeServer("application/json", ToJSONStr(JSONObj{ "Providers": []JSONObj{{ "Protocol": "transport-bitswap", "Schema": "bitswap", - "ID": prov, + "ID": provs[0], "Addrs": []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"}, }}, })) @@ -114,9 +116,39 @@ func TestHTTPDelegatedRouting(t *testing.T) { assert.Equal(t, res.Stdout.Trimmed(), server.URL) node.StartDaemon() + res = node.IPFS("routing", "findprovs", findProvsCID) + assert.Equal(t, provs[0], res.Stdout.Trimmed()) + }) + + node.StopDaemon() + + t.Run("adding HTTP delegated routing endpoint to Routing.Routers config works (streaming)", func(t *testing.T) { + server := fakeServer("application/x-ndjson", ToJSONStr(JSONObj{ + "Protocol": "transport-bitswap", + "Schema": "bitswap", + "ID": provs[1], + "Addrs": []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"}, + }), ToJSONStr(JSONObj{ + "Protocol": "transport-bitswap", + "Schema": "bitswap", + "ID": provs[0], + "Addrs": []string{"/ip4/0.0.0.0/tcp/4001", "/ip4/0.0.0.0/tcp/4002"}, + })) + t.Cleanup(server.Close) + + node.IPFS("config", "Routing.Routers.TestDelegatedRouter", "--json", ToJSONStr(JSONObj{ + "Type": "http", + "Parameters": JSONObj{ + "Endpoint": server.URL, + }, + })) + res := node.IPFS("config", "Routing.Routers.TestDelegatedRouter.Parameters.Endpoint") + assert.Equal(t, res.Stdout.Trimmed(), server.URL) + + node.StartDaemon() res = node.IPFS("routing", "findprovs", findProvsCID) - assert.Equal(t, prov, res.Stdout.Trimmed()) + assert.Equal(t, provs[1]+"\n"+provs[0], res.Stdout.Trimmed()) }) t.Run("HTTP client should emit OpenCensus metrics", func(t *testing.T) { From da28fbc65a2e0f1ce59f9923823326ae2bc4f713 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 16 May 2023 15:33:27 +0200 Subject: [PATCH 147/159] chore: use assert.NoError instead of assert.Nil for errors --- test/cli/gateway_range_test.go | 8 ++++---- test/cli/gateway_test.go | 4 ++-- test/cli/swarm_test.go | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/cli/gateway_range_test.go b/test/cli/gateway_range_test.go index 5db7cc00e3e5..7440e63411ae 100644 --- a/test/cli/gateway_range_test.go +++ b/test/cli/gateway_range_test.go @@ -31,10 +31,10 @@ func TestGatewayHAMTDirectory(t *testing.T) { // Import fixtures r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car") - assert.Nil(t, err) + assert.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) - assert.Nil(t, err) + assert.NoError(t, err) // Fetch HAMT directory succeeds with minimal refs resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid)) @@ -60,10 +60,10 @@ func TestGatewayMultiRange(t *testing.T) { // Import fixtures r, err := os.Open("./fixtures/TestGatewayMultiRange.car") - assert.Nil(t, err) + assert.NoError(t, err) defer r.Close() err = node.IPFSDagImport(r, fixtureCid) - assert.Nil(t, err) + assert.NoError(t, err) // Succeeds fetching a range of blocks we have resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) { diff --git a/test/cli/gateway_test.go b/test/cli/gateway_test.go index 67f7e136a24a..972544ae0012 100644 --- a/test/cli/gateway_test.go +++ b/test/cli/gateway_test.go @@ -28,7 +28,7 @@ func TestGateway(t *testing.T) { cid := node.IPFSAddStr("Hello Worlds!") peerID, err := peer.ToCid(node.PeerID()).StringOfBase(multibase.Base36) - assert.Nil(t, err) + assert.NoError(t, err) client := node.GatewayClient() client.TemplateData = map[string]string{ @@ -480,7 +480,7 @@ func TestGateway(t *testing.T) { }) node2PeerID, err := peer.ToCid(node2.PeerID()).StringOfBase(multibase.Base36) - assert.Nil(t, err) + assert.NoError(t, err) nodes.StartDaemons().Connect() diff --git a/test/cli/swarm_test.go b/test/cli/swarm_test.go index d3220810674d..d1a4b5c6ca32 100644 --- a/test/cli/swarm_test.go +++ b/test/cli/swarm_test.go @@ -35,7 +35,7 @@ func TestSwarm(t *testing.T) { res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify") var output expectedOutputType err := json.Unmarshal(res.Stdout.Bytes(), &output) - assert.Nil(t, err) + assert.NoError(t, err) assert.Equal(t, 0, len(output.Peers)) }) @@ -48,7 +48,7 @@ func TestSwarm(t *testing.T) { res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify") var output expectedOutputType err := json.Unmarshal(res.Stdout.Bytes(), &output) - assert.Nil(t, err) + assert.NoError(t, err) actualID := output.Peers[0].Identify.ID actualPublicKey := output.Peers[0].Identify.PublicKey actualAgentVersion := output.Peers[0].Identify.AgentVersion @@ -78,12 +78,12 @@ func TestSwarm(t *testing.T) { otherNodeIDResponse := otherNode.RunIPFS("id", "--enc=json") var otherNodeIDOutput identifyType err := json.Unmarshal(otherNodeIDResponse.Stdout.Bytes(), &otherNodeIDOutput) - assert.Nil(t, err) + assert.NoError(t, err) res := node.RunIPFS("swarm", "peers", "--enc=json", "--identify") var output expectedOutputType err = json.Unmarshal(res.Stdout.Bytes(), &output) - assert.Nil(t, err) + assert.NoError(t, err) outputIdentify := output.Peers[0].Identify assert.Equal(t, outputIdentify.ID, otherNodeIDOutput.ID) From eda19c8abeb9996ad963d439bc2545feb29fec86 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 23 May 2023 10:42:14 +0200 Subject: [PATCH 148/159] chore: fix nix install options (#9887) See: https://github.com/NixOS/nixpkgs/commit/eefaaf41d67ea28c3f70150958f6a29533dce709 Fixes: #9752 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 07f8a87c5e56..6b6154065077 100644 --- a/README.md +++ b/README.md @@ -187,10 +187,10 @@ $ ipfs get /ipns/dist.ipfs.tech/kubo/$VERSION/kubo_$VERSION_windows-amd64.zip With the purely functional package manager [Nix](https://nixos.org/nix/) you can install kubo (go-ipfs) like this: ``` -$ nix-env -i ipfs +$ nix-env -i kubo ``` -You can also install the Package by using its attribute name, which is also `ipfs`. +You can also install the Package by using its attribute name, which is also `kubo`. #### Solus @@ -251,10 +251,10 @@ $ sudo port install ipfs In macOS you can use the purely functional package manager [Nix](https://nixos.org/nix/): ``` -$ nix-env -i ipfs +$ nix-env -i kubo ``` -You can also install the Package by using its attribute name, which is also `ipfs`. +You can also install the Package by using its attribute name, which is also `kubo`. #### Homebrew From 63561f3baf63524ce7d147f67c0c4b4e0ddc5bc9 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Thu, 25 May 2023 09:39:49 -0300 Subject: [PATCH 149/159] feat(bootstrap): save connected peers as backup bootstrap peers (#8856) * feat(bootstrap): save connected peers as backup temporary bootstrap ones * fix: do not add duplicated oldSavedPeers, not using tags, reuse randomizeList * test: add regression test * chore: add changelog --------- Co-authored-by: Henrique Dias Co-authored-by: Marcin Rataj --- config/internal.go | 7 +- core/bootstrap/bootstrap.go | 229 +++++++++++++++++++++++------- core/bootstrap/bootstrap_test.go | 6 +- core/core.go | 61 +++++++- docs/changelogs/v0.21.md | 16 +++ test/cli/backup_bootstrap_test.go | 60 ++++++++ 6 files changed, 323 insertions(+), 56 deletions(-) create mode 100644 test/cli/backup_bootstrap_test.go diff --git a/config/internal.go b/config/internal.go index a860c02c9cca..40070b11b525 100644 --- a/config/internal.go +++ b/config/internal.go @@ -2,9 +2,10 @@ package config type Internal struct { // All marked as omitempty since we are expecting to make changes to all subcomponents of Internal - Bitswap *InternalBitswap `json:",omitempty"` - UnixFSShardingSizeThreshold *OptionalString `json:",omitempty"` - Libp2pForceReachability *OptionalString `json:",omitempty"` + Bitswap *InternalBitswap `json:",omitempty"` + UnixFSShardingSizeThreshold *OptionalString `json:",omitempty"` + Libp2pForceReachability *OptionalString `json:",omitempty"` + BackupBootstrapInterval *OptionalDuration `json:",omitempty"` } type InternalBitswap struct { diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index daa0a44d3d5d..b566e0e978e6 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -3,16 +3,16 @@ package bootstrap import ( "context" "errors" - "fmt" "io" "math/rand" "sync" + "sync/atomic" "time" logging "github.com/ipfs/go-log" "github.com/jbenet/goprocess" - "github.com/jbenet/goprocess/context" - "github.com/jbenet/goprocess/periodic" + goprocessctx "github.com/jbenet/goprocess/context" + periodicproc "github.com/jbenet/goprocess/periodic" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -50,13 +50,26 @@ type BootstrapConfig struct { // for the bootstrap process to use. This makes it possible for clients // to control the peers the process uses at any moment. BootstrapPeers func() []peer.AddrInfo + + // BackupBootstrapInterval governs the periodic interval at which the node will + // attempt to save connected nodes to use as temporary bootstrap peers. + BackupBootstrapInterval time.Duration + + // MaxBackupBootstrapSize controls the maximum number of peers we're saving + // as backup bootstrap peers. + MaxBackupBootstrapSize int + + SaveBackupBootstrapPeers func(context.Context, []peer.AddrInfo) + LoadBackupBootstrapPeers func(context.Context) []peer.AddrInfo } // DefaultBootstrapConfig specifies default sane parameters for bootstrapping. var DefaultBootstrapConfig = BootstrapConfig{ - MinPeerThreshold: 4, - Period: 30 * time.Second, - ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3 + MinPeerThreshold: 4, + Period: 30 * time.Second, + ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3 + BackupBootstrapInterval: 1 * time.Hour, + MaxBackupBootstrapSize: 20, } func BootstrapConfigWithPeers(pis []peer.AddrInfo) BootstrapConfig { @@ -90,6 +103,9 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf log.Debugf("%s bootstrap error: %s", id, err) } + // Exit the first call (triggered independently by `proc.Go`, not `Tick`) + // only after being done with the *single* Routing.Bootstrap call. Following + // periodic calls (`Tick`) will not block on this. <-doneWithRound } @@ -108,9 +124,100 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf doneWithRound <- struct{}{} close(doneWithRound) // it no longer blocks periodic + + startSavePeersAsTemporaryBootstrapProc(cfg, host, proc) + return proc, nil } +// Aside of the main bootstrap process we also run a secondary one that saves +// connected peers as a backup measure if we can't connect to the official +// bootstrap ones. These peers will serve as *temporary* bootstrap nodes. +func startSavePeersAsTemporaryBootstrapProc(cfg BootstrapConfig, host host.Host, bootstrapProc goprocess.Process) { + savePeersFn := func(worker goprocess.Process) { + ctx := goprocessctx.OnClosingContext(worker) + + if err := saveConnectedPeersAsTemporaryBootstrap(ctx, host, cfg); err != nil { + log.Debugf("saveConnectedPeersAsTemporaryBootstrap error: %s", err) + } + } + savePeersProc := periodicproc.Tick(cfg.BackupBootstrapInterval, savePeersFn) + + // When the main bootstrap process ends also terminate the 'save connected + // peers' ones. Coupling the two seems the easiest way to handle this backup + // process without additional complexity. + go func() { + <-bootstrapProc.Closing() + savePeersProc.Close() + }() + + // Run the first round now (after the first bootstrap process has finished) + // as the SavePeersPeriod can be much longer than bootstrap. + savePeersProc.Go(savePeersFn) +} + +func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, cfg BootstrapConfig) error { + // Randomize the list of connected peers, we don't prioritize anyone. + connectedPeers := randomizeList(host.Network().Peers()) + + bootstrapPeers := cfg.BootstrapPeers() + backupPeers := make([]peer.AddrInfo, 0, cfg.MaxBackupBootstrapSize) + + // Choose peers to save and filter out the ones that are already bootstrap nodes. + for _, p := range connectedPeers { + found := false + for _, bootstrapPeer := range bootstrapPeers { + if p == bootstrapPeer.ID { + found = true + break + } + } + if !found { + backupPeers = append(backupPeers, peer.AddrInfo{ + ID: p, + Addrs: host.Network().Peerstore().Addrs(p), + }) + } + + if len(backupPeers) >= cfg.MaxBackupBootstrapSize { + break + } + } + + // If we didn't reach the target number use previously stored connected peers. + if len(backupPeers) < cfg.MaxBackupBootstrapSize { + oldSavedPeers := cfg.LoadBackupBootstrapPeers(ctx) + log.Debugf("missing %d peers to reach backup bootstrap target of %d, trying from previous list of %d saved peers", + cfg.MaxBackupBootstrapSize-len(backupPeers), cfg.MaxBackupBootstrapSize, len(oldSavedPeers)) + + // Add some of the old saved peers. Ensure we don't duplicate them. + for _, p := range oldSavedPeers { + found := false + for _, sp := range backupPeers { + if p.ID == sp.ID { + found = true + break + } + } + + if !found { + backupPeers = append(backupPeers, p) + } + + if len(backupPeers) >= cfg.MaxBackupBootstrapSize { + break + } + } + } + + cfg.SaveBackupBootstrapPeers(ctx, backupPeers) + log.Debugf("saved %d peers (of %d target) as bootstrap backup in the config", len(backupPeers), cfg.MaxBackupBootstrapSize) + return nil +} + +// Connect to as many peers needed to reach the BootstrapConfig.MinPeerThreshold. +// Peers can be original bootstrap or temporary ones (drawn from a list of +// persisted previously connected peers). func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) error { ctx, cancel := context.WithTimeout(ctx, cfg.ConnectionTimeout) @@ -127,35 +234,58 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er id, len(connected), cfg.MinPeerThreshold) return nil } - numToDial := cfg.MinPeerThreshold - len(connected) + numToDial := cfg.MinPeerThreshold - len(connected) // numToDial > 0 - // filter out bootstrap nodes we are already connected to - var notConnected []peer.AddrInfo - for _, p := range peers { - if host.Network().Connectedness(p.ID) != network.Connected { - notConnected = append(notConnected, p) + if len(peers) > 0 { + numToDial -= int(peersConnect(ctx, host, peers, numToDial, true)) + if numToDial <= 0 { + return nil } } - // if connected to all bootstrap peer candidates, exit - if len(notConnected) < 1 { - log.Debugf("%s no more bootstrap peers to create %d connections", id, numToDial) - return ErrNotEnoughBootstrapPeers + log.Debugf("not enough bootstrap peers to fill the remaining target of %d connections, trying backup list", numToDial) + + tempBootstrapPeers := cfg.LoadBackupBootstrapPeers(ctx) + if len(tempBootstrapPeers) > 0 { + numToDial -= int(peersConnect(ctx, host, tempBootstrapPeers, numToDial, false)) + if numToDial <= 0 { + return nil + } } - // connect to a random susbset of bootstrap candidates - randSubset := randomSubsetOfPeers(notConnected, numToDial) + log.Debugf("tried both original bootstrap peers and temporary ones but still missing target of %d connections", numToDial) - log.Debugf("%s bootstrapping to %d nodes: %s", id, numToDial, randSubset) - return bootstrapConnect(ctx, host, randSubset) + return ErrNotEnoughBootstrapPeers } -func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo) error { - if len(peers) < 1 { - return ErrNotEnoughBootstrapPeers - } +// Attempt to make `needed` connections from the `availablePeers` list. Mark +// peers as either `permanent` or temporary when adding them to the Peerstore. +// Return the number of connections completed. We eagerly over-connect in parallel, +// so we might connect to more than needed. +// (We spawn as many routines and attempt connections as the number of availablePeers, +// but this list comes from restricted sets of original or temporary bootstrap +// nodes which will keep it under a sane value.) +func peersConnect(ctx context.Context, ph host.Host, availablePeers []peer.AddrInfo, needed int, permanent bool) uint64 { + peers := randomizeList(availablePeers) + + // Monitor the number of connections and stop if we reach the target. + var connected uint64 + ctx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + if int(atomic.LoadUint64(&connected)) >= needed { + cancel() + return + } + } + } + }() - errs := make(chan error, len(peers)) var wg sync.WaitGroup for _, p := range peers { @@ -164,45 +294,46 @@ func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo) // fail/abort due to an expiring context. // Also, performed asynchronously for dial speed. + if int(atomic.LoadUint64(&connected)) >= needed { + cancel() + break + } + wg.Add(1) go func(p peer.AddrInfo) { defer wg.Done() + + // Skip addresses belonging to a peer we're already connected to. + // (Not a guarantee but a best-effort policy.) + if ph.Network().Connectedness(p.ID) == network.Connected { + return + } log.Debugf("%s bootstrapping to %s", ph.ID(), p.ID) - ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) if err := ph.Connect(ctx, p); err != nil { - log.Debugf("failed to bootstrap with %v: %s", p.ID, err) - errs <- err + if ctx.Err() != context.Canceled { + log.Debugf("failed to bootstrap with %v: %s", p.ID, err) + } return } + if permanent { + // We're connecting to an original bootstrap peer, mark it as + // a permanent address (Connect will register it as TempAddrTTL). + ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) + } + log.Infof("bootstrapped with %v", p.ID) + atomic.AddUint64(&connected, 1) }(p) } wg.Wait() - // our failure condition is when no connection attempt succeeded. - // So drain the errs channel, counting the results. - close(errs) - count := 0 - var err error - for err = range errs { - if err != nil { - count++ - } - } - if count == len(peers) { - return fmt.Errorf("failed to bootstrap. %s", err) - } - return nil + return connected } -func randomSubsetOfPeers(in []peer.AddrInfo, max int) []peer.AddrInfo { - if max > len(in) { - max = len(in) - } - - out := make([]peer.AddrInfo, max) - for i, val := range rand.Perm(len(in))[:max] { +func randomizeList[T any](in []T) []T { + out := make([]T, len(in)) + for i, val := range rand.Perm(len(in)) { out[i] = in[val] } return out diff --git a/core/bootstrap/bootstrap_test.go b/core/bootstrap/bootstrap_test.go index 98a4a78272c0..39490a474c7e 100644 --- a/core/bootstrap/bootstrap_test.go +++ b/core/bootstrap/bootstrap_test.go @@ -7,9 +7,9 @@ import ( "github.com/libp2p/go-libp2p/core/test" ) -func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { +func TestRandomizeAddressList(t *testing.T) { var ps []peer.AddrInfo - sizeofSlice := 100 + sizeofSlice := 10 for i := 0; i < sizeofSlice; i++ { pid, err := test.RandPeerID() if err != nil { @@ -18,7 +18,7 @@ func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { ps = append(ps, peer.AddrInfo{ID: pid}) } - out := randomSubsetOfPeers(ps, 2*sizeofSlice) + out := randomizeList(ps) if len(out) != len(ps) { t.Fail() } diff --git a/core/core.go b/core/core.go index 448386444fd4..5c0a7ef2cba1 100644 --- a/core/core.go +++ b/core/core.go @@ -11,10 +11,13 @@ package core import ( "context" + "encoding/json" "io" + "time" "github.com/ipfs/boxo/filestore" pin "github.com/ipfs/boxo/pinning/pinner" + "github.com/ipfs/go-datastore" bserv "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" @@ -46,6 +49,7 @@ import ( "github.com/ipfs/boxo/namesys" ipnsrp "github.com/ipfs/boxo/namesys/republisher" + "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/bootstrap" "github.com/ipfs/kubo/core/node" "github.com/ipfs/kubo/core/node/libp2p" @@ -165,12 +169,40 @@ func (n *IpfsNode) Bootstrap(cfg bootstrap.BootstrapConfig) error { return ps } } + if cfg.SaveBackupBootstrapPeers == nil { + cfg.SaveBackupBootstrapPeers = func(ctx context.Context, peerList []peer.AddrInfo) { + err := n.saveTempBootstrapPeers(ctx, peerList) + if err != nil { + log.Warnf("saveTempBootstrapPeers failed: %s", err) + return + } + } + } + if cfg.LoadBackupBootstrapPeers == nil { + cfg.LoadBackupBootstrapPeers = func(ctx context.Context) []peer.AddrInfo { + peerList, err := n.loadTempBootstrapPeers(ctx) + if err != nil { + log.Warnf("loadTempBootstrapPeers failed: %s", err) + return nil + } + return peerList + } + } + + repoConf, err := n.Repo.Config() + if err != nil { + return err + } + if repoConf.Internal.BackupBootstrapInterval != nil { + cfg.BackupBootstrapInterval = repoConf.Internal.BackupBootstrapInterval.WithDefault(time.Hour) + } - var err error n.Bootstrapper, err = bootstrap.Bootstrap(n.Identity, n.PeerHost, n.Routing, cfg) return err } +var TempBootstrapPeersKey = datastore.NewKey("/local/temp_bootstrap_peers") + func (n *IpfsNode) loadBootstrapPeers() ([]peer.AddrInfo, error) { cfg, err := n.Repo.Config() if err != nil { @@ -180,6 +212,33 @@ func (n *IpfsNode) loadBootstrapPeers() ([]peer.AddrInfo, error) { return cfg.BootstrapPeers() } +func (n *IpfsNode) saveTempBootstrapPeers(ctx context.Context, peerList []peer.AddrInfo) error { + ds := n.Repo.Datastore() + bytes, err := json.Marshal(config.BootstrapPeerStrings(peerList)) + if err != nil { + return err + } + + if err := ds.Put(ctx, TempBootstrapPeersKey, bytes); err != nil { + return err + } + return ds.Sync(ctx, TempBootstrapPeersKey) +} + +func (n *IpfsNode) loadTempBootstrapPeers(ctx context.Context) ([]peer.AddrInfo, error) { + ds := n.Repo.Datastore() + bytes, err := ds.Get(ctx, TempBootstrapPeersKey) + if err != nil { + return nil, err + } + + var addrs []string + if err := json.Unmarshal(bytes, &addrs); err != nil { + return nil, err + } + return config.ParseBootstrapPeers(addrs) +} + type ConstructPeerHostOpts struct { AddrsFactory p2pbhost.AddrsFactory DisableNatPortMap bool diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index 57eb7002bad4..b4af6168b3b9 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -6,6 +6,7 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -13,6 +14,21 @@ ### ๐Ÿ”ฆ Highlights +#### Saving previously seen nodes for later bootstrapping + +Kubo now stores a subset of connected peers as backup bootstrap nodes ([kubo#8856](https://github.com/ipfs/kubo/pull/8856)). +These nodes are used in addition to the explicitly defined bootstrappers in the +[`Bootstrap`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bootstrap) configuration. + +This enhancement improves the resiliency of the system, as it eliminates the +necessity of relying solely on the default bootstrappers operated by Protocol +Labs for joining the public IPFS swarm. Previously, this level of robustness +was only available in LAN contexts with [mDNS peer discovery](https://github.com/ipfs/kubo/blob/master/docs/config.md#discoverymdns) +enabled. + +With this update, the same level of robustness is applied to peers that lack +mDNS peers and solely rely on the public DHT. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/test/cli/backup_bootstrap_test.go b/test/cli/backup_bootstrap_test.go new file mode 100644 index 000000000000..017499f3d631 --- /dev/null +++ b/test/cli/backup_bootstrap_test.go @@ -0,0 +1,60 @@ +package cli + +import ( + "fmt" + "testing" + "time" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/assert" +) + +func TestBackupBootstrapPeers(t *testing.T) { + nodes := harness.NewT(t).NewNodes(3).Init() + nodes.ForEachPar(func(n *harness.Node) { + n.UpdateConfig(func(cfg *config.Config) { + cfg.Bootstrap = []string{} + cfg.Addresses.Swarm = []string{fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", harness.NewRandPort())} + cfg.Discovery.MDNS.Enabled = false + cfg.Internal.BackupBootstrapInterval = config.NewOptionalDuration(250 * time.Millisecond) + }) + }) + + // Start all nodes and ensure they all have no peers. + nodes.StartDaemons() + nodes.ForEachPar(func(n *harness.Node) { + assert.Len(t, n.Peers(), 0) + }) + + // Connect nodes 0 and 1, ensure they know each other. + nodes[0].Connect(nodes[1]) + assert.Len(t, nodes[0].Peers(), 1) + assert.Len(t, nodes[1].Peers(), 1) + assert.Len(t, nodes[2].Peers(), 0) + + // Wait a bit to ensure that 0 and 1 saved their temporary bootstrap backups. + time.Sleep(time.Millisecond * 500) + nodes.StopDaemons() + + // Start 1 and 2. 2 does not know anyone yet. + nodes[1].StartDaemon() + nodes[2].StartDaemon() + assert.Len(t, nodes[1].Peers(), 0) + assert.Len(t, nodes[2].Peers(), 0) + + // Connect 1 and 2, ensure they know each other. + nodes[1].Connect(nodes[2]) + assert.Len(t, nodes[1].Peers(), 1) + assert.Len(t, nodes[2].Peers(), 1) + + // Start 0, wait a bit. Should connect to 1, and then discover 2 via the + // backup bootstrap peers. + nodes[0].StartDaemon() + time.Sleep(time.Millisecond * 500) + + // Check if they're all connected. + assert.Len(t, nodes[0].Peers(), 2) + assert.Len(t, nodes[1].Peers(), 2) + assert.Len(t, nodes[2].Peers(), 2) +} From e3126eb3e03f9e7b3ae291ace0f9cf7f6affea41 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 25 May 2023 15:36:33 +0200 Subject: [PATCH 150/159] chore: upgrade boxo and fix test function signature (#9896) --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/cli/content_routing_http_test.go | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9542831b51aa..9de8c6579d8b 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 + github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 4285b1b47871..11d274816214 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 h1:5EacqVA/Houq8q7jpecu80sWMB0Gvz0rLgRiK2Mfat0= -github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4= +github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index c61903c257c6..2799171dc9b6 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 + github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index c22b20251455..c9c874dc5a42 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816 h1:5EacqVA/Houq8q7jpecu80sWMB0Gvz0rLgRiK2Mfat0= -github.com/ipfs/boxo v0.8.2-0.20230516102723-5e94b9d91816/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4= +github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/cli/content_routing_http_test.go b/test/cli/content_routing_http_test.go index acdea7029dfc..c1ea64d3fb58 100644 --- a/test/cli/content_routing_http_test.go +++ b/test/cli/content_routing_http_test.go @@ -24,7 +24,7 @@ type fakeHTTPContentRouter struct { provideCalls int } -func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) { +func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) { r.m.Lock() defer r.m.Unlock() r.findProvidersCalls++ From 38556b8d21292c5c10f0bd7b5d874ca447111c23 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 29 May 2023 16:17:11 +0200 Subject: [PATCH 151/159] docs: Reprovider config (#9900) This updates docs to match new values from https://github.com/ipfs/kubo/pull/9326 --- docs/config.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/config.md b/docs/config.md index 91866a42323c..c571407e8f59 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1326,15 +1326,19 @@ Type: `array[peering]` ### `Reprovider.Interval` Sets the time between rounds of reproviding local content to the routing -system. If unset, it defaults to 12 hours. If set to the value `"0"` it will -disable content reproviding. +system. + +- If unset, it uses the implicit safe default. +- If set to the value `"0"` it will disable content reproviding. Note: disabling content reproviding will result in other nodes on the network not being able to discover that you have the objects that you have. If you want to have this disabled and keep the network aware of what you have, you must manually announce your content periodically. -Type: `duration` +Default: `22h` (`DefaultReproviderInterval`) + +Type: `optionalDuration` (unset for the default) ### `Reprovider.Strategy` @@ -1346,7 +1350,7 @@ Tells reprovider what should be announced. Valid strategies are: Default: `"all"` -Type: `string` (or unset for the default, which is "all") +Type: `optionalString` (unset for the default) ## `Routing` From c10b804449cd6e8e354a693f3d0c4bc7bd93fbca Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 30 May 2023 00:59:34 +0200 Subject: [PATCH 152/159] feat: Gateway.DeserializedResponses config flag (#9789) Co-authored-by: Marcin Rataj --- config/gateway.go | 16 ++++- core/corehttp/gateway.go | 65 +++++++++++--------- core/corehttp/gateway_test.go | 40 +++++++++++++ docs/changelogs/v0.21.md | 27 +++++++++ docs/config.md | 30 ++++++++-- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- test/cli/gateway_test.go | 83 ++++++++++++++++++++++++++ 10 files changed, 233 insertions(+), 40 deletions(-) diff --git a/config/gateway.go b/config/gateway.go index 8ae312b59aee..816b1f48d23e 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -1,6 +1,9 @@ package config -const DefaultInlineDNSLink = false +const ( + DefaultInlineDNSLink = false + DefaultDeserializedResponses = true +) type GatewaySpec struct { // Paths is explicit list of path prefixes that should be handled by @@ -25,6 +28,11 @@ type GatewaySpec struct { // (FQDN) into a single DNS label in order to interop with wildcard TLS certs // and Origin per CID isolation provided by rules like https://publicsuffix.org InlineDNSLink Flag + + // DeserializedResponses configures this gateway to respond to deserialized + // responses. Disabling this option enables a Trustless Gateway, as per: + // https://specs.ipfs.tech/http-gateways/trustless-gateway/. + DeserializedResponses Flag } // Gateway contains options for the HTTP gateway server. @@ -56,6 +64,12 @@ type Gateway struct { // This flag can be overridden per FQDN in PublicGateways. NoDNSLink bool + // DeserializedResponses configures this gateway to respond to deserialized + // requests. Disabling this option enables a Trustless only gateway, as per: + // https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can + // be overridden per FQDN in PublicGateways. + DeserializedResponses Flag + // PublicGateways configures behavior of known public gateways. // Each key is a fully qualified domain name (FQDN). PublicGateways map[string]*GatewaySpec diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index a9c42f185739..0f86d4da7b1e 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -28,22 +28,11 @@ import ( func GatewayOption(paths ...string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - cfg, err := n.Repo.Config() + gwConfig, err := getGatewayConfig(n) if err != nil { return nil, err } - headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders)) - for h, v := range cfg.Gateway.HTTPHeaders { - headers[http.CanonicalHeaderKey(h)] = v - } - - gateway.AddAccessControlHeaders(headers) - - gwConfig := gateway.Config{ - Headers: headers, - } - gwAPI, err := newGatewayBackend(n) if err != nil { return nil, err @@ -65,7 +54,7 @@ func GatewayOption(paths ...string) ServeOption { func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - cfg, err := n.Repo.Config() + gwConfig, err := getGatewayConfig(n) if err != nil { return nil, err } @@ -75,9 +64,8 @@ func HostnameOption() ServeOption { return nil, err } - publicGateways := convertPublicGateways(cfg.Gateway.PublicGateways) childMux := http.NewServeMux() - mux.HandleFunc("/", gateway.WithHostname(childMux, gwAPI, publicGateways, cfg.Gateway.NoDNSLink).ServeHTTP) + mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP) return childMux, nil } } @@ -212,30 +200,49 @@ var defaultKnownGateways = map[string]*gateway.Specification{ "localhost": subdomainGatewaySpec, } -func convertPublicGateways(publicGateways map[string]*config.GatewaySpec) map[string]*gateway.Specification { - gws := map[string]*gateway.Specification{} +func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) { + cfg, err := n.Repo.Config() + if err != nil { + return gateway.Config{}, err + } + + // Parse configuration headers and add the default Access Control Headers. + headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders)) + for h, v := range cfg.Gateway.HTTPHeaders { + headers[http.CanonicalHeaderKey(h)] = v + } + gateway.AddAccessControlHeaders(headers) + + // Initialize gateway configuration, with empty PublicGateways, handled after. + gwCfg := gateway.Config{ + Headers: headers, + DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses), + NoDNSLink: cfg.Gateway.NoDNSLink, + PublicGateways: map[string]*gateway.Specification{}, + } - // First, implicit defaults such as subdomain gateway on localhost + // Add default implicit known gateways, such as subdomain gateway on localhost. for hostname, gw := range defaultKnownGateways { - gws[hostname] = gw + gwCfg.PublicGateways[hostname] = gw } - // Then apply values from Gateway.PublicGateways, if present in the config - for hostname, gw := range publicGateways { + // Apply values from cfg.Gateway.PublicGateways if they exist. + for hostname, gw := range cfg.Gateway.PublicGateways { if gw == nil { // Remove any implicit defaults, if present. This is useful when one - // wants to disable subdomain gateway on localhost etc. - delete(gws, hostname) + // wants to disable subdomain gateway on localhost, etc. + delete(gwCfg.PublicGateways, hostname) continue } - gws[hostname] = &gateway.Specification{ - Paths: gw.Paths, - NoDNSLink: gw.NoDNSLink, - UseSubdomains: gw.UseSubdomains, - InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink), + gwCfg.PublicGateways[hostname] = &gateway.Specification{ + Paths: gw.Paths, + NoDNSLink: gw.NoDNSLink, + UseSubdomains: gw.UseSubdomains, + InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink), + DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses), } } - return gws + return gwCfg, nil } diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 522d92f13c64..cfc2451376aa 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -14,6 +14,7 @@ import ( core "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/coreapi" repo "github.com/ipfs/kubo/repo" + "github.com/stretchr/testify/assert" iface "github.com/ipfs/boxo/coreiface" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" @@ -173,3 +174,42 @@ func TestVersion(t *testing.T) { t.Fatalf("response doesn't contain protocol version:\n%s", s) } } + +func TestDeserializedResponsesInheritance(t *testing.T) { + for _, testCase := range []struct { + globalSetting config.Flag + gatewaySetting config.Flag + expectedGatewaySetting bool + }{ + {config.True, config.Default, true}, + {config.False, config.Default, false}, + {config.False, config.True, true}, + {config.True, config.False, false}, + } { + c := config.Config{ + Identity: config.Identity{ + PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node + }, + Gateway: config.Gateway{ + DeserializedResponses: testCase.globalSetting, + PublicGateways: map[string]*config.GatewaySpec{ + "example.com": { + DeserializedResponses: testCase.gatewaySetting, + }, + }, + }, + } + r := &repo.Mock{ + C: c, + D: syncds.MutexWrap(datastore.NewMapDatastore()), + } + n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) + assert.NoError(t, err) + + gwCfg, err := getGatewayConfig(n) + assert.NoError(t, err) + + assert.Contains(t, gwCfg.PublicGateways, "example.com") + assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses) + } +} diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index b4af6168b3b9..d4a69514ff5d 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -7,6 +7,7 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) + - [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -29,6 +30,32 @@ enabled. With this update, the same level of robustness is applied to peers that lack mDNS peers and solely rely on the public DHT. + +#### `Gateway.DeserializedResponses` config flag + +This release introduces the +[`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses) +configuration flag. + +With this flag, one can explicitly configure whether the gateway responds to +deserialized requests or not. By default, this flag is enabled. + +Disabling deserialized responses allows the +gateway to operate +as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) +limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) +response types: +[application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw), +[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car), +and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record). + +With deserialized responses disabled, the Kubo gateway can serve as a block +backend for other software (like +[bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme), +[IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md) +etc) without the usual risks associated with hosting deserialized data behind +third-party CIDs. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index c571407e8f59..bf2b7750c641 100644 --- a/docs/config.md +++ b/docs/config.md @@ -50,6 +50,7 @@ config file at runtime. - [`Gateway`](#gateway) - [`Gateway.NoFetch`](#gatewaynofetch) - [`Gateway.NoDNSLink`](#gatewaynodnslink) + - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses) - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) - [`Gateway.RootRedirect`](#gatewayrootredirect) - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold) @@ -60,6 +61,7 @@ config file at runtime. - [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains) - [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink) - [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink) + - [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses) - [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways) - [`Gateway` recipes](#gateway-recipes) - [`Identity`](#identity) @@ -236,7 +238,7 @@ documented in `ipfs config profile --help`. smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in your IPFS node, and disk usage is more critical than performance, consider using `flatfs`. - - This datastore uses up to several gigabytes of memory. + - This datastore uses up to several gigabytes of memory. - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. - The current implementation is based on old badger 1.x which is no longer supported by the upstream team. @@ -646,6 +648,16 @@ Default: `false` Type: `bool` +#### `Gateway.DeserializedResponses` + +An optional flag to explicitly configure whether this gateway responds to deserialized +requests, or not. By default, it is enabled. When disabling this option, the gateway +operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/. + +Default: `true` + +Type: `flag` + ### `Gateway.HTTPHeaders` Headers to set on gateway responses. @@ -790,6 +802,16 @@ Default: `false` Type: `flag` +#### `Gateway.PublicGateways: DeserializedResponses` + +An optional flag to explicitly configure whether this gateway responds to deserialized +requests, or not. By default, it is enabled. When disabling this option, the gateway +operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/. + +Default: same as global `Gateway.DeserializedResponses` + +Type: `flag` + #### Implicit defaults of `Gateway.PublicGateways` Default entries for `localhost` hostname and loopback IPs are always present. @@ -895,7 +917,7 @@ Type: `string` (base64 encoded) ## `Internal` -This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo. +This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo. **Be aware that making informed change here requires in-depth knowledge and most users should leave these untouched. All knobs listed here are subject to breaking changes between versions.** @@ -971,7 +993,7 @@ Type: `optionalInteger` (byte count, `null` means default which is 1MB) ### `Internal.Bitswap.ProviderSearchDelay` This parameter determines how long to wait before looking for providers outside of bitswap. -Other routing systems like the DHT are able to provide results in less than a second, so lowering +Other routing systems like the DHT are able to provide results in less than a second, so lowering this number will allow faster peers lookups in some cases. Type: `optionalDuration` (`null` means default which is 1s) @@ -1552,7 +1574,7 @@ another node, even if this other node is on a different network. This may trigger netscan alerts on some hosting providers or cause strain in some setups. The `server` configuration profile fills up this list with sensible defaults, -preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, +preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, which is the multiaddress representation of `192.168.0.0/16`) but you should always check settings against your own network and/or hosting provider. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9de8c6579d8b..e21648bd45be 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 + github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 11d274816214..d6dd8467980c 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 2799171dc9b6..006c3ab58e51 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 + github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index c9c874dc5a42..2717767c09c9 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4= -github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0= +github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/cli/gateway_test.go b/test/cli/gateway_test.go index 972544ae0012..eb955b593b30 100644 --- a/test/cli/gateway_test.go +++ b/test/cli/gateway_test.go @@ -513,4 +513,87 @@ func TestGateway(t *testing.T) { }) }) }) + + t.Run("DeserializedResponses", func(t *testing.T) { + type testCase struct { + globalValue config.Flag + gatewayValue config.Flag + deserializedGlobalStatusCode int + deserializedGatewayStaticCode int + message string + } + + setHost := func(r *http.Request) { + r.Host = "example.com" + } + + withAccept := func(accept string) func(r *http.Request) { + return func(r *http.Request) { + r.Header.Set("Accept", accept) + } + } + + withHostAndAccept := func(accept string) func(r *http.Request) { + return func(r *http.Request) { + setHost(r) + withAccept(accept)(r) + } + } + + makeTest := func(test *testCase) func(t *testing.T) { + return func(t *testing.T) { + t.Parallel() + + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Gateway.DeserializedResponses = test.globalValue + cfg.Gateway.PublicGateways = map[string]*config.GatewaySpec{ + "example.com": { + Paths: []string{"/ipfs", "/ipns"}, + DeserializedResponses: test.gatewayValue, + }, + } + }) + node.StartDaemon() + + cidFoo := node.IPFSAddStr("foo") + client := node.GatewayClient() + + deserializedPath := "/ipfs/" + cidFoo + + blockPath := deserializedPath + "?format=raw" + carPath := deserializedPath + "?format=car" + + // Global Check (Gateway.DeserializedResponses) + assert.Equal(t, http.StatusOK, client.Get(blockPath).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.raw")).StatusCode) + + assert.Equal(t, http.StatusOK, client.Get(carPath).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.car")).StatusCode) + + assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath).StatusCode) + assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath, withAccept("application/json")).StatusCode) + + // Public Gateway (example.com) Check (Gateway.PublicGateways[example.com].DeserializedResponses) + assert.Equal(t, http.StatusOK, client.Get(blockPath, setHost).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.raw")).StatusCode) + + assert.Equal(t, http.StatusOK, client.Get(carPath, setHost).StatusCode) + assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.car")).StatusCode) + + assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, setHost).StatusCode) + assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, withHostAndAccept("application/json")).StatusCode) + + } + } + + for _, test := range []*testCase{ + {config.True, config.Default, http.StatusOK, http.StatusOK, "when Gateway.DeserializedResponses is globally enabled, leaving implicit default for Gateway.PublicGateways[example.com] should inherit the global setting (enabled)"}, + {config.False, config.Default, http.StatusNotAcceptable, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally disabled, leaving implicit default on Gateway.PublicGateways[example.com] should inherit the global setting (disabled)"}, + {config.False, config.True, http.StatusNotAcceptable, http.StatusOK, "when Gateway.DeserializedResponses is globally disabled, explicitly enabling on Gateway.PublicGateways[example.com] should override global (enabled)"}, + {config.True, config.False, http.StatusOK, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally enabled, explicitly disabling on Gateway.PublicGateways[example.com] should override global (disabled)"}, + } { + t.Run(test.message, makeTest(test)) + } + }) } From 12f343d837850b0cb198ccf2b40630bdbf1ec715 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 31 May 2023 14:29:59 +0200 Subject: [PATCH 153/159] client/rpc: rename package name to match rpc and edit migration story --- client/rpc/README.md | 6 +++--- client/rpc/api.go | 2 +- client/rpc/api_test.go | 2 +- client/rpc/apifile.go | 2 +- client/rpc/block.go | 2 +- client/rpc/dag.go | 2 +- client/rpc/dht.go | 2 +- client/rpc/errors.go | 2 +- client/rpc/errors_test.go | 2 +- client/rpc/key.go | 2 +- client/rpc/name.go | 2 +- client/rpc/object.go | 2 +- client/rpc/path.go | 2 +- client/rpc/pin.go | 2 +- client/rpc/pubsub.go | 2 +- client/rpc/request.go | 2 +- client/rpc/requestbuilder.go | 2 +- client/rpc/response.go | 2 +- client/rpc/routing.go | 2 +- client/rpc/swarm.go | 2 +- client/rpc/unixfs.go | 2 +- docs/changelogs/v0.21.md | 3 ++- docs/http-rpc-clients.md | 4 ++-- 23 files changed, 27 insertions(+), 26 deletions(-) diff --git a/client/rpc/README.md b/client/rpc/README.md index e6b534b90e92..a020aa9da277 100644 --- a/client/rpc/README.md +++ b/client/rpc/README.md @@ -1,4 +1,4 @@ -# `httpapi` +# `coreiface.CoreAPI` over http `rpc` > IPFS CoreAPI implementation using HTTP API @@ -19,13 +19,13 @@ import ( "context" "fmt" - ipfsClient "github.com/ipfs/kubo/client/rpc" + "github.com/ipfs/kubo/client/rpc" path "github.com/ipfs/boxo/coreiface/path" ) func main() { // "Connect" to local node - node, err := ipfsClient.NewLocalApi() + node, err := rpc.NewLocalApi() if err != nil { fmt.Printf(err) return diff --git a/client/rpc/api.go b/client/rpc/api.go index 5584de85d998..4df4dfde1237 100644 --- a/client/rpc/api.go +++ b/client/rpc/api.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "errors" diff --git a/client/rpc/api_test.go b/client/rpc/api_test.go index 2fd8a65dc97e..51f8cf89d8e3 100644 --- a/client/rpc/api_test.go +++ b/client/rpc/api_test.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/apifile.go b/client/rpc/apifile.go index 25fd7c3b3718..24e93a8347f0 100644 --- a/client/rpc/apifile.go +++ b/client/rpc/apifile.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/block.go b/client/rpc/block.go index 2a794c26f529..a11e5e6550a2 100644 --- a/client/rpc/block.go +++ b/client/rpc/block.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "bytes" diff --git a/client/rpc/dag.go b/client/rpc/dag.go index 795e1d78d9e3..62b54697e63f 100644 --- a/client/rpc/dag.go +++ b/client/rpc/dag.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "bytes" diff --git a/client/rpc/dht.go b/client/rpc/dht.go index a2910fef6d37..1d3b246438a8 100644 --- a/client/rpc/dht.go +++ b/client/rpc/dht.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/errors.go b/client/rpc/errors.go index 59e4ad705432..dc5946be3272 100644 --- a/client/rpc/errors.go +++ b/client/rpc/errors.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "errors" diff --git a/client/rpc/errors_test.go b/client/rpc/errors_test.go index 0e6cfe0dcc2e..c14738740faa 100644 --- a/client/rpc/errors_test.go +++ b/client/rpc/errors_test.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "errors" diff --git a/client/rpc/key.go b/client/rpc/key.go index 434e98fe5214..487c14c3d1b1 100644 --- a/client/rpc/key.go +++ b/client/rpc/key.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/name.go b/client/rpc/name.go index f82f69f3ac42..e6c726c5811c 100644 --- a/client/rpc/name.go +++ b/client/rpc/name.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/object.go b/client/rpc/object.go index 4e3b9ef6be18..5860c7661a1c 100644 --- a/client/rpc/object.go +++ b/client/rpc/object.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "bytes" diff --git a/client/rpc/path.go b/client/rpc/path.go index d69d425ab664..6edf0e797c2f 100644 --- a/client/rpc/path.go +++ b/client/rpc/path.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/pin.go b/client/rpc/pin.go index 30a3d7b7a902..9ce92b4cf155 100644 --- a/client/rpc/pin.go +++ b/client/rpc/pin.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/pubsub.go b/client/rpc/pubsub.go index 28f1ef8e609c..a386c80a1d27 100644 --- a/client/rpc/pubsub.go +++ b/client/rpc/pubsub.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "bytes" diff --git a/client/rpc/request.go b/client/rpc/request.go index dd5293b7a6ed..67571770714b 100644 --- a/client/rpc/request.go +++ b/client/rpc/request.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/requestbuilder.go b/client/rpc/requestbuilder.go index 476aed7866d1..63206be40407 100644 --- a/client/rpc/requestbuilder.go +++ b/client/rpc/requestbuilder.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "bytes" diff --git a/client/rpc/response.go b/client/rpc/response.go index 189b43671614..bad274777674 100644 --- a/client/rpc/response.go +++ b/client/rpc/response.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "encoding/json" diff --git a/client/rpc/routing.go b/client/rpc/routing.go index 550308329033..a6c4f5c0865c 100644 --- a/client/rpc/routing.go +++ b/client/rpc/routing.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "bytes" diff --git a/client/rpc/swarm.go b/client/rpc/swarm.go index 9b073078da0a..49ece0d0722e 100644 --- a/client/rpc/swarm.go +++ b/client/rpc/swarm.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/client/rpc/unixfs.go b/client/rpc/unixfs.go index b9c34c59ffcf..2099f190b30d 100644 --- a/client/rpc/unixfs.go +++ b/client/rpc/unixfs.go @@ -1,4 +1,4 @@ -package httpapi +package rpc import ( "context" diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index d88a48fa6555..ce08e3dcc297 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -66,7 +66,8 @@ updated the CoreAPI with new Kubo features but forgot to port thoses to the http-client, making it impossible to use them together with the same coreapi version. -**TODO(@Jorropo)**: add link to `boxo-migrate` once support for rewriting this import path has been added +For smooth transition `v0.7.0` of `go-ipfs-http-client` provides updated stubs +for Kubo `v0.21`. ### ๐Ÿ“ Changelog diff --git a/docs/http-rpc-clients.md b/docs/http-rpc-clients.md index 0a6ea369ddda..0b4baa1b0d1f 100644 --- a/docs/http-rpc-clients.md +++ b/docs/http-rpc-clients.md @@ -4,11 +4,11 @@ Kubo provides official HTTP RPC (`/api/v0`) clients for selected languages: - [`js-kubo-rpc-client`](https://github.com/ipfs/js-kubo-rpc-client) - Official JS client for talking to Kubo RPC over HTTP - [`go-ipfs-api`](https://github.com/ipfs/go-ipfs-api) - The go interface to ipfs's HTTP RPC - Follow https://github.com/ipfs/kubo/issues/9124 for coming changes. -- [`httpapi`](./client/rpc) (previously `go-ipfs-http-client`)) - IPFS CoreAPI implementation using HTTP RPC +- [`httpapi`](./client/rpc) (previously `go-ipfs-http-client`) - [`coreiface.CoreAPI`](https://pkg.go.dev/github.com/ipfs/boxo/coreiface#CoreAPI) implementation using HTTP RPC ## Recommended clients | Language | Package Name | Github Repository | |:--------:|:-------------------:|--------------------------------------------| | JS | kubo-rpc-client | https://github.com/ipfs/js-kubo-rpc-client | -| Go | `httpapi` | [`./client/rpc`](./client/rpc) | +| Go | `rpc` | [`./client/rpc`](./client/rpc) | From 5a993cf20dc251adc196a230e0c1fd43d8f9c33d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 08:18:48 +0000 Subject: [PATCH 154/159] chore(deps): bump go.uber.org/dig from 1.16.1 to 1.17.0 Bumps [go.uber.org/dig](https://github.com/uber-go/dig) from 1.16.1 to 1.17.0. - [Release notes](https://github.com/uber-go/dig/releases) - [Changelog](https://github.com/uber-go/dig/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/dig/compare/v1.16.1...v1.17.0) --- updated-dependencies: - dependency-name: go.uber.org/dig dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index e03472af88ce..00cb50575b8b 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -174,7 +174,7 @@ require ( go.opentelemetry.io/otel/trace v1.14.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/dig v1.16.1 // indirect + go.uber.org/dig v1.17.0 // indirect go.uber.org/fx v1.19.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 542d2e512728..e19356484f95 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -840,8 +840,8 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8= -go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= diff --git a/go.mod b/go.mod index e34d6461ff44..87e7dfea1f9b 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/sdk v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 - go.uber.org/dig v1.16.1 + go.uber.org/dig v1.17.0 go.uber.org/fx v1.19.2 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index 1c2f1079b02b..b8dcfb8ff3f8 100644 --- a/go.sum +++ b/go.sum @@ -970,8 +970,8 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8= -go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= From 99fdaa1b4dc46e369a72989d2e1b3a6bfbf1a11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Wed, 31 May 2023 17:49:21 +0200 Subject: [PATCH 155/159] plugin: fix non-deterministic loading order fix #9909 --- plugin/loader/loader.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/plugin/loader/loader.go b/plugin/loader/loader.go index 0742e6a491ce..f891bb570f4a 100644 --- a/plugin/loader/loader.go +++ b/plugin/loader/loader.go @@ -87,7 +87,7 @@ func (ls loaderState) String() string { // 5. Call Close to close all plugins. type PluginLoader struct { state loaderState - plugins map[string]plugin.Plugin + plugins []plugin.Plugin started []plugin.Plugin config config.Plugins repo string @@ -95,7 +95,7 @@ type PluginLoader struct { // NewPluginLoader creates new plugin loader func NewPluginLoader(repo string) (*PluginLoader, error) { - loader := &PluginLoader{plugins: make(map[string]plugin.Plugin, len(preloadPlugins)), repo: repo} + loader := &PluginLoader{plugins: make([]plugin.Plugin, 0, len(preloadPlugins)), repo: repo} if repo != "" { cfg, err := cserialize.Load(filepath.Join(repo, config.DefaultConfigFile)) switch err { @@ -106,6 +106,7 @@ func NewPluginLoader(repo string) (*PluginLoader, error) { return nil, err } } + for _, v := range preloadPlugins { if err := loader.Load(v); err != nil { return nil, err @@ -140,18 +141,22 @@ func (loader *PluginLoader) Load(pl plugin.Plugin) error { } name := pl.Name() - if ppl, ok := loader.plugins[name]; ok { - // plugin is already loaded - return fmt.Errorf( - "plugin: %s, is duplicated in version: %s, "+ - "while trying to load dynamically: %s", - name, ppl.Version(), pl.Version()) + + for _, p := range loader.plugins { + if p.Name() == name { + // plugin is already loaded + return fmt.Errorf( + "plugin: %s, is duplicated in version: %s, "+ + "while trying to load dynamically: %s", + name, p.Version(), pl.Version()) + } } + if loader.config.Plugins[name].Disabled { log.Infof("not loading disabled plugin %s", name) return nil } - loader.plugins[name] = pl + loader.plugins = append(loader.plugins, pl) return nil } @@ -219,10 +224,10 @@ func (loader *PluginLoader) Initialize() error { if err := loader.transition(loaderLoading, loaderInitializing); err != nil { return err } - for name, p := range loader.plugins { + for _, p := range loader.plugins { err := p.Init(&plugin.Environment{ Repo: loader.repo, - Config: loader.config.Plugins[name].Config, + Config: loader.config.Plugins[p.Name()].Config, }) if err != nil { loader.state = loaderFailed From 67e1a173fcde1b7c4b09464184aea8ef86bedab2 Mon Sep 17 00:00:00 2001 From: imthe1 Date: Thu, 20 Apr 2023 20:54:43 +0530 Subject: [PATCH 156/159] feat: adds secp256k1 keypair type to key gen command, adds test cases --- core/commands/keystore.go | 6 +++--- core/coreapi/key.go | 8 ++++++++ test/sharness/lib/test-lib.sh | 16 ++++++++++++++++ test/sharness/t0027-rotate.sh | 7 +++++++ test/sharness/t0165-keystore.sh | 31 +++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/core/commands/keystore.go b/core/commands/keystore.go index ed0d5d4e9c58..cf3e75b7de3d 100644 --- a/core/commands/keystore.go +++ b/core/commands/keystore.go @@ -83,7 +83,7 @@ var keyGenCmd = &cmds.Command{ Tagline: "Create a new keypair", }, Options: []cmds.Option{ - cmds.StringOption(keyStoreTypeOptionName, "t", "type of the key to create: rsa, ed25519").WithDefault(keyStoreAlgorithmDefault), + cmds.StringOption(keyStoreTypeOptionName, "t", "type of the key to create: rsa, ed25519, secp256k1").WithDefault(keyStoreAlgorithmDefault), cmds.IntOption(keyStoreSizeOptionName, "s", "size of the key to generate"), ke.OptionIPNSBase, }, @@ -398,7 +398,7 @@ The PEM format allows for key generation outside of the IPFS node: allowAnyKeyType, _ := req.Options[keyAllowAnyTypeOptionName].(bool) if !allowAnyKeyType { switch t := sk.(type) { - case *crypto.RsaPrivateKey, *crypto.Ed25519PrivateKey: + case *crypto.RsaPrivateKey, *crypto.Ed25519PrivateKey, *crypto.Secp256k1PrivateKey: default: return fmt.Errorf("key type %T is not allowed to be imported, only RSA or Ed25519;"+ " use flag --%s if you are sure of what you're doing", @@ -604,7 +604,7 @@ environment variable: Arguments: []cmds.Argument{}, Options: []cmds.Option{ cmds.StringOption(oldKeyOptionName, "o", "Keystore name to use for backing up your existing identity"), - cmds.StringOption(keyStoreTypeOptionName, "t", "type of the key to create: rsa, ed25519").WithDefault(keyStoreAlgorithmDefault), + cmds.StringOption(keyStoreTypeOptionName, "t", "type of the key to create: rsa, ed25519, secp256k1").WithDefault(keyStoreAlgorithmDefault), cmds.IntOption(keyStoreSizeOptionName, "s", "size of the key to generate"), }, NoRemote: true, diff --git a/core/coreapi/key.go b/core/coreapi/key.go index 925748a37d28..743f2076ee24 100644 --- a/core/coreapi/key.go +++ b/core/coreapi/key.go @@ -82,6 +82,14 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key return nil, err } + sk = priv + pk = pub + case "secp256k1": + priv, pub, err := crypto.GenerateSecp256k1Key(rand.Reader) + if err != nil { + return nil, err + } + sk = priv pk = pub default: diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index bd8f7de9b90b..35c4ae835483 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -486,6 +486,14 @@ test_check_ed25519_b58mh_peerid() { } } +test_check_secp256k1_b58mh_peerid() { + peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && + test "$peeridlen" = "53" || { + echo "Bad SECP256K1 B58MH peerid '$1' with len '$peeridlen'" + return 1 + } +} + test_check_rsa2048_base36_peerid() { peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && test "$peeridlen" = "56" || { @@ -502,6 +510,14 @@ test_check_ed25519_base36_peerid() { } } +test_check_secp256k1_base36_peerid() { + peeridlen=$(echo "$1" | tr -dC "[:alnum:]" | wc -c | tr -d " ") && + test "$peeridlen" = "63" || { + echo "Bad SECP256K1 B36CID peerid '$1' with len '$peeridlen'" + return 1 + } +} + convert_tcp_maddr() { echo $1 | awk -F'/' '{ printf "%s:%s", $3, $5 }' } diff --git a/test/sharness/t0027-rotate.sh b/test/sharness/t0027-rotate.sh index b3e748e90d1b..982b70a922df 100755 --- a/test/sharness/t0027-rotate.sh +++ b/test/sharness/t0027-rotate.sh @@ -87,12 +87,19 @@ test_rotate() { } test_rotate 'rsa' '' test_rotate 'ed25519' '' +test_rotate 'secp256k1' '' test_rotate '' '' test_rotate 'rsa' 'rsa' test_rotate 'ed25519' 'rsa' +test_rotate 'secp256k1' 'rsa' test_rotate '' 'rsa' test_rotate 'rsa' 'ed25519' test_rotate 'ed25519' 'ed25519' +test_rotate 'secp256k1' 'ed25519' test_rotate '' 'ed25519' +test_rotate 'rsa' 'secp256k1' +test_rotate 'ed25519' 'secp256k1' +test_rotate 'secp256k1' 'secp256k1' +test_rotate '' 'secp256k1' test_done diff --git a/test/sharness/t0165-keystore.sh b/test/sharness/t0165-keystore.sh index 60089ecd7f2a..2fc7c2e67cad 100755 --- a/test/sharness/t0165-keystore.sh +++ b/test/sharness/t0165-keystore.sh @@ -55,6 +55,29 @@ PEERID=$(ipfs key list --ipns-base=base36 -l | grep key_ed25519 | head -n 1 | cu test_check_ed25519_base36_peerid $PEERID && ipfs key rm key_ed25519 ' + +test_expect_success "create an SECP256k1 key and test B58MH/B36CID output formats" ' +PEERID=$(ipfs key gen --ipns-base=b58mh --type=secp256k1 key_secp256k1) && +test_check_secp256k1_b58mh_peerid $PEERID && +ipfs key rm key_secp256k1 && +PEERID=$(ipfs key gen --ipns-base=base36 --type=secp256k1 key_secp256k1) && +test_check_secp256k1_base36_peerid $PEERID +' + +test_expect_success "test SECP256k1 key sk export format" ' +ipfs key export key_secp256k1 && +test_check_ed25519_sk key_secp256k1.key && +rm key_secp256k1.key +' + +test_expect_success "test SECP256k1 key B58MH/B36CID multihash format" ' +PEERID=$(ipfs key list --ipns-base=b58mh -l | grep key_secp256k1 | head -n 1 | cut -d " " -f1) && +test_check_secp256k1_b58mh_peerid $PEERID && +PEERID=$(ipfs key list --ipns-base=base36 -l | grep key_secp256k1 | head -n 1 | cut -d " " -f1) && +test_check_secp256k1_base36_peerid $PEERID && +ipfs key rm key_secp256k1 +' + # end of format test @@ -72,6 +95,11 @@ ipfs key rm key_ed25519 test_key_import_export_all_formats ed25519_key + test_expect_success "create a new secp256k1 key" ' + k1hash=$(ipfs key gen generated_secp256k1_key --type=secp256k1) + echo $k1hash > secp256k1_key_id + ' + test_openssl_compatibility_all_types INVALID_KEY=../t0165-keystore-data/openssl_secp384r1.pem @@ -116,6 +144,7 @@ ipfs key rm key_ed25519 test_expect_success "all keys show up in list output" ' echo generated_ed25519_key > list_exp && echo generated_rsa_key >> list_exp && + echo generated_secp256k1_key >> list_exp && echo quxel >> list_exp && echo self >> list_exp ipfs key list > list_out && @@ -135,6 +164,7 @@ ipfs key rm key_ed25519 test_expect_success "key rm remove a key" ' ipfs key rm generated_rsa_key echo generated_ed25519_key > list_exp && + echo generated_secp256k1_key >> list_exp && echo quxel >> list_exp && echo self >> list_exp ipfs key list > list_out && @@ -149,6 +179,7 @@ ipfs key rm key_ed25519 test_expect_success "key rename rename a key" ' ipfs key rename generated_ed25519_key fooed echo fooed > list_exp && + echo generated_secp256k1_key >> list_exp && echo quxel >> list_exp && echo self >> list_exp ipfs key list > list_out && From dfd244816a7701c61dc2e0fe944cfc92395c5652 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 1 Jun 2023 01:43:11 +0200 Subject: [PATCH 157/159] feat(gateway): human error pages, dag-cbor/dag-json preview (#9904) Co-authored-by: Marcin Rataj --- docs/changelogs/v0.21.md | 19 +++++++++++++++++-- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/sharness/t0123-gateway-json-cbor.sh | 4 ++-- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index ce08e3dcc297..dccdb505d9e4 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -8,7 +8,8 @@ - [๐Ÿ”ฆ Highlights](#-highlights) - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) - [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag) - - [`client/rpc` migration of go-ipfs-http-client](#clientrpc-migration-of-go-ipfs-http-client) + - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) + - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -59,7 +60,7 @@ third-party CIDs. #### `client/rpc` migration of `go-ipfs-http-client` The [`go-ipfs-http-client`](https://github.com/ipfs/go-ipfs-http-client) RPC has -been migrated into [`client/rpc`](./client/rpc). +been migrated into [`kubo/client/rpc`](../../client/rpc). With this change the two will be kept in sync, in some previous releases we updated the CoreAPI with new Kubo features but forgot to port thoses to the @@ -69,6 +70,20 @@ version. For smooth transition `v0.7.0` of `go-ipfs-http-client` provides updated stubs for Kubo `v0.21`. +#### Gateway: DAG-CBOR/-JSON previews and improved error pages + +In this release, we improved the HTML templates of our HTTP gateway: + +1. You can now preview the contents of a DAG-CBOR and DAG-JSON document from your browser, as well as follow any IPLD Links ([CBOR Tag 42](https://github.com/ipld/cid-cbor/)) contained within them. +2. The HTML directory listings now contain [updated, higher-definition icons](https://user-images.githubusercontent.com/5447088/241224419-5385793a-d3bb-40aa-8cb0-0382b5bc56a0.png). +3. On gateway error, instead of a plain text error message, web browsers will now get a friendly HTML response with more details regarding the problem. + +HTML responses are returned when request's `Accept` header includes `text/html`. + +| DAG-CBOR Preview | Error Page | +| ---- | ---- | +| ![DAG-CBOR Preview](https://github.com/ipfs/boxo/assets/5447088/973f05d1-5731-4469-9da5-d1d776891899) | ![Error Page](https://github.com/ipfs/boxo/assets/5447088/14c453df-adbc-4634-b038-133121914550) | + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 00cb50575b8b..4ce4275acb89 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230530175401-c23df3837fbe + github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index e19356484f95..406780e47d90 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230530175401-c23df3837fbe h1:H6ETX0d5BiNTd1XIK8VARmEn0hpYpYdKKKp3rUJBLjY= -github.com/ipfs/boxo v0.8.2-0.20230530175401-c23df3837fbe/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 h1:jkmzkt/eRxC+tAOgUYOoQh50Bvfnun/Dy3n72tRSkmo= +github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 87e7dfea1f9b..5e980dc2bcf6 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230530175401-c23df3837fbe + github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index b8dcfb8ff3f8..5e3e2c99af98 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230530175401-c23df3837fbe h1:H6ETX0d5BiNTd1XIK8VARmEn0hpYpYdKKKp3rUJBLjY= -github.com/ipfs/boxo v0.8.2-0.20230530175401-c23df3837fbe/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 h1:jkmzkt/eRxC+tAOgUYOoQh50Bvfnun/Dy3n72tRSkmo= +github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 745e984db98d..b4b0446ffb8a 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -349,7 +349,7 @@ test_native_dag () { # As this is generated, we don't return immutable Cache-Control, even on /ipfs (same as for dir-index-html) test_expect_success "GET $name on /ipfs with Accept: text/html returns HTML (dag-index-html)" ' - curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID/" > curl_output 2>&1 && test_should_not_contain "Content-Disposition" curl_output && test_should_not_contain "Cache-Control" curl_output && test_should_contain "Etag: \"DagIndex-" curl_output && @@ -358,7 +358,7 @@ test_native_dag () { ' test_expect_success "GET $name on /ipns with Accept: text/html returns HTML (dag-index-html)" ' - curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" > curl_output 2>&1 && + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID/" > curl_output 2>&1 && test_should_not_contain "Content-Disposition" curl_output && test_should_not_contain "Cache-Control" curl_output && test_should_contain "Etag: \"DagIndex-" curl_output && From 6eef0b4eefd6813dd6169d5a78ae57b9aa1cd90b Mon Sep 17 00:00:00 2001 From: Nikhilesh Susarla Date: Thu, 1 Jun 2023 15:49:29 +0530 Subject: [PATCH 158/159] fix: 'ipfs routing findpeer' explicitly fails when searching for self (#9903) --- core/commands/root.go | 1 + core/commands/routing.go | 4 ++++ test/cli/routing_dht_test.go | 16 ++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/core/commands/root.go b/core/commands/root.go index cfa2708e6421..76e110e2b9ba 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -17,6 +17,7 @@ import ( var log = logging.Logger("core/commands") var ErrNotOnline = errors.New("this command must be run in online mode. Try running 'ipfs daemon' first") +var ErrSelfUnsupported = errors.New("finding your own node in the DHT is currently not supported") const ( RepoDirOption = "repo-dir" diff --git a/core/commands/routing.go b/core/commands/routing.go index 76672c6f2fab..44ca2e29f1d7 100644 --- a/core/commands/routing.go +++ b/core/commands/routing.go @@ -301,6 +301,10 @@ var findPeerRoutingCmd = &cmds.Command{ return err } + if pid == nd.Identity { + return ErrSelfUnsupported + } + ctx, cancel := context.WithCancel(req.Context) ctx, events := routing.RegisterForQueryEvents(ctx) diff --git a/test/cli/routing_dht_test.go b/test/cli/routing_dht_test.go index 7dc0ddfcb45d..3a3adc51c517 100644 --- a/test/cli/routing_dht_test.go +++ b/test/cli/routing_dht_test.go @@ -117,7 +117,23 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) { }) } +func testSelfFindDHT(t *testing.T) { + t.Run("ipfs routing findpeer fails for self", func(t *testing.T) { + t.Parallel() + nodes := harness.NewT(t).NewNodes(1).Init() + nodes.ForEachPar(func(node *harness.Node) { + node.IPFS("config", "Routing.Type", "dht") + }) + + nodes.StartDaemons() + + res := nodes[0].RunIPFS("dht", "findpeer", nodes[0].PeerID().String()) + assert.Equal(t, 1, res.ExitCode()) + }) +} + func TestRoutingDHT(t *testing.T) { testRoutingDHT(t, false) testRoutingDHT(t, true) + testSelfFindDHT(t) } From eb265f7cd89443f8d9260749377aa2cd6b9d7547 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 2 Jun 2023 08:14:19 +0200 Subject: [PATCH 159/159] fix(gateway)!: no duplicate payload during subdomain redirects (#9913) Co-authored-by: Marcin Rataj --- docs/changelogs/v0.21.md | 22 ++++++++++++++++++++-- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/sharness/t0114-gateway-subdomains.sh | 10 ++++------ 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index dccdb505d9e4..fe369ef90701 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -7,9 +7,10 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping) - - [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag) + - [Gateway: `DeserializedResponses` config flag](#gateway-deserializedresponses-config-flag) - [`client/rpc` migration of `go-ipfs-http-client`](#clientrpc-migration-of-go-ipfs-http-client) - [Gateway: DAG-CBOR/-JSON previews and improved error pages](#gateway-dag-cbor-json-previews-and-improved-error-pages) + - [Gateway: subdomain redirects are now `text/html`](#gateway-subdomain-redirects-are-now-texthtml) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -32,7 +33,7 @@ enabled. With this update, the same level of robustness is applied to peers that lack mDNS peers and solely rely on the public DHT. -#### `Gateway.DeserializedResponses` config flag +#### Gateway: `DeserializedResponses` config flag This release introduces the [`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses) @@ -84,6 +85,23 @@ HTML responses are returned when request's `Accept` header includes `text/html`. | ---- | ---- | | ![DAG-CBOR Preview](https://github.com/ipfs/boxo/assets/5447088/973f05d1-5731-4469-9da5-d1d776891899) | ![Error Page](https://github.com/ipfs/boxo/assets/5447088/14c453df-adbc-4634-b038-133121914550) | +#### Gateway: subdomain redirects are now `text/html` + +HTTP 301 redirects [from path to subdomain](https://specs.ipfs.tech/http-gateways/subdomain-gateway/#migrating-from-path-to-subdomain-gateway) +no longer include the target data in the body. +The data is returned only once, with the final HTTP 200 returned from the +target subdomain. + +The HTTP 301 body now includes human-readable `text/html` message +for clients that do not follow redirects by default: + +```console +$ curl "https://subdomain-gw.example.net/ipfs/${cid}/" +Moved Permanently. +``` + +Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913). + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 4ce4275acb89..81caa1258a6c 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 + github.com/ipfs/boxo v0.8.2-0.20230602025754-4c5c98b94b21 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 406780e47d90..a51153459239 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 h1:jkmzkt/eRxC+tAOgUYOoQh50Bvfnun/Dy3n72tRSkmo= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230602025754-4c5c98b94b21 h1:efxZK66VVeQ1cGczX7ufjd4DW6X2nb/PbAJ5k85YC98= +github.com/ipfs/boxo v0.8.2-0.20230602025754-4c5c98b94b21/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 5e980dc2bcf6..78765c0b6ff5 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 + github.com/ipfs/boxo v0.8.2-0.20230602025754-4c5c98b94b21 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 5e3e2c99af98..9157775d77d8 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076 h1:jkmzkt/eRxC+tAOgUYOoQh50Bvfnun/Dy3n72tRSkmo= -github.com/ipfs/boxo v0.8.2-0.20230531151409-d1b8d1d6d076/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230602025754-4c5c98b94b21 h1:efxZK66VVeQ1cGczX7ufjd4DW6X2nb/PbAJ5k85YC98= +github.com/ipfs/boxo v0.8.2-0.20230602025754-4c5c98b94b21/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index bf5070985901..1d19a6570692 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -162,14 +162,12 @@ test_localhost_gateway_response_should_contain \ "http://localhost:$GWAY_PORT/ipfs/$DIR_CID/" \ "Location: http://$DIR_CID.ipfs.localhost:$GWAY_PORT/" -# We return body with HTTP 301 so existing cli scripts that use path-based -# gateway do not break (curl doesn't auto-redirect without passing -L; wget -# does not span across hostnames by default) -# Context: https://github.com/ipfs/go-ipfs/issues/6975 +# We return human-readable body with HTTP 301 so existing cli scripts that use path-based +# gateway are informed to enable following HTTP redirects test_localhost_gateway_response_should_contain \ - "request for localhost/ipfs/{CIDv1} includes valid payload in body for CLI tools like curl" \ + "request for localhost/ipfs/{CIDv1} includes human-readable link and redirect info in HTTP 301 body" \ "http://localhost:$GWAY_PORT/ipfs/$CIDv1" \ - "$CID_VAL" + ">Moved Permanently" test_localhost_gateway_response_should_contain \ "request for localhost/ipfs/{CIDv0} redirects to CIDv1 representation in subdomain" \