From 202081523fdc8bc54b16e5e4447b221586febef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:12:10 +0100 Subject: [PATCH 001/112] Setup repo This commit was moved from ipfs/go-ipfs-http-client@4e7edce41d91825f0ea21dec426de2e128db5b38 --- client/httpapi/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 client/httpapi/README.md diff --git a/client/httpapi/README.md b/client/httpapi/README.md new file mode 100644 index 000000000000..ab37a4cf8aa8 --- /dev/null +++ b/client/httpapi/README.md @@ -0,0 +1,27 @@ +# go-ipfs-http-api + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) +[![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) + +> IPFS CoreAPI implementation using HTTP API + +## Documentation + +https://godoc.org/github.com/ipfs/go-ipfs-http-api + +## Contribute + +Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +### Want to hack on IPFS? + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) + +## License + +MIT From 6d85aff407d3fb3eb6d2549fbac94615fd147b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:13:15 +0100 Subject: [PATCH 002/112] Initial structure, path stuff This commit was moved from ipfs/go-ipfs-http-client@93943f7f5671948b0d2aee19c30cf689307faf13 --- client/httpapi/api.go | 127 +++++++++++++++++++++++++++++ client/httpapi/name.go | 35 ++++++++ client/httpapi/path.go | 48 +++++++++++ client/httpapi/request.go | 34 ++++++++ client/httpapi/requestbuilder.go | 100 +++++++++++++++++++++++ client/httpapi/response.go | 132 +++++++++++++++++++++++++++++++ 6 files changed, 476 insertions(+) create mode 100644 client/httpapi/api.go create mode 100644 client/httpapi/name.go create mode 100644 client/httpapi/path.go create mode 100644 client/httpapi/request.go create mode 100644 client/httpapi/requestbuilder.go create mode 100644 client/httpapi/response.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go new file mode 100644 index 000000000000..82e030350f72 --- /dev/null +++ b/client/httpapi/api.go @@ -0,0 +1,127 @@ +package httpapi + +import ( + "github.com/pkg/errors" + "io/ioutil" + gohttp "net/http" + "os" + "path" + "strings" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + homedir "github.com/mitchellh/go-homedir" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" + ) + +const ( + DefaultPathName = ".ipfs" + DefaultPathRoot = "~/" + DefaultPathName + DefaultApiFile = "api" + EnvDir = "IPFS_PATH" +) + +var ErrNotImplemented = errors.New("not implemented") + +type HttpApi struct { + url string + httpcli *gohttp.Client +} + +func NewLocalApi() iface.CoreAPI { + baseDir := os.Getenv(EnvDir) + if baseDir == "" { + baseDir = DefaultPathRoot + } + + baseDir, err := homedir.Expand(baseDir) + if err != nil { + return nil + } + + apiFile := path.Join(baseDir, DefaultApiFile) + + if _, err := os.Stat(apiFile); err != nil { + return nil + } + + api, err := ioutil.ReadFile(apiFile) + if err != nil { + return nil + } + + return NewApi(strings.TrimSpace(string(api))) +} + +func NewApi(url string) *HttpApi { + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + } + + return NewApiWithClient(url, c) +} + +func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { + if a, err := ma.NewMultiaddr(url); err == nil { + _, host, err := manet.DialArgs(a) + if err == nil { + url = host + } + } + + return &HttpApi{ + url: url, + httpcli: c, + } +} + +func (api *HttpApi) request(command string, args ...string) *RequestBuilder { + return &RequestBuilder{ + command: command, + args: args, + shell: api, + } +} + +func (api *HttpApi) Unixfs() iface.UnixfsAPI { + return nil +} + +func (api *HttpApi) Block() iface.BlockAPI { + return nil +} + +func (api *HttpApi) Dag() iface.DagAPI { + return nil +} + +func (api *HttpApi) Name() iface.NameAPI { + return (*NameAPI)(api) +} + +func (api *HttpApi) Key() iface.KeyAPI { + return nil +} + +func (api *HttpApi) Pin() iface.PinAPI { + return nil +} + +func (api *HttpApi) Object() iface.ObjectAPI { + return nil +} + +func (api *HttpApi) Dht() iface.DhtAPI { + return nil +} + +func (api *HttpApi) Swarm() iface.SwarmAPI { + return nil +} + +func (api *HttpApi) PubSub() iface.PubSubAPI { + return nil +} diff --git a/client/httpapi/name.go b/client/httpapi/name.go new file mode 100644 index 000000000000..41426ef57c9c --- /dev/null +++ b/client/httpapi/name.go @@ -0,0 +1,35 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type NameAPI HttpApi + +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { + // TODO: options! + + req := api.core().request("name/resolve") + req.Arguments(name) + + var out struct{ Path string } + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + + return iface.ParsePath(out.Path) +} + +func (api *NameAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/path.go b/client/httpapi/path.go new file mode 100644 index 000000000000..28656fbd4c03 --- /dev/null +++ b/client/httpapi/path.go @@ -0,0 +1,48 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { + var out struct { + Cid cid.Cid + RemPath string + } + + //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help + + var err error + if path.Namespace() == "ipns" { + if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + return nil, err + } + } + + if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + // TODO: + ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + if err != nil { + return nil, err + } + + root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + if err != nil { + return nil, err + } + + return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil +} + +func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { + return nil, ErrNotImplemented +} diff --git a/client/httpapi/request.go b/client/httpapi/request.go new file mode 100644 index 000000000000..58c61ac67805 --- /dev/null +++ b/client/httpapi/request.go @@ -0,0 +1,34 @@ +package httpapi + +import ( + "context" + "io" + "strings" +) + +type Request struct { + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string +} + +func NewRequest(ctx context.Context, url, command string, args ...string) *Request { + if !strings.HasPrefix(url, "http") { + url = "http://" + url + } + + opts := map[string]string{ + "encoding": "json", + "stream-channels": "true", + } + return &Request{ + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), + } +} diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go new file mode 100644 index 000000000000..9ccc8cf9720e --- /dev/null +++ b/client/httpapi/requestbuilder.go @@ -0,0 +1,100 @@ +package httpapi + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "strings" +) + +// RequestBuilder is an IPFS commands request builder. +type RequestBuilder struct { + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader + + shell *HttpApi +} + +// Arguments adds the arguments to the args. +func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { + r.args = append(r.args, args...) + return r +} + +// BodyString sets the request body to the given string. +func (r *RequestBuilder) BodyString(body string) *RequestBuilder { + return r.Body(strings.NewReader(body)) +} + +// BodyBytes sets the request body to the given buffer. +func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { + return r.Body(bytes.NewReader(body)) +} + +// Body sets the request body to the given reader. +func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { + r.body = body + return r +} + +// Option sets the given option. +func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { + var s string + switch v := value.(type) { + case bool: + s = strconv.FormatBool(v) + case string: + s = v + case []byte: + s = string(v) + default: + // slow case. + s = fmt.Sprint(value) + } + if r.opts == nil { + r.opts = make(map[string]string, 1) + } + r.opts[key] = s + return r +} + +// Header sets the given header. +func (r *RequestBuilder) Header(name, value string) *RequestBuilder { + if r.headers == nil { + r.headers = make(map[string]string, 1) + } + r.headers[name] = value + return r +} + +// Send sends the request and return the response. +func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + req := NewRequest(ctx, r.shell.url, r.command, r.args...) + req.Opts = r.opts + req.Headers = r.headers + req.Body = r.body + return req.Send(r.shell.httpcli) +} + +// Exec sends the request a request and decodes the response. +func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { + httpRes, err := r.Send(ctx) + if err != nil { + return err + } + + if res == nil { + httpRes.Close() + if httpRes.Error != nil { + return httpRes.Error + } + return nil + } + + return httpRes.Decode(res) +} diff --git a/client/httpapi/response.go b/client/httpapi/response.go new file mode 100644 index 000000000000..27709769b1f6 --- /dev/null +++ b/client/httpapi/response.go @@ -0,0 +1,132 @@ +package httpapi + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + files "github.com/ipfs/go-ipfs-files" +) + +type Response struct { + Output io.ReadCloser + Error *Error +} + +func (r *Response) Close() error { + if r.Output != nil { + // always drain output (response body) + ioutil.ReadAll(r.Output) + return r.Output.Close() + } + return nil +} + +func (r *Response) Decode(dec interface{}) error { + defer r.Close() + if r.Error != nil { + return r.Error + } + + return json.NewDecoder(r.Output).Decode(dec) +} + +type Error struct { + Command string + Message string + Code int +} + +func (e *Error) Error() string { + var out string + if e.Command != "" { + out = e.Command + ": " + } + if e.Code != 0 { + out = fmt.Sprintf("%s%d: ", out, e.Code) + } + return out + e.Message +} + +func (r *Request) Send(c *http.Client) (*Response, error) { + url := r.getURL() + req, err := http.NewRequest("POST", url, r.Body) + if err != nil { + return nil, err + } + + // Add any headers that were supplied via the RequestBuilder. + for k, v := range r.Headers { + req.Header.Add(k, v) + } + + if fr, ok := r.Body.(*files.MultiFileReader); ok { + req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) + req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + } + + resp, err := c.Do(req) + if err != nil { + return nil, err + } + + contentType := resp.Header.Get("Content-Type") + parts := strings.Split(contentType, ";") + contentType = parts[0] + + nresp := new(Response) + + nresp.Output = resp.Body + if resp.StatusCode >= http.StatusBadRequest { + e := &Error{ + Command: r.Command, + } + switch { + case resp.StatusCode == http.StatusNotFound: + e.Message = "command not found" + case contentType == "text/plain": + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = string(out) + case contentType == "application/json": + if err = json.NewDecoder(resp.Body).Decode(e); err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) + } + default: + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out) + } + nresp.Error = e + nresp.Output = nil + + // drain body and close + ioutil.ReadAll(resp.Body) + resp.Body.Close() + } + + return nresp, nil +} + +func (r *Request) getURL() string { + + values := make(url.Values) + for _, arg := range r.Args { + values.Add("arg", arg) + } + for k, v := range r.Opts { + values.Add(k, v) + } + + return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode()) +} From 35c271e3d88725a9ab25be40c1837435de110a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:40:02 +0100 Subject: [PATCH 003/112] wip notice in readme This commit was moved from ipfs/go-ipfs-http-client@e823507553ca82115e303e1ba26f76b83f5bd12c --- client/httpapi/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index ab37a4cf8aa8..dd09142e32bc 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,6 +8,8 @@ > IPFS CoreAPI implementation using HTTP API +This project is WIP, use https://github.com/ipfs/go-ipfs-api for now + ## Documentation https://godoc.org/github.com/ipfs/go-ipfs-http-api From 6c927fd9624807e124f0376445c41daaa975fbd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:38:11 +0100 Subject: [PATCH 004/112] Partial ipld node impl This commit was moved from ipfs/go-ipfs-http-client@df916c7849d6cfd7f479c4fe80c0f75d6705a0fc --- client/httpapi/api.go | 6 +-- client/httpapi/block.go | 40 ++++++++++++++ client/httpapi/ipldnode.go | 107 +++++++++++++++++++++++++++++++++++++ client/httpapi/path.go | 2 +- 4 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/block.go create mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 82e030350f72..cd3fb9fd021e 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,7 @@ package httpapi import ( - "github.com/pkg/errors" + "errors" "io/ioutil" gohttp "net/http" "os" @@ -12,7 +12,7 @@ import ( homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" - ) +) const ( DefaultPathName = ".ipfs" @@ -91,7 +91,7 @@ func (api *HttpApi) Unixfs() iface.UnixfsAPI { } func (api *HttpApi) Block() iface.BlockAPI { - return nil + return (*BlockAPI)(api) } func (api *HttpApi) Dag() iface.DagAPI { diff --git a/client/httpapi/block.go b/client/httpapi/block.go new file mode 100644 index 000000000000..8bdb4c502ebc --- /dev/null +++ b/client/httpapi/block.go @@ -0,0 +1,40 @@ +package httpapi + +import ( + "context" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type BlockAPI HttpApi + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + if err != nil { + return nil, err + } + + //TODO: is close on the reader enough? + //defer resp.Close() + + //TODO: make blockApi return ReadCloser + return resp.Output, resp.Error +} + +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { + return ErrNotImplemented +} + +func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go new file mode 100644 index 000000000000..a3dd6204b2b6 --- /dev/null +++ b/client/httpapi/ipldnode.go @@ -0,0 +1,107 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + "io/ioutil" + "strconv" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +type ipldNode struct { + ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? + path iface.ResolvedPath + api *HttpApi +} + +func (n *ipldNode) RawData() []byte { + r, err := n.api.Block().Get(n.ctx, n.path) + if err != nil { + panic(err) // TODO: eww, should we add errors too / better ideas? + } + + b, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + + return b +} + +func (n *ipldNode) Cid() cid.Cid { + return n.path.Cid() +} + +func (n *ipldNode) String() string { + return fmt.Sprintf("[Block %s]", n.Cid()) +} + +func (n *ipldNode) Loggable() map[string]interface{} { + return nil //TODO: we can't really do better here, can we? +} + +// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. +// We can also give people a choice +func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { + p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) + + var out interface{} + n.api.request("dag/get", p).Exec(n.ctx, &out) + + // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack + for len(path) > 0 { + switch o := out.(type) { + case map[string]interface{}: + v, ok := o[path[0]] + if !ok { + // TODO: ipld links + return nil, nil, errors.New("no element under this path") + } + out = v + case []interface{}: + n, err := strconv.ParseUint(path[0], 10, 32) + if err != nil { + return nil, nil, err + } + if len(o) < int(n) { + return nil, nil, errors.New("no element under this path") + } + out = o[n] + } + path = path[1:] + } + + return out, path, nil +} + +func (n *ipldNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (n *ipldNode) Copy() ipld.Node { + panic("implement me") +} + +func (n *ipldNode) Links() []*ipld.Link { + panic("implement me") +} + +func (n *ipldNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (n *ipldNode) Size() (uint64, error) { + panic("implement me") +} + +var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 28656fbd4c03..6b6e4b027bc7 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -30,7 +30,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } // TODO: - ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } From 1cd2ec05b7524e921f58750a4b9c8f48ad1a2e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Dec 2018 16:04:19 +0100 Subject: [PATCH 005/112] Init gx This commit was moved from ipfs/go-ipfs-http-client@e06cddbedd360b1982e414571dd08ab25df38546 --- client/httpapi/ipldnode.go | 8 ++++---- client/httpapi/name.go | 1 + client/httpapi/path.go | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index a3dd6204b2b6..b8e6fba01806 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -2,16 +2,16 @@ package httpapi import ( "context" + "errors" "fmt" - "github.com/pkg/errors" - "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" "io/ioutil" "strconv" "github.com/ipfs/go-ipfs/core/coreapi/interface" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) type ipldNode struct { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 41426ef57c9c..7315ac2c3918 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 6b6e4b027bc7..5701326fc560 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -5,9 +5,9 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { From 16e97bf5e8f80b681c5a8d4b96f406b4b83a8e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 21 Dec 2018 18:44:34 +0100 Subject: [PATCH 006/112] It builds This commit was moved from ipfs/go-ipfs-http-client@a23d794e5fceadb9115d8d0c8466c10e9c5d0891 --- client/httpapi/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cd3fb9fd021e..dc9acb9f6067 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -78,6 +79,10 @@ func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { } } +func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { + return nil, ErrNotImplemented +} + func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, From fe4c9fd8033c94c5ea707b09e76479a6d018635d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 30 Dec 2018 04:24:09 +0100 Subject: [PATCH 007/112] Skeleton for tests This commit was moved from ipfs/go-ipfs-http-client@dfbe0026ad9438d4e795bbb43debeffb6bb9ca1a --- client/httpapi/api.go | 24 ++++++++--- client/httpapi/api_test.go | 84 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 client/httpapi/api_test.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dc9acb9f6067..63ea3aad7d33 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -35,7 +35,11 @@ func NewLocalApi() iface.CoreAPI { baseDir = DefaultPathRoot } - baseDir, err := homedir.Expand(baseDir) + return NewPathApi(baseDir) +} + +func NewPathApi(p string) iface.CoreAPI { + baseDir, err := homedir.Expand(p) if err != nil { return nil } @@ -51,10 +55,15 @@ func NewLocalApi() iface.CoreAPI { return nil } - return NewApi(strings.TrimSpace(string(api))) + maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) + if err != nil { + return nil + } + + return NewApi(maddr) } -func NewApi(url string) *HttpApi { +func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -62,10 +71,15 @@ func NewApi(url string) *HttpApi { }, } - return NewApiWithClient(url, c) + return NewApiWithClient(a, c) } -func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { + _, url, err := manet.DialArgs(a) + if err != nil { + return nil // TODO: return that error + } + if a, err := ma.NewMultiaddr(url); err == nil { _, host, err := manet.DialArgs(a) if err == nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go new file mode 100644 index 000000000000..b925b5c348f2 --- /dev/null +++ b/client/httpapi/api_test.go @@ -0,0 +1,84 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/iptb/testbed/interfaces" + "io/ioutil" + "os" + "path" + "strconv" + "testing" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" + + local "github.com/ipfs/iptb-plugins/local" + "github.com/ipfs/iptb/cli" + "github.com/ipfs/iptb/testbed" +) + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ + From: "", + NewNode: local.NewNode, + GetAttrList: local.GetAttrList, + GetAttrDesc: local.GetAttrDesc, + PluginName: local.PluginName, + BuiltIn: true, + }, false) + if err != nil { + return nil, err + } + + dir, err := ioutil.TempDir("", "httpapi-tb-") + if err != nil { + return nil, err + } + + c := cli.NewCli() + if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + + defer os.Remove(dir) + + defer func() { + _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + }() + }() + + apis := make([]iface.CoreAPI, n) + + for i := range apis { + tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) + + node, err := tb.Node(i) + if err != nil { + return nil, err + } + + attrNode, ok := node.(testbedi.Attribute) + if !ok { + return nil, fmt.Errorf("node does not implement attributes") + } + + pth, err := attrNode.Attr("path") + if err != nil { + return nil, err + } + + apis[i] = NewPathApi(pth) + } + + return apis, nil +} + +func TestHttpApi(t *testing.T) { + tests.TestApi(&NodeProvider{})(t) +} From e8da6e2cf9a8fd40c5fd960a60ce1b1b27ada713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 7 Jan 2019 23:44:43 +0100 Subject: [PATCH 008/112] Block API This commit was moved from ipfs/go-ipfs-http-client@d0c98b870e55bed78fa777793e520f08157deb4f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 11 +++- client/httpapi/block.go | 109 ++++++++++++++++++++++++++++--- client/httpapi/requestbuilder.go | 12 ++++ 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 63ea3aad7d33..1768e8477497 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return (*NameAPI)(api) + return nil } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b925b5c348f2..e32c456e5a36 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/iptb/testbed/interfaces" "io/ioutil" "os" "path" @@ -16,6 +15,7 @@ import ( local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" + "github.com/ipfs/iptb/testbed/interfaces" ) type NodeProvider struct{} @@ -39,7 +39,14 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := cli.NewCli() - if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + + initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} + if err := c.Run(initArgs); err != nil { + return nil, err + } + + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 8bdb4c502ebc..185aa0a42be0 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -1,17 +1,69 @@ package httpapi import ( + "bytes" "context" + "errors" + "fmt" "io" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + mh "github.com/multiformats/go-multihash" ) type BlockAPI HttpApi -func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { - return nil, ErrNotImplemented +type blockStat struct { + Key string + BSize int `json:"Size"` +} + +func (s *blockStat) Size() int { + return s.BSize +} + +func (s *blockStat) valid() (iface.ResolvedPath, error) { + c, err := cid.Parse(s.Key) + if err != nil { + return nil, err + } + + return iface.IpldPath(c), nil +} + +func (s *blockStat) Path() iface.ResolvedPath { + p, _ := s.valid() + return p +} + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { + options, _, err := caopts.BlockPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("block/put"). + Option("mhtype", mht). + Option("mhlen", options.MhLength). + Option("format", options.Codec). + FileBody(r) + + var out blockStat + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { @@ -19,20 +71,57 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } - //TODO: is close on the reader enough? - //defer resp.Close() + //TODO: make get return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } - //TODO: make blockApi return ReadCloser - return resp.Output, resp.Error + return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { - return ErrNotImplemented +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { + options, err := caopts.BlockRmOptions(opts...) + if err != nil { + return err + } + + removedBlock := struct { + Hash string `json:",omitempty"` + Error string `json:",omitempty"` + }{} + + req := api.core().request("block/rm"). + Option("force", options.Force). + Arguments(p.String()) + + if err := req.Exec(ctx, &removedBlock); err != nil { + return err + } + + if removedBlock.Error != "" { + return errors.New(removedBlock.Error) + } + + return nil } func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { - return nil, ErrNotImplemented + var out blockStat + err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) core() *HttpApi { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 9ccc8cf9720e..6e5a89ebdb13 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,8 +5,11 @@ import ( "context" "fmt" "io" + "io/ioutil" "strconv" "strings" + + "github.com/ipfs/go-ipfs-files" ) // RequestBuilder is an IPFS commands request builder. @@ -42,6 +45,15 @@ func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { return r } +// FileBody sets the request body to the given reader wrapped into multipartreader. +func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { + pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + d := files.NewMapDirectory(map[string]files.Node{"": pr}) + r.body = files.NewMultiFileReader(d, false) + + return r +} + // Option sets the given option. func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { var s string From ab89e0abf99a0bc42ee2d8fb55a9b3de79573635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 00:30:22 +0100 Subject: [PATCH 009/112] Partial Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@44696b84f59f6f707858787c9a0750f224596635 --- client/httpapi/api.go | 4 +- client/httpapi/unixfs.go | 95 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/unixfs.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 1768e8477497..099b45123113 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -106,7 +106,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { } func (api *HttpApi) Unixfs() iface.UnixfsAPI { - return nil + return (*UnixfsAPI)(api) } func (api *HttpApi) Block() iface.BlockAPI { @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return nil + return (*NameAPI)(api) } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go new file mode 100644 index 000000000000..567f60284d44 --- /dev/null +++ b/client/httpapi/unixfs.go @@ -0,0 +1,95 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type addEvent struct { + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + +type UnixfsAPI HttpApi + +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { + options, _, err := caopts.UnixfsAddOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("add"). + Option("hash", mht). + Option("chunker", options.Chunker). + Option("cid-version", options.CidVersion). + //Option("", options.Events). + Option("fscache", options.FsCache). + Option("hidden", options.Hidden). + Option("inline", options.Inline). + Option("inline-limit", options.InlineLimit). + Option("nocopy", options.NoCopy). + Option("only-hash", options.OnlyHash). + Option("pin", options.Pin). + //Option("", options.Progress). + Option("silent", options.Silent). + Option("stdin-name", options.StdinName). + Option("wrap-with-directory", options.Wrap). + Option("quieter", true) // TODO: rm after event impl + + if options.RawLeavesSet { + req.Option("raw-leaves", options.RawLeaves) + } + + switch options.Layout { + case caopts.BalancedLayout: + // noop, default + case caopts.TrickleLayout: + req.Option("trickle", true) + } + + switch c := f.(type) { + case files.Directory: + req.Body(files.NewMultiFileReader(c, false)) + case files.File: + req.Body(c) + } + + var out addEvent + if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil +} + +func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { + panic("implement me") +} + +func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *UnixfsAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cf74d391603457ad626b7f6b27ff0655bc946ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:07:21 +0100 Subject: [PATCH 010/112] Partial Key API, ApiAddr funcion This commit was moved from ipfs/go-ipfs-http-client@0ffdef159261e19cfd76edabb394179aa9295003 --- client/httpapi/api.go | 13 ++++++- client/httpapi/key.go | 86 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/key.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 099b45123113..d104d98dffb9 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,6 +29,7 @@ type HttpApi struct { httpcli *gohttp.Client } +//TODO: Return errors here func NewLocalApi() iface.CoreAPI { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -39,6 +40,14 @@ func NewLocalApi() iface.CoreAPI { } func NewPathApi(p string) iface.CoreAPI { + a := ApiAddr(p) + if a == nil { + return nil + } + return NewApi(a) +} + +func ApiAddr(p string) ma.Multiaddr { baseDir, err := homedir.Expand(p) if err != nil { return nil @@ -60,7 +69,7 @@ func NewPathApi(p string) iface.CoreAPI { return nil } - return NewApi(maddr) + return maddr } func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? @@ -122,7 +131,7 @@ func (api *HttpApi) Name() iface.NameAPI { } func (api *HttpApi) Key() iface.KeyAPI { - return nil + return (*KeyAPI)(api) } func (api *HttpApi) Pin() iface.PinAPI { diff --git a/client/httpapi/key.go b/client/httpapi/key.go new file mode 100644 index 000000000000..87b573f98a56 --- /dev/null +++ b/client/httpapi/key.go @@ -0,0 +1,86 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type KeyAPI HttpApi + +type keyOutput struct { + JName string `json:"Name"` + Id string +} + +func (k *keyOutput) Name() string { + return k.JName +} + +func (k *keyOutput) Path() iface.Path { + p, _ := iface.ParsePath("/ipns/" + k.Id) + return p +} + +func (k *keyOutput) ID() peer.ID { + p, _ := peer.IDB58Decode(k.Id) + return p +} + +func (k *keyOutput) valid() error { + _, err := peer.IDB58Decode(k.Id) + return err +} + + +func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { + options, err := caopts.KeyGenerateOptions(opts...) + if err != nil { + return nil, err + } + + var out keyOutput + err = api.core().request("key/gen", name). + Option("type", options.Algorithm). + Option("size", options.Size). + Exec(ctx, &out) + if err != nil { + return nil, err + } + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { + panic("implement me") +} + +func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { + var id struct{ID string} + if err := api.core().request("id").Exec(ctx, &id); err != nil { + return nil, err + } + + out := keyOutput{JName: "self", Id: id.ID} + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c236393733bf3491c45002d67189ba78303332a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:08:18 +0100 Subject: [PATCH 011/112] Connect test swarms, don't compress api calls This commit was moved from ipfs/go-ipfs-http-client@c6472d9b8286c932492db4d6bf03f890bf97759c --- client/httpapi/api_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index e32c456e5a36..02c7830bb650 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io/ioutil" + gohttp "net/http" "os" "path" "strconv" @@ -50,6 +51,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + if n > 1 { + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + if err := c.Run(connectArgs); err != nil { + return nil, err + } + } + go func() { <-ctx.Done() @@ -80,7 +88,18 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - apis[i] = NewPathApi(pth) + a := ApiAddr(pth) + if a == nil { + return nil, fmt.Errorf("nil addr for node") + } + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(a, c) } return apis, nil From 7861315f49398e20e7e1189671e42d0a16adbef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:00 +0100 Subject: [PATCH 012/112] Wrap single files in Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@16f77b24a1b150e7ec624d5675ded478ac6cb853 --- client/httpapi/unixfs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 567f60284d44..21f75f85ae90 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -66,7 +66,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix case files.Directory: req.Body(files.NewMultiFileReader(c, false)) case files.File: - req.Body(c) + d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) } var out addEvent From f638bae3a960a74d284503c30086f0bbffbe73c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:17 +0100 Subject: [PATCH 013/112] implement .Name This commit was moved from ipfs/go-ipfs-http-client@e19e5f54e48d0909796add6d57e3e74e84eba85e --- client/httpapi/name.go | 70 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 7315ac2c3918..fbc440b96664 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,26 +2,80 @@ package httpapi import ( "context" + "fmt" + "github.com/ipfs/go-ipfs/namesys/opts" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) type NameAPI HttpApi -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { - return nil, ErrNotImplemented +type ipnsEntry struct { + JName string `json:"Name"` + JValue string `json:"Value"` +} + +func (e *ipnsEntry) valid() (iface.Path, error) { + return iface.ParsePath(e.JValue) +} + +func (e *ipnsEntry) Name() string { + return e.JName +} + +func (e *ipnsEntry) Value() iface.Path { + p, _ := e.valid() + return p } -func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { + options, err := caopts.NamePublishOptions(opts...) + if err != nil { + return nil, err + } + + req := api.core().request("name/publish", p.String()). + Option("key", options.Key). + Option("allow-offline", options.AllowOffline). + Option("lifetime", options.ValidTime.String()). + Option("resolve", false) + + if options.TTL != nil { + req.Option("ttl", options.TTL.String()) + } + + var out ipnsEntry + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { return nil, ErrNotImplemented } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { - // TODO: options! +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } + + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } - req := api.core().request("name/resolve") - req.Arguments(name) + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout.String()) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { From 1acf4163902ae3b9e0e182761deafee5e67415ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:53:24 +0100 Subject: [PATCH 014/112] Implement .Unixfs.Ls() This commit was moved from ipfs/go-ipfs-http-client@eb1944fae32f4ccc43969ecd632681c2f2cbfa00 --- client/httpapi/response.go | 14 ++++++++++++- client/httpapi/unixfs.go | 43 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 27709769b1f6..5749ca29eea1 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -33,7 +33,19 @@ func (r *Response) Decode(dec interface{}) error { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + n := 0 + var err error + for { + err = json.NewDecoder(r.Output).Decode(dec) + if err != nil { + break + } + n++ + } + if n > 0 && err == io.EOF { + err = nil + } + return err } type Error struct { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 21f75f85ae90..75e565a4a910 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -11,6 +12,7 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { @@ -87,8 +89,45 @@ func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { panic("implement me") } -func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +type lsLink struct { + Name, Hash string + Size uint64 + Type unixfspb.Data_DataType +} + +type lsObject struct { + Hash string + Links []lsLink +} + +type lsOutput struct { + Objects []lsObject +} + +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out lsOutput + err := api.core().request("ls", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + + if len(out.Objects) != 1 { + return nil, errors.New("unexpected objects len") + } + + links := make([]*format.Link, len(out.Objects[0].Links)) + for i, l := range out.Objects[0].Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + links[i] = &format.Link{ + Name: l.Name, + Size: l.Size, + Cid: c, + } + } + return links, nil } func (api *UnixfsAPI) core() *HttpApi { From a6636aac599a1a8a4802ccadb885909f0ab88629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:18:21 +0100 Subject: [PATCH 015/112] Imprement partian Pin API This commit was moved from ipfs/go-ipfs-http-client@dbf90eac67d95722e88056ca16fa769a32dcd48f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 16 +++++++- client/httpapi/pin.go | 78 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pin.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d104d98dffb9..d5741da089d4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -135,7 +135,7 @@ func (api *HttpApi) Key() iface.KeyAPI { } func (api *HttpApi) Pin() iface.PinAPI { - return nil + return (*PinAPI)(api) } func (api *HttpApi) Object() iface.ObjectAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 02c7830bb650..5fdf1835241f 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -39,7 +40,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - c := cli.NewCli() + c := cli.NewCli() //TODO: is there a better way? initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} if err := c.Run(initArgs); err != nil { @@ -100,6 +101,19 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) }, } apis[i] = NewApiWithClient(a, c) + + // node cleanup + // TODO: pass --empty-repo somehow (how?) + pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + for _, pin := range pins { //TODO: parallel + if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { + return nil, err + } + } + } return apis, nil diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go new file mode 100644 index 000000000000..02dcc4222473 --- /dev/null +++ b/client/httpapi/pin.go @@ -0,0 +1,78 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type PinAPI HttpApi + +type pinRefKeyObject struct { + Type string +} + +type pinRefKeyList struct { + Keys map[string]pinRefKeyObject +} + +type pin struct { + path iface.ResolvedPath + typ string +} + +func (p *pin) Path() iface.ResolvedPath { + return p.path +} + +func (p *pin) Type() string { + return p.typ +} + + +func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { + panic("implement me") +} + +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { + options, err := caopts.PinLsOptions(opts...) + if err != nil { + return nil, err + } + + var out pinRefKeyList + err = api.core().request("pin/ls"). + Option("type", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + pins := make([]iface.Pin, 0, len(out.Keys)) + for hash, p := range out.Keys { + c, err := cid.Parse(hash) + if err != nil { + return nil, err + } + pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + } + + return pins, nil +} + +func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { + return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +} + +func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { + panic("implement me") +} + +func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { + panic("implement me") +} + +func (api *PinAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 0f7c83956b2d769c801c270fe01d4819cb070881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:20:04 +0100 Subject: [PATCH 016/112] Import missing unixfs dep This commit was moved from ipfs/go-ipfs-http-client@6169321d1df6495cbd1c06cca9f2e0777a00f638 --- client/httpapi/unixfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 75e565a4a910..5a1495c24b45 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -11,8 +11,8 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" - unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { From af197cb6d90da1b51525e68f97a30ebfc322e134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 14:46:52 +0100 Subject: [PATCH 017/112] api.WithOption This commit was moved from ipfs/go-ipfs-http-client@634b00bf1a42dc7f2a2dc75f85c77e5bb5ef727c --- client/httpapi/api.go | 21 ++++++++++++++++++--- client/httpapi/api_test.go | 4 ++-- client/httpapi/requestbuilder.go | 2 ++ client/httpapi/response.go | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d5741da089d4..3df45f38c242 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -27,6 +27,8 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string httpcli *gohttp.Client + + applyGlobal func(*RequestBuilder) } //TODO: Return errors here @@ -99,11 +101,24 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return &HttpApi{ url: url, httpcli: c, + applyGlobal: func(*RequestBuilder) {}, } } -func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { - return nil, ErrNotImplemented +func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { + options, err := caopts.ApiOptions(opts...) + if err != nil { + return nil, err + } + + subApi := *api + subApi.applyGlobal = func(req *RequestBuilder) { + if options.Offline { + req.Option("offline", options.Offline) + } + } + + return &subApi, nil } func (api *HttpApi) request(command string, args ...string) *RequestBuilder { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5fdf1835241f..67a2dbdce533 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -104,7 +104,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) // node cleanup // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 6e5a89ebdb13..831e6d71c0ba 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -86,6 +86,8 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + r.shell.applyGlobal(r) + req := NewRequest(ctx, r.shell.url, r.command, r.args...) req.Opts = r.opts req.Headers = r.headers diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 5749ca29eea1..f6e7f3ab7099 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -21,7 +21,7 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { // always drain output (response body) - ioutil.ReadAll(r.Output) + //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data return r.Output.Close() } return nil From 6d9dea62825012678ab0f117ec3cfe28c96fdaf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 17:47:22 +0100 Subject: [PATCH 018/112] Implement Unixfs.Get() This commit was moved from ipfs/go-ipfs-http-client@b31bee083d1fc5002779d755b6a624b517d2b10c --- client/httpapi/apifile.go | 231 ++++++++++++++++++++++++++++++++++++++ client/httpapi/unixfs.go | 4 - 2 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/apifile.go diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go new file mode 100644 index 000000000000..541189b609e6 --- /dev/null +++ b/client/httpapi/apifile.go @@ -0,0 +1,231 @@ +package httpapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + "github.com/ipfs/go-ipfs-files" + unixfspb "github.com/ipfs/go-unixfs/pb" +) + +func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { + if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS + var err error + p, err = api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + } + + var stat struct{ + Hash string + Type string + Size int64 // unixfs size + } + err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + if err != nil { + return nil, err + } + + switch stat.Type { + case "file": + return api.getFile(ctx, p, stat.Size) + case "directory": + return api.getDir(ctx, p, stat.Size) + default: + return nil, fmt.Errorf("unsupported file type '%s'", stat.Type) + } +} + +type apiFile struct { + ctx context.Context + core *HttpApi + size int64 + path iface.Path + + r io.ReadCloser + at int64 +} + +func (f *apiFile) reset() error { + if f.r != nil { + f.r.Close() + } + req := f.core.request("cat", f.path.String()) + if f.at != 0 { + req.Option("offset", f.at) + } + resp, err := req.Send(f.ctx) + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + f.r = resp.Output + return nil +} + +func (f *apiFile) Read(p []byte) (int, error) { + n, err := f.r.Read(p) + if n > 0 { + f.at += int64(n) + } + return n, err +} + +func (f *apiFile) Seek(offset int64, whence int) (int64, error) { + panic("implement me") //TODO +} + +func (f *apiFile) Close() error { + if f.r != nil { + return f.r.Close() + } + return nil +} + +func (f *apiFile) Size() (int64, error) { + return f.size, nil +} + +func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + f := &apiFile{ + ctx: ctx, + core: api.core(), + size: size, + path: p, + } + + return f, f.reset() +} + +type apiIter struct { + ctx context.Context + core *UnixfsAPI + + err error + + dec *json.Decoder + curFile files.Node + cur lsLink +} + +func (it *apiIter) Err() error { + return it.err +} + +func (it *apiIter) Name() string { + return it.cur.Name +} + +func (it *apiIter) Next() bool { + var out lsOutput + if err := it.dec.Decode(&out); err != nil { + if err != io.EOF { + it.err = err + } + return false + } + + if len(out.Objects) != 1 { + it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + return false + } + + if len(out.Objects[0].Links) != 1 { + it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + return false + } + + it.cur = out.Objects[0].Links[0] + c, err := cid.Parse(it.cur.Hash) + if err != nil { + it.err = err + return false + } + + switch it.cur.Type { + case unixfspb.Data_HAMTShard: + fallthrough + case unixfspb.Data_Metadata: + fallthrough + case unixfspb.Data_Directory: + it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + case unixfspb.Data_File: + it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + default: + it.err = fmt.Errorf("file type %d not supported", it.cur.Type) + return false + } + return true +} + +func (it *apiIter) Node() files.Node { + return it.curFile +} + +type apiDir struct { + ctx context.Context + core *UnixfsAPI + size int64 + path iface.Path + + dec *json.Decoder +} + +func (d *apiDir) Close() error { + return nil +} + +func (d *apiDir) Size() (int64, error) { + return d.size, nil +} + +func (d *apiDir) Entries() files.DirIterator { + return &apiIter{ + ctx: d.ctx, + core: d.core, + dec: d.dec, + } +} + +func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + resp, err := api.core().request("ls", p.String()). + Option("resolve-size", true). + Option("stream", true).Send(ctx) + + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + d := &apiDir{ + ctx: ctx, + core: api, + size: size, + path: p, + + dec: json.NewDecoder(resp.Output), + } + + return d, nil +} + +var _ files.File = &apiFile{} +var _ files.Directory = &apiDir{} diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5a1495c24b45..77daf7b9b7e1 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -85,10 +85,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return iface.IpfsPath(c), nil } -func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { - panic("implement me") -} - type lsLink struct { Name, Hash string Size uint64 From 00597e6dff81846a2dbf742dd1ab9bf546b856e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:04:22 +0100 Subject: [PATCH 019/112] Dag.Put This commit was moved from ipfs/go-ipfs-http-client@3217104469020f0c66d8457842f8221b64289d74 --- client/httpapi/api.go | 2 +- client/httpapi/dag.go | 69 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dag.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3df45f38c242..c2ec1ad6719e 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { } func (api *HttpApi) Dag() iface.DagAPI { - return nil + return (*DagAPI)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go new file mode 100644 index 000000000000..c3b29e7ee0bc --- /dev/null +++ b/client/httpapi/dag.go @@ -0,0 +1,69 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type DagAPI HttpApi + +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { + options, err := caopts.DagPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + codec, ok := cid.CodecToStr[options.Codec] + if !ok { + return nil, fmt.Errorf("unknowm codec %d", options.MhType) + } + + if options.MhLength != -1 { + return nil, fmt.Errorf("setting hash len is not supported yet") + } + + var out struct{ + Cid cid.Cid + } + err = api.core().request("dht/put"). + Option("hash", mht). + Option("format", codec). + Option("input-enc", options.InputEnc). + FileBody(src). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + return iface.IpldPath(out.Cid), nil +} + +func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { + panic("implement me") +} + +func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { + panic("implement me") +} + +func (api *DagAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 101910f04ed0575be19efaacb44b1bce73717033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:22:10 +0100 Subject: [PATCH 020/112] Object.New This commit was moved from ipfs/go-ipfs-http-client@60321ed42fd8f42e964e37ab5e96839fb8fd77b6 --- client/httpapi/api.go | 2 +- client/httpapi/ipldnode.go | 8 ++++ client/httpapi/object.go | 82 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/object.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c2ec1ad6719e..cf9a633bc8b4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -154,7 +154,7 @@ func (api *HttpApi) Pin() iface.PinAPI { } func (api *HttpApi) Object() iface.ObjectAPI { - return nil + return (*ObjectAPI)(api) } func (api *HttpApi) Dht() iface.DhtAPI { diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index b8e6fba01806..1f584ac0700a 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -20,6 +20,14 @@ type ipldNode struct { api *HttpApi } +func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { + return &ipldNode{ + ctx: ctx, + path: p, + api: a, + } +} + func (n *ipldNode) RawData() []byte { r, err := n.api.Block().Get(n.ctx, n.path) if err != nil { diff --git a/client/httpapi/object.go b/client/httpapi/object.go new file mode 100644 index 000000000000..9c1063af9938 --- /dev/null +++ b/client/httpapi/object.go @@ -0,0 +1,82 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" +) + +type ObjectAPI HttpApi + +type objectOut struct { + Hash string +} + +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { + options, err := caopts.ObjectNewOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/new", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil +} + +func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { + panic("implement me") +} + +func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { + panic("implement me") +} + +func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { + panic("implement me") +} + +func (api *ObjectAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 31a4c3754b87e68f60525533ba9c6bb9c0b1021e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:47:32 +0100 Subject: [PATCH 021/112] Fix Dag.Put This commit was moved from ipfs/go-ipfs-http-client@5b2c99abdedaf464f061511bc7b24b1ade66546f --- client/httpapi/dag.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index c3b29e7ee0bc..152d2c6b0a74 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -3,30 +3,25 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-cid" "io" + "math" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" ) type DagAPI HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { options, err := caopts.DagPutOptions(opts...) if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) - } - codec, ok := cid.CodecToStr[options.Codec] if !ok { return nil, fmt.Errorf("unknowm codec %d", options.MhType) @@ -39,12 +34,19 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPu var out struct{ Cid cid.Cid } - err = api.core().request("dht/put"). - Option("hash", mht). + req := api.core().request("dag/put"). Option("format", codec). - Option("input-enc", options.InputEnc). - FileBody(src). - Exec(ctx, &out) + Option("input-enc", options.InputEnc) + + if options.MhType != math.MaxUint64 { + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + req.Option("hash", mht) + } + + err = req.FileBody(src).Exec(ctx, &out) if err != nil { return nil, err } @@ -56,7 +58,7 @@ func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error panic("implement me") } -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { panic("implement me") } From c213e2654235ffbdb6ce2f3a7df3c027ba53e02f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 21:55:45 +0100 Subject: [PATCH 022/112] Unixfs.Add progress events This commit was moved from ipfs/go-ipfs-http-client@2f3a77b686ee8c4194ee6c21cbdecd57f2081519 --- client/httpapi/api.go | 4 +-- client/httpapi/api_test.go | 6 ++--- client/httpapi/apifile.go | 24 +++++++++--------- client/httpapi/block.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/ipldnode.go | 4 +-- client/httpapi/key.go | 5 ++-- client/httpapi/object.go | 2 +- client/httpapi/pin.go | 3 +-- client/httpapi/unixfs.go | 51 ++++++++++++++++++++++++++++++++++---- 10 files changed, 71 insertions(+), 32 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cf9a633bc8b4..c686a8d0dac4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -99,8 +99,8 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } return &HttpApi{ - url: url, - httpcli: c, + url: url, + httpcli: c, applyGlobal: func(*RequestBuilder) {}, } } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 67a2dbdce533..b822378f73a2 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -53,7 +53,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} if err := c.Run(connectArgs); err != nil { return nil, err } @@ -95,8 +95,8 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := &gohttp.Client{ Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, DisableCompression: true, }, } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 541189b609e6..d9da2397598f 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -22,12 +22,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } } - var stat struct{ + var stat struct { Hash string Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -43,12 +43,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } type apiFile struct { - ctx context.Context + ctx context.Context core *HttpApi size int64 path iface.Path - r io.ReadCloser + r io.ReadCloser at int64 } @@ -96,7 +96,7 @@ func (f *apiFile) Size() (int64, error) { func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { f := &apiFile{ - ctx: ctx, + ctx: ctx, core: api.core(), size: size, path: p, @@ -106,14 +106,14 @@ func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (fi } type apiIter struct { - ctx context.Context + ctx context.Context core *UnixfsAPI err error - dec *json.Decoder + dec *json.Decoder curFile files.Node - cur lsLink + cur lsLink } func (it *apiIter) Err() error { @@ -179,7 +179,7 @@ func (it *apiIter) Node() files.Node { } type apiDir struct { - ctx context.Context + ctx context.Context core *UnixfsAPI size int64 path iface.Path @@ -197,9 +197,9 @@ func (d *apiDir) Size() (int64, error) { func (d *apiDir) Entries() files.DirIterator { return &apiIter{ - ctx: d.ctx, + ctx: d.ctx, core: d.core, - dec: d.dec, + dec: d.dec, } } @@ -216,7 +216,7 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (fil } d := &apiDir{ - ctx: ctx, + ctx: ctx, core: api, size: size, path: p, diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 185aa0a42be0..f933d3c4afaf 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -16,7 +16,7 @@ import ( type BlockAPI HttpApi type blockStat struct { - Key string + Key string BSize int `json:"Size"` } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 152d2c6b0a74..20c233196d83 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -31,7 +31,7 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPut return nil, fmt.Errorf("setting hash len is not supported yet") } - var out struct{ + var out struct { Cid cid.Cid } req := api.core().request("dag/put"). diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 1f584ac0700a..8e028e1a7ed7 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -22,9 +22,9 @@ type ipldNode struct { func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ - ctx: ctx, + ctx: ctx, path: p, - api: a, + api: a, } } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 87b573f98a56..3653526ebef6 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -13,7 +13,7 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` - Id string + Id string } func (k *keyOutput) Name() string { @@ -35,7 +35,6 @@ func (k *keyOutput) valid() error { return err } - func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { options, err := caopts.KeyGenerateOptions(opts...) if err != nil { @@ -65,7 +64,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { - var id struct{ID string} + var id struct{ ID string } if err := api.core().request("id").Exec(ctx, &id); err != nil { return nil, err } diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 9c1063af9938..6259473c7ce9 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,7 +14,7 @@ import ( type ObjectAPI HttpApi type objectOut struct { - Hash string + Hash string } func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 02dcc4222473..ea34dc5409d6 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -20,7 +20,7 @@ type pinRefKeyList struct { type pin struct { path iface.ResolvedPath - typ string + typ string } func (p *pin) Path() iface.ResolvedPath { @@ -31,7 +31,6 @@ func (p *pin) Type() string { return p.typ } - func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { panic("implement me") } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 77daf7b9b7e1..7e63d241f878 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -2,9 +2,11 @@ package httpapi import ( "context" + "encoding/json" "fmt" "github.com/ipfs/go-cid" "github.com/pkg/errors" + "io" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -39,7 +41,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). - //Option("", options.Events). Option("fscache", options.FsCache). Option("hidden", options.Hidden). Option("inline", options.Inline). @@ -47,11 +48,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). - //Option("", options.Progress). Option("silent", options.Silent). Option("stdin-name", options.StdinName). Option("wrap-with-directory", options.Wrap). - Option("quieter", true) // TODO: rm after event impl + Option("progress", options.Progress) if options.RawLeavesSet { req.Option("raw-leaves", options.RawLeaves) @@ -73,9 +73,50 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix } var out addEvent - if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + resp, err := req.Send(ctx) + if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Output.Close() + dec := json.NewDecoder(resp.Output) +loop: + for { + var evt addEvent + switch err := dec.Decode(&evt); err { + case nil: + case io.EOF: + break loop + default: + return nil, err + } + out = evt + + if options.Events != nil { + ifevt := &iface.AddEvent{ + Name: out.Name, + Size: out.Size, + Bytes: out.Bytes, + } + + if out.Hash != "" { + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + ifevt.Path = iface.IpfsPath(c) + } + + select { + case options.Events <- ifevt: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } c, err := cid.Parse(out.Hash) if err != nil { @@ -120,7 +161,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, err links[i] = &format.Link{ Name: l.Name, Size: l.Size, - Cid: c, + Cid: c, } } return links, nil From 266c2f92c50df9b50bbc5aa093c793bb5d1a2265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 14:33:03 +0100 Subject: [PATCH 023/112] Implement Key API This commit was moved from ipfs/go-ipfs-http-client@281b2bf65b19b17e4f0c047ac8f9901306bf171f --- client/httpapi/key.go | 57 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 3653526ebef6..417415ddbf4f 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -49,18 +50,47 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { - panic("implement me") + options, err := caopts.KeyRenameOptions(opts...) + if err != nil { + return nil, false, err + } + + var out struct{ + Was string + Now string + Id string + Overwrite bool + } + err = api.core().request("key/rename", oldName, newName). + Option("force", options.Force). + Exec(ctx, &out) + if err != nil { + return nil, false, err + } + + id := &keyOutput{JName: out.Now, Id: out.Id} + return id, out.Overwrite, id.valid() } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { - panic("implement me") + var out struct{ Keys []*keyOutput } + if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]iface.Key, len(out.Keys)) + for i, k := range out.Keys { + if err := k.valid(); err != nil { + return nil, err + } + res[i] = k + } + + return res, nil } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { @@ -70,14 +100,19 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { } out := keyOutput{JName: "self", Id: id.ID} - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { - panic("implement me") + var out struct{ Keys []keyOutput } + if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + return nil, err + } + if len(out.Keys) != 1 { + return nil, errors.New("got unexpected number of keys back") + } + + return &out.Keys[0], out.Keys[0].valid() } func (api *KeyAPI) core() *HttpApi { From 9d647d011efa551bae28c50cafb402a35d33b04a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 15:02:15 +0100 Subject: [PATCH 024/112] Implement Pin API This commit was moved from ipfs/go-ipfs-http-client@5bb7a581323aad4dd765a0916f6552cd5902d1e2 --- client/httpapi/pin.go | 92 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ea34dc5409d6..34cafad71b67 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -31,8 +33,14 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { - panic("implement me") +func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { + options, err := caopts.PinAddOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/add", p.String()). + Option("recursive", options.Recursive).Exec(ctx, nil) } func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { @@ -65,11 +73,85 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { - panic("implement me") + options, err := caopts.PinUpdateOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/update"). + Option("unpin", options.Unpin).Exec(ctx, nil) +} + +type pinVerifyRes struct { + Cid string + JOk bool `json:"Ok"` + JBadNodes []*badNode `json:"BadNodes,omitempty"` +} + +func (r *pinVerifyRes) Ok() bool { + return r.JOk +} + +func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { + out := make([]iface.BadPinNode, len(r.JBadNodes)) + for i, n := range r.JBadNodes { + out[i] = n + } + return out +} + +type badNode struct { + Cid string + JErr string `json:"Err"` } -func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { - panic("implement me") +func (n *badNode) Path() iface.ResolvedPath { + c, err := cid.Parse(n.Cid) + if err != nil { + return nil // todo: handle this better + } + return iface.IpldPath(c) +} + +func (n *badNode) Err() error { + if n.JErr != "" { + return errors.New(n.JErr) + } + if _, err := cid.Parse(n.Cid); err != nil { + return err + } + return nil +} + +func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { + resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan iface.PinStatus) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + for { + var out pinVerifyRes + if err := dec.Decode(&out); err != nil { + return // todo: handle non io.EOF somehow + } + + select { + case res <- &out: + case <-ctx.Done(): + return + } + } + }() + + return res, nil } func (api *PinAPI) core() *HttpApi { From eaa19388d4d86eebf09d93a3a637e0690df722bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 16:31:07 +0100 Subject: [PATCH 025/112] Implement Object API This commit was moved from ipfs/go-ipfs-http-client@38149e46c8c3a1544dc3e3108ef72ffbc885e410 --- client/httpapi/block.go | 2 +- client/httpapi/ipldnode.go | 7 +- client/httpapi/key.go | 8 +- client/httpapi/object.go | 216 +++++++++++++++++++++++++++++++++---- client/httpapi/pin.go | 6 +- 5 files changed, 208 insertions(+), 31 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f933d3c4afaf..f8334a10980d 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -67,7 +67,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + resp, err := api.core().request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 8e028e1a7ed7..2294a95b42bd 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "errors" - "fmt" "io/ioutil" "strconv" @@ -20,11 +19,11 @@ type ipldNode struct { api *HttpApi } -func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { +func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ ctx: ctx, path: p, - api: a, + api: api, } } @@ -47,7 +46,7 @@ func (n *ipldNode) Cid() cid.Cid { } func (n *ipldNode) String() string { - return fmt.Sprintf("[Block %s]", n.Cid()) + return n.Cid().String() } func (n *ipldNode) Loggable() map[string]interface{} { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 417415ddbf4f..5087b6d99893 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -59,10 +59,10 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o return nil, false, err } - var out struct{ - Was string - Now string - Id string + var out struct { + Was string + Now string + Id string Overwrite bool } err = api.core().request("key/rename", oldName, newName). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6259473c7ce9..4bc78799166e 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -1,14 +1,17 @@ package httpapi import ( + "bytes" "context" - "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" ) type ObjectAPI HttpApi @@ -37,44 +40,219 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil } -func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + options, err := caopts.ObjectPutOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/put"). + Option("inputenc", options.InputEnc). + Option("datafieldenc", options.DataType). + Option("pin", options.Pin). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { - panic("implement me") +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { + r, err := api.core().Block().Get(ctx, p) + if err != nil { + return nil, err + } + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { - panic("implement me") +func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("object/data", p.String()).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + //TODO: make Data return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } + + return b, nil } -func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out struct { + Links []struct { + Name string + Hash string + Size uint64 + } + } + if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]*format.Link, len(out.Links)) + for i, l := range out.Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + + res[i] = &format.Link{ + Cid: c, + Name: l.Name, + Size: l.Size, + } + } + + return res, nil } -func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { - panic("implement me") +func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { + var out struct { + Hash string + NumLinks int + BlockSize int + LinksSize int + DataSize int + CumulativeSize int + } + if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return &iface.ObjectStat{ + Cid: c, + NumLinks: out.NumLinks, + BlockSize: out.BlockSize, + LinksSize: out.LinksSize, + DataSize: out.DataSize, + CumulativeSize: out.CumulativeSize, + }, nil } func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { - panic("implement me") + options, err := caopts.ObjectAddLinkOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + Option("create", options.Create). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { - panic("implement me") + var out objectOut + err := api.core().request("object/patch/rm-link", base.String(), link). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil +} + +func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/append-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/set-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +type change struct { + Type iface.ChangeType + Path string + Before cid.Cid + After cid.Cid } -func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { - panic("implement me") +func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { + var out struct { + Changes []change + } + if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]iface.ObjectChange, len(out.Changes)) + for i, ch := range out.Changes { + res[i] = iface.ObjectChange{ + Type: ch.Type, + Path: ch.Path, + } + if ch.Before != cid.Undef { + res[i].Before = iface.IpfsPath(ch.Before) + } + if ch.After != cid.Undef { + res[i].After = iface.IpfsPath(ch.After) + } + } + return res, nil } func (api *ObjectAPI) core() *HttpApi { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 34cafad71b67..1624b7867580 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -83,8 +83,8 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` + Cid string + JOk bool `json:"Ok"` JBadNodes []*badNode `json:"BadNodes,omitempty"` } @@ -101,7 +101,7 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { } type badNode struct { - Cid string + Cid string JErr string `json:"Err"` } From ed9f2dd091693e914e9de072b2c23c2f991210a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:01:33 +0100 Subject: [PATCH 026/112] Implement DHT Api This commit was moved from ipfs/go-ipfs-http-client@c77355067a22300a8abe6b93b354de0ae0a3b548 --- client/httpapi/api.go | 2 +- client/httpapi/dht.go | 98 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dht.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c686a8d0dac4..208b958e9312 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -158,7 +158,7 @@ func (api *HttpApi) Object() iface.ObjectAPI { } func (api *HttpApi) Dht() iface.DhtAPI { - return nil + return (*DhtAPI)(api) } func (api *HttpApi) Swarm() iface.SwarmAPI { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go new file mode 100644 index 000000000000..17d3b0500c2b --- /dev/null +++ b/client/httpapi/dht.go @@ -0,0 +1,98 @@ +package httpapi + +import ( + "context" + "encoding/json" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + notif "github.com/libp2p/go-libp2p-routing/notifications" +) + +type DhtAPI HttpApi + +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + if err != nil { + return peerstore.PeerInfo{}, err + } + if resp.Error != nil { + return peerstore.PeerInfo{}, resp.Error + } + defer resp.Close() + dec := json.NewDecoder(resp.Output) + for { + if err := dec.Decode(&out); err != nil { + return peerstore.PeerInfo{}, err + } + if out.Type == notif.FinalPeer { + return out.Responses[0], nil + } + } +} + +func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { + options, err := caopts.DhtFindProvidersOptions(opts...) + if err != nil { + return nil, err + } + resp, err := api.core().request("dht/findprovs", p.String()). + Option("num-providers", options.NumProviders). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan peerstore.PeerInfo) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + + for { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + + if err := dec.Decode(&out); err != nil { + return // todo: handle this somehow + } + if out.Type == notif.Provider { + for _, pi := range out.Responses { + select { + case res <- pi: + case <-ctx.Done(): + return + } + } + } + } + }() + + return res, nil +} + +func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { + options, err := caopts.DhtProvideOptions(opts...) + if err != nil { + return err + } + + return api.core().request("dht/provide", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) +} + +func (api *DhtAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cae0ff2379f29a205426f7a8199c398885f33897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:36:35 +0100 Subject: [PATCH 027/112] Implement Swarm Api This commit was moved from ipfs/go-ipfs-http-client@01105690d2da6e3f3581fa364c6284248a251a35 --- client/httpapi/api.go | 2 +- client/httpapi/swarm.go | 176 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/swarm.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 208b958e9312..a5b56eff6347 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -162,7 +162,7 @@ func (api *HttpApi) Dht() iface.DhtAPI { } func (api *HttpApi) Swarm() iface.SwarmAPI { - return nil + return (*SwarmAPI)(api) } func (api *HttpApi) PubSub() iface.PubSubAPI { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go new file mode 100644 index 000000000000..cf0ee2a9b369 --- /dev/null +++ b/client/httpapi/swarm.go @@ -0,0 +1,176 @@ +package httpapi + +import ( + "context" + "github.com/libp2p/go-libp2p-protocol" + "time" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + inet "github.com/libp2p/go-libp2p-net" + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + "github.com/multiformats/go-multiaddr" +) + +type SwarmAPI HttpApi + +func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + saddrs := make([]string, len(pi.Addrs)) + for i, addr := range pi.Addrs { + saddrs[i] = addr.String() + } + + return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) +} + +func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { + return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) +} + +type streamInfo struct { + Protocol string +} + +type connInfo struct { + Addr string + Peer string + JLatency time.Duration `json:"Latency"` + Muxer string + JDirection inet.Direction `json:"Direction"` + JStreams []streamInfo `json:"Streams"` +} + +func (c *connInfo) valid() error { + _, err := multiaddr.NewMultiaddr(c.Addr) + if err != nil { + return err + } + + _, err = peer.IDB58Decode(c.Peer) + return err +} + +func (c *connInfo) ID() peer.ID { + id, _ := peer.IDB58Decode(c.Peer) + return id +} + +func (c *connInfo) Address() multiaddr.Multiaddr { + a, _ := multiaddr.NewMultiaddr(c.Addr) + return a +} + +func (c *connInfo) Direction() inet.Direction { + return c.JDirection +} + +func (c *connInfo) Latency() (time.Duration, error) { + return c.JLatency, nil +} + +func (c *connInfo) Streams() ([]protocol.ID, error) { + res := make([]protocol.ID, len(c.JStreams)) + for i, stream := range c.JStreams { + res[i] = protocol.ID(stream.Protocol) + } + return res, nil +} + +func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { + var out struct { + Peers []*connInfo + } + + err := api.core().request("swarm/peers"). + Option("streams", true). + Option("latency", true). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + res := make([]iface.ConnectionInfo, len(out.Peers)) + for i, conn := range out.Peers { + if err := conn.valid(); err != nil { + return nil, err + } + res[i] = conn + } + + return res, nil +} + +func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Multiaddr, error) { + var out struct { + Addrs map[string][]string + } + if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + return nil, err + } + res := map[peer.ID][]multiaddr.Multiaddr{} + for spid, saddrs := range out.Addrs { + addrs := make([]multiaddr.Multiaddr, len(saddrs)) + + for i, addr := range saddrs { + a, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + addrs[i] = a + } + + pid, err := peer.IDB58Decode(spid) + if err != nil { + return nil, err + } + + res[pid] = addrs + } + + return res, nil +} + +func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c2e0872f6dfba6f6d3f74a4b9eb755053a3f483e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:19:50 +0100 Subject: [PATCH 028/112] Implement PubSub Api This commit was moved from ipfs/go-ipfs-http-client@7abddda1d33595dffb0e0da782b52660147f6855 --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 2 +- client/httpapi/pubsub.go | 129 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pubsub.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index a5b56eff6347..3a8e0e0031fe 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -166,5 +166,5 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { } func (api *HttpApi) PubSub() iface.PubSubAPI { - return nil + return (*PubsubAPI)(api) } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b822378f73a2..1acece88c3c6 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,7 +47,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go new file mode 100644 index 000000000000..3c5e3c50d606 --- /dev/null +++ b/client/httpapi/pubsub.go @@ -0,0 +1,129 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type PubsubAPI HttpApi + +func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + return nil, err + } + + return out.Strings, nil +} + +func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { + options, err := caopts.PubSubPeersOptions(opts...) + if err != nil { + return nil, err + } + + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]peer.ID, len(out.Strings)) + for i, sid := range out.Strings { + id, err := peer.IDB58Decode(sid) + if err != nil { + return nil, err + } + res[i] = id + } + return res, nil +} + +func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { + return api.core().request("pubsub/pub", topic). + FileBody(bytes.NewReader(message)). + Exec(ctx, nil) +} + +type pubsubSub struct { + io.Closer + dec *json.Decoder +} + +type pubsubMessage struct { + JFrom []byte `json:"from,omitempty"` + JData []byte `json:"data,omitempty"` + JSeqno []byte `json:"seqno,omitempty"` + JTopicIDs []string `json:"topicIDs,omitempty"` +} + +func (msg *pubsubMessage) valid() error { + _, err := peer.IDFromBytes(msg.JFrom) + return err +} + +func (msg *pubsubMessage) From() peer.ID { + id, _ := peer.IDFromBytes(msg.JFrom) + return id +} + +func (msg *pubsubMessage) Data() []byte { + return msg.JData +} + +func (msg *pubsubMessage) Seq() []byte { + return msg.JSeqno +} + +func (msg *pubsubMessage) Topics() []string { + return msg.JTopicIDs +} + +func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { + // TODO: handle ctx + + var msg pubsubMessage + if err := s.dec.Decode(&msg); err != nil { + return nil, err + } + return &msg, msg.valid() +} + +func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + options, err := caopts.PubSubSubscribeOptions(opts...) + if err != nil { + return nil, err + } + + resp, err := api.core().request("pubsub/sub", topic). + Option("discover", options.Discover). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + return &pubsubSub{ + Closer: resp, + dec: json.NewDecoder(resp.Output), + }, nil +} + +func (api *PubsubAPI) core() *HttpApi { + return (*HttpApi)(api) +} + From 9b24cf0aafa94a7b4cb4401b6fbb9c959bde09b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:49:11 +0100 Subject: [PATCH 029/112] Use cids in DHT calls This commit was moved from ipfs/go-ipfs-http-client@163b25f8b88eb98ddae50978440e3940070fabc0 --- client/httpapi/dht.go | 20 ++++++++++++++++++-- client/httpapi/pubsub.go | 3 +-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 17d3b0500c2b..8fedabbf83de 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -42,7 +42,13 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err != nil { return nil, err } - resp, err := api.core().request("dht/findprovs", p.String()). + + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + resp, err := api.core().request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -60,6 +66,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop for { var out struct { + Extra string Type notif.QueryEventType Responses []peerstore.PeerInfo } @@ -67,6 +74,10 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } + if out.Type == notif.QueryError { + return // usually a 'not found' error + // todo: handle other errors + } if out.Type == notif.Provider { for _, pi := range out.Responses { select { @@ -88,7 +99,12 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", p.String()). + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return err + } + + return api.core().request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 3c5e3c50d606..fb9bb74601c5 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -119,11 +119,10 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return &pubsubSub{ Closer: resp, - dec: json.NewDecoder(resp.Output), + dec: json.NewDecoder(resp.Output), }, nil } func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } - From b7e258cc1015e518198c50b02fcc6ebdd8767375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Jan 2019 21:31:09 +0100 Subject: [PATCH 030/112] Reimplement DAG as DAGService This commit was moved from ipfs/go-ipfs-http-client@f34a5f6d2569bb21f022d3231e811b3db6262ca8 --- client/httpapi/api.go | 6 ++- client/httpapi/apifile.go | 5 ++ client/httpapi/dag.go | 103 +++++++++++++++++++++++++------------- 3 files changed, 77 insertions(+), 37 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3a8e0e0031fe..7d4fbd0e0811 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -10,6 +10,8 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -137,8 +139,8 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() iface.DagAPI { - return (*DagAPI)(api) +func (api *HttpApi) Dag() format.DAGService { + return (*HttpDagServ)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index d9da2397598f..99ae72059c8d 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -125,6 +125,11 @@ func (it *apiIter) Name() string { } func (it *apiIter) Next() bool { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + var out lsOutput if err := it.dec.Decode(&out); err != nil { if err != io.EOF { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 20c233196d83..de2934c2f115 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -1,71 +1,104 @@ package httpapi import ( + "bytes" "context" "fmt" - "io" - "math" + "io/ioutil" + "sync" "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - mh "github.com/multiformats/go-multihash" ) -type DagAPI HttpApi +type HttpDagServ HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { - options, err := caopts.DagPutOptions(opts...) +func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { + r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) if err != nil { return nil, err } - codec, ok := cid.CodecToStr[options.Codec] - if !ok { - return nil, fmt.Errorf("unknowm codec %d", options.MhType) + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err } - if options.MhLength != -1 { - return nil, fmt.Errorf("setting hash len is not supported yet") + blk, err := blocks.NewBlockWithCid(data, c) + if err != nil { + return nil, err } - var out struct { - Cid cid.Cid + return format.DefaultBlockDecoder.Decode(blk) +} + +func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { + out := make(chan *format.NodeOption) + wg := sync.WaitGroup{} + wg.Add(len(cids)) + + for _, c := range cids { + // TODO: Consider limiting concurrency of this somehow + go func() { + defer wg.Done() + n, err := api.Get(ctx, c) + + select { + case out <- &format.NodeOption{Node: n, Err: err}: + case <-ctx.Done(): + } + }() } - req := api.core().request("dag/put"). - Option("format", codec). - Option("input-enc", options.InputEnc) - - if options.MhType != math.MaxUint64 { - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) - } - req.Option("hash", mht) + return out +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + c := nd.Cid() + prefix := c.Prefix() + format := cid.CodecToStr[prefix.Codec] + if prefix.Version == 0 { + format = "v0" } - err = req.FileBody(src).Exec(ctx, &out) + stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), + options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) if err != nil { - return nil, err + return err } - - return iface.IpldPath(out.Cid), nil + if !stat.Path().Cid().Equals(c) { + return fmt.Errorf("cids didn't match - local %s, remote %s", c.String(), stat.Path().Cid().String()) + } + return nil } -func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { - panic("implement me") +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + for _, nd := range nds { + // TODO: optimize + if err := api.Add(ctx, nd); err != nil { + return err + } + } + return nil } -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { - panic("implement me") +func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { + return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? } -func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { - panic("implement me") +func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { + for _, c := range cids { + // TODO: optimize + if err := api.Remove(ctx, c); err != nil { + return err + } + } + return nil } -func (api *DagAPI) core() *HttpApi { +func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From c3f2970f8a7302344154e6d7fe74485f0dd272c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:42:32 +0100 Subject: [PATCH 031/112] block: Pin option This commit was moved from ipfs/go-ipfs-http-client@def66919dfbee162595852334468719f57158121 --- client/httpapi/block.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f8334a10980d..24df9629ed81 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -53,6 +53,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). + Option("pin", options.Pin). FileBody(r) var out blockStat From af2edd12eb44f477ef2e0dc22ae31a75df645b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:00 +0100 Subject: [PATCH 032/112] tests: enable filestore This commit was moved from ipfs/go-ipfs-http-client@904e8eeeb1b677d458c819122d91264040120d3b --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 1acece88c3c6..bf1d6b9b04b9 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,6 +47,11 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} + if err := c.Run(filestoreArgs); err != nil { + return nil, err + } + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err From 75cf2be1026a7f206e61d164342dae4807d36bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:26 +0100 Subject: [PATCH 033/112] apifile: Implement Seek This commit was moved from ipfs/go-ipfs-http-client@69cc3e8106552605ecfdfa5c6f352996dc6f9497 --- client/httpapi/apifile.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 99ae72059c8d..ec916d8e2990 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,13 +6,15 @@ import ( "fmt" "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs-files" - unixfspb "github.com/ipfs/go-unixfs/pb" ) +const forwardSeekLimit = 1 << 14 //16k + func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error @@ -80,7 +82,24 @@ func (f *apiFile) Read(p []byte) (int, error) { } func (f *apiFile) Seek(offset int64, whence int) (int64, error) { - panic("implement me") //TODO + switch whence { + case io.SeekEnd: + offset = f.size + offset + case io.SeekCurrent: + offset = f.at + offset + } + if f.at == offset { //noop + return offset, nil + } + + if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + + f.at += r + return f.at, err + } + f.at = offset + return f.at, f.reset() } func (f *apiFile) Close() error { @@ -156,17 +175,17 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case unixfspb.Data_HAMTShard: + case iface.THAMTShard: fallthrough - case unixfspb.Data_Metadata: + case iface.TMetadata: fallthrough - case unixfspb.Data_Directory: + case iface.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case unixfspb.Data_File: + case iface.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err From bb83ccb15feb4854c7bdb4821fbcf90b89f36163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:44:48 +0100 Subject: [PATCH 034/112] response: read trailing error headers This commit was moved from ipfs/go-ipfs-http-client@83dfd84ba8bf2341478bb98d437ae1f14fe29e18 --- client/httpapi/response.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f6e7f3ab7099..745c9a2a99da 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -2,6 +2,7 @@ package httpapi import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -13,6 +14,24 @@ import ( files "github.com/ipfs/go-ipfs-files" ) +type trailerReader struct { + resp *http.Response +} + +func (r *trailerReader) Read(b []byte) (int, error) { + n, err := r.resp.Body.Read(b) + if err != nil { + if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + err = errors.New(e) + } + } + return n, err +} + +func (r *trailerReader) Close() error { + return r.resp.Body.Close() +} + type Response struct { Output io.ReadCloser Error *Error @@ -56,9 +75,6 @@ type Error struct { func (e *Error) Error() string { var out string - if e.Command != "" { - out = e.Command + ": " - } if e.Code != 0 { out = fmt.Sprintf("%s%d: ", out, e.Code) } @@ -93,7 +109,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.Output = resp.Body + nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ Command: r.Command, From 88139ddc50e111e39721990bb86bb1c018c488b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:45:24 +0100 Subject: [PATCH 035/112] dag: Interface updates This commit was moved from ipfs/go-ipfs-http-client@7bea2efb45cba072b4e24c20b46a4372be3a716d --- client/httpapi/api.go | 3 +-- client/httpapi/dag.go | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 7d4fbd0e0811..2c7a97c99265 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -11,7 +11,6 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -139,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() format.DAGService { +func (api *HttpApi) Dag() iface.APIDagService { return (*HttpDagServ)(api) } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index de2934c2f115..eacf631b92cf 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -15,7 +15,9 @@ import ( "github.com/ipfs/go-ipld-format" ) -type HttpDagServ HttpApi +type httpNodeAdder HttpApi +type HttpDagServ httpNodeAdder +type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) @@ -56,7 +58,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for return out } -func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { +func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() format := cid.CodecToStr[prefix.Codec] @@ -65,7 +67,9 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), - options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) + options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.Format(format), + options.Block.Pin(pin)) if err != nil { return err } @@ -75,16 +79,36 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { return nil } -func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { +func (api *httpNodeAdder) addMany(ctx context.Context, nds []format.Node, pin bool) error { for _, nd := range nds { // TODO: optimize - if err := api.Add(ctx, nd); err != nil { + if err := api.add(ctx, nd, pin); err != nil { return err } } return nil } +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, false) +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, false) +} + +func (api *pinningHttpNodeAdder) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, true) +} + +func (api *pinningHttpNodeAdder) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, true) +} + +func (api *HttpDagServ) Pinning() format.NodeAdder { + return (*pinningHttpNodeAdder)(api) +} + func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? } @@ -99,6 +123,10 @@ func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { return nil } +func (api *httpNodeAdder) core() *HttpApi { + return (*HttpApi)(api) +} + func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From 93bfcf91cf979b267758c04aa14c6435a687fc36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:46:00 +0100 Subject: [PATCH 036/112] unixfs: updated ls This commit was moved from ipfs/go-ipfs-http-client@bb8d9d1a60e4161c4cf3a33c468746050f267a12 --- client/httpapi/unixfs.go | 94 ++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 7e63d241f878..67d05309b295 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,7 +13,6 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" - unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" ) @@ -129,7 +128,7 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type unixfspb.Data_DataType + Type iface.FileType } type lsObject struct { @@ -141,30 +140,87 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { - var out lsOutput - err := api.core().request("ls", p.String()).Exec(ctx, &out) +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { + options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err } - if len(out.Objects) != 1 { - return nil, errors.New("unexpected objects len") + resp, err := api.core().request("ls", p.String()). + Option("resolve-type", options.ResolveChildren). + Option("size", options.ResolveChildren). + Option("stream", true). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error } - links := make([]*format.Link, len(out.Objects[0].Links)) - for i, l := range out.Objects[0].Links { - c, err := cid.Parse(l.Hash) - if err != nil { - return nil, err - } - links[i] = &format.Link{ - Name: l.Name, - Size: l.Size, - Cid: c, + dec := json.NewDecoder(resp.Output) + out := make(chan iface.LsLink) + + go func() { + defer resp.Close() + defer close(out) + + for { + var link lsOutput + if err := dec.Decode(&link); err != nil { + if err == io.EOF { + return + } + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects[0].Links) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case <-ctx.Done(): + } + return + } + + l0 := link.Objects[0].Links[0] + + c, err := cid.Decode(l0.Hash) + if err != nil { + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + select { + case out <- iface.LsLink{ + Link: &format.Link{ + Cid: c, + Name: l0.Name, + Size: l0.Size, + }, + Size: l0.Size, + Type: l0.Type, + }: + case <-ctx.Done(): + } } - } - return links, nil + }() + + return out, nil } func (api *UnixfsAPI) core() *HttpApi { From 1bc854bf2169f82d2275232e123adae721b8dc0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:53:02 +0100 Subject: [PATCH 037/112] pin: handle Rm options This commit was moved from ipfs/go-ipfs-http-client@e85e856ea2841cd3e0cf4cb7b87441539af42bbe --- client/httpapi/pin.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 1624b7867580..237b3232976d 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -68,8 +68,15 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { - return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { + options, err := caopts.PinRmOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/rm", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { From c543354b1772323cb2cfbf41c06522297bf27667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Feb 2019 22:36:30 +0100 Subject: [PATCH 038/112] Check for redirects This commit was moved from ipfs/go-ipfs-http-client@19c65db4f0fd1549fc6f224efd868d4f00da997d --- client/httpapi/api.go | 18 +++++++++++++----- client/httpapi/requestbuilder.go | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 2c7a97c99265..da0836d4a6f8 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -2,6 +2,7 @@ package httpapi import ( "errors" + "fmt" "io/ioutil" gohttp "net/http" "os" @@ -27,7 +28,7 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string - httpcli *gohttp.Client + httpcli gohttp.Client applyGlobal func(*RequestBuilder) } @@ -50,8 +51,8 @@ func NewPathApi(p string) iface.CoreAPI { return NewApi(a) } -func ApiAddr(p string) ma.Multiaddr { - baseDir, err := homedir.Expand(p) +func ApiAddr(ipfspath string) ma.Multiaddr { + baseDir, err := homedir.Expand(ipfspath) if err != nil { return nil } @@ -99,11 +100,18 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } } - return &HttpApi{ + api := &HttpApi{ url: url, - httpcli: c, + httpcli: *c, applyGlobal: func(*RequestBuilder) {}, } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 831e6d71c0ba..628ad03cdf39 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -92,7 +92,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { req.Opts = r.opts req.Headers = r.headers req.Body = r.body - return req.Send(r.shell.httpcli) + return req.Send(&r.shell.httpcli) } // Exec sends the request a request and decodes the response. From ef5bf40df1d0d790aeb213147c53412145894bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 11 Feb 2019 18:40:38 +0100 Subject: [PATCH 039/112] Update imports to use extracted interface This commit was moved from ipfs/go-ipfs-http-client@fad467bc437a1282ed3813d828c6f8ec5b7b2d7a --- client/httpapi/api.go | 5 ++--- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 9 ++++----- client/httpapi/block.go | 4 ++-- client/httpapi/dag.go | 5 ++--- client/httpapi/dht.go | 4 ++-- client/httpapi/ipldnode.go | 3 +-- client/httpapi/key.go | 5 ++--- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 5 ++--- client/httpapi/path.go | 3 +-- client/httpapi/pin.go | 6 +++--- client/httpapi/pubsub.go | 5 ++--- client/httpapi/swarm.go | 5 ++--- client/httpapi/unixfs.go | 11 +++++------ 15 files changed, 36 insertions(+), 47 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index da0836d4a6f8..698e36524e7d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,9 +9,8 @@ import ( "path" "strings" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + iface "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index bf1d6b9b04b9..5621ab87f0f3 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,10 +10,9 @@ import ( "strconv" "testing" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec916d8e2990..1e5f61a9a7b3 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -4,13 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/ipfs/go-cid" "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -92,8 +91,8 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { return offset, nil } - if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 24df9629ed81..45c73472c599 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -8,8 +8,8 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index eacf631b92cf..d0059698ab60 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -7,12 +7,11 @@ import ( "io/ioutil" "sync" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/options" ) type httpNodeAdder HttpApi diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 8fedabbf83de..dc7dd6beaf6f 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -3,9 +3,9 @@ package httpapi import ( "context" "encoding/json" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 2294a95b42bd..43fa5d50de94 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -6,11 +6,10 @@ import ( "io/ioutil" "strconv" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) type ipldNode struct { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 5087b6d99893..dc2adf8b7fe1 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,8 @@ import ( "context" "errors" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index fbc440b96664..d760e6188aeb 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -3,10 +3,10 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-ipfs/namesys/opts" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/options/namesys" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 4bc78799166e..3b648d82bc0a 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,12 +6,11 @@ import ( "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 5701326fc560..68dc2633dd5d 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,11 +3,10 @@ package httpapi import ( "context" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 237b3232976d..4c4e5713ce64 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,11 +3,11 @@ package httpapi import ( "context" "encoding/json" + "github.com/ipfs/go-cid" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/pkg/errors" - - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) type PinAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index fb9bb74601c5..3345097806db 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,9 +6,8 @@ import ( "encoding/json" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index cf0ee2a9b369..d179b6540fe2 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -2,14 +2,13 @@ package httpapi import ( "context" - "github.com/libp2p/go-libp2p-protocol" "time" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/interface-go-ipfs-core" inet "github.com/libp2p/go-libp2p-net" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" + "github.com/libp2p/go-libp2p-protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 67d05309b295..1f340b6573a2 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -3,16 +3,15 @@ package httpapi import ( "context" "encoding/json" + "errors" "fmt" - "github.com/ipfs/go-cid" - "github.com/pkg/errors" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -208,7 +207,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf select { case out <- iface.LsLink{ Link: &format.Link{ - Cid: c, + Cid: c, Name: l0.Name, Size: l0.Size, }, From 0813d808b54341414441a56e5cd3a7722a516a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:29:31 +0100 Subject: [PATCH 040/112] Fix govet warning in Dag This commit was moved from ipfs/go-ipfs-http-client@62552b33959401c7a9d9463dffba433e522b2f15 --- client/httpapi/dag.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index d0059698ab60..3f54ced34f02 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -44,7 +44,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for for _, c := range cids { // TODO: Consider limiting concurrency of this somehow - go func() { + go func(c cid.Cid) { defer wg.Done() n, err := api.Get(ctx, c) @@ -52,7 +52,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for case out <- &format.NodeOption{Node: n, Err: err}: case <-ctx.Done(): } - }() + }(c) } return out } From abc30e384fafe157490df0dae80a22ddeb6255a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:48:04 +0100 Subject: [PATCH 041/112] Register iptb plugin once This commit was moved from ipfs/go-ipfs-http-client@d6c8cbd5e643d2131477120a76159074d1a07cd3 --- client/httpapi/api_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5621ab87f0f3..fa38866e1a24 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -19,9 +19,7 @@ import ( "github.com/ipfs/iptb/testbed/interfaces" ) -type NodeProvider struct{} - -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", NewNode: local.NewNode, @@ -31,8 +29,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) BuiltIn: true, }, false) if err != nil { - return nil, err + panic(err) } +} + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { From 9a6ee6f5e153b96afda0f4ca1ee893cf28aa4f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 17:30:04 +0100 Subject: [PATCH 042/112] Improve test node spawning This commit was moved from ipfs/go-ipfs-http-client@6bb2a287a6f5090792e07f3e3a74871c459b395f --- client/httpapi/api_test.go | 198 ++++++++++++++++++++++++------------- 1 file changed, 131 insertions(+), 67 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fa38866e1a24..af2180384d62 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,23 +2,23 @@ package httpapi import ( "context" - "fmt" "io/ioutil" gohttp "net/http" "os" - "path" "strconv" + "sync" "testing" "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" - "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" "github.com/ipfs/iptb/testbed/interfaces" + ma "github.com/multiformats/go-multiaddr" ) +const parallelSpeculativeNodes = 15 // 15 seems to work best + func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", @@ -33,99 +33,163 @@ func init() { } } -type NodeProvider struct{} +type NodeProvider struct { + simple <-chan func(context.Context) ([]iface.CoreAPI, error) +} + +func newNodeProvider(ctx context.Context) *NodeProvider { + simpleNodes := make(chan func(context.Context) ([]iface.CoreAPI, error), parallelSpeculativeNodes) + + np := &NodeProvider{ + simple: simpleNodes, + } + + // start basic nodes speculatively in parallel + for i := 0; i < parallelSpeculativeNodes; i++ { + go func() { + for { + ctx, cancel := context.WithCancel(ctx) + + snd, err := np.makeAPISwarm(ctx, false, 1) + + res := func(ctx context.Context) ([]iface.CoreAPI, error) { + if err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + cancel() + }() + + return snd, nil + } + + select { + case simpleNodes <- res: + case <-ctx.Done(): + return + } + } + }() + } + + return np +} + +func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + if !fullIdentity && n == 1 { + return (<-np.simple)(ctx) + } + return np.makeAPISwarm(ctx, fullIdentity, n) +} -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { return nil, err } - c := cli.NewCli() //TODO: is there a better way? + tb := testbed.NewTestbed(dir) - initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} - if err := c.Run(initArgs); err != nil { + specs, err := testbed.BuildSpecs(tb.Dir(), n, "localipfs", nil) + if err != nil { return nil, err } - filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} - if err := c.Run(filestoreArgs); err != nil { + if err := testbed.WriteNodeSpecs(tb.Dir(), specs); err != nil { return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} - if err := c.Run(startArgs); err != nil { + nodes, err := tb.Nodes() + if err != nil { return nil, err } - if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} - if err := c.Run(connectArgs); err != nil { - return nil, err - } + apis := make([]iface.CoreAPI, n) + + wg := sync.WaitGroup{} + zero := sync.WaitGroup{} + + wg.Add(len(nodes)) + zero.Add(1) + + for i, nd := range nodes { + go func(i int, nd testbedi.Core) { + defer wg.Done() + + if _, err := nd.Init(ctx, "--empty-repo"); err != nil { + panic(err) + } + + if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { + panic(err) + } + + if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { + panic(err) + } + + if i > 0 { + zero.Wait() + if err := nd.Connect(ctx, nodes[0]); err != nil { + panic(err) + } + } else { + zero.Done() + } + + addr, err := nd.APIAddr() + if err != nil { + panic(err) + } + + maddr, err := ma.NewMultiaddr(addr) + if err != nil { + panic(err) + } + + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(maddr, c) + + // empty node is pinned even with --empty-repo, we don't want that + emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + if err != nil { + panic(err) + } + if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { + panic(err) + } + }(i, nd) } + wg.Wait() + go func() { <-ctx.Done() defer os.Remove(dir) defer func() { - _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + for _, nd := range nodes { + _ = nd.Stop(context.Background()) + } }() }() - apis := make([]iface.CoreAPI, n) - - for i := range apis { - tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) - - node, err := tb.Node(i) - if err != nil { - return nil, err - } - - attrNode, ok := node.(testbedi.Attribute) - if !ok { - return nil, fmt.Errorf("node does not implement attributes") - } - - pth, err := attrNode.Attr("path") - if err != nil { - return nil, err - } - - a := ApiAddr(pth) - if a == nil { - return nil, fmt.Errorf("nil addr for node") - } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - DisableCompression: true, - }, - } - apis[i] = NewApiWithClient(a, c) - - // node cleanup - // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) - if err != nil { - return nil, err - } - for _, pin := range pins { //TODO: parallel - if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { - return nil, err - } - } - - } - return apis, nil } func TestHttpApi(t *testing.T) { - tests.TestApi(&NodeProvider{})(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tests.TestApi(newNodeProvider(ctx))(t) } From f7dd0c690960ea79a7aad7f40b11362512d173bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:12:02 +0100 Subject: [PATCH 043/112] Improve apifile error messages This commit was moved from ipfs/go-ipfs-http-client@3393b8379021695126da83afdf3bf54eaff828f4 --- client/httpapi/apifile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1e5f61a9a7b3..8647606844b3 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -157,12 +157,12 @@ func (it *apiIter) Next() bool { } if len(out.Objects) != 1 { - it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + it.err = fmt.Errorf("ls returned more objects than expected (%d)", len(out.Objects)) return false } if len(out.Objects[0].Links) != 1 { - it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + it.err = fmt.Errorf("ls returned more links than expected (%d)", len(out.Objects[0].Links)) return false } From e1b14d78c72edf0a782e362afb0e818e032c31cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:45:19 +0100 Subject: [PATCH 044/112] Implement missing methods This commit was moved from ipfs/go-ipfs-http-client@b76413dfe55fd33d39a680458282d85565cc1c69 --- client/httpapi/api.go | 3 -- client/httpapi/name.go | 63 +++++++++++++++++++++++++++++++++++++++--- client/httpapi/path.go | 9 ++++-- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 698e36524e7d..e106d8d87a63 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,6 @@ package httpapi import ( - "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -23,8 +22,6 @@ const ( EnvDir = "IPFS_PATH" ) -var ErrNotImplemented = errors.New("not implemented") - type HttpApi struct { url string httpcli gohttp.Client diff --git a/client/httpapi/name.go b/client/httpapi/name.go index d760e6188aeb..58fb59249b9a 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "fmt" + "io" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -38,11 +40,11 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na req := api.core().request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). - Option("lifetime", options.ValidTime.String()). + Option("lifetime", options.ValidTime). Option("resolve", false) if options.TTL != nil { - req.Option("ttl", options.TTL.String()) + req.Option("ttl", options.TTL) } var out ipnsEntry @@ -57,7 +59,60 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { - return nil, ErrNotImplemented + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } + + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } + + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout). + Option("stream", true) + resp, err := req.Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + res := make(chan iface.IpnsResult) + + go func() { + defer close(res) + defer resp.Close() + + dec := json.NewDecoder(resp.Output) + + for { + var out struct{ Path string } + err := dec.Decode(&out) + if err == io.EOF { + return + } + var ires iface.IpnsResult + if err == nil { + ires.Path, err = iface.ParsePath(out.Path) + } + + select { + case res <- ires: + case <-ctx.Done(): + } + if err != nil { + return + } + } + }() + + return res, nil } func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { @@ -75,7 +130,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). - Option("dht-timeout", ropts.DhtTimeout.String()) + Option("dht-timeout", ropts.DhtTimeout) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 68dc2633dd5d..8c819121ad16 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -42,6 +42,11 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { - return nil, ErrNotImplemented +func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { + rp, err := api.ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + return api.Dag().Get(ctx, rp.Cid()) } From 42273cab06348b19a813dee77f7cbe5ed7cc78d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:05:17 +0100 Subject: [PATCH 045/112] Don't use valid() pattern This commit was moved from ipfs/go-ipfs-http-client@4d07c48f98b0af0b18c6c13f80f04c966fc43bdc --- client/httpapi/block.go | 20 +++++++------------- client/httpapi/key.go | 28 ++++++++++++++++------------ client/httpapi/name.go | 14 ++++---------- client/httpapi/pubsub.go | 12 +++++------- 4 files changed, 32 insertions(+), 42 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 45c73472c599..fd4d9bab9711 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -18,24 +18,16 @@ type BlockAPI HttpApi type blockStat struct { Key string BSize int `json:"Size"` + + cid cid.Cid } func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) valid() (iface.ResolvedPath, error) { - c, err := cid.Parse(s.Key) - if err != nil { - return nil, err - } - - return iface.IpldPath(c), nil -} - func (s *blockStat) Path() iface.ResolvedPath { - p, _ := s.valid() - return p + return iface.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +52,8 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } @@ -118,7 +111,8 @@ func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, e if err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index dc2adf8b7fe1..a16c30d8ef89 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -14,6 +14,8 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` Id string + + pid peer.ID } func (k *keyOutput) Name() string { @@ -26,13 +28,7 @@ func (k *keyOutput) Path() iface.Path { } func (k *keyOutput) ID() peer.ID { - p, _ := peer.IDB58Decode(k.Id) - return p -} - -func (k *keyOutput) valid() error { - _, err := peer.IDB58Decode(k.Id) - return err + return k.pid } func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { @@ -49,7 +45,8 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { @@ -72,7 +69,8 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - return id, out.Overwrite, id.valid() + id.pid, err = peer.IDB58Decode(id.Id) + return id, out.Overwrite, err } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { @@ -83,7 +81,9 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { - if err := k.valid(); err != nil { + var err error + k.pid, err = peer.IDB58Decode(k.Id) + if err != nil { return nil, err } res[i] = k @@ -98,8 +98,10 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { return nil, err } + var err error out := keyOutput{JName: "self", Id: id.ID} - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { @@ -111,7 +113,9 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { return nil, errors.New("got unexpected number of keys back") } - return &out.Keys[0], out.Keys[0].valid() + var err error + out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + return &out.Keys[0], err } func (api *KeyAPI) core() *HttpApi { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 58fb59249b9a..b848aa819783 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -16,10 +16,8 @@ type NameAPI HttpApi type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` -} -func (e *ipnsEntry) valid() (iface.Path, error) { - return iface.ParsePath(e.JValue) + path iface.Path } func (e *ipnsEntry) Name() string { @@ -27,8 +25,7 @@ func (e *ipnsEntry) Name() string { } func (e *ipnsEntry) Value() iface.Path { - p, _ := e.valid() - return p + return e.path } func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { @@ -51,11 +48,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { - return nil, err - } - - return &out, nil + out.path, err = iface.ParsePath(out.JValue) + return &out, err } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 3345097806db..e27cb565557d 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -66,16 +66,12 @@ type pubsubMessage struct { JData []byte `json:"data,omitempty"` JSeqno []byte `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` -} -func (msg *pubsubMessage) valid() error { - _, err := peer.IDFromBytes(msg.JFrom) - return err + from peer.ID } func (msg *pubsubMessage) From() peer.ID { - id, _ := peer.IDFromBytes(msg.JFrom) - return id + return msg.from } func (msg *pubsubMessage) Data() []byte { @@ -97,7 +93,9 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if err := s.dec.Decode(&msg); err != nil { return nil, err } - return &msg, msg.valid() + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { From 93f684617a3adbd7cba751ee421e32b75cd3dafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:07:45 +0100 Subject: [PATCH 046/112] dag: remove unused waitgroup This commit was moved from ipfs/go-ipfs-http-client@934fc60a7c41a14b679fd89fec3b349650a76c81 --- client/httpapi/dag.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 3f54ced34f02..a613d18dec5b 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -39,13 +39,10 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { out := make(chan *format.NodeOption) - wg := sync.WaitGroup{} - wg.Add(len(cids)) for _, c := range cids { // TODO: Consider limiting concurrency of this somehow go func(c cid.Cid) { - defer wg.Done() n, err := api.Get(ctx, c) select { From cc964b4ab8ea5d059be2c05e64ecdca851dcd8b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:15:21 +0100 Subject: [PATCH 047/112] Simplify Object.New, remove ipldnode.go This commit was moved from ipfs/go-ipfs-http-client@0752a6ee63a2ec42aabb6c9acac6ef6c4a94af38 --- client/httpapi/dag.go | 1 - client/httpapi/ipldnode.go | 113 ------------------------------------- client/httpapi/object.go | 34 +++++------ 3 files changed, 18 insertions(+), 130 deletions(-) delete mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index a613d18dec5b..669b5f893ad8 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io/ioutil" - "sync" "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go deleted file mode 100644 index 43fa5d50de94..000000000000 --- a/client/httpapi/ipldnode.go +++ /dev/null @@ -1,113 +0,0 @@ -package httpapi - -import ( - "context" - "errors" - "io/ioutil" - "strconv" - - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" -) - -type ipldNode struct { - ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? - path iface.ResolvedPath - api *HttpApi -} - -func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { - return &ipldNode{ - ctx: ctx, - path: p, - api: api, - } -} - -func (n *ipldNode) RawData() []byte { - r, err := n.api.Block().Get(n.ctx, n.path) - if err != nil { - panic(err) // TODO: eww, should we add errors too / better ideas? - } - - b, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - - return b -} - -func (n *ipldNode) Cid() cid.Cid { - return n.path.Cid() -} - -func (n *ipldNode) String() string { - return n.Cid().String() -} - -func (n *ipldNode) Loggable() map[string]interface{} { - return nil //TODO: we can't really do better here, can we? -} - -// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. -// We can also give people a choice -func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { - p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) - - var out interface{} - n.api.request("dag/get", p).Exec(n.ctx, &out) - - // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack - for len(path) > 0 { - switch o := out.(type) { - case map[string]interface{}: - v, ok := o[path[0]] - if !ok { - // TODO: ipld links - return nil, nil, errors.New("no element under this path") - } - out = v - case []interface{}: - n, err := strconv.ParseUint(path[0], 10, 32) - if err != nil { - return nil, nil, err - } - if len(o) < int(n) { - return nil, nil, errors.New("no element under this path") - } - out = o[n] - } - path = path[1:] - } - - return out, path, nil -} - -func (n *ipldNode) Tree(path string, depth int) []string { - panic("implement me") -} - -func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { - panic("implement me") -} - -func (n *ipldNode) Copy() ipld.Node { - panic("implement me") -} - -func (n *ipldNode) Links() []*ipld.Link { - panic("implement me") -} - -func (n *ipldNode) Stat() (*ipld.NodeStat, error) { - panic("implement me") -} - -func (n *ipldNode) Size() (uint64, error) { - panic("implement me") -} - -var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 3b648d82bc0a..5a06f74d99aa 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -3,12 +3,15 @@ package httpapi import ( "bytes" "context" + "fmt" "io" "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + dag "github.com/ipfs/go-merkledag" + ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" ) @@ -19,24 +22,23 @@ type objectOut struct { Hash string } -func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (ipld.Node, error) { options, err := caopts.ObjectNewOptions(opts...) if err != nil { return nil, err } - var out objectOut - err = api.core().request("object/new", options.Type).Exec(ctx, &out) - if err != nil { - return nil, err - } - - c, err := cid.Parse(out.Hash) - if err != nil { - return nil, err + var n ipld.Node + switch options.Type { + case "empty": + n = new(dag.ProtoNode) + case "unixfs-dir": + n = ft.EmptyDirNode() + default: + return nil, fmt.Errorf("unknown object type: %s", options.Type) } - return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil + return n, nil } func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { @@ -64,7 +66,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -96,7 +98,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -107,14 +109,14 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } - res := make([]*format.Link, len(out.Links)) + res := make([]*ipld.Link, len(out.Links)) for i, l := range out.Links { c, err := cid.Parse(l.Hash) if err != nil { return nil, err } - res[i] = &format.Link{ + res[i] = &ipld.Link{ Cid: c, Name: l.Name, Size: l.Size, From a23da822de54922faf1f82a86c25660e8cca8bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:35:32 +0100 Subject: [PATCH 048/112] swarm: attach peerid if needed This commit was moved from ipfs/go-ipfs-http-client@aa88b18ac497400d16b36af6489af50a003c298c --- client/httpapi/swarm.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index d179b6540fe2..0bb36aca3faa 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -17,6 +17,13 @@ type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { + if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + addr = addr.Encapsulate(pidma) + } saddrs[i] = addr.String() } From f005b8dc5666916ff2a0ff1eef58b7795670ec73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 16:39:00 +0100 Subject: [PATCH 049/112] pin: verify: parse bad node cids early This commit was moved from ipfs/go-ipfs-http-client@7032dfc9b016f2c584e2102e80fb60c3ecf573ea --- client/httpapi/pin.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 4c4e5713ce64..b11efd35c5db 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -110,23 +110,18 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { type badNode struct { Cid string JErr string `json:"Err"` + + cid cid.Cid } func (n *badNode) Path() iface.ResolvedPath { - c, err := cid.Parse(n.Cid) - if err != nil { - return nil // todo: handle this better - } - return iface.IpldPath(c) + return iface.IpldPath(n.cid) } func (n *badNode) Err() error { if n.JErr != "" { return errors.New(n.JErr) } - if _, err := cid.Parse(n.Cid); err != nil { - return err - } return nil } @@ -150,6 +145,13 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { return // todo: handle non io.EOF somehow } + for i, n := range out.JBadNodes { + out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + if err != nil { + return + } + } + select { case res <- &out: case <-ctx.Done(): From 4a6d36d98b6527e9c13918ac901c4cbc68630b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:02:13 +0100 Subject: [PATCH 050/112] pubsub: handle ctx This commit was moved from ipfs/go-ipfs-http-client@d451a4943c5311d7e32fcf86f56bdc1eda977fc7 --- client/httpapi/pubsub.go | 70 ++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index e27cb565557d..edc1a9709ef7 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -57,8 +57,10 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) } type pubsubSub struct { - io.Closer - dec *json.Decoder + messages chan pubsubMessage + + done chan struct{} + rcloser io.Closer } type pubsubMessage struct { @@ -68,6 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID + err error } func (msg *pubsubMessage) From() peer.ID { @@ -87,15 +90,20 @@ func (msg *pubsubMessage) Topics() []string { } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { - // TODO: handle ctx - - var msg pubsubMessage - if err := s.dec.Decode(&msg); err != nil { - return nil, err + select { + case msg, ok := <-s.messages: + if !ok { + return nil, io.EOF + } + if msg.err != nil { + return nil, msg.err + } + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err + case <-ctx.Done(): + return nil, ctx.Err() } - var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { @@ -114,10 +122,44 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, resp.Error } - return &pubsubSub{ - Closer: resp, - dec: json.NewDecoder(resp.Output), - }, nil + sub := &pubsubSub{ + messages: make(chan pubsubMessage), + done: make(chan struct{}), + } + + dec := json.NewDecoder(resp.Output) + + go func() { + defer close(sub.messages) + + for { + var msg pubsubMessage + if err := dec.Decode(&msg); err != nil { + if err == io.EOF { + return + } + msg.err = err + } + + select { + case sub.messages <- msg: + case <-sub.done: + return + case <-ctx.Done(): + return + } + } + }() + + return sub, nil +} + +func (s*pubsubSub) Close() error { + if s.done != nil { + close(s.done) + s.done = nil + } + return s.rcloser.Close() } func (api *PubsubAPI) core() *HttpApi { From 7229dbbf73fd925da390fc5db5a25caf398b1fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:04:33 +0100 Subject: [PATCH 051/112] don't read all and then throw away the buffer This commit was moved from ipfs/go-ipfs-http-client@5c96c2954a5d45dd907dca9942d2773d76b8a71b --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 745c9a2a99da..b9e83eb3d491 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -139,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - ioutil.ReadAll(resp.Body) + io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } From b6ace8dd401321b842c5f0054d9ed025fd19cd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:12:37 +0100 Subject: [PATCH 052/112] response: handle late errors This commit was moved from ipfs/go-ipfs-http-client@139e9e5ff1b52a92c169d28bf56ff58e36b49ca3 --- client/httpapi/requestbuilder.go | 4 ++-- client/httpapi/response.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 628ad03cdf39..af43ce236baa 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -103,11 +103,11 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { } if res == nil { - httpRes.Close() + lateErr := httpRes.Close() if httpRes.Error != nil { return httpRes.Error } - return nil + return lateErr } return httpRes.Decode(res) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index b9e83eb3d491..f773130e83ce 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -39,9 +39,14 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data - return r.Output.Close() + + // always drain output (response body) //TODO: make optional for things like cat + _, err1 := io.Copy(ioutil.Discard, r.Output) + err2 := r.Output.Close() + if err1 != nil { + return err1 + } + return err2 } return nil } From 7be8d01ee732be71f08f4482e0d5c5b61731ee81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:27:12 +0100 Subject: [PATCH 053/112] response: option to disable output draining This commit was moved from ipfs/go-ipfs-http-client@9f3d9635fa4e977b6ec202c9e4f78ca1c64fc0dd --- client/httpapi/api.go | 1 + client/httpapi/apifile.go | 2 +- client/httpapi/pubsub.go | 4 ++-- client/httpapi/request.go | 2 ++ client/httpapi/requestbuilder.go | 7 +++++++ client/httpapi/response.go | 10 ++++++++-- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e106d8d87a63..f74300216e35 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -131,6 +131,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, + drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 8647606844b3..e3cb85ea460d 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,7 +57,7 @@ func (f *apiFile) reset() error { if f.r != nil { f.r.Close() } - req := f.core.request("cat", f.path.String()) + req := f.core.request("cat", f.path.String()).NoDrain() if f.at != 0 { req.Option("offset", f.at) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index edc1a9709ef7..b3e45ed36c2c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -113,8 +113,8 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover). - Send(ctx) + Option("discover", options.Discover).NoDrain().Send(ctx) + if err != nil { return nil, err } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac67805..18cfb7fd0c5e 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,6 +13,7 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string + DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -30,5 +31,6 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), + DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236baa..8b040522e2f9 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,6 +19,7 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader + drainOut bool shell *HttpApi } @@ -84,6 +85,12 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } +// NoDrain disables output draining in response closer +func (r *RequestBuilder) NoDrain() *RequestBuilder { + r.drainOut = false + return r +} + // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f773130e83ce..cd3cc2b71e7c 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -35,13 +35,18 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error + + drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) //TODO: make optional for things like cat - _, err1 := io.Copy(ioutil.Discard, r.Output) + // always drain output (response body) + var err1 error + if r.drainOutput { + _, err1 = io.Copy(ioutil.Discard, r.Output) + } err2 := r.Output.Close() if err1 != nil { return err1 @@ -114,6 +119,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) + nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ From 3088776f8150fe6967a245cf0360f5219e2dc9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:30:02 +0100 Subject: [PATCH 054/112] swarm: always append peer IDs This commit was moved from ipfs/go-ipfs-http-client@49267901c71f77f098deaf8966aa2dcecf22c2df --- client/httpapi/swarm.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0bb36aca3faa..13a814bc4133 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -15,16 +15,14 @@ import ( type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { - if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { - pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) - if err != nil { - return err - } - addr = addr.Encapsulate(pidma) - } - saddrs[i] = addr.String() + saddrs[i] = addr.Encapsulate(pidma).String() } return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) From 745bf92506bd488bde4f695c4d644e7d3b58f0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:32:40 +0100 Subject: [PATCH 055/112] gofmt This commit was moved from ipfs/go-ipfs-http-client@f3c2c350861470c05617cf74e7b78d06782bb706 --- client/httpapi/api.go | 6 +++--- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f74300216e35..6dbfe3b6a81f 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -128,9 +128,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, drainOut: true, } } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index b3e45ed36c2c..49c58ab88225 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -59,7 +59,7 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) type pubsubSub struct { messages chan pubsubMessage - done chan struct{} + done chan struct{} rcloser io.Closer } @@ -70,7 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID - err error + err error } func (msg *pubsubMessage) From() peer.ID { @@ -124,7 +124,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), - done: make(chan struct{}), + done: make(chan struct{}), } dec := json.NewDecoder(resp.Output) @@ -154,7 +154,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return sub, nil } -func (s*pubsubSub) Close() error { +func (s *pubsubSub) Close() error { if s.done != nil { close(s.done) s.done = nil diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 18cfb7fd0c5e..d65d638350e6 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string DrainOut bool // if set, resp.Close will read all remaining data } @@ -26,11 +26,11 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 8b040522e2f9..c5bc44124f27 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader drainOut bool shell *HttpApi From 27aa13fe44561e8d9469ce537e64baa60d17a705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 21:53:17 +0100 Subject: [PATCH 056/112] return errors from constructor methods This commit was moved from ipfs/go-ipfs-http-client@dbee4e27aaf9ef8e3fa7e25ac1f4f28df8a31e0d --- client/httpapi/api.go | 39 ++++++++++++++++---------------------- client/httpapi/api_test.go | 5 ++++- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6dbfe3b6a81f..e4603bd380de 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,8 +29,7 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } -//TODO: Return errors here -func NewLocalApi() iface.CoreAPI { +func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -39,40 +38,34 @@ func NewLocalApi() iface.CoreAPI { return NewPathApi(baseDir) } -func NewPathApi(p string) iface.CoreAPI { - a := ApiAddr(p) - if a == nil { - return nil +func NewPathApi(p string) (iface.CoreAPI, error) { + a, err := ApiAddr(p) + if err != nil { + if err == os.ErrNotExist { + err = nil + } + return nil, err } return NewApi(a) } -func ApiAddr(ipfspath string) ma.Multiaddr { +func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { - return nil + return nil, err } apiFile := path.Join(baseDir, DefaultApiFile) - if _, err := os.Stat(apiFile); err != nil { - return nil - } - api, err := ioutil.ReadFile(apiFile) if err != nil { - return nil - } - - maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) - if err != nil { - return nil + return nil, err } - return maddr + return ma.NewMultiaddr(strings.TrimSpace(string(api))) } -func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? +func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -83,10 +76,10 @@ func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? return NewApiWithClient(a, c) } -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { - return nil // TODO: return that error + return nil, err } if a, err := ma.NewMultiaddr(url); err == nil { @@ -107,7 +100,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return fmt.Errorf("unexpected redirect") } - return api + return api, nil } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index af2180384d62..2f6d47c58eda 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -157,7 +157,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) DisableCompression: true, }, } - apis[i] = NewApiWithClient(maddr, c) + apis[i], err = NewApiWithClient(maddr, c) + if err != nil { + panic(err) + } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") From fc299e7e849d46e364204234f9ea3d3aa52e42cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:19:56 +0100 Subject: [PATCH 057/112] pin verify: use temporary struct This commit was moved from ipfs/go-ipfs-http-client@e400fa3b380b589c2ae385a66a007d685147e4c7 --- client/httpapi/pin.go | 57 +++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index b11efd35c5db..f667a54875c2 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -90,27 +90,20 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` - JBadNodes []*badNode `json:"BadNodes,omitempty"` + ok bool + badNodes []iface.BadPinNode } func (r *pinVerifyRes) Ok() bool { - return r.JOk + return r.ok } func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { - out := make([]iface.BadPinNode, len(r.JBadNodes)) - for i, n := range r.JBadNodes { - out[i] = n - } - return out + return r.badNodes } type badNode struct { - Cid string - JErr string `json:"Err"` - + err error cid cid.Cid } @@ -119,10 +112,7 @@ func (n *badNode) Path() iface.ResolvedPath { } func (n *badNode) Err() error { - if n.JErr != "" { - return errors.New(n.JErr) - } - return nil + return n.err } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { @@ -140,20 +130,45 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { defer close(res) dec := json.NewDecoder(resp.Output) for { - var out pinVerifyRes + var out struct { + Cid string + Ok bool + + BadNodes []struct{ + Cid string + Err string + } + } if err := dec.Decode(&out); err != nil { return // todo: handle non io.EOF somehow } - for i, n := range out.JBadNodes { - out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + badNodes := make([]iface.BadPinNode, len(out.BadNodes)) + for i, n := range out.BadNodes { + c, err := cid.Decode(n.Cid) if err != nil { - return + badNodes[i] = &badNode{ + cid: c, + err: err, + } + continue + } + + if n.Err != "" { + err = errors.New(n.Err) + } + badNodes[i] = &badNode{ + cid: c, + err: err, } } select { - case res <- &out: + case res <- &pinVerifyRes{ + ok: out.Ok, + + badNodes: badNodes, + }: case <-ctx.Done(): return } From a058e7d31a32d1bca98e41886a310149b872af84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:27:48 +0100 Subject: [PATCH 058/112] response: Document zero-result Decode behaviour This commit was moved from ipfs/go-ipfs-http-client@902bc5ef8c1b9595e97728dca3da0d0da124f142 --- client/httpapi/response.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index cd3cc2b71e7c..26f549c9af7b 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -71,6 +71,9 @@ func (r *Response) Decode(dec interface{}) error { } n++ } + + // Decode expects at least one result. For calls where zero results are valid, + // use Send and construct json Decoder manually. if n > 0 && err == io.EOF { err = nil } From db1af499c9d661a70ac6fcce9cc7347b934385e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:33:57 +0100 Subject: [PATCH 059/112] use mime.ParseMediaType for Content-Type response parsing This commit was moved from ipfs/go-ipfs-http-client@217d1945237321c12f96abef82707217a30813b0 --- client/httpapi/response.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 26f549c9af7b..708d2714fb78 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,14 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" + "mime" "net/http" "net/url" "os" - "strings" - - files "github.com/ipfs/go-ipfs-files" ) type trailerReader struct { @@ -116,9 +115,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType := resp.Header.Get("Content-Type") - parts := strings.Split(contentType, ";") - contentType = parts[0] + + contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return nil, err + } nresp := new(Response) From f34788e6ee61aad0ee1970a4f3d004bdebdbcd27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:11:08 +0100 Subject: [PATCH 060/112] cleanup Swarm.Peers This commit was moved from ipfs/go-ipfs-http-client@fd7858dc57b86761599b0e844f3536f2d4a0d953 --- client/httpapi/api.go | 1 - client/httpapi/apifile.go | 14 +++--- client/httpapi/pubsub.go | 9 ++-- client/httpapi/request.go | 2 - client/httpapi/requestbuilder.go | 7 --- client/httpapi/response.go | 19 +++++--- client/httpapi/swarm.go | 83 +++++++++++++++++--------------- 7 files changed, 69 insertions(+), 66 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e4603bd380de..c5a706d05741 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -124,7 +124,6 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, - drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e3cb85ea460d..a8eb0de1af24 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -49,15 +49,15 @@ type apiFile struct { size int64 path iface.Path - r io.ReadCloser + r *Response at int64 } func (f *apiFile) reset() error { if f.r != nil { - f.r.Close() + f.r.Cancel() } - req := f.core.request("cat", f.path.String()).NoDrain() + req := f.core.request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -68,12 +68,12 @@ func (f *apiFile) reset() error { if resp.Error != nil { return resp.Error } - f.r = resp.Output + f.r = resp return nil } func (f *apiFile) Read(p []byte) (int, error) { - n, err := f.r.Read(p) + n, err := f.r.Output.Read(p) if n > 0 { f.at += int64(n) } @@ -92,7 +92,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) + r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err @@ -103,7 +103,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { func (f *apiFile) Close() error { if f.r != nil { - return f.r.Close() + return f.r.Cancel() } return nil } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 49c58ab88225..2ac04b53ce36 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -60,7 +60,7 @@ type pubsubSub struct { messages chan pubsubMessage done chan struct{} - rcloser io.Closer + rcloser func() error } type pubsubMessage struct { @@ -113,7 +113,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover).NoDrain().Send(ctx) + Option("discover", options.Discover).Send(ctx) if err != nil { return nil, err @@ -125,6 +125,9 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), done: make(chan struct{}), + rcloser: func() error { + return resp.Cancel() + }, } dec := json.NewDecoder(resp.Output) @@ -159,7 +162,7 @@ func (s *pubsubSub) Close() error { close(s.done) s.done = nil } - return s.rcloser.Close() + return s.rcloser() } func (api *PubsubAPI) core() *HttpApi { diff --git a/client/httpapi/request.go b/client/httpapi/request.go index d65d638350e6..f6f7ee486ef0 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,7 +13,6 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string - DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -31,6 +30,5 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), - DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index c5bc44124f27..ba407217f216 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,7 +19,6 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader - drainOut bool shell *HttpApi } @@ -85,12 +84,6 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } -// NoDrain disables output draining in response closer -func (r *RequestBuilder) NoDrain() *RequestBuilder { - r.drainOut = false - return r -} - // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 708d2714fb78..c9f178f9ce8c 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -34,18 +34,13 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error - - drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - var err1 error - if r.drainOutput { - _, err1 = io.Copy(ioutil.Discard, r.Output) - } + // drain output (response body) + _, err1 := io.Copy(ioutil.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -55,6 +50,15 @@ func (r *Response) Close() error { return nil } +// Cancel aborts running request (without draining request body) +func (r *Response) Cancel() error { + if r.Output != nil { + return r.Output.Close() + } + + return nil +} + func (r *Response) Decode(dec interface{}) error { defer r.Close() if r.Error != nil { @@ -123,7 +127,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 13a814bc4133..8827119178a7 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -32,74 +32,81 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) } -type streamInfo struct { - Protocol string -} - type connInfo struct { - Addr string - Peer string - JLatency time.Duration `json:"Latency"` - Muxer string - JDirection inet.Direction `json:"Direction"` - JStreams []streamInfo `json:"Streams"` -} - -func (c *connInfo) valid() error { - _, err := multiaddr.NewMultiaddr(c.Addr) - if err != nil { - return err - } - - _, err = peer.IDB58Decode(c.Peer) - return err + addr multiaddr.Multiaddr + peer peer.ID + latency time.Duration + muxer string + direction inet.Direction + streams []protocol.ID } func (c *connInfo) ID() peer.ID { - id, _ := peer.IDB58Decode(c.Peer) - return id + return c.peer } func (c *connInfo) Address() multiaddr.Multiaddr { - a, _ := multiaddr.NewMultiaddr(c.Addr) - return a + return c.addr } func (c *connInfo) Direction() inet.Direction { - return c.JDirection + return c.direction } func (c *connInfo) Latency() (time.Duration, error) { - return c.JLatency, nil + return c.latency, nil } func (c *connInfo) Streams() ([]protocol.ID, error) { - res := make([]protocol.ID, len(c.JStreams)) - for i, stream := range c.JStreams { - res[i] = protocol.ID(stream.Protocol) - } - return res, nil + return c.streams, nil } func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { - var out struct { - Peers []*connInfo + var resp struct { + Peers []struct{ + Addr string + Peer string + Latency time.Duration + Muxer string + Direction inet.Direction + Streams []struct { + Protocol string + } + } } err := api.core().request("swarm/peers"). Option("streams", true). Option("latency", true). - Exec(ctx, &out) + Exec(ctx, &resp) if err != nil { return nil, err } - res := make([]iface.ConnectionInfo, len(out.Peers)) - for i, conn := range out.Peers { - if err := conn.valid(); err != nil { + res := make([]iface.ConnectionInfo, len(resp.Peers)) + for i, conn := range resp.Peers { + out := &connInfo{ + latency: conn.Latency, + muxer: conn.Muxer, + direction: conn.Direction, + } + + out.peer, err = peer.IDB58Decode(conn.Peer) + if err != nil { return nil, err } - res[i] = conn + + out.addr, err = multiaddr.NewMultiaddr(conn.Addr) + if err != nil { + return nil, err + } + + out.streams = make([]protocol.ID, len(conn.Streams)) + for i, p := range conn.Streams { + out.streams[i] = protocol.ID(p.Protocol) + } + + res[i] = out } return res, nil From aa4d6e16f13bd753b77d0a8fc2174e41af89310e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:26:34 +0100 Subject: [PATCH 061/112] Add some docs to constructors This commit was moved from ipfs/go-ipfs-http-client@e34cd600e690bfa9bd20d6089b9335ac6090d4dd --- client/httpapi/api.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c5a706d05741..9e8d55188178 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -22,6 +22,11 @@ const ( EnvDir = "IPFS_PATH" ) +// HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using +// IPFS HTTP API. +// +// For interface docs see +// https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string httpcli gohttp.Client @@ -29,6 +34,11 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } +// NewLocalApi tries to construct new HttpApi instance communicating with local +// IPFS daemon +// +// Daemon api address is pulled from the $IPFS_PATH/api file. +// If $IPFS_PATH env var is not present, it defaults to ~/.ipfs func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -38,8 +48,10 @@ func NewLocalApi() (iface.CoreAPI, error) { return NewPathApi(baseDir) } -func NewPathApi(p string) (iface.CoreAPI, error) { - a, err := ApiAddr(p) +// NewPathApi constructs new HttpApi by pulling api address from specified +// ipfspath. Api file should be located at $ipfspath/api +func NewPathApi(ipfspath string) (iface.CoreAPI, error) { + a, err := ApiAddr(ipfspath) if err != nil { if err == os.ErrNotExist { err = nil @@ -49,6 +61,7 @@ func NewPathApi(p string) (iface.CoreAPI, error) { return NewApi(a) } +// ApiAddr reads api file in specified ipfs path func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { @@ -65,6 +78,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return ma.NewMultiaddr(strings.TrimSpace(string(api))) } +// NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ @@ -76,6 +90,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { return NewApiWithClient(a, c) } +// NewApiWithClient constructs HttpApi with specified endpoint and custom http client func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { From f6a3a4fb2d715ffc0394408287b23eae0d322631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:28:39 +0100 Subject: [PATCH 062/112] gofmt This commit was moved from ipfs/go-ipfs-http-client@65cd935e13a2ea05befbc69d72d64a89056952ea --- client/httpapi/api.go | 6 +++--- client/httpapi/pin.go | 4 ++-- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- client/httpapi/response.go | 1 - client/httpapi/swarm.go | 18 +++++++++--------- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 9e8d55188178..b67fb0b483ac 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -136,9 +136,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, } } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f667a54875c2..0111d626a091 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -132,9 +132,9 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { for { var out struct { Cid string - Ok bool + Ok bool - BadNodes []struct{ + BadNodes []struct { Cid string Err string } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index f6f7ee486ef0..58c61ac67805 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -25,10 +25,10 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index ba407217f216..af43ce236baa 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader shell *HttpApi } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index c9f178f9ce8c..84f28214dcfa 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -119,7 +119,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil { return nil, err diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 8827119178a7..0814debee3c4 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -33,10 +33,10 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e } type connInfo struct { - addr multiaddr.Multiaddr - peer peer.ID + addr multiaddr.Multiaddr + peer peer.ID latency time.Duration - muxer string + muxer string direction inet.Direction streams []protocol.ID } @@ -63,11 +63,11 @@ func (c *connInfo) Streams() ([]protocol.ID, error) { func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { var resp struct { - Peers []struct{ - Addr string - Peer string + Peers []struct { + Addr string + Peer string Latency time.Duration - Muxer string + Muxer string Direction inet.Direction Streams []struct { Protocol string @@ -86,8 +86,8 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { out := &connInfo{ - latency: conn.Latency, - muxer: conn.Muxer, + latency: conn.Latency, + muxer: conn.Muxer, direction: conn.Direction, } From ad844e3d0b0221333c04896ed80f11afed301495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 16:50:38 +0100 Subject: [PATCH 063/112] response: simplify Decode This commit was moved from ipfs/go-ipfs-http-client@adbfda4c1c6ee037df3f91ca73f60de1f4620d67 --- client/httpapi/api.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 20 +++----------------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b67fb0b483ac..17a289cc5973 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -53,7 +53,7 @@ func NewLocalApi() (iface.CoreAPI, error) { func NewPathApi(ipfspath string) (iface.CoreAPI, error) { a, err := ApiAddr(ipfspath) if err != nil { - if err == os.ErrNotExist { + if os.IsNotExist(err) { err = nil } return nil, err diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236baa..2ffed7a0ae7d 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -110,5 +110,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return lateErr } - return httpRes.Decode(res) + return httpRes.decode(res) } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 84f28214dcfa..794cd14704fa 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -59,28 +59,14 @@ func (r *Response) Cancel() error { return nil } -func (r *Response) Decode(dec interface{}) error { +// Decode reads request body and decodes it as json +func (r *Response) decode(dec interface{}) error { defer r.Close() if r.Error != nil { return r.Error } - n := 0 - var err error - for { - err = json.NewDecoder(r.Output).Decode(dec) - if err != nil { - break - } - n++ - } - - // Decode expects at least one result. For calls where zero results are valid, - // use Send and construct json Decoder manually. - if n > 0 && err == io.EOF { - err = nil - } - return err + return json.NewDecoder(r.Output).Decode(dec) } type Error struct { From 19d91fbb259220e91c8887ee8a09218865418720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 22:08:02 +0100 Subject: [PATCH 064/112] response: pass close error in decode This commit was moved from ipfs/go-ipfs-http-client@4a87232ecaa96246f038d15eb7ee6094216a201e --- client/httpapi/response.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 794cd14704fa..624a488d7d79 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -61,12 +61,17 @@ func (r *Response) Cancel() error { // Decode reads request body and decodes it as json func (r *Response) decode(dec interface{}) error { - defer r.Close() if r.Error != nil { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + err := json.NewDecoder(r.Output).Decode(dec) + err2 := r.Close() + if err != nil { + return err + } + + return err2 } type Error struct { From cc9968d5dda71c21b3c90eb376458cacadb59e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:39:12 +0100 Subject: [PATCH 065/112] request: fix Content-Disposition header in Send This commit was moved from ipfs/go-ipfs-http-client@b7db17c63b69dc4fbb726dc720e9c594fa9ed495 --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 624a488d7d79..339c73658619 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -102,7 +102,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { if fr, ok := r.Body.(*files.MultiFileReader); ok { req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) - req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + req.Header.Set("Content-Disposition", "form-data; name=\"files\"") } resp, err := c.Do(req) From 47b820150bddb9868274ed2c6faf79c6b8cbe369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:46:18 +0100 Subject: [PATCH 066/112] test: don't panic on errors in async node construction This commit was moved from ipfs/go-ipfs-http-client@95ce0f3949da47b2db92508b8c7b0a00f502682b --- client/httpapi/api_test.go | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2f6d47c58eda..df45c15af508 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -114,27 +114,32 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) wg.Add(len(nodes)) zero.Add(1) + errs := make(chan error, len(nodes)) for i, nd := range nodes { go func(i int, nd testbedi.Core) { defer wg.Done() if _, err := nd.Init(ctx, "--empty-repo"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { - panic(err) + errs <- err + return } if i > 0 { zero.Wait() if err := nd.Connect(ctx, nodes[0]); err != nil { - panic(err) + errs <- err + return } } else { zero.Done() @@ -142,12 +147,14 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) addr, err := nd.APIAddr() if err != nil { - panic(err) + errs <- err + return } maddr, err := ma.NewMultiaddr(addr) if err != nil { - panic(err) + errs <- err + return } c := &gohttp.Client{ @@ -159,16 +166,19 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } apis[i], err = NewApiWithClient(maddr, c) if err != nil { - panic(err) + errs <- err + return } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err != nil { - panic(err) + errs <- err + return } if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { - panic(err) + errs <- err + return } }(i, nd) } @@ -187,7 +197,12 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) }() }() - return apis, nil + select { + case err = <-errs: + default: + } + + return apis, err } func TestHttpApi(t *testing.T) { From 0e800c3d8431fdcdc22ee2a2ac3337cb009553c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 18:15:37 +0000 Subject: [PATCH 067/112] update interface-go-ipfs-core and handle breaking changes * No more Hidden, StdinName, and Wrap options. * LsLink -> DirEntry with file types that don't expose internals. This commit was moved from ipfs/go-ipfs-http-client@5a7161eeab7127da4348dda6a6dc16f07b051f36 --- client/httpapi/apifile.go | 13 ++++----- client/httpapi/unixfs.go | 58 ++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index a8eb0de1af24..e0c0c9e1b97b 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -8,8 +8,9 @@ import ( "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + iface "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -174,17 +175,13 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case iface.THAMTShard: - fallthrough - case iface.TMetadata: - fallthrough - case iface.TDirectory: + case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case iface.TFile: + case unixfs.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 1f340b6573a2..4f717b1291bc 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,9 +8,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + unixfs_pb "github.com/ipfs/go-unixfs/pb" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -40,15 +41,12 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). Option("fscache", options.FsCache). - Option("hidden", options.Hidden). Option("inline", options.Inline). Option("inline-limit", options.InlineLimit). Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). Option("silent", options.Silent). - Option("stdin-name", options.StdinName). - Option("wrap-with-directory", options.Wrap). Option("progress", options.Progress) if options.RawLeavesSet { @@ -62,13 +60,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix req.Option("trickle", true) } - switch c := f.(type) { - case files.Directory: - req.Body(files.NewMultiFileReader(c, false)) - case files.File: - d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side - req.Body(files.NewMultiFileReader(d, false)) - } + d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) var out addEvent resp, err := req.Send(ctx) @@ -127,7 +120,8 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type iface.FileType + Type unixfs_pb.Data_DataType + Target string } type lsObject struct { @@ -139,7 +133,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err @@ -158,7 +152,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf } dec := json.NewDecoder(resp.Output) - out := make(chan iface.LsLink) + out := make(chan iface.DirEntry) go func() { defer resp.Close() @@ -171,7 +165,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return } select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return @@ -179,7 +173,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Objects len")}: case <-ctx.Done(): } return @@ -187,7 +181,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects[0].Links) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Links len")}: case <-ctx.Done(): } return @@ -198,21 +192,29 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf c, err := cid.Decode(l0.Hash) if err != nil { select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return } + var ftype iface.FileType + switch l0.Type { + case unixfs.TRaw, unixfs.TFile: + ftype = iface.TFile + case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: + ftype = iface.TDirectory + case unixfs.TSymlink: + ftype = iface.TSymlink + } + select { - case out <- iface.LsLink{ - Link: &format.Link{ - Cid: c, - Name: l0.Name, - Size: l0.Size, - }, - Size: l0.Size, - Type: l0.Type, + case out <- iface.DirEntry{ + Name: l0.Name, + Cid: c, + Size: l0.Size, + Type: ftype, + Target: l0.Target, }: case <-ctx.Done(): } From e76ecd30384799dfe388671ebd153e88903d9388 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 16:54:05 +0000 Subject: [PATCH 068/112] nit: return concrete types This is generally considered the "better" way to do things in go. It also allows us to expose functions like `request` at some point without changing the API interface itself. This commit was moved from ipfs/go-ipfs-http-client@524e5d82e6bcd2c4956151a4811addad656d7bf0 --- client/httpapi/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 17a289cc5973..ef6eff3e8234 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -39,7 +39,7 @@ type HttpApi struct { // // Daemon api address is pulled from the $IPFS_PATH/api file. // If $IPFS_PATH env var is not present, it defaults to ~/.ipfs -func NewLocalApi() (iface.CoreAPI, error) { +func NewLocalApi() (*HttpApi, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -50,7 +50,7 @@ func NewLocalApi() (iface.CoreAPI, error) { // NewPathApi constructs new HttpApi by pulling api address from specified // ipfspath. Api file should be located at $ipfspath/api -func NewPathApi(ipfspath string) (iface.CoreAPI, error) { +func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { From c6f704b453cd72734ca03793cd370021f4a39db9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:02:04 +0000 Subject: [PATCH 069/112] return an error when the API file isn't found fixes #7 This commit was moved from ipfs/go-ipfs-http-client@897d1b1bcdbf9f6616991366e66ac55e18a99a9c --- client/httpapi/api.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ef6eff3e8234..6c41ef3ab8f1 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,6 +1,7 @@ package httpapi import ( + "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -22,6 +23,9 @@ const ( EnvDir = "IPFS_PATH" ) +// ErrApiNotFound if we fail to find a running daemon. +var ErrApiNotFound = errors.New("ipfs api address could not be found") + // HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using // IPFS HTTP API. // @@ -54,7 +58,7 @@ func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { - err = nil + err = ErrApiNotFound } return nil, err } From 40f40b5a6593af189b81f43e393316b381bd541d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:04:33 +0000 Subject: [PATCH 070/112] fix golang-ci lint nits This commit was moved from ipfs/go-ipfs-http-client@ccf20b0ef5f440aca754f94325fca7202275ba8b --- client/httpapi/apifile.go | 2 +- client/httpapi/response.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e0c0c9e1b97b..1fd751ede7b2 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -56,7 +56,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { - f.r.Cancel() + _ = f.r.Cancel() } req := f.core.request("cat", f.path.String()) if f.at != 0 { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c73658619..ed3033886070 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -147,8 +147,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() + _, _ = io.Copy(ioutil.Discard, resp.Body) + _ = resp.Body.Close() } return nresp, nil From c791d73355e33139e2face5fc96884c5444f96c7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:05:02 +0000 Subject: [PATCH 071/112] avoid canceling multiple times This commit was moved from ipfs/go-ipfs-http-client@fe16408225d4c9b688528f67ffabe08198b53ab5 --- client/httpapi/apifile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1fd751ede7b2..9e6df95c24b5 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,6 +57,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { _ = f.r.Cancel() + f.r = nil } req := f.core.request("cat", f.path.String()) if f.at != 0 { From 1c07bed90465210c27dba3cda850913cf397b5c5 Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Mon, 1 Apr 2019 17:13:39 -0700 Subject: [PATCH 072/112] Export (*HttpApi).request to enable building custom requests This commit was moved from ipfs/go-ipfs-http-client@8c9ed7dbc284b4d112f6805914a43e63f234df78 --- client/httpapi/api.go | 2 +- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 8 ++++---- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 10 +++++----- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 18 +++++++++--------- client/httpapi/path.go | 2 +- client/httpapi/pin.go | 10 +++++----- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/swarm.go | 12 ++++++------ client/httpapi/unixfs.go | 4 ++-- 12 files changed, 46 insertions(+), 46 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6c41ef3ab8f1..dad205fd09d4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, args: args, diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 9e6df95c24b5..6e3433a51bf3 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -29,7 +29,7 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) + err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -59,7 +59,7 @@ func (f *apiFile) reset() error { _ = f.r.Cancel() f.r = nil } - req := f.core.request("cat", f.path.String()) + req := f.core.Request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -225,7 +225,7 @@ func (d *apiDir) Entries() files.DirIterator { } func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index fd4d9bab9711..3389f4c317ed 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -41,7 +41,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("block/put"). + req := api.core().Request("block/put"). Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). @@ -61,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(ctx) + resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR Error string `json:",omitempty"` }{} - req := api.core().request("block/rm"). + req := api.core().Request("block/rm"). Option("force", options.Force). Arguments(p.String()) @@ -107,7 +107,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { var out blockStat - err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { return nil, err } diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index dc7dd6beaf6f..f7417cb7fc3a 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -18,7 +18,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, Type notif.QueryEventType Responses []peerstore.PeerInfo } - resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { return peerstore.PeerInfo{}, err } @@ -48,7 +48,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return nil, err } - resp, err := api.core().request("dht/findprovs", rp.Cid().String()). + resp, err := api.core().Request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -104,7 +104,7 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", rp.Cid().String()). + return api.core().Request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index a16c30d8ef89..1571a3350bf8 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -38,7 +38,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key } var out keyOutput - err = api.core().request("key/gen", name). + err = api.core().Request("key/gen", name). Option("type", options.Algorithm). Option("size", options.Size). Exec(ctx, &out) @@ -61,7 +61,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o Id string Overwrite bool } - err = api.core().request("key/rename", oldName, newName). + err = api.core().Request("key/rename", oldName, newName). Option("force", options.Force). Exec(ctx, &out) if err != nil { @@ -75,7 +75,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { var out struct{ Keys []*keyOutput } - if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + if err := api.core().Request("key/list").Exec(ctx, &out); err != nil { return nil, err } @@ -94,7 +94,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var id struct{ ID string } - if err := api.core().request("id").Exec(ctx, &id); err != nil { + if err := api.core().Request("id").Exec(ctx, &id); err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { var out struct{ Keys []keyOutput } - if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + if err := api.core().Request("key/rm", name).Exec(ctx, &out); err != nil { return nil, err } if len(out.Keys) != 1 { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b848aa819783..0641d1a22d60 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -34,7 +34,7 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na return nil, err } - req := api.core().request("name/publish", p.String()). + req := api.core().Request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). Option("lifetime", options.ValidTime). @@ -63,7 +63,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). @@ -120,7 +120,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 5a06f74d99aa..a1ee486b32c4 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -48,7 +48,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec } var out objectOut - err = api.core().request("object/put"). + err = api.core().Request("object/put"). Option("inputenc", options.InputEnc). Option("datafieldenc", options.DataType). Option("pin", options.Pin). @@ -80,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) } func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("object/data", p.String()).Send(ctx) + resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er Size uint64 } } - if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]*ipld.Link, len(out.Links)) @@ -135,7 +135,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat DataSize int CumulativeSize int } - if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/stat", p.String()).Exec(ctx, &out); err != nil { return nil, err } @@ -161,7 +161,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, } var out objectOut - err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + err = api.core().Request("object/patch/add-link", base.String(), name, child.String()). Option("create", options.Create). Exec(ctx, &out) if err != nil { @@ -178,7 +178,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/rm-link", base.String(), link). + err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) if err != nil { return nil, err @@ -194,7 +194,7 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/append-data", p.String()). + err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -211,7 +211,7 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/set-data", p.String()). + err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -237,7 +237,7 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i var out struct { Changes []change } - if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]iface.ObjectChange, len(out.Changes)) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 8c819121ad16..639ca2f8e20a 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -24,7 +24,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } } - if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 0111d626a091..f779640f1740 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -39,7 +39,7 @@ func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddO return err } - return api.core().request("pin/add", p.String()). + return api.core().Request("pin/add", p.String()). Option("recursive", options.Recursive).Exec(ctx, nil) } @@ -50,7 +50,7 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. } var out pinRefKeyList - err = api.core().request("pin/ls"). + err = api.core().Request("pin/ls"). Option("type", options.Type).Exec(ctx, &out) if err != nil { return nil, err @@ -74,7 +74,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt return err } - return api.core().request("pin/rm", p.String()). + return api.core().Request("pin/rm", p.String()). Option("recursive", options.Recursive). Exec(ctx, nil) } @@ -85,7 +85,7 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o return err } - return api.core().request("pin/update"). + return api.core().Request("pin/update"). Option("unpin", options.Unpin).Exec(ctx, nil) } @@ -116,7 +116,7 @@ func (n *badNode) Err() error { } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { - resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + resp, err := api.core().Request("pin/verify").Option("verbose", true).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 2ac04b53ce36..380b933dc95b 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -18,7 +18,7 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { Strings []string } - if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } @@ -35,7 +35,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +51,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", topic). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -112,7 +112,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, err } - resp, err := api.core().request("pubsub/sub", topic). + resp, err := api.core().Request("pubsub/sub", topic). Option("discover", options.Discover).Send(ctx) if err != nil { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0814debee3c4..0e47df2ab36c 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -25,11 +25,11 @@ func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs[i] = addr.Encapsulate(pidma).String() } - return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) + return api.core().Request("swarm/connect", saddrs...).Exec(ctx, nil) } func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { - return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) + return api.core().Request("swarm/disconnect", addr.String()).Exec(ctx, nil) } type connInfo struct { @@ -75,7 +75,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) } } - err := api.core().request("swarm/peers"). + err := api.core().Request("swarm/peers"). Option("streams", true). Option("latency", true). Exec(ctx, &resp) @@ -116,7 +116,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu var out struct { Addrs map[string][]string } - if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs").Exec(ctx, &out); err != nil { return nil, err } res := map[peer.ID][]multiaddr.Multiaddr{} @@ -147,7 +147,7 @@ func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, err Strings []string } - if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/local").Exec(ctx, &out); err != nil { return nil, err } @@ -167,7 +167,7 @@ func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, er Strings []string } - if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/listen").Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 4f717b1291bc..84609194bff9 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -36,7 +36,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("add"). + req := api.core().Request("add"). Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). @@ -139,7 +139,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return nil, err } - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-type", options.ResolveChildren). Option("size", options.ResolveChildren). Option("stream", true). From 70158edd5950671a26631788d15204350d174f52 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 4 Apr 2019 00:36:32 -0700 Subject: [PATCH 073/112] feed through context see https://github.com/ipfs/go-ipfs-api/pull/173 all credit to @optman This commit was moved from ipfs/go-ipfs-http-client@7b75a8c577988e4728474a02144bf5bf83260527 --- client/httpapi/request.go | 2 ++ client/httpapi/response.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac67805..dd5293b7a6ed 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,6 +7,7 @@ import ( ) type Request struct { + Ctx context.Context ApiBase string Command string Args []string @@ -25,6 +26,7 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ + Ctx: ctx, ApiBase: url + "/api/v0", Command: command, Args: args, diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c73658619..29ae38c69ee0 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -95,6 +95,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } + req = req.WithContext(r.Ctx) + // Add any headers that were supplied via the RequestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) From 3b02ea46462d1f87a64e3fe23f73ee0f186b40d5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 9 Apr 2019 19:53:07 -0700 Subject: [PATCH 074/112] api: add authenticated transport, and direct connection This commit was moved from ipfs/go-ipfs-http-client@d04afa02b059d2c6a631f23a6923eb57ecb21015 --- client/httpapi/api.go | 36 ++++++++++++++++++++++++++++++++++++ client/httpapi/api_test.go | 25 +++++++++++++++++++++++-- client/httpapi/transport.go | 21 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dad205fd09d4..c0eacd7cbfd2 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -122,6 +122,42 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } +// NewDirectAPIClient is used to instantiate a HttpApi client +// that connects to an endpoint which leverages additional http paths. +// +// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 +// you should use NewDirectAPIClient. +func NewDirectAPIClient(url string) (*HttpApi, error) { + api := &HttpApi{ + url: url, + httpcli: gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }, + applyGlobal: func(*RequestBuilder) {}, + } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api, nil +} + +// WithAuthorization is used to wrap an instance of HttpApi +// with an authenticated transport, such as JWT +func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { + return &HttpApi{ + url: api.url, + httpcli: gohttp.Client{ + Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), + }, + } +} + func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af508..f1b1bf269426 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -9,11 +9,11 @@ import ( "sync" "testing" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" - "github.com/ipfs/iptb/testbed/interfaces" + testbedi "github.com/ipfs/iptb/testbed/interfaces" ma "github.com/multiformats/go-multiaddr" ) @@ -211,3 +211,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } + +func TestDirectAPI(t *testing.T) { + type args struct { + url, header, value string + } + tests := []struct { + name string + args args + }{ + {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + api, err := NewDirectAPIClient(tt.args.url) + if err != nil { + t.Fatal(err) + } + api = api.WithAuthorization(tt.args.header, tt.args.value) + }) + } +} diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go new file mode 100644 index 000000000000..3fb0e6e00a2a --- /dev/null +++ b/client/httpapi/transport.go @@ -0,0 +1,21 @@ +package httpapi + +import "net/http" + +type transport struct { + header, value string + httptr http.RoundTripper +} + +func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { + return &transport{ + header: header, + value: value, + httptr: tr, + } +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set(t.header, t.value) + return t.httptr.RoundTrip(req) +} From 3b67615201cd38aceb6de61e87da1552be7689fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 26 Apr 2019 20:30:55 +0200 Subject: [PATCH 075/112] gomod: update iface with path refactor This commit was moved from ipfs/go-ipfs-http-client@0bd65a67ee2994966538fae6d034ddb9fac466e6 --- client/httpapi/api_test.go | 7 ++----- client/httpapi/apifile.go | 16 ++++++++-------- client/httpapi/block.go | 11 ++++++----- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 17 +++++++++-------- client/httpapi/object.go | 35 ++++++++++++++++++----------------- client/httpapi/path.go | 18 +++++++++--------- client/httpapi/pin.go | 17 +++++++++-------- client/httpapi/unixfs.go | 9 +++++---- 11 files changed, 75 insertions(+), 73 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af508..b1bae5565e9e 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" @@ -171,11 +172,7 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } // empty node is pinned even with --empty-repo, we don't want that - emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - errs <- err - return - } + emptyNode := path.New("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { errs <- err return diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 6e3433a51bf3..afd9934e61fa 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -10,12 +10,12 @@ import ( "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" unixfs "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k -func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { +func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error p, err = api.core().ResolvePath(ctx, p) @@ -48,7 +48,7 @@ type apiFile struct { ctx context.Context core *HttpApi size int64 - path iface.Path + path path.Path r *Response at int64 @@ -114,7 +114,7 @@ func (f *apiFile) Size() (int64, error) { return f.size, nil } -func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) { f := &apiFile{ ctx: ctx, core: api.core(), @@ -177,13 +177,13 @@ func (it *apiIter) Next() bool { switch it.cur.Type { case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: - it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getDir(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } case unixfs.TFile: - it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getFile(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false @@ -203,7 +203,7 @@ type apiDir struct { ctx context.Context core *UnixfsAPI size int64 - path iface.Path + path path.Path dec *json.Decoder } @@ -224,7 +224,7 @@ func (d *apiDir) Entries() files.DirIterator { } } -func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) { resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 3389f4c317ed..640f186f5044 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -26,8 +27,8 @@ func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) Path() iface.ResolvedPath { - return iface.IpldPath(s.cid) +func (s *blockStat) Path() path.Resolved { + return path.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return &out, nil } -func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { +func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRmOption) error { options, err := caopts.BlockRmOptions(opts...) if err != nil { return err @@ -105,7 +106,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR return nil } -func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { +func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 669b5f893ad8..15d9a9c62ed2 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -9,8 +9,8 @@ import ( "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type httpNodeAdder HttpApi @@ -18,7 +18,7 @@ type HttpDagServ httpNodeAdder type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { - r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) + r, err := api.core().Block().Get(ctx, path.IpldPath(c)) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (api *HttpDagServ) Pinning() format.NodeAdder { } func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { - return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? + return api.core().Block().Rm(ctx, path.IpldPath(c)) //TODO: should we force rm? } func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index f7417cb7fc3a..e12caa1c5341 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" @@ -37,7 +37,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, } } -func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -93,7 +93,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return res, nil } -func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { +func (api *DhtAPI) Provide(ctx context.Context, p path.Path, opts ...caopts.DhtProvideOption) error { options, err := caopts.DhtProvideOptions(opts...) if err != nil { return err diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 1571a3350bf8..9a47e2ed010f 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" ) @@ -22,9 +23,8 @@ func (k *keyOutput) Name() string { return k.JName } -func (k *keyOutput) Path() iface.Path { - p, _ := iface.ParsePath("/ipns/" + k.Id) - return p +func (k *keyOutput) Path() path.Path { + return path.New("/ipns/" + k.Id) } func (k *keyOutput) ID() peer.ID { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 0641d1a22d60..47227a2ab893 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/options/namesys" + "github.com/ipfs/interface-go-ipfs-core/path" ) type NameAPI HttpApi @@ -17,18 +18,18 @@ type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` - path iface.Path + path path.Path } func (e *ipnsEntry) Name() string { return e.JName } -func (e *ipnsEntry) Value() iface.Path { +func (e *ipnsEntry) Value() path.Path { return e.path } -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { +func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { options, err := caopts.NamePublishOptions(opts...) if err != nil { return nil, err @@ -48,8 +49,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - out.path, err = iface.ParsePath(out.JValue) - return &out, err + out.path = path.New(out.JValue) + return &out, out.path.IsValid() } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { @@ -93,7 +94,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name } var ires iface.IpnsResult if err == nil { - ires.Path, err = iface.ParsePath(out.Path) + ires.Path = path.New(out.Path) } select { @@ -109,7 +110,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return res, nil } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (path.Path, error) { options, err := caopts.NameResolveOptions(opts...) if err != nil { return nil, err @@ -131,7 +132,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, err } - return iface.ParsePath(out.Path) + return path.New(out.Path), nil } func (api *NameAPI) core() *HttpApi { diff --git a/client/httpapi/object.go b/client/httpapi/object.go index a1ee486b32c4..280b2490f93d 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,6 +14,7 @@ import ( ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi @@ -41,7 +42,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return n, nil } -func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (path.Resolved, error) { options, err := caopts.ObjectPutOptions(opts...) if err != nil { return nil, err @@ -63,10 +64,10 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *ObjectAPI) Data(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err @@ -98,7 +99,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p path.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -126,7 +127,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er return res, nil } -func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { +func (api *ObjectAPI) Stat(ctx context.Context, p path.Path) (*iface.ObjectStat, error) { var out struct { Hash string NumLinks int @@ -154,7 +155,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat }, nil } -func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...caopts.ObjectAddLinkOption) (path.Resolved, error) { options, err := caopts.ObjectAddLinkOptions(opts...) if err != nil { return nil, err @@ -173,10 +174,10 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { +func (api *ObjectAPI) RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) @@ -189,10 +190,10 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AppendData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). @@ -206,10 +207,10 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) SetData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). @@ -223,7 +224,7 @@ func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (i return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type change struct { @@ -233,7 +234,7 @@ type change struct { After cid.Cid } -func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { +func (api *ObjectAPI) Diff(ctx context.Context, a path.Path, b path.Path) ([]iface.ObjectChange, error) { var out struct { Changes []change } @@ -247,10 +248,10 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i Path: ch.Path, } if ch.Before != cid.Undef { - res[i].Before = iface.IpfsPath(ch.Before) + res[i].Before = path.IpfsPath(ch.Before) } if ch.After != cid.Undef { - res[i].After = iface.IpfsPath(ch.After) + res[i].After = path.IpfsPath(ch.After) } } return res, nil diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 639ca2f8e20a..619dd0e0d2cb 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -6,10 +6,10 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) -func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { +func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { var out struct { Cid cid.Cid RemPath string @@ -18,31 +18,31 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help var err error - if path.Namespace() == "ipns" { - if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + if p.Namespace() == "ipns" { + if p, err = api.Name().Resolve(ctx, p.String()); err != nil { return nil, err } } - if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", p.String()).Exec(ctx, &out); err != nil { return nil, err } // TODO: - ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+p.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } - root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + root, err := cid.Parse(ipfspath.Path(p.String()).Segments()[1]) if err != nil { return nil, err } - return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil + return path.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *HttpApi) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) { rp, err := api.ResolvePath(ctx, p) if err != nil { return nil, err diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f779640f1740..ec0fc91a8c2d 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) @@ -21,11 +22,11 @@ type pinRefKeyList struct { } type pin struct { - path iface.ResolvedPath + path path.Resolved typ string } -func (p *pin) Path() iface.ResolvedPath { +func (p *pin) Path() path.Resolved { return p.path } @@ -33,7 +34,7 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { +func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error { options, err := caopts.PinAddOptions(opts...) if err != nil { return err @@ -62,13 +63,13 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. if err != nil { return nil, err } - pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) } return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { +func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { options, err := caopts.PinRmOptions(opts...) if err != nil { return err @@ -79,7 +80,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt Exec(ctx, nil) } -func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { +func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error { options, err := caopts.PinUpdateOptions(opts...) if err != nil { return err @@ -107,8 +108,8 @@ type badNode struct { cid cid.Cid } -func (n *badNode) Path() iface.ResolvedPath { - return iface.IpldPath(n.cid) +func (n *badNode) Path() path.Resolved { + return path.IpldPath(n.cid) } func (n *badNode) Err() error { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 84609194bff9..5e27fb0367f1 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,6 +13,7 @@ import ( unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -25,7 +26,7 @@ type addEvent struct { type UnixfsAPI HttpApi -func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (path.Resolved, error) { options, _, err := caopts.UnixfsAddOptions(opts...) if err != nil { return nil, err @@ -98,7 +99,7 @@ loop: return nil, err } - ifevt.Path = iface.IpfsPath(c) + ifevt.Path = path.IpfsPath(c) } select { @@ -114,7 +115,7 @@ loop: return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type lsLink struct { @@ -133,7 +134,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err From 11a990c7b12db59d9d03cbd9b28c2b8c75abfa26 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:13:59 -0700 Subject: [PATCH 076/112] api: expose headers via HttpApi, copy headers during request build This commit was moved from ipfs/go-ipfs-http-client@033befdef3f06a4a80fb9cf7c3afecc1a383700c --- client/httpapi/api.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c0eacd7cbfd2..712801d53233 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" gohttp "net/http" "os" "path" @@ -32,9 +33,9 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // For interface docs see // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { - url string - httpcli gohttp.Client - + url string + httpcli gohttp.Client + Headers http.Header applyGlobal func(*RequestBuilder) } @@ -175,10 +176,17 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { + var headers map[string]string + if api.Headers != nil { + for k := range api.Headers { + headers[k] = api.Headers.Get(k) + } + } return &RequestBuilder{ command: command, args: args, shell: api, + headers: headers, } } From d8c286bb15e628dbe7ddd2370c77ef02a5dc16b2 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:15 -0700 Subject: [PATCH 077/112] api: remove WithAuthorization, DirectAPI client, instantiate header map This commit was moved from ipfs/go-ipfs-http-client@9c7d495b03db4d130a8fd87217093beb33e6fdb7 --- client/httpapi/api.go | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 712801d53233..ffcc85f2b517 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -112,6 +112,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -123,20 +124,11 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } -// NewDirectAPIClient is used to instantiate a HttpApi client -// that connects to an endpoint which leverages additional http paths. -// -// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 -// you should use NewDirectAPIClient. -func NewDirectAPIClient(url string) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ - url: url, - httpcli: gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - }, - }, + url: url, + httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -144,21 +136,9 @@ func NewDirectAPIClient(url string) (*HttpApi, error) { api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { return fmt.Errorf("unexpected redirect") } - return api, nil } -// WithAuthorization is used to wrap an instance of HttpApi -// with an authenticated transport, such as JWT -func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { - return &HttpApi{ - url: api.url, - httpcli: gohttp.Client{ - Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), - }, - } -} - func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { @@ -176,7 +156,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { - var headers map[string]string + headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) From 89600e62ce160cf5f3aa2a752b3806a273d1d142 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:37 -0700 Subject: [PATCH 078/112] api: add wip api test This commit was moved from ipfs/go-ipfs-http-client@2b48dd36ad170149bd4c62a4194587bf7c704f7e --- client/httpapi/api_test.go | 62 ++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index f1b1bf269426..7b519ce7dd66 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,11 +3,13 @@ package httpapi import ( "context" "io/ioutil" + "net/http" gohttp "net/http" "os" "strconv" "sync" "testing" + "time" iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +214,51 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func TestDirectAPI(t *testing.T) { - type args struct { - url, header, value string +func Test_NewURLApiWithClient(t *testing.T) { + t.Skip() + var ( + url = "127.0.0.1:65501" + headerToTest = "Test-Header" + expectedHeaderValue = "thisisaheadertest" + ) + server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) + if err != nil { + t.Fatal(err) } - tests := []struct { - name string - args args - }{ - {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + defer server.Close() + go func() { + server.ListenAndServe() + }() + time.Sleep(time.Second * 2) + api, err := NewURLApiWithClient(url, &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }) + if err != nil { + t.Fatal(err) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - api, err := NewDirectAPIClient(tt.args.url) - if err != nil { - t.Fatal(err) - } - api = api.WithAuthorization(tt.args.header, tt.args.value) - }) + api.Headers.Set(headerToTest, expectedHeaderValue) + if _, err := api.Pin().Ls(context.Background()); err != nil { + t.Fatal(err) } } + +/// testHTTPServer spins up a test go http server +// used to check headers +func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { + r := http.NewServeMux() + r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }) + return &http.Server{ + Handler: r, + Addr: url, + }, nil +} From efe3b6054e8f62191e74e6664adec06c306c06aa Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:00:48 -0700 Subject: [PATCH 079/112] api: call NewURLApiWithClient from NewApiWithClient This commit was moved from ipfs/go-ipfs-http-client@aea6890f36001f15d24a4b373979bb1b9bf81483 --- client/httpapi/api.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ffcc85f2b517..eb99230eb1e4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -109,19 +109,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { } } - api := &HttpApi{ - url: url, - httpcli: *c, - Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, - } - - // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { - return fmt.Errorf("unexpected redirect") - } - - return api, nil + return NewURLApiWithClient(url, c) } func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { From fd5010bd2a852ea8fa3a2b08e199312fb3b71097 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:13:47 -0700 Subject: [PATCH 080/112] api: cleanup header and NewURLApiWithClient test This commit was moved from ipfs/go-ipfs-http-client@b8b55cb89a5dd830915f83e43e8afa7a01597905 --- client/httpapi/api_test.go | 46 +++++++++++++------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 52fa67d0e2ca..fadd9663e9dd 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -5,13 +5,14 @@ import ( "io/ioutil" "net/http" gohttp "net/http" + "net/http/httptest" "os" "strconv" "sync" "testing" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +213,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func Test_NewURLApiWithClient(t *testing.T) { - t.Skip() +func Test_NewURLApiWithClient_With_Headers(t *testing.T) { var ( - url = "127.0.0.1:65501" headerToTest = "Test-Header" expectedHeaderValue = "thisisaheadertest" ) - server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) - if err != nil { - t.Fatal(err) - } - defer server.Close() - go func() { - server.ListenAndServe() - }() + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }), + ) + defer ts.Close() time.Sleep(time.Second * 2) - api, err := NewURLApiWithClient(url, &http.Client{ + api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, @@ -242,21 +244,3 @@ func Test_NewURLApiWithClient(t *testing.T) { t.Fatal(err) } } - -/// testHTTPServer spins up a test go http server -// used to check headers -func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { - r := http.NewServeMux() - r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { - val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { - w.WriteHeader(400) - return - } - w.WriteHeader(200) - }) - return &http.Server{ - Handler: r, - Addr: url, - }, nil -} From daf1b72809a3d1af819d7ff44aabcc6bc6471fe8 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 17:19:35 -0700 Subject: [PATCH 081/112] api: fix failing test This commit was moved from ipfs/go-ipfs-http-client@ea507216d821455c351440ba33775f564ea18ea8 --- client/httpapi/api_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fadd9663e9dd..5f012b476721 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "strconv" + "strings" "sync" "testing" "time" @@ -221,15 +222,14 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { ts := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { + if val != expectedHeaderValue { w.WriteHeader(400) return } - w.WriteHeader(200) + http.ServeContent(w, r, "", time.Now(), strings.NewReader("test")) }), ) defer ts.Close() - time.Sleep(time.Second * 2) api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -240,7 +240,7 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } api.Headers.Set(headerToTest, expectedHeaderValue) - if _, err := api.Pin().Ls(context.Background()); err != nil { + if err := api.Pin().Rm(context.Background(), path.New("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv")); err != nil { t.Fatal(err) } } From 8b9e189169b51c46956a18e08f9dc7bc668401b4 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 19:14:15 -0700 Subject: [PATCH 082/112] remove unused transport.go file This commit was moved from ipfs/go-ipfs-http-client@320130421f4727d72d68a3c6a5c0e633e1dabc38 --- client/httpapi/transport.go | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go deleted file mode 100644 index 3fb0e6e00a2a..000000000000 --- a/client/httpapi/transport.go +++ /dev/null @@ -1,21 +0,0 @@ -package httpapi - -import "net/http" - -type transport struct { - header, value string - httptr http.RoundTripper -} - -func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { - return &transport{ - header: header, - value: value, - httptr: tr, - } -} - -func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header.Set(t.header, t.value) - return t.httptr.RoundTrip(req) -} From 8d953a7359c488cde5299ca704594bd8e1349ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 10 May 2019 15:38:20 +0200 Subject: [PATCH 083/112] Use interface for RequestBuilder This commit was moved from ipfs/go-ipfs-http-client@0534b7ca8319329459c505a546f3025862a735f5 --- client/httpapi/api.go | 10 ++++----- client/httpapi/requestbuilder.go | 36 ++++++++++++++++++++++---------- client/httpapi/response.go | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index eb99230eb1e4..5c2eb76d6d06 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -36,7 +36,7 @@ type HttpApi struct { url string httpcli gohttp.Client Headers http.Header - applyGlobal func(*RequestBuilder) + applyGlobal func(*requestBuilder) } // NewLocalApi tries to construct new HttpApi instance communicating with local @@ -117,7 +117,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { url: url, httpcli: *c, Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, + applyGlobal: func(*requestBuilder) {}, } // We don't support redirects. @@ -134,7 +134,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } subApi := *api - subApi.applyGlobal = func(req *RequestBuilder) { + subApi.applyGlobal = func(req *requestBuilder) { if options.Offline { req.Option("offline", options.Offline) } @@ -143,14 +143,14 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) RequestBuilder { headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) } } - return &RequestBuilder{ + return &requestBuilder{ command: command, args: args, shell: api, diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 2ffed7a0ae7d..7012a8935bfa 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -12,8 +12,20 @@ import ( "github.com/ipfs/go-ipfs-files" ) -// RequestBuilder is an IPFS commands request builder. -type RequestBuilder struct { +type RequestBuilder interface { + Arguments(args ...string) RequestBuilder + BodyString(body string) RequestBuilder + BodyBytes(body []byte) RequestBuilder + Body(body io.Reader) RequestBuilder + FileBody(body io.Reader) RequestBuilder + Option(key string, value interface{}) RequestBuilder + Header(name, value string) RequestBuilder + Send(ctx context.Context) (*Response, error) + Exec(ctx context.Context, res interface{}) error +} + +// requestBuilder is an IPFS commands request builder. +type requestBuilder struct { command string args []string opts map[string]string @@ -24,29 +36,29 @@ type RequestBuilder struct { } // Arguments adds the arguments to the args. -func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { +func (r *requestBuilder) Arguments(args ...string) RequestBuilder { r.args = append(r.args, args...) return r } // BodyString sets the request body to the given string. -func (r *RequestBuilder) BodyString(body string) *RequestBuilder { +func (r *requestBuilder) BodyString(body string) RequestBuilder { return r.Body(strings.NewReader(body)) } // BodyBytes sets the request body to the given buffer. -func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { +func (r *requestBuilder) BodyBytes(body []byte) RequestBuilder { return r.Body(bytes.NewReader(body)) } // Body sets the request body to the given reader. -func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { +func (r *requestBuilder) Body(body io.Reader) RequestBuilder { r.body = body return r } // FileBody sets the request body to the given reader wrapped into multipartreader. -func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { +func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) @@ -55,7 +67,7 @@ func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { } // Option sets the given option. -func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { +func (r *requestBuilder) Option(key string, value interface{}) RequestBuilder { var s string switch v := value.(type) { case bool: @@ -76,7 +88,7 @@ func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { } // Header sets the given header. -func (r *RequestBuilder) Header(name, value string) *RequestBuilder { +func (r *requestBuilder) Header(name, value string) RequestBuilder { if r.headers == nil { r.headers = make(map[string]string, 1) } @@ -85,7 +97,7 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { } // Send sends the request and return the response. -func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { +func (r *requestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) req := NewRequest(ctx, r.shell.url, r.command, r.args...) @@ -96,7 +108,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { } // Exec sends the request a request and decodes the response. -func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { +func (r *requestBuilder) Exec(ctx context.Context, res interface{}) error { httpRes, err := r.Send(ctx) if err != nil { return err @@ -112,3 +124,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return httpRes.decode(res) } + +var _ RequestBuilder = &requestBuilder{} diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 2d6af49d2ca6..db22a66faf4f 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -97,7 +97,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { req = req.WithContext(r.Ctx) - // Add any headers that were supplied via the RequestBuilder. + // Add any headers that were supplied via the requestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) } From 1b81f2ef10c40b5ba5a50ae545a62cc050c11813 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 11 Jun 2019 17:41:49 -0700 Subject: [PATCH 084/112] add extended error handling fixes #19 This commit was moved from ipfs/go-ipfs-http-client@8e3552ac1e6ea18ffd5f49918eef578de2b224fb --- client/httpapi/response.go | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index db22a66faf4f..056b2d68d8bb 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -13,6 +13,48 @@ import ( "os" ) +// Error codes adapted from go-ipfs-cmds. We should find a better solution. + +// ErrorType signfies a category of errors +type ErrorType uint + +// ErrorTypes convey what category of error ocurred +const ( + // ErrNormal is a normal error. The command failed for some reason that's not a bug. + ErrNormal ErrorType = iota + // ErrClient means the client made an invalid request. + ErrClient + // ErrImplementation means there's a bug in the implementation. + ErrImplementation + // ErrRateLimited is returned when the operation has been rate-limited. + ErrRateLimited + // ErrForbidden is returned when the client doesn't have permission to + // perform the requested operation. + ErrForbidden +) + +func (e ErrorType) Error() string { + return e.String() +} + +func (e ErrorType) String() string { + switch e { + case ErrNormal: + return "command failed" + case ErrClient: + return "invalid argument" + case ErrImplementation: + return "internal error" + case ErrRateLimited: + return "rate limited" + case ErrForbidden: + return "request forbidden" + default: + return "unknown error code" + } + +} + type trailerReader struct { resp *http.Response } @@ -77,7 +119,13 @@ func (r *Response) decode(dec interface{}) error { type Error struct { Command string Message string - Code int + Code ErrorType +} + +// Unwrap returns the base error (an ErrorType). Works with go 1.14 error +// helpers. +func (e *Error) Unwrap() error { + return e.Code } func (e *Error) Error() string { @@ -133,11 +181,23 @@ func (r *Request) Send(c *http.Client) (*Response, error) { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } e.Message = string(out) + + // set special status codes. + switch resp.StatusCode { + case http.StatusNotFound, http.StatusBadRequest: + e.Code = ErrClient + case http.StatusTooManyRequests: + e.Code = ErrRateLimited + case http.StatusForbidden: + e.Code = ErrForbidden + } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) } default: + // This is a server-side bug (probably). + e.Code = ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 2612353d85385a595e889dfa247363a8a4ab02c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 13 Jun 2019 12:00:58 -0700 Subject: [PATCH 085/112] use error from go-ipfs-cmds note: this drops the Command field This commit was moved from ipfs/go-ipfs-http-client@523a26f0a853a2d6779a21d7bad239bf2012833b --- client/httpapi/response.go | 79 +++++--------------------------------- 1 file changed, 10 insertions(+), 69 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 056b2d68d8bb..95cbf13ec1bf 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,56 +4,19 @@ import ( "encoding/json" "errors" "fmt" - "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" "mime" "net/http" "net/url" "os" -) -// Error codes adapted from go-ipfs-cmds. We should find a better solution. - -// ErrorType signfies a category of errors -type ErrorType uint - -// ErrorTypes convey what category of error ocurred -const ( - // ErrNormal is a normal error. The command failed for some reason that's not a bug. - ErrNormal ErrorType = iota - // ErrClient means the client made an invalid request. - ErrClient - // ErrImplementation means there's a bug in the implementation. - ErrImplementation - // ErrRateLimited is returned when the operation has been rate-limited. - ErrRateLimited - // ErrForbidden is returned when the client doesn't have permission to - // perform the requested operation. - ErrForbidden + cmds "github.com/ipfs/go-ipfs-cmds" + cmdhttp "github.com/ipfs/go-ipfs-cmds/http" + files "github.com/ipfs/go-ipfs-files" ) -func (e ErrorType) Error() string { - return e.String() -} - -func (e ErrorType) String() string { - switch e { - case ErrNormal: - return "command failed" - case ErrClient: - return "invalid argument" - case ErrImplementation: - return "internal error" - case ErrRateLimited: - return "rate limited" - case ErrForbidden: - return "request forbidden" - default: - return "unknown error code" - } - -} +type Error = cmds.Error type trailerReader struct { resp *http.Response @@ -62,7 +25,7 @@ type trailerReader struct { func (r *trailerReader) Read(b []byte) (int, error) { n, err := r.resp.Body.Read(b) if err != nil { - if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + if e := r.resp.Trailer.Get(cmdhttp.StreamErrHeader); e != "" { err = errors.New(e) } } @@ -116,26 +79,6 @@ func (r *Response) decode(dec interface{}) error { return err2 } -type Error struct { - Command string - Message string - Code ErrorType -} - -// Unwrap returns the base error (an ErrorType). Works with go 1.14 error -// helpers. -func (e *Error) Unwrap() error { - return e.Code -} - -func (e *Error) Error() string { - var out string - if e.Code != 0 { - out = fmt.Sprintf("%s%d: ", out, e.Code) - } - return out + e.Message -} - func (r *Request) Send(c *http.Client) (*Response, error) { url := r.getURL() req, err := http.NewRequest("POST", url, r.Body) @@ -169,9 +112,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { - e := &Error{ - Command: r.Command, - } + e := new(Error) switch { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" @@ -185,11 +126,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // set special status codes. switch resp.StatusCode { case http.StatusNotFound, http.StatusBadRequest: - e.Code = ErrClient + e.Code = cmds.ErrClient case http.StatusTooManyRequests: - e.Code = ErrRateLimited + e.Code = cmds.ErrRateLimited case http.StatusForbidden: - e.Code = ErrForbidden + e.Code = cmds.ErrForbidden } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { @@ -197,7 +138,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { } default: // This is a server-side bug (probably). - e.Code = ErrImplementation + e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 9345cfa7b83d3fb2fb470fa1e77e7af89087452b Mon Sep 17 00:00:00 2001 From: godcong Date: Sun, 21 Jul 2019 12:33:35 +0800 Subject: [PATCH 086/112] update to consolidated libp2p interface package (#21) and fix parsing of connection latencies This commit was moved from ipfs/go-ipfs-http-client@fd5cce4cbc9d95617ded33849c437a492f5e9d96 --- client/httpapi/dht.go | 31 +++++++++++++++---------------- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/swarm.go | 20 ++++++++++---------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index e12caa1c5341..0a78514f9739 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,38 +6,37 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - notif "github.com/libp2p/go-libp2p-routing/notifications" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/routing" ) type DhtAPI HttpApi -func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) { var out struct { - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } if resp.Error != nil { - return peerstore.PeerInfo{}, resp.Error + return peer.AddrInfo{}, resp.Error } defer resp.Close() dec := json.NewDecoder(resp.Output) for { if err := dec.Decode(&out); err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } - if out.Type == notif.FinalPeer { + if out.Type == routing.FinalPeer { return out.Responses[0], nil } } } -func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peer.AddrInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -57,7 +56,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt if resp.Error != nil { return nil, resp.Error } - res := make(chan peerstore.PeerInfo) + res := make(chan peer.AddrInfo) go func() { defer resp.Close() @@ -67,18 +66,18 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt for { var out struct { Extra string - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } - if out.Type == notif.QueryError { + if out.Type == routing.QueryError { return // usually a 'not found' error // todo: handle other errors } - if out.Type == notif.Provider { + if out.Type == routing.Provider { for _, pi := range out.Responses { select { case res <- pi: diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 9a47e2ed010f..cf22c61089c8 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 380b933dc95b..81ddb5211c3c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type PubsubAPI HttpApi diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0e47df2ab36c..10280c39f080 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,16 +5,15 @@ import ( "time" "github.com/ipfs/interface-go-ipfs-core" - inet "github.com/libp2p/go-libp2p-net" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - "github.com/libp2p/go-libp2p-protocol" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/multiformats/go-multiaddr" ) type SwarmAPI HttpApi -func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { +func (api *SwarmAPI) Connect(ctx context.Context, pi peer.AddrInfo) error { pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) if err != nil { return err @@ -37,7 +36,7 @@ type connInfo struct { peer peer.ID latency time.Duration muxer string - direction inet.Direction + direction network.Direction streams []protocol.ID } @@ -49,7 +48,7 @@ func (c *connInfo) Address() multiaddr.Multiaddr { return c.addr } -func (c *connInfo) Direction() inet.Direction { +func (c *connInfo) Direction() network.Direction { return c.direction } @@ -66,9 +65,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) Peers []struct { Addr string Peer string - Latency time.Duration + Latency string Muxer string - Direction inet.Direction + Direction network.Direction Streams []struct { Protocol string } @@ -85,8 +84,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { + latency, _ := time.ParseDuration(conn.Latency) out := &connInfo{ - latency: conn.Latency, + latency: latency, muxer: conn.Muxer, direction: conn.Direction, } From be8f80237d4ba8ed8d2ef278180b2b3d446c60f0 Mon Sep 17 00:00:00 2001 From: godcong Date: Tue, 13 Aug 2019 07:16:47 +0800 Subject: [PATCH 087/112] fix path miss in windows/add a new api create function (#23) * fix path was not always correct in windows * add new api create function for if the port was different with api file. * Update api.go remove no need commit This commit was moved from ipfs/go-ipfs-http-client@8c87debb1233f2af7a0cd102a934b630d8a12147 --- client/httpapi/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 5c2eb76d6d06..ec16c4b2da83 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -7,12 +7,12 @@ import ( "net/http" gohttp "net/http" "os" - "path" + "path/filepath" "strings" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - homedir "github.com/mitchellh/go-homedir" + "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) @@ -73,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return nil, err } - apiFile := path.Join(baseDir, DefaultApiFile) + apiFile := filepath.Join(baseDir, DefaultApiFile) api, err := ioutil.ReadFile(apiFile) if err != nil { From b6b2ae355bf86c4859f234240b44b35656685ece Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 30 Aug 2019 18:44:24 -0700 Subject: [PATCH 088/112] doc: mark this package experimental instead of WIP (#30) We're at a point where this package works pretty well and users should consider using it. This commit was moved from ipfs/go-ipfs-http-client@3e8506bbecd9b77ae30d04c7b997d9d612654beb --- client/httpapi/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index dd09142e32bc..e5ab1c612273 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,7 +8,10 @@ > IPFS CoreAPI implementation using HTTP API -This project is WIP, use https://github.com/ipfs/go-ipfs-api for now +This package is experimental and subject to change. If you need to depend on +something less likely to change, please use +[go-ipfs-api](https://github.com/ipfs/go-ipfs-api). If you'd like the latest and +greatest features, please use _this_ package. ## Documentation From d48f8fce64d6496a18b442c74021829554151823 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 10 Sep 2019 18:07:48 -0700 Subject: [PATCH 089/112] file: implement ReadAt This commit was moved from ipfs/go-ipfs-http-client@88a9b615205816ea20ea38fbee170b3256e51b94 --- client/httpapi/apifile.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index afd9934e61fa..ec2f7588d591 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -82,6 +82,25 @@ func (f *apiFile) Read(p []byte) (int, error) { return n, err } +func (f *apiFile) ReadAt(p []byte, off int64) (int, error) { + // Always make a new request. This method should be parallel-safe. + resp, err := f.core.Request("cat", f.path.String()). + Option("offset", off).Option("length", len(p)).Send(f.ctx) + if err != nil { + return 0, err + } + if resp.Error != nil { + return 0, resp.Error + } + defer resp.Output.Close() + + n, err := io.ReadFull(resp.Output, p) + if err == io.ErrUnexpectedEOF { + err = io.EOF + } + return n, err +} + func (f *apiFile) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekEnd: From 8f9306304efdc60147eccbb49380b83e3c7cb4ea Mon Sep 17 00:00:00 2001 From: Joel Gustafson Date: Sat, 7 Dec 2019 01:08:48 -0500 Subject: [PATCH 090/112] Update pin.go This commit was moved from ipfs/go-ipfs-http-client@31b4e60d961ada25cb7b40317b18011b284436a7 --- client/httpapi/pin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ec0fc91a8c2d..6026c71b38ab 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -86,7 +86,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt return err } - return api.core().Request("pin/update"). + return api.core().Request("pin/update", from.String(), to.String()). Option("unpin", options.Unpin).Exec(ctx, nil) } From d9004a9061070263dcc01976334836acb9493530 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 3 May 2020 12:04:04 -0700 Subject: [PATCH 091/112] IDB58Decode -> Decode Allow Non-RSA keys to be decoded by the HTTP client This commit was moved from ipfs/go-ipfs-http-client@b0549d86231d69a933b0e4b771c350cf1bf5c27b --- client/httpapi/key.go | 12 ++++++------ client/httpapi/pubsub.go | 4 ++-- client/httpapi/swarm.go | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index cf22c61089c8..78b67517c011 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-core/peer" @@ -45,7 +45,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -69,7 +69,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - id.pid, err = peer.IDB58Decode(id.Id) + id.pid, err = peer.Decode(id.Id) return id, out.Overwrite, err } @@ -82,7 +82,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { var err error - k.pid, err = peer.IDB58Decode(k.Id) + k.pid, err = peer.Decode(k.Id) if err != nil { return nil, err } @@ -100,7 +100,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var err error out := keyOutput{JName: "self", Id: id.ID} - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -114,7 +114,7 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { } var err error - out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + out.Keys[0].pid, err = peer.Decode(out.Keys[0].Id) return &out.Keys[0], err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 81ddb5211c3c..da7c59ef1d2f 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,7 +6,7 @@ import ( "encoding/json" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" ) @@ -41,7 +41,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio res := make([]peer.ID, len(out.Strings)) for i, sid := range out.Strings { - id, err := peer.IDB58Decode(sid) + id, err := peer.Decode(sid) if err != nil { return nil, err } diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 10280c39f080..1cb0d91df7c6 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" @@ -91,7 +91,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) direction: conn.Direction, } - out.peer, err = peer.IDB58Decode(conn.Peer) + out.peer, err = peer.Decode(conn.Peer) if err != nil { return nil, err } @@ -131,7 +131,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu addrs[i] = a } - pid, err := peer.IDB58Decode(spid) + pid, err := peer.Decode(spid) if err != nil { return nil, err } From 7ebef96ba12bfdfc550412748ee16eadcaffd25d Mon Sep 17 00:00:00 2001 From: Bryan Stenson Date: Tue, 12 May 2020 20:09:16 +0000 Subject: [PATCH 092/112] update contributing link This commit was moved from ipfs/go-ipfs-http-client@ef2a556e19e8d9ebafcb24b3574724c6d7718bc8 --- client/httpapi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index e5ab1c612273..376e6590e85d 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -25,7 +25,7 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ### Want to hack on IPFS? -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) ## License From 247cb05fbc4fa3863542ecd9e0e5109c927790c6 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 31 Jul 2020 10:20:59 -0700 Subject: [PATCH 093/112] Bump interface-go-ipfs-core version (#117) * fix issues #26 Implement an IsPinned function * fix issues #113 Implement interface v0.3.0 * ci: update to use go-ipfs v0.6.0 * ci: bump go version Co-authored-by: godcong Co-authored-by: Adin Schmahmann This commit was moved from ipfs/go-ipfs-http-client@4e8a6c8b981f808062caa4ef7481babc0a39f298 --- client/httpapi/pin.go | 54 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 6026c71b38ab..bd3e5423bc36 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,9 +3,10 @@ package httpapi import ( "context" "encoding/json" + "strings" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" @@ -24,6 +25,11 @@ type pinRefKeyList struct { type pin struct { path path.Resolved typ string + err error +} + +func (p *pin) Err() error { + return p.err } func (p *pin) Path() path.Resolved { @@ -44,7 +50,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp Option("recursive", options.Recursive).Exec(ctx, nil) } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan iface.Pin, error) { options, err := caopts.PinLsOptions(opts...) if err != nil { return nil, err @@ -57,16 +63,46 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return nil, err } - pins := make([]iface.Pin, 0, len(out.Keys)) - for hash, p := range out.Keys { - c, err := cid.Parse(hash) - if err != nil { - return nil, err + pins := make(chan iface.Pin) + go func(ch chan<- iface.Pin) { + defer close(ch) + for hash, p := range out.Keys { + c, e := cid.Parse(hash) + if e != nil { + ch <- &pin{typ: p.Type, err: e} + return + } + ch <- &pin{typ: p.Type, path: path.IpldPath(c), err: e} } - pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) + }(pins) + return pins, nil +} + +// IsPinned returns whether or not the given cid is pinned +// and an explanation of why its pinned +func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { + options, err := caopts.PinIsPinnedOptions(opts...) + if err != nil { + return "", false, err + } + var out pinRefKeyList + err = api.core().Request("pin/ls"). + Option("type", options.WithType). + Option("arg", p.String()). + Exec(ctx, &out) + if err != nil { + // TODO: This error-type discrimination based on sub-string matching is brittle. + // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 + if strings.Index(err.Error(), "is not pinned") != -1 { + return "", false, nil + } + return "", false, err } - return pins, nil + for _, obj := range out.Keys { + return obj.Type, true, nil + } + return "", false, errors.New("http api returned no error and no results") } func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { From ea2a35686db523b544245cbaea5592021d0cd163 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 15 May 2021 18:21:24 -0700 Subject: [PATCH 094/112] fix staticcheck This commit was moved from ipfs/go-ipfs-http-client@4461a0bb952ce0a05a5bab9728cc64b658375068 --- client/httpapi/api.go | 17 ++++++++--------- client/httpapi/api_test.go | 7 +++---- client/httpapi/object.go | 3 +-- client/httpapi/pin.go | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ec16c4b2da83..f589e4267a47 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "net/http" - gohttp "net/http" "os" "path/filepath" "strings" @@ -14,7 +13,7 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) const ( @@ -34,7 +33,7 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string - httpcli gohttp.Client + httpcli http.Client Headers http.Header applyGlobal func(*requestBuilder) } @@ -85,9 +84,9 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { // NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, }, } @@ -96,7 +95,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { } // NewApiWithClient constructs HttpApi with specified endpoint and custom http client -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { +func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { return nil, err @@ -112,7 +111,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return NewURLApiWithClient(url, c) } -func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, @@ -121,7 +120,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { } // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("unexpected redirect") } return api, nil diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5f012b476721..35a57d2c089a 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,7 +4,6 @@ import ( "context" "io/ioutil" "net/http" - gohttp "net/http" "net/http/httptest" "os" "strconv" @@ -163,9 +162,9 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) return } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, DisableCompression: true, }, diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 280b2490f93d..6ec7f5503cd0 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -10,7 +10,6 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" - dag "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -32,7 +31,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( var n ipld.Node switch options.Type { case "empty": - n = new(dag.ProtoNode) + n = new(merkledag.ProtoNode) case "unixfs-dir": n = ft.EmptyDirNode() default: diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index bd3e5423bc36..13de2d3893de 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -93,7 +93,7 @@ func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.Pin if err != nil { // TODO: This error-type discrimination based on sub-string matching is brittle. // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 - if strings.Index(err.Error(), "is not pinned") != -1 { + if strings.Contains(err.Error(), "is not pinned") { return "", false, nil } return "", false, err From 5322cf89e45197afb74cc36104ee581ca179c1d6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 29 Nov 2021 22:24:27 +0100 Subject: [PATCH 095/112] feat: pubsub http rpc with multibase (#151) * feat: pubsub http rpc with multibase This updates HTTP RPC wire format to one from https://github.com/ipfs/go-ipfs/pull/8183 * chore: use updated go-ipfs * chore: switch ci to go-ipfs master This commit was moved from ipfs/go-ipfs-http-client@c832fc0ce1d8235c830fecd0dd44dda04a9fba37 --- client/httpapi/README.md | 6 ++-- client/httpapi/pubsub.go | 78 +++++++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index 376e6590e85d..d3e14ac3f1be 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -1,8 +1,8 @@ # go-ipfs-http-api -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](https://protocol.ai) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.io/) +[![](https://img.shields.io/badge/matrix-%23ipfs-blue.svg?style=flat-square)](https://app.element.io/#/room/#ipfs:matrix.org) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index da7c59ef1d2f..72f5923761f2 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -9,6 +9,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" + mbase "github.com/multiformats/go-multibase" ) type PubsubAPI HttpApi @@ -21,8 +22,15 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } - - return out.Strings, nil + topics := make([]string, len(out.Strings)) + for n, mb := range out.Strings { + _, topic, err := mbase.Decode(mb) + if err != nil { + return nil, err + } + topics[n] = string(topic) + } + return topics, nil } func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { @@ -35,7 +43,11 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + var optionalTopic string + if len(options.Topic) > 0 { + optionalTopic = toMultibase([]byte(options.Topic)) + } + if err := api.core().Request("pubsub/peers", optionalTopic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +63,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().Request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", toMultibase([]byte(topic))). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -64,13 +76,18 @@ type pubsubSub struct { } type pubsubMessage struct { - JFrom []byte `json:"from,omitempty"` - JData []byte `json:"data,omitempty"` - JSeqno []byte `json:"seqno,omitempty"` + JFrom string `json:"from,omitempty"` + JData string `json:"data,omitempty"` + JSeqno string `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` - from peer.ID - err error + // real values after unpacking from text/multibase envelopes + from peer.ID + data []byte + seqno []byte + topics []string + + err error } func (msg *pubsubMessage) From() peer.ID { @@ -78,15 +95,17 @@ func (msg *pubsubMessage) From() peer.ID { } func (msg *pubsubMessage) Data() []byte { - return msg.JData + return msg.data } func (msg *pubsubMessage) Seq() []byte { - return msg.JSeqno + return msg.seqno } +// TODO: do we want to keep this interface as []string, +// or change to more correct [][]byte? func (msg *pubsubMessage) Topics() []string { - return msg.JTopicIDs + return msg.topics } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { @@ -98,22 +117,41 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if msg.err != nil { return nil, msg.err } + // unpack values from text/multibase envelopes var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err + msg.from, err = peer.Decode(msg.JFrom) + if err != nil { + return nil, err + } + _, msg.data, err = mbase.Decode(msg.JData) + if err != nil { + return nil, err + } + _, msg.seqno, err = mbase.Decode(msg.JSeqno) + if err != nil { + return nil, err + } + for _, mbt := range msg.JTopicIDs { + _, topic, err := mbase.Decode(mbt) + if err != nil { + return nil, err + } + msg.topics = append(msg.topics, string(topic)) + } + return &msg, nil case <-ctx.Done(): return nil, ctx.Err() } } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + /* right now we have no options (discover got deprecated) options, err := caopts.PubSubSubscribeOptions(opts...) if err != nil { return nil, err } - - resp, err := api.core().Request("pubsub/sub", topic). - Option("discover", options.Discover).Send(ctx) + */ + resp, err := api.core().Request("pubsub/sub", toMultibase([]byte(topic))).Send(ctx) if err != nil { return nil, err @@ -168,3 +206,9 @@ func (s *pubsubSub) Close() error { func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } + +// Encodes bytes into URL-safe multibase that can be sent over HTTP RPC (URL or body) +func toMultibase(data []byte) string { + mb, _ := mbase.Encode(mbase.Base64url, data) + return mb +} From b52c80ca4da5730f47674946025c9490fcd50e1f Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 14 Jan 2022 16:16:26 +0100 Subject: [PATCH 096/112] skip TestHttpApi on Windows This commit was moved from ipfs/go-ipfs-http-client@a41c7956c0da2a4c51e15794b22dea09395cc7ef --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 35a57d2c089a..d5f27f0ec9d8 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "os" + "runtime" "strconv" "strings" "sync" @@ -207,6 +208,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } func TestHttpApi(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping due to #142") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() From b85aa9639f861d659b49e0396b409398462226a2 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 27 Mar 2022 14:11:47 +0200 Subject: [PATCH 097/112] fix: make Block().* return correct ABI based ipld.ErrNotFound errors This commit was moved from ipfs/go-ipfs-http-client@4f5f8e9b144d1f38e0fc37c5080845611dcbcf1e --- client/httpapi/abyfy_errors.go | 104 ++++++++++++++++++++++++++++ client/httpapi/abyfy_errors_test.go | 60 ++++++++++++++++ client/httpapi/block.go | 11 +-- 3 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 client/httpapi/abyfy_errors.go create mode 100644 client/httpapi/abyfy_errors_test.go diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/abyfy_errors.go new file mode 100644 index 000000000000..e4f4afb7cb3d --- /dev/null +++ b/client/httpapi/abyfy_errors.go @@ -0,0 +1,104 @@ +package httpapi + +import ( + "errors" + "strings" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +type prePostWrappedNotFoundError struct { + pre string + post string + + wrapped ipld.ErrNotFound +} + +func (e prePostWrappedNotFoundError) String() string { + return e.Error() +} + +func (e prePostWrappedNotFoundError) Error() string { + return e.pre + e.wrapped.Error() + e.post +} + +func (e prePostWrappedNotFoundError) Unwrap() error { + return e.wrapped +} + +func abyfyIpldNotFoundFallbackToMSG(msg string) error { + err, handled := abyfyIpldNotFound(msg) + if handled { + return err + } + + return errors.New(msg) +} + +func abyfyIpldNotFoundFallbackToError(msg error) error { + err, handled := abyfyIpldNotFound(msg.Error()) + if handled { + return err + } + + return msg +} + +// This file handle parsing and returning the correct ABI based errors from error messages +//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type +func abyfyIpldNotFound(msg string) (error, bool) { + if msg == "" { + return nil, true // Fast path + } + + // The patern we search for is: + // node not found (fallback) + // or + // CID not found (here we parse the CID) + notFoundIndex := strings.LastIndex(msg, " not found") + + if notFoundIndex == -1 { + // Unknown, ot found not found + return nil, false + } + + preNotFound := msg[:notFoundIndex] + + var c cid.Cid + var preIndex int + if strings.HasSuffix(preNotFound, "node") { + // Fallback case + c = cid.Undef + preIndex = notFoundIndex - len("node") + } else { + // Assume that CIDs does not include whitespace to pull out the CID + preIndex = strings.LastIndexByte(preNotFound, ' ') + // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space + preIndex++ + var err error + c, err = cid.Decode(preNotFound[preIndex:]) + if err != nil { + // Unknown + return nil, false + } + } + + postIndex := notFoundIndex + len(" not found") + + err := ipld.ErrNotFound{Cid: c} + + pre := msg[:preIndex] + post := msg[postIndex:] + + if len(pre) > 0 || len(post) > 0 { + // We have some text to wrap arround the ErrNotFound one + return prePostWrappedNotFoundError{ + pre: pre, + post: post, + wrapped: err, + }, true + } + + return err, true +} diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/abyfy_errors_test.go new file mode 100644 index 000000000000..fe1cf0707bf7 --- /dev/null +++ b/client/httpapi/abyfy_errors_test.go @@ -0,0 +1,60 @@ +package httpapi + +import ( + "errors" + "fmt" + "testing" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} + +func doAbyfyIpldNotFoundTest(t *testing.T, original error) { + originalMsg := original.Error() + + rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + + rebuiltMsg := rebuilt.Error() + + if originalMsg != rebuiltMsg { + t.Errorf("expected message to be %q; got %q", originalMsg, rebuiltMsg) + } + + originalNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(original) + if originalNotFound != rebuiltNotFound { + t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + } +} + +func TestAbyfyIpldNotFound(t *testing.T) { + if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { + t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + for _, err := range [...]error{ + errors.New("file not found"), + errors.New(" not found"), + errors.New("Bad_CID not found"), + errors.New("network connection timeout"), + ipld.ErrNotFound{Cid: cid.Undef}, + ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, + ipld.ErrNotFound{Cid: cid.NewCidV1(cid.Raw, randomSha256MH)}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doAbyfyIpldNotFoundTest(t, err) + } + } +} diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 640f186f5044..e78ba2281759 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -3,7 +3,6 @@ package httpapi import ( "bytes" "context" - "errors" "fmt" "io" @@ -67,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, resp.Error + return nil, abyfyIpldNotFoundFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -99,18 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - if removedBlock.Error != "" { - return errors.New(removedBlock.Error) - } - - return nil + return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, err + return nil, abyfyIpldNotFoundFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { From 5a8e2b77c0e4361a0dd6e57a7f4129adba2f4d8a Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 30 Mar 2022 04:32:12 +0200 Subject: [PATCH 098/112] chore: rename abyfyIpldErrNotFound to parseIPLDErrNotFound This commit was moved from ipfs/go-ipfs-http-client@7aa002992970058e37bb82f9837a442227562d33 --- client/httpapi/block.go | 6 +++--- client/httpapi/{abyfy_errors.go => errors.go} | 10 +++++----- .../httpapi/{abyfy_errors_test.go => errors_test.go} | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) rename client/httpapi/{abyfy_errors.go => errors.go} (88%) rename client/httpapi/{abyfy_errors_test.go => errors_test.go} (84%) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index e78ba2281759..0ee838e8326a 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, abyfyIpldNotFoundFallbackToError(resp.Error) + return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) + return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, abyfyIpldNotFoundFallbackToError(err) + return nil, parseIPLDNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/errors.go similarity index 88% rename from client/httpapi/abyfy_errors.go rename to client/httpapi/errors.go index e4f4afb7cb3d..a527e545232d 100644 --- a/client/httpapi/abyfy_errors.go +++ b/client/httpapi/errors.go @@ -27,8 +27,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func abyfyIpldNotFoundFallbackToMSG(msg string) error { - err, handled := abyfyIpldNotFound(msg) +func parseIPLDNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseIPLDNotFound(msg) if handled { return err } @@ -36,8 +36,8 @@ func abyfyIpldNotFoundFallbackToMSG(msg string) error { return errors.New(msg) } -func abyfyIpldNotFoundFallbackToError(msg error) error { - err, handled := abyfyIpldNotFound(msg.Error()) +func parseIPLDNotFoundWithFallbackToError(msg error) error { + err, handled := parseIPLDNotFound(msg.Error()) if handled { return err } @@ -47,7 +47,7 @@ func abyfyIpldNotFoundFallbackToError(msg error) error { // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func abyfyIpldNotFound(msg string) (error, bool) { +func parseIPLDNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/errors_test.go similarity index 84% rename from client/httpapi/abyfy_errors_test.go rename to client/httpapi/errors_test.go index fe1cf0707bf7..08def204dc8c 100644 --- a/client/httpapi/abyfy_errors_test.go +++ b/client/httpapi/errors_test.go @@ -12,10 +12,10 @@ import ( var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} -func doAbyfyIpldNotFoundTest(t *testing.T, original error) { +func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -30,8 +30,8 @@ func doAbyfyIpldNotFoundTest(t *testing.T, original error) { } } -func TestAbyfyIpldNotFound(t *testing.T) { - if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { +func TestParseIPLDNotFound(t *testing.T) { + if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -54,7 +54,7 @@ func TestAbyfyIpldNotFound(t *testing.T) { err = fmt.Errorf(wrap, err) } - doAbyfyIpldNotFoundTest(t, err) + doParseIpldNotFoundTest(t, err) } } } From a2a60768ea7e5d5dad1ad1f0dcf8811cca24d3eb Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:36:11 +0200 Subject: [PATCH 099/112] fix: parseIPLDErrNotFound test This commit was moved from ipfs/go-ipfs-http-client@2e09c4b3ab580f71c652c82926ba7b1580a2b495 --- client/httpapi/errors_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 08def204dc8c..502c10966dfe 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -24,9 +24,9 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } originalNotFound := ipld.IsNotFound(original) - rebuiltNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(rebuilt) if originalNotFound != rebuiltNotFound { - t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + t.Errorf("for %q expected Ipld.IsNotFound to be %t; got %t", originalMsg, originalNotFound, rebuiltNotFound) } } From ddd36645b25c97651deb81c092c7669efabd959e Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:43:27 +0200 Subject: [PATCH 100/112] feat: update the error parsing for go-ipld-format to v0.4.0 This commit was moved from ipfs/go-ipfs-http-client@296534fd1647a1ec9cae08314daf41ec96d55763 --- client/httpapi/errors.go | 54 +++++++++++++++++++++-------------- client/httpapi/errors_test.go | 14 +++++++-- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index a527e545232d..f42d9257523c 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -45,6 +45,18 @@ func parseIPLDNotFoundWithFallbackToError(msg error) error { return msg } +// Use a string to move it into RODATA +// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) +const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + +func notAsciiLetterOrDigits(r rune) bool { + if r > 'z' { + return true + } + + return notAsciiLetterOrDigitsLUT[r] > 0 +} + // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseIPLDNotFound(msg string) (error, bool) { @@ -53,46 +65,46 @@ func parseIPLDNotFound(msg string) (error, bool) { } // The patern we search for is: - // node not found (fallback) - // or - // CID not found (here we parse the CID) - notFoundIndex := strings.LastIndex(msg, " not found") + const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ + // We try to parse the CID, if it's invalid we give up and return a simple text error. + // We also accept "node" in place of the CID because that means it's an Undefined CID. - if notFoundIndex == -1 { - // Unknown, ot found not found + keyIndex := strings.Index(msg, ipldErrNotFoundKey) + + if keyIndex < 0 { // Unknown error return nil, false } - preNotFound := msg[:notFoundIndex] + cidStart := keyIndex + len(ipldErrNotFoundKey) + msgPostKey := msg[cidStart:] var c cid.Cid - var preIndex int - if strings.HasSuffix(preNotFound, "node") { + var postIndex int + if strings.HasPrefix(msgPostKey, "node") { // Fallback case c = cid.Undef - preIndex = notFoundIndex - len("node") + postIndex = len("node") } else { - // Assume that CIDs does not include whitespace to pull out the CID - preIndex = strings.LastIndexByte(preNotFound, ' ') - // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space - preIndex++ + // Assume that CIDs only contain a-zA-Z0-9 characters. + // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. + postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + if postIndex < 0 { + postIndex = len(msgPostKey) + } + var err error - c, err = cid.Decode(preNotFound[preIndex:]) + c, err = cid.Decode(msgPostKey[:postIndex]) if err != nil { // Unknown return nil, false } } - postIndex := notFoundIndex + len(" not found") - err := ipld.ErrNotFound{Cid: c} - - pre := msg[:preIndex] - post := msg[postIndex:] + pre := msg[:keyIndex] + post := msgPostKey[postIndex:] if len(pre) > 0 || len(post) > 0 { - // We have some text to wrap arround the ErrNotFound one return prePostWrappedNotFoundError{ pre: pre, post: post, diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 502c10966dfe..1b7de8798268 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,6 +3,7 @@ package httpapi import ( "errors" "fmt" + "strings" "testing" "github.com/ipfs/go-cid" @@ -42,9 +43,8 @@ func TestParseIPLDNotFound(t *testing.T) { "%w is wrong", } { for _, err := range [...]error{ - errors.New("file not found"), - errors.New(" not found"), - errors.New("Bad_CID not found"), + errors.New("ipld: could not find "), + errors.New("ipld: could not find Bad_CID"), errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, @@ -58,3 +58,11 @@ func TestParseIPLDNotFound(t *testing.T) { } } } + +func TestNotAsciiLetterOrDigits(t *testing.T) { + for i := rune(0); i <= 256; i++ { + if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { + t.Errorf("%q is incorrectly identified", i) + } + } +} From 775bcb7f09de0d13a64b013ccaefa253e44c3403 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Fri, 1 Apr 2022 00:23:55 +0200 Subject: [PATCH 101/112] feat: add blockstore: block not found matching too This commit was moved from ipfs/go-ipfs-http-client@a3354f062c97bcb6412e74ecfb24e84208edc479 --- client/httpapi/block.go | 6 ++-- client/httpapi/errors.go | 56 +++++++++++++++++++++++++++++++---- client/httpapi/errors_test.go | 25 ++++++++++++++-- 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 0ee838e8326a..c074f79407e2 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) + return nil, parseErrNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) + return parseErrNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, parseIPLDNotFoundWithFallbackToError(err) + return nil, parseErrNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f42d9257523c..b2a6f86de274 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -8,6 +8,8 @@ import ( ipld "github.com/ipfs/go-ipld-format" ) +// This file handle parsing and returning the correct ABI based errors from error messages + type prePostWrappedNotFoundError struct { pre string post string @@ -27,8 +29,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func parseIPLDNotFoundWithFallbackToMSG(msg string) error { - err, handled := parseIPLDNotFound(msg) +func parseErrNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseErrNotFound(msg) if handled { return err } @@ -36,8 +38,8 @@ func parseIPLDNotFoundWithFallbackToMSG(msg string) error { return errors.New(msg) } -func parseIPLDNotFoundWithFallbackToError(msg error) error { - err, handled := parseIPLDNotFound(msg.Error()) +func parseErrNotFoundWithFallbackToError(msg error) error { + err, handled := parseErrNotFound(msg.Error()) if handled { return err } @@ -57,13 +59,25 @@ func notAsciiLetterOrDigits(r rune) bool { return notAsciiLetterOrDigitsLUT[r] > 0 } -// This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func parseIPLDNotFound(msg string) (error, bool) { +func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } + if err, handled := parseIPLDErrNotFound(msg); handled { + return err, true + } + + if err, handled := parseBlockstoreNotFound(msg); handled { + return err, true + } + + return nil, false +} + +//lint:ignore ST1008 using error as values +func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ // We try to parse the CID, if it's invalid we give up and return a simple text error. @@ -114,3 +128,33 @@ func parseIPLDNotFound(msg string) (error, bool) { return err, true } + +// This is a simple error type that just return msg as Error(). +// But that also match ipld.ErrNotFound when called with Is(err). +// That is needed to keep compatiblity with code that use string.Contains(err.Error(), "blockstore: block not found") +// and code using ipld.ErrNotFound +type blockstoreNotFoundMatchingIPLDErrNotFound struct { + msg string +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) String() string { + return e.Error() +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Error() string { + return e.msg +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { + _, ok := err.(ipld.ErrNotFound) + return ok +} + +//lint:ignore ST1008 using error as values +func parseBlockstoreNotFound(msg string) (error, bool) { + if !strings.Contains(msg, "blockstore: block not found") { + return nil, false + } + + return blockstoreNotFoundMatchingIPLDErrNotFound{msg: msg}, true +} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 1b7de8798268..09437bfbfcce 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -16,7 +16,7 @@ var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1 func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) + rebuilt := parseErrNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -32,7 +32,7 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } func TestParseIPLDNotFound(t *testing.T) { - if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { + if err := parseErrNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -59,6 +59,27 @@ func TestParseIPLDNotFound(t *testing.T) { } } +func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { + if !ipld.IsNotFound(blockstoreNotFoundMatchingIPLDErrNotFound{}) { + t.Fatalf("expected blockstoreNotFoundMatchingIPLDErrNotFound to match ipld.IsNotFound; got false") + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doParseIpldNotFoundTest(t, err) + } +} + func TestNotAsciiLetterOrDigits(t *testing.T) { for i := rune(0); i <= 256; i++ { if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { From 54f6e90870c61a686f57b0bd0f1ad4923615897b Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 03:25:39 +0200 Subject: [PATCH 102/112] fix: change CID breaking logic when parsing ipld.ErrNotFound This commit was moved from ipfs/go-ipfs-http-client@a3b49352bfd1b885567018092a06f303de18b7aa --- client/httpapi/errors.go | 24 +++++++++--------------- client/httpapi/errors_test.go | 18 +++++++----------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index b2a6f86de274..f1a57a049600 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -47,18 +47,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -// Use a string to move it into RODATA -// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) -const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - -func notAsciiLetterOrDigits(r rune) bool { - if r > 'z' { - return true - } - - return notAsciiLetterOrDigitsLUT[r] > 0 -} - //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { @@ -76,6 +64,12 @@ func parseErrNotFound(msg string) (error, bool) { return nil, false } +// Assume CIDs break on: +// - Whitespaces: " \t\n\r\v\f" +// - Semicolon: ";" this is to parse ipld.ErrNotFound wrapped in multierr +// - Double Quotes: "\"" this is for parsing %q and %#v formating +const cidBreakSet = " \t\n\r\v\f;\"" + //lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: @@ -99,9 +93,9 @@ func parseIPLDErrNotFound(msg string) (error, bool) { c = cid.Undef postIndex = len("node") } else { - // Assume that CIDs only contain a-zA-Z0-9 characters. - // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. - postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + postIndex = strings.IndexFunc(msgPostKey, func(r rune) bool { + return strings.ContainsAny(string(r), cidBreakSet) + }) if postIndex < 0 { postIndex = len(msgPostKey) } diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 09437bfbfcce..86f91cdf5f7b 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "strings" "testing" "github.com/ipfs/go-cid" @@ -36,12 +35,17 @@ func TestParseIPLDNotFound(t *testing.T) { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } - for _, wrap := range [...]string{ + cidBreaks := make([]string, len(cidBreakSet)) + for i, v := range cidBreakSet { + cidBreaks[i] = "%w" + string(v) + } + + for _, wrap := range append(cidBreaks, "", "merkledag: %w", "testing: %w the test", "%w is wrong", - } { + ) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), @@ -79,11 +83,3 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { doParseIpldNotFoundTest(t, err) } } - -func TestNotAsciiLetterOrDigits(t *testing.T) { - for i := rune(0); i <= 256; i++ { - if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { - t.Errorf("%q is incorrectly identified", i) - } - } -} From f2f2109bc1e8f5b9c636597eb7478e0a4ddca427 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 04:24:12 +0200 Subject: [PATCH 103/112] test: add a false case test to blockstore parsing This commit was moved from ipfs/go-ipfs-http-client@75f597aa16c512ec02f549836e998f1fc29a3846 --- client/httpapi/errors_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 86f91cdf5f7b..7709fc9c53a2 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -74,12 +74,15 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { "testing: %w the test", "%w is wrong", } { - var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + for _, err := range [...]error{ + errors.New("network connection timeout"), + blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } - if wrap != "" { - err = fmt.Errorf(wrap, err) + doParseIpldNotFoundTest(t, err) } - - doParseIpldNotFoundTest(t, err) } } From 60a548b621ff229a648a268c4d06f6e206ad6099 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 5 Apr 2022 20:11:06 +0200 Subject: [PATCH 104/112] correctness: only match CIDs matching go-cid.Cid.String output This commit was moved from ipfs/go-ipfs-http-client@34cc489461985cf4139819e589fdd09bdc034c24 --- client/httpapi/errors.go | 19 +++++++++++++++++-- client/httpapi/errors_test.go | 7 +++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f1a57a049600..1ccf6182c7b4 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -3,9 +3,11 @@ package httpapi import ( "errors" "strings" + "unicode/utf8" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" ) // This file handle parsing and returning the correct ABI based errors from error messages @@ -97,15 +99,28 @@ func parseIPLDErrNotFound(msg string) (error, bool) { return strings.ContainsAny(string(r), cidBreakSet) }) if postIndex < 0 { + // no breakage meaning the string look like this something + "ipld: could not find bafy" postIndex = len(msgPostKey) } + cidStr := msgPostKey[:postIndex] + var err error - c, err = cid.Decode(msgPostKey[:postIndex]) + c, err = cid.Decode(cidStr) if err != nil { - // Unknown + // failed to decode CID give up return nil, false } + + // check that the CID is either a CIDv0 or a base32 multibase + // because that what ipld.ErrNotFound.Error() -> cid.Cid.String() do currently + if c.Version() != 0 { + baseRune, _ := utf8.DecodeRuneInString(cidStr) + if baseRune == utf8.RuneError || baseRune != mbase.Base32 { + // not a multibase we expect, give up + return nil, false + } + } } err := ipld.ErrNotFound{Cid: c} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 7709fc9c53a2..c8b98d08eb24 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" mh "github.com/multiformats/go-multihash" ) @@ -40,6 +41,11 @@ func TestParseIPLDNotFound(t *testing.T) { cidBreaks[i] = "%w" + string(v) } + base58BTCEncoder, err := mbase.NewEncoder(mbase.Base58BTC) + if err != nil { + t.Fatalf("expected to find Base58BTC encoder; got error %q", err.Error()) + } + for _, wrap := range append(cidBreaks, "", "merkledag: %w", @@ -49,6 +55,7 @@ func TestParseIPLDNotFound(t *testing.T) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), + errors.New("ipld: could not find " + cid.NewCidV1(cid.Raw, randomSha256MH).Encode(base58BTCEncoder)), // Test that we only accept CIDv0 and base32 CIDs errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, From fd209019f27c7a16f94bfc41ebc7b815daee9903 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 21 Apr 2022 22:18:19 +0200 Subject: [PATCH 105/112] fix: interop with 'block put' from go-ipfs 0.13 (#158) * chore: interop with go-ipfs 0.13 Applies necessary changes to ensure 'block/put' works and is backward-compatible. Context: https://github.com/ipfs/go-ipfs/pull/8568 * chore: 0.3.1 bumping as patch because we bumped to 0.3.0 recently, as part of other (unreleased) go-ipfs 0.13 work This commit was moved from ipfs/go-ipfs-http-client@ecf364c9898d78034e3232574aae70d1bf85357c --- client/httpapi/block.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index c074f79407e2..7b4cf0edab56 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -7,9 +7,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) @@ -31,20 +32,33 @@ func (s *blockStat) Path() path.Resolved { } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { - options, _, err := caopts.BlockPutOptions(opts...) + options, err := caopts.BlockPutOptions(opts...) + px := options.CidPrefix if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] + mht, ok := mh.Codes[px.MhType] if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + return nil, fmt.Errorf("unknowm mhType %d", px.MhType) + } + + var cidOptKey, cidOptVal string + switch { + case px.Version == 0 && px.Codec == cid.DagProtobuf: + // ensure legacy --format=v0 passes as BlockPutOption still works + cidOptKey = "format" + cidOptVal = "v0" + default: + // pass codec as string + cidOptKey = "cid-codec" + cidOptVal = mc.Code(px.Codec).String() } req := api.core().Request("block/put"). Option("mhtype", mht). - Option("mhlen", options.MhLength). - Option("format", options.Codec). + Option("mhlen", px.MhLength). + Option(cidOptKey, cidOptVal). Option("pin", options.Pin). FileBody(r) From 1cdb9adf898a8a2b9d2cd0d3fe8ba6467a822166 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 23 Jun 2022 22:39:01 +0200 Subject: [PATCH 106/112] fix: interop with go-ipfs 0.13 (#160) This ensures cid-codec introduced in https://github.com/ipfs/go-ipfs/pull/8568 gets correctly passed (+ we maintain backward-compatibility with CIDv0) This commit was moved from ipfs/go-ipfs-http-client@9c9f43fd9ca36c7ebdc664bb6936a6fe9881255b --- client/httpapi/dag.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 15d9a9c62ed2..879c1e499327 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,12 @@ import ( "fmt" "io/ioutil" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + multicodec "github.com/multiformats/go-multicodec" ) type httpNodeAdder HttpApi @@ -56,13 +57,21 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() - format := cid.CodecToStr[prefix.Codec] + + // preserve 'cid-codec' when sent over HTTP + cidCodec := multicodec.Code(prefix.Codec).String() + + // 'format' got replaced by 'cid-codec' in https://github.com/ipfs/interface-go-ipfs-core/pull/80 + // but we still support it here for backward-compatibility with use of CIDv0 + format := "" if prefix.Version == 0 { + cidCodec = "" format = "v0" } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.CidCodec(cidCodec), options.Block.Format(format), options.Block.Pin(pin)) if err != nil { From 0fff1d5d8e4cb2debaff1ee4e85b1266907101af Mon Sep 17 00:00:00 2001 From: web3-bot Date: Fri, 23 Sep 2022 07:43:33 +0000 Subject: [PATCH 107/112] stop using the deprecated io/ioutil package This commit was moved from ipfs/go-ipfs-http-client@026ba730a1fe9a4a25a22449bec8d69c2262ccfc --- client/httpapi/api.go | 3 +-- client/httpapi/api_test.go | 3 +-- client/httpapi/apifile.go | 3 +-- client/httpapi/dag.go | 4 ++-- client/httpapi/name.go | 4 ++-- client/httpapi/object.go | 5 ++--- client/httpapi/requestbuilder.go | 5 ++--- client/httpapi/response.go | 9 ++++----- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f589e4267a47..97440a72436d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "io/ioutil" "net/http" "os" "path/filepath" @@ -74,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { apiFile := filepath.Join(baseDir, DefaultApiFile) - api, err := ioutil.ReadFile(apiFile) + api, err := os.ReadFile(apiFile) if err != nil { return nil, err } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index d5f27f0ec9d8..5960ea2c0270 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,7 +2,6 @@ package httpapi import ( "context" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -92,7 +91,7 @@ func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n i func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { - dir, err := ioutil.TempDir("", "httpapi-tb-") + dir, err := os.MkdirTemp("", "httpapi-tb-") if err != nil { return nil, err } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec2f7588d591..c4884b924f08 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" @@ -113,7 +112,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) + r, err := io.CopyN(io.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 879c1e499327..f32c67c42efa 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "io" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -24,7 +24,7 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) return nil, err } - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 47227a2ab893..b6a7836033ab 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/options/namesys" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6ec7f5503cd0..894369223fc1 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -5,13 +5,12 @@ import ( "context" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -71,7 +70,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { if err != nil { return nil, err } - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 7012a8935bfa..039bca036410 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,11 +5,10 @@ import ( "context" "fmt" "io" - "io/ioutil" "strconv" "strings" - "github.com/ipfs/go-ipfs-files" + files "github.com/ipfs/go-ipfs-files" ) type RequestBuilder interface { @@ -59,7 +58,7 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder { // FileBody sets the request body to the given reader wrapped into multipartreader. func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { - pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 95cbf13ec1bf..8a491ab73169 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime" "net/http" "net/url" @@ -45,7 +44,7 @@ func (r *Response) Close() error { if r.Output != nil { // drain output (response body) - _, err1 := io.Copy(ioutil.Discard, r.Output) + _, err1 := io.Copy(io.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -117,7 +116,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" case contentType == "text/plain": - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } @@ -140,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // This is a server-side bug (probably). e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) } @@ -150,7 +149,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() } From 1193b2978f7859ed8e1df4be50e980ff1bde7015 Mon Sep 17 00:00:00 2001 From: galargh Date: Sat, 1 Oct 2022 17:37:23 +0200 Subject: [PATCH 108/112] chore: remove unused linter directives This commit was moved from ipfs/go-ipfs-http-client@c4fc1a7740bfaad2abcba2563d569422e525324c --- client/httpapi/errors.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index 1ccf6182c7b4..59e4ad705432 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -49,7 +49,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path @@ -72,7 +71,6 @@ func parseErrNotFound(msg string) (error, bool) { // - Double Quotes: "\"" this is for parsing %q and %#v formating const cidBreakSet = " \t\n\r\v\f;\"" -//lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ @@ -159,7 +157,6 @@ func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { return ok } -//lint:ignore ST1008 using error as values func parseBlockstoreNotFound(msg string) (error, bool) { if !strings.Contains(msg, "blockstore: block not found") { return nil, false From 75b6f45f765237d611168a5e1c09915f380e7bd7 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 6 Feb 2023 15:32:57 +0100 Subject: [PATCH 109/112] chore: bumps for Kubo 0.18 and Routing API with stub for Put This commit was moved from ipfs/go-ipfs-http-client@c076c3cb71d08fb9448ae9bf6916dab137322722 --- client/httpapi/api.go | 4 +++ client/httpapi/apifile.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/dht.go | 4 +-- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/routing.go | 55 ++++++++++++++++++++++++++++++++ client/httpapi/swarm.go | 6 ++-- client/httpapi/unixfs.go | 2 +- 11 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 client/httpapi/routing.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 97440a72436d..10df94603f75 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -195,3 +195,7 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { func (api *HttpApi) PubSub() iface.PubSubAPI { return (*PubsubAPI)(api) } + +func (api *HttpApi) Routing() iface.RoutingAPI { + return (*RoutingAPI)(api) +} diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index c4884b924f08..80fd13cd4fe9 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -7,7 +7,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index f32c67c42efa..2fbfea1ff29c 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 0a78514f9739..ebecfcaeb3ae 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,8 +6,8 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" ) type DhtAPI HttpApi diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 78b67517c011..b785bbf453dc 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 72f5923761f2..ec45d1a51bef 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 039bca036410..bc6f4c8a94f2 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 8a491ab73169..99932ca0da7e 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -12,7 +12,7 @@ import ( cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/routing.go b/client/httpapi/routing.go new file mode 100644 index 000000000000..18f5332766fb --- /dev/null +++ b/client/httpapi/routing.go @@ -0,0 +1,55 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + + "github.com/libp2p/go-libp2p/core/routing" +) + +type RoutingAPI HttpApi + +func (api *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) { + resp, err := api.core().Request("routing/get", key).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Close() + + var out routing.QueryEvent + + dec := json.NewDecoder(resp.Output) + if err := dec.Decode(&out); err != nil { + return nil, err + } + + res, err := base64.StdEncoding.DecodeString(out.Extra) + if err != nil { + return nil, err + } + + return res, nil +} + +func (api *RoutingAPI) Put(ctx context.Context, key string, value []byte) error { + resp, err := api.core().Request("routing/put", key). + FileBody(bytes.NewReader(value)). + Send(ctx) + + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + return nil +} + +func (api *RoutingAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 1cb0d91df7c6..c5dfbe564523 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,9 +5,9 @@ import ( "time" iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5e27fb0367f1..38f23d5b9b4c 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,7 +8,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" From 5c21cf0ce95d7c2710d5bb5dd58e2871c0fbc7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= <92101826+PedrobyJoao@users.noreply.github.com> Date: Tue, 28 Mar 2023 07:38:24 -0300 Subject: [PATCH 110/112] docs: adding example of connection and pinning to README (#173) Co-authored-by: JPexplorer This commit was moved from ipfs/go-ipfs-http-client@b0de2b028e6ef03d4dba3ed105641ad299d4f7be --- client/httpapi/README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index d3e14ac3f1be..da25f24feb00 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -17,6 +17,41 @@ greatest features, please use _this_ package. https://godoc.org/github.com/ipfs/go-ipfs-http-api +### Example + +Pin file on your local IPFS node based on its CID: + +```go +package main + +import ( + "context" + "fmt" + + ipfsClient "github.com/ipfs/go-ipfs-http-client" + path "github.com/ipfs/interface-go-ipfs-core/path" +) + +func main() { + // "Connect" to local node + node, err := ipfsClient.NewLocalApi() + if err != nil { + fmt.Printf(err) + return + } + // Pin a given file by its CID + ctx := context.Background() + cid := "bafkreidtuosuw37f5xmn65b3ksdiikajy7pwjjslzj2lxxz2vc4wdy3zku" + p := path.New(cid) + err = node.Pin().Add(ctx, p) + if err != nil { + fmt.Printf(err) + return + } + return +} +``` + ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! From 8bebfc00b7192de63b79082bd2666d8d1bf76126 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 16 Apr 2023 12:31:09 +0200 Subject: [PATCH 111/112] chore: migrate from go-libipfs to boxo This commit was moved from ipfs/go-ipfs-http-client@ae996cbe5a91708d245100267c45094ab52c42ad --- client/httpapi/api.go | 4 ++-- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 6 +++--- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 4 ++-- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 8 ++++---- client/httpapi/object.go | 10 +++++----- client/httpapi/path.go | 4 ++-- client/httpapi/pin.go | 6 +++--- client/httpapi/pubsub.go | 4 ++-- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/swarm.go | 2 +- client/httpapi/unixfs.go | 12 ++++++------ 16 files changed, 44 insertions(+), 45 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 10df94603f75..b7cfb2b520d4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strings" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5960ea2c0270..2832d722d374 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -12,10 +12,9 @@ import ( "testing" "time" - iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/ipfs/interface-go-ipfs-core/path" - - "github.com/ipfs/interface-go-ipfs-core/tests" + iface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/coreiface/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" testbedi "github.com/ipfs/iptb/testbed/interfaces" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 80fd13cd4fe9..25fd7c3b3718 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 7b4cf0edab56..2a794c26f529 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 2fbfea1ff29c..795e1d78d9e3 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,11 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" - "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" ) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index ebecfcaeb3ae..a2910fef6d37 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index b785bbf453dc..434e98fe5214 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,9 @@ import ( "context" "errors" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b6a7836033ab..f82f69f3ac42 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + nsopts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/coreiface/path" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 894369223fc1..4e3b9ef6be18 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,13 +6,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipld/merkledag" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - ft "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 619dd0e0d2cb..d69d425ab664 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,10 +3,10 @@ package httpapi import ( "context" + "github.com/ipfs/boxo/coreiface/path" + ipfspath "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core/path" ) func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 13de2d3893de..30a3d7b7a902 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -5,10 +5,10 @@ import ( "encoding/json" "strings" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index ec45d1a51bef..28f1ef8e609c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,8 +6,8 @@ import ( "encoding/json" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index bc6f4c8a94f2..476aed7866d1 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/boxo/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 99932ca0da7e..189b43671614 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -10,9 +10,9 @@ import ( "net/url" "os" + "github.com/ipfs/boxo/files" cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index c5dfbe564523..9b073078da0a 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - iface "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/boxo/coreiface" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 38f23d5b9b4c..b9c34c59ffcf 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -7,13 +7,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" + unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - unixfs_pb "github.com/ipfs/go-unixfs/pb" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) From 7d8262307cb40c1f21fd79fe52cdfd36a7dcb5cd Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 15 May 2023 17:18:51 +0100 Subject: [PATCH 112/112] fix: use https URI when multiaddress specifies tls (#177) Currently any clients created through `NewApiWithClient` will make a HTTP request to the api, even if the multiaddress specifies TLS or (the deprecated multiaddr option) https. This commit addresses this by having NewApiWithClient iterate the available protocols for the multiaddress, specifying the URL proto as https if it finds TLS or HTTPS is specified. The default continues to be http for those multiaddresses that do not specify these options. Should resolve #176 This commit was moved from ipfs/go-ipfs-http-client@7e1de1f7ccfa8cd1f616e94664886fe04d70806e --- client/httpapi/api.go | 16 +++++++++++++++- client/httpapi/api_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b7cfb2b520d4..5584de85d998 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -107,7 +107,21 @@ func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { } } - return NewURLApiWithClient(url, c) + proto := "http://" + + // By default, DialArgs is going to provide details suitable for connecting + // a socket to, but not really suitable for making an informed choice of http + // protocol. For multiaddresses specifying tls and/or https we want to make + // a https request instead of a http request. + protocols := a.Protocols() + for _, p := range protocols { + if p.Code == ma.P_HTTPS || p.Code == ma.P_TLS { + proto = "https://" + break + } + } + + return NewURLApiWithClient(proto+url, c) } func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2832d722d374..a6d684b0d075 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -246,3 +246,31 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } } + +func Test_NewURLApiWithClient_HTTP_Variant(t *testing.T) { + testcases := []struct { + address string + expected string + }{ + {address: "/ip4/127.0.0.1/tcp/80", expected: "http://127.0.0.1:80"}, + {address: "/ip4/127.0.0.1/tcp/443/tls", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/https", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/tls/http", expected: "https://127.0.0.1:443"}, + } + + for _, tc := range testcases { + address, err := ma.NewMultiaddr(tc.address) + if err != nil { + t.Fatal(err) + } + + api, err := NewApiWithClient(address, &http.Client{}) + if err != nil { + t.Fatal(err) + } + + if api.url != tc.expected { + t.Errorf("Expected = %s; got %s", tc.expected, api.url) + } + } +}