From 202081523fdc8bc54b16e5e4447b221586febef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:12:10 +0100 Subject: [PATCH 001/112] Setup repo This commit was moved from ipfs/go-ipfs-http-client@4e7edce41d91825f0ea21dec426de2e128db5b38 --- client/httpapi/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 client/httpapi/README.md diff --git a/client/httpapi/README.md b/client/httpapi/README.md new file mode 100644 index 00000000000..ab37a4cf8aa --- /dev/null +++ b/client/httpapi/README.md @@ -0,0 +1,27 @@ +# go-ipfs-http-api + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) +[![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) + +> IPFS CoreAPI implementation using HTTP API + +## Documentation + +https://godoc.org/github.com/ipfs/go-ipfs-http-api + +## Contribute + +Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +### Want to hack on IPFS? + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) + +## License + +MIT From 6d85aff407d3fb3eb6d2549fbac94615fd147b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 13:13:15 +0100 Subject: [PATCH 002/112] Initial structure, path stuff This commit was moved from ipfs/go-ipfs-http-client@93943f7f5671948b0d2aee19c30cf689307faf13 --- client/httpapi/api.go | 127 +++++++++++++++++++++++++++++ client/httpapi/name.go | 35 ++++++++ client/httpapi/path.go | 48 +++++++++++ client/httpapi/request.go | 34 ++++++++ client/httpapi/requestbuilder.go | 100 +++++++++++++++++++++++ client/httpapi/response.go | 132 +++++++++++++++++++++++++++++++ 6 files changed, 476 insertions(+) create mode 100644 client/httpapi/api.go create mode 100644 client/httpapi/name.go create mode 100644 client/httpapi/path.go create mode 100644 client/httpapi/request.go create mode 100644 client/httpapi/requestbuilder.go create mode 100644 client/httpapi/response.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go new file mode 100644 index 00000000000..82e030350f7 --- /dev/null +++ b/client/httpapi/api.go @@ -0,0 +1,127 @@ +package httpapi + +import ( + "github.com/pkg/errors" + "io/ioutil" + gohttp "net/http" + "os" + "path" + "strings" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + homedir "github.com/mitchellh/go-homedir" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" + ) + +const ( + DefaultPathName = ".ipfs" + DefaultPathRoot = "~/" + DefaultPathName + DefaultApiFile = "api" + EnvDir = "IPFS_PATH" +) + +var ErrNotImplemented = errors.New("not implemented") + +type HttpApi struct { + url string + httpcli *gohttp.Client +} + +func NewLocalApi() iface.CoreAPI { + baseDir := os.Getenv(EnvDir) + if baseDir == "" { + baseDir = DefaultPathRoot + } + + baseDir, err := homedir.Expand(baseDir) + if err != nil { + return nil + } + + apiFile := path.Join(baseDir, DefaultApiFile) + + if _, err := os.Stat(apiFile); err != nil { + return nil + } + + api, err := ioutil.ReadFile(apiFile) + if err != nil { + return nil + } + + return NewApi(strings.TrimSpace(string(api))) +} + +func NewApi(url string) *HttpApi { + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + } + + return NewApiWithClient(url, c) +} + +func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { + if a, err := ma.NewMultiaddr(url); err == nil { + _, host, err := manet.DialArgs(a) + if err == nil { + url = host + } + } + + return &HttpApi{ + url: url, + httpcli: c, + } +} + +func (api *HttpApi) request(command string, args ...string) *RequestBuilder { + return &RequestBuilder{ + command: command, + args: args, + shell: api, + } +} + +func (api *HttpApi) Unixfs() iface.UnixfsAPI { + return nil +} + +func (api *HttpApi) Block() iface.BlockAPI { + return nil +} + +func (api *HttpApi) Dag() iface.DagAPI { + return nil +} + +func (api *HttpApi) Name() iface.NameAPI { + return (*NameAPI)(api) +} + +func (api *HttpApi) Key() iface.KeyAPI { + return nil +} + +func (api *HttpApi) Pin() iface.PinAPI { + return nil +} + +func (api *HttpApi) Object() iface.ObjectAPI { + return nil +} + +func (api *HttpApi) Dht() iface.DhtAPI { + return nil +} + +func (api *HttpApi) Swarm() iface.SwarmAPI { + return nil +} + +func (api *HttpApi) PubSub() iface.PubSubAPI { + return nil +} diff --git a/client/httpapi/name.go b/client/httpapi/name.go new file mode 100644 index 00000000000..41426ef57c9 --- /dev/null +++ b/client/httpapi/name.go @@ -0,0 +1,35 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type NameAPI HttpApi + +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { + return nil, ErrNotImplemented +} + +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { + // TODO: options! + + req := api.core().request("name/resolve") + req.Arguments(name) + + var out struct{ Path string } + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + + return iface.ParsePath(out.Path) +} + +func (api *NameAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/path.go b/client/httpapi/path.go new file mode 100644 index 00000000000..28656fbd4c0 --- /dev/null +++ b/client/httpapi/path.go @@ -0,0 +1,48 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { + var out struct { + Cid cid.Cid + RemPath string + } + + //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help + + var err error + if path.Namespace() == "ipns" { + if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + return nil, err + } + } + + if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + // TODO: + ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + if err != nil { + return nil, err + } + + root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + if err != nil { + return nil, err + } + + return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil +} + +func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { + return nil, ErrNotImplemented +} diff --git a/client/httpapi/request.go b/client/httpapi/request.go new file mode 100644 index 00000000000..58c61ac6780 --- /dev/null +++ b/client/httpapi/request.go @@ -0,0 +1,34 @@ +package httpapi + +import ( + "context" + "io" + "strings" +) + +type Request struct { + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string +} + +func NewRequest(ctx context.Context, url, command string, args ...string) *Request { + if !strings.HasPrefix(url, "http") { + url = "http://" + url + } + + opts := map[string]string{ + "encoding": "json", + "stream-channels": "true", + } + return &Request{ + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), + } +} diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go new file mode 100644 index 00000000000..9ccc8cf9720 --- /dev/null +++ b/client/httpapi/requestbuilder.go @@ -0,0 +1,100 @@ +package httpapi + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "strings" +) + +// RequestBuilder is an IPFS commands request builder. +type RequestBuilder struct { + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader + + shell *HttpApi +} + +// Arguments adds the arguments to the args. +func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { + r.args = append(r.args, args...) + return r +} + +// BodyString sets the request body to the given string. +func (r *RequestBuilder) BodyString(body string) *RequestBuilder { + return r.Body(strings.NewReader(body)) +} + +// BodyBytes sets the request body to the given buffer. +func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { + return r.Body(bytes.NewReader(body)) +} + +// Body sets the request body to the given reader. +func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { + r.body = body + return r +} + +// Option sets the given option. +func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { + var s string + switch v := value.(type) { + case bool: + s = strconv.FormatBool(v) + case string: + s = v + case []byte: + s = string(v) + default: + // slow case. + s = fmt.Sprint(value) + } + if r.opts == nil { + r.opts = make(map[string]string, 1) + } + r.opts[key] = s + return r +} + +// Header sets the given header. +func (r *RequestBuilder) Header(name, value string) *RequestBuilder { + if r.headers == nil { + r.headers = make(map[string]string, 1) + } + r.headers[name] = value + return r +} + +// Send sends the request and return the response. +func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + req := NewRequest(ctx, r.shell.url, r.command, r.args...) + req.Opts = r.opts + req.Headers = r.headers + req.Body = r.body + return req.Send(r.shell.httpcli) +} + +// Exec sends the request a request and decodes the response. +func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { + httpRes, err := r.Send(ctx) + if err != nil { + return err + } + + if res == nil { + httpRes.Close() + if httpRes.Error != nil { + return httpRes.Error + } + return nil + } + + return httpRes.Decode(res) +} diff --git a/client/httpapi/response.go b/client/httpapi/response.go new file mode 100644 index 00000000000..27709769b1f --- /dev/null +++ b/client/httpapi/response.go @@ -0,0 +1,132 @@ +package httpapi + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + files "github.com/ipfs/go-ipfs-files" +) + +type Response struct { + Output io.ReadCloser + Error *Error +} + +func (r *Response) Close() error { + if r.Output != nil { + // always drain output (response body) + ioutil.ReadAll(r.Output) + return r.Output.Close() + } + return nil +} + +func (r *Response) Decode(dec interface{}) error { + defer r.Close() + if r.Error != nil { + return r.Error + } + + return json.NewDecoder(r.Output).Decode(dec) +} + +type Error struct { + Command string + Message string + Code int +} + +func (e *Error) Error() string { + var out string + if e.Command != "" { + out = e.Command + ": " + } + if e.Code != 0 { + out = fmt.Sprintf("%s%d: ", out, e.Code) + } + return out + e.Message +} + +func (r *Request) Send(c *http.Client) (*Response, error) { + url := r.getURL() + req, err := http.NewRequest("POST", url, r.Body) + if err != nil { + return nil, err + } + + // Add any headers that were supplied via the RequestBuilder. + for k, v := range r.Headers { + req.Header.Add(k, v) + } + + if fr, ok := r.Body.(*files.MultiFileReader); ok { + req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) + req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + } + + resp, err := c.Do(req) + if err != nil { + return nil, err + } + + contentType := resp.Header.Get("Content-Type") + parts := strings.Split(contentType, ";") + contentType = parts[0] + + nresp := new(Response) + + nresp.Output = resp.Body + if resp.StatusCode >= http.StatusBadRequest { + e := &Error{ + Command: r.Command, + } + switch { + case resp.StatusCode == http.StatusNotFound: + e.Message = "command not found" + case contentType == "text/plain": + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = string(out) + case contentType == "application/json": + if err = json.NewDecoder(resp.Body).Decode(e); err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) + } + default: + fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) + out, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) + } + e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out) + } + nresp.Error = e + nresp.Output = nil + + // drain body and close + ioutil.ReadAll(resp.Body) + resp.Body.Close() + } + + return nresp, nil +} + +func (r *Request) getURL() string { + + values := make(url.Values) + for _, arg := range r.Args { + values.Add("arg", arg) + } + for k, v := range r.Opts { + values.Add(k, v) + } + + return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode()) +} From 35c271e3d88725a9ab25be40c1837435de110a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:40:02 +0100 Subject: [PATCH 003/112] wip notice in readme This commit was moved from ipfs/go-ipfs-http-client@e823507553ca82115e303e1ba26f76b83f5bd12c --- client/httpapi/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index ab37a4cf8aa..dd09142e32b 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,6 +8,8 @@ > IPFS CoreAPI implementation using HTTP API +This project is WIP, use https://github.com/ipfs/go-ipfs-api for now + ## Documentation https://godoc.org/github.com/ipfs/go-ipfs-http-api From 6c927fd9624807e124f0376445c41daaa975fbd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Nov 2018 14:38:11 +0100 Subject: [PATCH 004/112] Partial ipld node impl This commit was moved from ipfs/go-ipfs-http-client@df916c7849d6cfd7f479c4fe80c0f75d6705a0fc --- client/httpapi/api.go | 6 +-- client/httpapi/block.go | 40 ++++++++++++++ client/httpapi/ipldnode.go | 107 +++++++++++++++++++++++++++++++++++++ client/httpapi/path.go | 2 +- 4 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/block.go create mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 82e030350f7..cd3fb9fd021 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,7 @@ package httpapi import ( - "github.com/pkg/errors" + "errors" "io/ioutil" gohttp "net/http" "os" @@ -12,7 +12,7 @@ import ( homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" - ) +) const ( DefaultPathName = ".ipfs" @@ -91,7 +91,7 @@ func (api *HttpApi) Unixfs() iface.UnixfsAPI { } func (api *HttpApi) Block() iface.BlockAPI { - return nil + return (*BlockAPI)(api) } func (api *HttpApi) Dag() iface.DagAPI { diff --git a/client/httpapi/block.go b/client/httpapi/block.go new file mode 100644 index 00000000000..8bdb4c502eb --- /dev/null +++ b/client/httpapi/block.go @@ -0,0 +1,40 @@ +package httpapi + +import ( + "context" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type BlockAPI HttpApi + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + if err != nil { + return nil, err + } + + //TODO: is close on the reader enough? + //defer resp.Close() + + //TODO: make blockApi return ReadCloser + return resp.Output, resp.Error +} + +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { + return ErrNotImplemented +} + +func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { + return nil, ErrNotImplemented +} + +func (api *BlockAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go new file mode 100644 index 00000000000..a3dd6204b2b --- /dev/null +++ b/client/httpapi/ipldnode.go @@ -0,0 +1,107 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" + "io/ioutil" + "strconv" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" + ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" +) + +type ipldNode struct { + ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? + path iface.ResolvedPath + api *HttpApi +} + +func (n *ipldNode) RawData() []byte { + r, err := n.api.Block().Get(n.ctx, n.path) + if err != nil { + panic(err) // TODO: eww, should we add errors too / better ideas? + } + + b, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + + return b +} + +func (n *ipldNode) Cid() cid.Cid { + return n.path.Cid() +} + +func (n *ipldNode) String() string { + return fmt.Sprintf("[Block %s]", n.Cid()) +} + +func (n *ipldNode) Loggable() map[string]interface{} { + return nil //TODO: we can't really do better here, can we? +} + +// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. +// We can also give people a choice +func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { + p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) + + var out interface{} + n.api.request("dag/get", p).Exec(n.ctx, &out) + + // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack + for len(path) > 0 { + switch o := out.(type) { + case map[string]interface{}: + v, ok := o[path[0]] + if !ok { + // TODO: ipld links + return nil, nil, errors.New("no element under this path") + } + out = v + case []interface{}: + n, err := strconv.ParseUint(path[0], 10, 32) + if err != nil { + return nil, nil, err + } + if len(o) < int(n) { + return nil, nil, errors.New("no element under this path") + } + out = o[n] + } + path = path[1:] + } + + return out, path, nil +} + +func (n *ipldNode) Tree(path string, depth int) []string { + panic("implement me") +} + +func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { + panic("implement me") +} + +func (n *ipldNode) Copy() ipld.Node { + panic("implement me") +} + +func (n *ipldNode) Links() []*ipld.Link { + panic("implement me") +} + +func (n *ipldNode) Stat() (*ipld.NodeStat, error) { + panic("implement me") +} + +func (n *ipldNode) Size() (uint64, error) { + panic("implement me") +} + +var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 28656fbd4c0..6b6e4b027bc 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -30,7 +30,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } // TODO: - ipath, err := ipfspath.FromSegments("/" +path.Namespace() + "/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } From 1cd2ec05b7524e921f58750a4b9c8f48ad1a2e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 20 Dec 2018 16:04:19 +0100 Subject: [PATCH 005/112] Init gx This commit was moved from ipfs/go-ipfs-http-client@e06cddbedd360b1982e414571dd08ab25df38546 --- client/httpapi/ipldnode.go | 8 ++++---- client/httpapi/name.go | 1 + client/httpapi/path.go | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index a3dd6204b2b..b8e6fba0180 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -2,16 +2,16 @@ package httpapi import ( "context" + "errors" "fmt" - "github.com/pkg/errors" - "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" "io/ioutil" "strconv" "github.com/ipfs/go-ipfs/core/coreapi/interface" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) type ipldNode struct { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 41426ef57c9..7315ac2c391 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 6b6e4b027bc..5701326fc56 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -5,9 +5,9 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" - ipfspath "gx/ipfs/QmRG3XuGwT7GYuAqgWDJBKTzdaHMwAnc1x7J2KHEXNHxzG/go-path" - ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + ipfspath "github.com/ipfs/go-path" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { From 16e97bf5e8f80b681c5a8d4b96f406b4b83a8e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 21 Dec 2018 18:44:34 +0100 Subject: [PATCH 006/112] It builds This commit was moved from ipfs/go-ipfs-http-client@a23d794e5fceadb9115d8d0c8466c10e9c5d0891 --- client/httpapi/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cd3fb9fd021..dc9acb9f606 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -78,6 +79,10 @@ func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { } } +func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { + return nil, ErrNotImplemented +} + func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, From fe4c9fd8033c94c5ea707b09e76479a6d018635d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 30 Dec 2018 04:24:09 +0100 Subject: [PATCH 007/112] Skeleton for tests This commit was moved from ipfs/go-ipfs-http-client@dfbe0026ad9438d4e795bbb43debeffb6bb9ca1a --- client/httpapi/api.go | 24 ++++++++--- client/httpapi/api_test.go | 84 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 client/httpapi/api_test.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dc9acb9f606..63ea3aad7d3 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -35,7 +35,11 @@ func NewLocalApi() iface.CoreAPI { baseDir = DefaultPathRoot } - baseDir, err := homedir.Expand(baseDir) + return NewPathApi(baseDir) +} + +func NewPathApi(p string) iface.CoreAPI { + baseDir, err := homedir.Expand(p) if err != nil { return nil } @@ -51,10 +55,15 @@ func NewLocalApi() iface.CoreAPI { return nil } - return NewApi(strings.TrimSpace(string(api))) + maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) + if err != nil { + return nil + } + + return NewApi(maddr) } -func NewApi(url string) *HttpApi { +func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -62,10 +71,15 @@ func NewApi(url string) *HttpApi { }, } - return NewApiWithClient(url, c) + return NewApiWithClient(a, c) } -func NewApiWithClient(url string, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { + _, url, err := manet.DialArgs(a) + if err != nil { + return nil // TODO: return that error + } + if a, err := ma.NewMultiaddr(url); err == nil { _, host, err := manet.DialArgs(a) if err == nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go new file mode 100644 index 00000000000..b925b5c348f --- /dev/null +++ b/client/httpapi/api_test.go @@ -0,0 +1,84 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/iptb/testbed/interfaces" + "io/ioutil" + "os" + "path" + "strconv" + "testing" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" + + local "github.com/ipfs/iptb-plugins/local" + "github.com/ipfs/iptb/cli" + "github.com/ipfs/iptb/testbed" +) + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ + From: "", + NewNode: local.NewNode, + GetAttrList: local.GetAttrList, + GetAttrDesc: local.GetAttrDesc, + PluginName: local.PluginName, + BuiltIn: true, + }, false) + if err != nil { + return nil, err + } + + dir, err := ioutil.TempDir("", "httpapi-tb-") + if err != nil { + return nil, err + } + + c := cli.NewCli() + if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + + defer os.Remove(dir) + + defer func() { + _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + }() + }() + + apis := make([]iface.CoreAPI, n) + + for i := range apis { + tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) + + node, err := tb.Node(i) + if err != nil { + return nil, err + } + + attrNode, ok := node.(testbedi.Attribute) + if !ok { + return nil, fmt.Errorf("node does not implement attributes") + } + + pth, err := attrNode.Attr("path") + if err != nil { + return nil, err + } + + apis[i] = NewPathApi(pth) + } + + return apis, nil +} + +func TestHttpApi(t *testing.T) { + tests.TestApi(&NodeProvider{})(t) +} From e8da6e2cf9a8fd40c5fd960a60ce1b1b27ada713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 7 Jan 2019 23:44:43 +0100 Subject: [PATCH 008/112] Block API This commit was moved from ipfs/go-ipfs-http-client@d0c98b870e55bed78fa777793e520f08157deb4f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 11 +++- client/httpapi/block.go | 109 ++++++++++++++++++++++++++++--- client/httpapi/requestbuilder.go | 12 ++++ 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 63ea3aad7d3..1768e847749 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return (*NameAPI)(api) + return nil } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b925b5c348f..e32c456e5a3 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/iptb/testbed/interfaces" "io/ioutil" "os" "path" @@ -16,6 +15,7 @@ import ( local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" + "github.com/ipfs/iptb/testbed/interfaces" ) type NodeProvider struct{} @@ -39,7 +39,14 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := cli.NewCli() - if err := c.Run([]string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10), "--start"}); err != nil { + + initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} + if err := c.Run(initArgs); err != nil { + return nil, err + } + + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 8bdb4c502eb..185aa0a42be 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -1,17 +1,69 @@ package httpapi import ( + "bytes" "context" + "errors" + "fmt" "io" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + mh "github.com/multiformats/go-multihash" ) type BlockAPI HttpApi -func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (iface.BlockStat, error) { - return nil, ErrNotImplemented +type blockStat struct { + Key string + BSize int `json:"Size"` +} + +func (s *blockStat) Size() int { + return s.BSize +} + +func (s *blockStat) valid() (iface.ResolvedPath, error) { + c, err := cid.Parse(s.Key) + if err != nil { + return nil, err + } + + return iface.IpldPath(c), nil +} + +func (s *blockStat) Path() iface.ResolvedPath { + p, _ := s.valid() + return p +} + +func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { + options, _, err := caopts.BlockPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("block/put"). + Option("mhtype", mht). + Option("mhlen", options.MhLength). + Option("format", options.Codec). + FileBody(r) + + var out blockStat + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { @@ -19,20 +71,57 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } - //TODO: is close on the reader enough? - //defer resp.Close() + //TODO: make get return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } - //TODO: make blockApi return ReadCloser - return resp.Output, resp.Error + return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...options.BlockRmOption) error { - return ErrNotImplemented +func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { + options, err := caopts.BlockRmOptions(opts...) + if err != nil { + return err + } + + removedBlock := struct { + Hash string `json:",omitempty"` + Error string `json:",omitempty"` + }{} + + req := api.core().request("block/rm"). + Option("force", options.Force). + Arguments(p.String()) + + if err := req.Exec(ctx, &removedBlock); err != nil { + return err + } + + if removedBlock.Error != "" { + return errors.New(removedBlock.Error) + } + + return nil } func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { - return nil, ErrNotImplemented + var out blockStat + err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil } func (api *BlockAPI) core() *HttpApi { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 9ccc8cf9720..6e5a89ebdb1 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,8 +5,11 @@ import ( "context" "fmt" "io" + "io/ioutil" "strconv" "strings" + + "github.com/ipfs/go-ipfs-files" ) // RequestBuilder is an IPFS commands request builder. @@ -42,6 +45,15 @@ func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { return r } +// FileBody sets the request body to the given reader wrapped into multipartreader. +func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { + pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + d := files.NewMapDirectory(map[string]files.Node{"": pr}) + r.body = files.NewMultiFileReader(d, false) + + return r +} + // Option sets the given option. func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { var s string From ab89e0abf99a0bc42ee2d8fb55a9b3de79573635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 00:30:22 +0100 Subject: [PATCH 009/112] Partial Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@44696b84f59f6f707858787c9a0750f224596635 --- client/httpapi/api.go | 4 +- client/httpapi/unixfs.go | 95 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/unixfs.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 1768e847749..099b4512311 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -106,7 +106,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { } func (api *HttpApi) Unixfs() iface.UnixfsAPI { - return nil + return (*UnixfsAPI)(api) } func (api *HttpApi) Block() iface.BlockAPI { @@ -118,7 +118,7 @@ func (api *HttpApi) Dag() iface.DagAPI { } func (api *HttpApi) Name() iface.NameAPI { - return nil + return (*NameAPI)(api) } func (api *HttpApi) Key() iface.KeyAPI { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go new file mode 100644 index 00000000000..567f60284d4 --- /dev/null +++ b/client/httpapi/unixfs.go @@ -0,0 +1,95 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type addEvent struct { + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + +type UnixfsAPI HttpApi + +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { + options, _, err := caopts.UnixfsAddOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + req := api.core().request("add"). + Option("hash", mht). + Option("chunker", options.Chunker). + Option("cid-version", options.CidVersion). + //Option("", options.Events). + Option("fscache", options.FsCache). + Option("hidden", options.Hidden). + Option("inline", options.Inline). + Option("inline-limit", options.InlineLimit). + Option("nocopy", options.NoCopy). + Option("only-hash", options.OnlyHash). + Option("pin", options.Pin). + //Option("", options.Progress). + Option("silent", options.Silent). + Option("stdin-name", options.StdinName). + Option("wrap-with-directory", options.Wrap). + Option("quieter", true) // TODO: rm after event impl + + if options.RawLeavesSet { + req.Option("raw-leaves", options.RawLeaves) + } + + switch options.Layout { + case caopts.BalancedLayout: + // noop, default + case caopts.TrickleLayout: + req.Option("trickle", true) + } + + switch c := f.(type) { + case files.Directory: + req.Body(files.NewMultiFileReader(c, false)) + case files.File: + req.Body(c) + } + + var out addEvent + if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil +} + +func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { + panic("implement me") +} + +func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *UnixfsAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cf74d391603457ad626b7f6b27ff0655bc946ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:07:21 +0100 Subject: [PATCH 010/112] Partial Key API, ApiAddr funcion This commit was moved from ipfs/go-ipfs-http-client@0ffdef159261e19cfd76edabb394179aa9295003 --- client/httpapi/api.go | 13 ++++++- client/httpapi/key.go | 86 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/key.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 099b4512311..d104d98dffb 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,6 +29,7 @@ type HttpApi struct { httpcli *gohttp.Client } +//TODO: Return errors here func NewLocalApi() iface.CoreAPI { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -39,6 +40,14 @@ func NewLocalApi() iface.CoreAPI { } func NewPathApi(p string) iface.CoreAPI { + a := ApiAddr(p) + if a == nil { + return nil + } + return NewApi(a) +} + +func ApiAddr(p string) ma.Multiaddr { baseDir, err := homedir.Expand(p) if err != nil { return nil @@ -60,7 +69,7 @@ func NewPathApi(p string) iface.CoreAPI { return nil } - return NewApi(maddr) + return maddr } func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? @@ -122,7 +131,7 @@ func (api *HttpApi) Name() iface.NameAPI { } func (api *HttpApi) Key() iface.KeyAPI { - return nil + return (*KeyAPI)(api) } func (api *HttpApi) Pin() iface.PinAPI { diff --git a/client/httpapi/key.go b/client/httpapi/key.go new file mode 100644 index 00000000000..87b573f98a5 --- /dev/null +++ b/client/httpapi/key.go @@ -0,0 +1,86 @@ +package httpapi + +import ( + "context" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type KeyAPI HttpApi + +type keyOutput struct { + JName string `json:"Name"` + Id string +} + +func (k *keyOutput) Name() string { + return k.JName +} + +func (k *keyOutput) Path() iface.Path { + p, _ := iface.ParsePath("/ipns/" + k.Id) + return p +} + +func (k *keyOutput) ID() peer.ID { + p, _ := peer.IDB58Decode(k.Id) + return p +} + +func (k *keyOutput) valid() error { + _, err := peer.IDB58Decode(k.Id) + return err +} + + +func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { + options, err := caopts.KeyGenerateOptions(opts...) + if err != nil { + return nil, err + } + + var out keyOutput + err = api.core().request("key/gen", name). + Option("type", options.Algorithm). + Option("size", options.Size). + Exec(ctx, &out) + if err != nil { + return nil, err + } + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { + panic("implement me") +} + +func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { + var id struct{ID string} + if err := api.core().request("id").Exec(ctx, &id); err != nil { + return nil, err + } + + out := keyOutput{JName: "self", Id: id.ID} + if err := out.valid(); err != nil { + return nil, err + } + return &out, nil +} + +func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { + panic("implement me") +} + +func (api *KeyAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c236393733bf3491c45002d67189ba78303332a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:08:18 +0100 Subject: [PATCH 011/112] Connect test swarms, don't compress api calls This commit was moved from ipfs/go-ipfs-http-client@c6472d9b8286c932492db4d6bf03f890bf97759c --- client/httpapi/api_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index e32c456e5a3..02c7830bb65 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io/ioutil" + gohttp "net/http" "os" "path" "strconv" @@ -50,6 +51,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + if n > 1 { + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + if err := c.Run(connectArgs); err != nil { + return nil, err + } + } + go func() { <-ctx.Done() @@ -80,7 +88,18 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - apis[i] = NewPathApi(pth) + a := ApiAddr(pth) + if a == nil { + return nil, fmt.Errorf("nil addr for node") + } + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(a, c) } return apis, nil From 7861315f49398e20e7e1189671e42d0a16adbef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:00 +0100 Subject: [PATCH 012/112] Wrap single files in Unixfs.Add This commit was moved from ipfs/go-ipfs-http-client@16f77b24a1b150e7ec624d5675ded478ac6cb853 --- client/httpapi/unixfs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 567f60284d4..21f75f85ae9 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -66,7 +66,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix case files.Directory: req.Body(files.NewMultiFileReader(c, false)) case files.File: - req.Body(c) + d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) } var out addEvent From f638bae3a960a74d284503c30086f0bbffbe73c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:09:17 +0100 Subject: [PATCH 013/112] implement .Name This commit was moved from ipfs/go-ipfs-http-client@e19e5f54e48d0909796add6d57e3e74e84eba85e --- client/httpapi/name.go | 70 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 7315ac2c391..fbc440b9666 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,26 +2,80 @@ package httpapi import ( "context" + "fmt" + "github.com/ipfs/go-ipfs/namesys/opts" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) type NameAPI HttpApi -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...options.NamePublishOption) (iface.IpnsEntry, error) { - return nil, ErrNotImplemented +type ipnsEntry struct { + JName string `json:"Name"` + JValue string `json:"Value"` +} + +func (e *ipnsEntry) valid() (iface.Path, error) { + return iface.ParsePath(e.JValue) +} + +func (e *ipnsEntry) Name() string { + return e.JName +} + +func (e *ipnsEntry) Value() iface.Path { + p, _ := e.valid() + return p } -func (api *NameAPI) Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan iface.IpnsResult, error) { +func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { + options, err := caopts.NamePublishOptions(opts...) + if err != nil { + return nil, err + } + + req := api.core().request("name/publish", p.String()). + Option("key", options.Key). + Option("allow-offline", options.AllowOffline). + Option("lifetime", options.ValidTime.String()). + Option("resolve", false) + + if options.TTL != nil { + req.Option("ttl", options.TTL.String()) + } + + var out ipnsEntry + if err := req.Exec(ctx, &out); err != nil { + return nil, err + } + if _, err := out.valid(); err != nil { + return nil, err + } + + return &out, nil +} + +func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { return nil, ErrNotImplemented } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (iface.Path, error) { - // TODO: options! +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } + + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } - req := api.core().request("name/resolve") - req.Arguments(name) + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout.String()) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { From 1acf4163902ae3b9e0e182761deafee5e67415ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 02:53:24 +0100 Subject: [PATCH 014/112] Implement .Unixfs.Ls() This commit was moved from ipfs/go-ipfs-http-client@eb1944fae32f4ccc43969ecd632681c2f2cbfa00 --- client/httpapi/response.go | 14 ++++++++++++- client/httpapi/unixfs.go | 43 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 27709769b1f..5749ca29eea 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -33,7 +33,19 @@ func (r *Response) Decode(dec interface{}) error { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + n := 0 + var err error + for { + err = json.NewDecoder(r.Output).Decode(dec) + if err != nil { + break + } + n++ + } + if n > 0 && err == io.EOF { + err = nil + } + return err } type Error struct { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 21f75f85ae9..75e565a4a91 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -11,6 +12,7 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { @@ -87,8 +89,45 @@ func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { panic("implement me") } -func (api *UnixfsAPI) Ls(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +type lsLink struct { + Name, Hash string + Size uint64 + Type unixfspb.Data_DataType +} + +type lsObject struct { + Hash string + Links []lsLink +} + +type lsOutput struct { + Objects []lsObject +} + +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out lsOutput + err := api.core().request("ls", p.String()).Exec(ctx, &out) + if err != nil { + return nil, err + } + + if len(out.Objects) != 1 { + return nil, errors.New("unexpected objects len") + } + + links := make([]*format.Link, len(out.Objects[0].Links)) + for i, l := range out.Objects[0].Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + links[i] = &format.Link{ + Name: l.Name, + Size: l.Size, + Cid: c, + } + } + return links, nil } func (api *UnixfsAPI) core() *HttpApi { From a6636aac599a1a8a4802ccadb885909f0ab88629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:18:21 +0100 Subject: [PATCH 015/112] Imprement partian Pin API This commit was moved from ipfs/go-ipfs-http-client@dbf90eac67d95722e88056ca16fa769a32dcd48f --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 16 +++++++- client/httpapi/pin.go | 78 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pin.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d104d98dffb..d5741da089d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -135,7 +135,7 @@ func (api *HttpApi) Key() iface.KeyAPI { } func (api *HttpApi) Pin() iface.PinAPI { - return nil + return (*PinAPI)(api) } func (api *HttpApi) Object() iface.ObjectAPI { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 02c7830bb65..5fdf1835241 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -39,7 +40,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - c := cli.NewCli() + c := cli.NewCli() //TODO: is there a better way? initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} if err := c.Run(initArgs); err != nil { @@ -100,6 +101,19 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) }, } apis[i] = NewApiWithClient(a, c) + + // node cleanup + // TODO: pass --empty-repo somehow (how?) + pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + if err != nil { + return nil, err + } + for _, pin := range pins { //TODO: parallel + if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { + return nil, err + } + } + } return apis, nil diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go new file mode 100644 index 00000000000..02dcc422247 --- /dev/null +++ b/client/httpapi/pin.go @@ -0,0 +1,78 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" +) + +type PinAPI HttpApi + +type pinRefKeyObject struct { + Type string +} + +type pinRefKeyList struct { + Keys map[string]pinRefKeyObject +} + +type pin struct { + path iface.ResolvedPath + typ string +} + +func (p *pin) Path() iface.ResolvedPath { + return p.path +} + +func (p *pin) Type() string { + return p.typ +} + + +func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { + panic("implement me") +} + +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { + options, err := caopts.PinLsOptions(opts...) + if err != nil { + return nil, err + } + + var out pinRefKeyList + err = api.core().request("pin/ls"). + Option("type", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + pins := make([]iface.Pin, 0, len(out.Keys)) + for hash, p := range out.Keys { + c, err := cid.Parse(hash) + if err != nil { + return nil, err + } + pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + } + + return pins, nil +} + +func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { + return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +} + +func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { + panic("implement me") +} + +func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { + panic("implement me") +} + +func (api *PinAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 0f7c83956b2d769c801c270fe01d4819cb070881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 03:20:04 +0100 Subject: [PATCH 016/112] Import missing unixfs dep This commit was moved from ipfs/go-ipfs-http-client@6169321d1df6495cbd1c06cca9f2e0777a00f638 --- client/httpapi/unixfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 75e565a4a91..5a1495c24b4 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -11,8 +11,8 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" - unixfspb "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/pb" ) type addEvent struct { From af197cb6d90da1b51525e68f97a30ebfc322e134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 14:46:52 +0100 Subject: [PATCH 017/112] api.WithOption This commit was moved from ipfs/go-ipfs-http-client@634b00bf1a42dc7f2a2dc75f85c77e5bb5ef727c --- client/httpapi/api.go | 21 ++++++++++++++++++--- client/httpapi/api_test.go | 4 ++-- client/httpapi/requestbuilder.go | 2 ++ client/httpapi/response.go | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index d5741da089d..3df45f38c24 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -27,6 +27,8 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string httpcli *gohttp.Client + + applyGlobal func(*RequestBuilder) } //TODO: Return errors here @@ -99,11 +101,24 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return &HttpApi{ url: url, httpcli: c, + applyGlobal: func(*RequestBuilder) {}, } } -func (api *HttpApi) WithOptions(...options.ApiOption) (iface.CoreAPI, error) { - return nil, ErrNotImplemented +func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { + options, err := caopts.ApiOptions(opts...) + if err != nil { + return nil, err + } + + subApi := *api + subApi.applyGlobal = func(req *RequestBuilder) { + if options.Offline { + req.Option("offline", options.Offline) + } + } + + return &subApi, nil } func (api *HttpApi) request(command string, args ...string) *RequestBuilder { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5fdf1835241..67a2dbdce53 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" local "github.com/ipfs/iptb-plugins/local" @@ -104,7 +104,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) // node cleanup // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, options.Pin.Type.Recursive()) + pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 6e5a89ebdb1..831e6d71c0b 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -86,6 +86,8 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { + r.shell.applyGlobal(r) + req := NewRequest(ctx, r.shell.url, r.command, r.args...) req.Opts = r.opts req.Headers = r.headers diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 5749ca29eea..f6e7f3ab709 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -21,7 +21,7 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { // always drain output (response body) - ioutil.ReadAll(r.Output) + //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data return r.Output.Close() } return nil From 6d9dea62825012678ab0f117ec3cfe28c96fdaf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 8 Jan 2019 17:47:22 +0100 Subject: [PATCH 018/112] Implement Unixfs.Get() This commit was moved from ipfs/go-ipfs-http-client@b31bee083d1fc5002779d755b6a624b517d2b10c --- client/httpapi/apifile.go | 231 ++++++++++++++++++++++++++++++++++++++ client/httpapi/unixfs.go | 4 - 2 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 client/httpapi/apifile.go diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go new file mode 100644 index 00000000000..541189b609e --- /dev/null +++ b/client/httpapi/apifile.go @@ -0,0 +1,231 @@ +package httpapi + +import ( + "context" + "encoding/json" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + "github.com/ipfs/go-ipfs-files" + unixfspb "github.com/ipfs/go-unixfs/pb" +) + +func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { + if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS + var err error + p, err = api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + } + + var stat struct{ + Hash string + Type string + Size int64 // unixfs size + } + err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + if err != nil { + return nil, err + } + + switch stat.Type { + case "file": + return api.getFile(ctx, p, stat.Size) + case "directory": + return api.getDir(ctx, p, stat.Size) + default: + return nil, fmt.Errorf("unsupported file type '%s'", stat.Type) + } +} + +type apiFile struct { + ctx context.Context + core *HttpApi + size int64 + path iface.Path + + r io.ReadCloser + at int64 +} + +func (f *apiFile) reset() error { + if f.r != nil { + f.r.Close() + } + req := f.core.request("cat", f.path.String()) + if f.at != 0 { + req.Option("offset", f.at) + } + resp, err := req.Send(f.ctx) + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + f.r = resp.Output + return nil +} + +func (f *apiFile) Read(p []byte) (int, error) { + n, err := f.r.Read(p) + if n > 0 { + f.at += int64(n) + } + return n, err +} + +func (f *apiFile) Seek(offset int64, whence int) (int64, error) { + panic("implement me") //TODO +} + +func (f *apiFile) Close() error { + if f.r != nil { + return f.r.Close() + } + return nil +} + +func (f *apiFile) Size() (int64, error) { + return f.size, nil +} + +func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + f := &apiFile{ + ctx: ctx, + core: api.core(), + size: size, + path: p, + } + + return f, f.reset() +} + +type apiIter struct { + ctx context.Context + core *UnixfsAPI + + err error + + dec *json.Decoder + curFile files.Node + cur lsLink +} + +func (it *apiIter) Err() error { + return it.err +} + +func (it *apiIter) Name() string { + return it.cur.Name +} + +func (it *apiIter) Next() bool { + var out lsOutput + if err := it.dec.Decode(&out); err != nil { + if err != io.EOF { + it.err = err + } + return false + } + + if len(out.Objects) != 1 { + it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + return false + } + + if len(out.Objects[0].Links) != 1 { + it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + return false + } + + it.cur = out.Objects[0].Links[0] + c, err := cid.Parse(it.cur.Hash) + if err != nil { + it.err = err + return false + } + + switch it.cur.Type { + case unixfspb.Data_HAMTShard: + fallthrough + case unixfspb.Data_Metadata: + fallthrough + case unixfspb.Data_Directory: + it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + case unixfspb.Data_File: + it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + if err != nil { + it.err = err + return false + } + default: + it.err = fmt.Errorf("file type %d not supported", it.cur.Type) + return false + } + return true +} + +func (it *apiIter) Node() files.Node { + return it.curFile +} + +type apiDir struct { + ctx context.Context + core *UnixfsAPI + size int64 + path iface.Path + + dec *json.Decoder +} + +func (d *apiDir) Close() error { + return nil +} + +func (d *apiDir) Size() (int64, error) { + return d.size, nil +} + +func (d *apiDir) Entries() files.DirIterator { + return &apiIter{ + ctx: d.ctx, + core: d.core, + dec: d.dec, + } +} + +func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { + resp, err := api.core().request("ls", p.String()). + Option("resolve-size", true). + Option("stream", true).Send(ctx) + + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + d := &apiDir{ + ctx: ctx, + core: api, + size: size, + path: p, + + dec: json.NewDecoder(resp.Output), + } + + return d, nil +} + +var _ files.File = &apiFile{} +var _ files.Directory = &apiDir{} diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5a1495c24b4..77daf7b9b7e 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -85,10 +85,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return iface.IpfsPath(c), nil } -func (api *UnixfsAPI) Get(context.Context, iface.Path) (files.Node, error) { - panic("implement me") -} - type lsLink struct { Name, Hash string Size uint64 From 00597e6dff81846a2dbf742dd1ab9bf546b856e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:04:22 +0100 Subject: [PATCH 019/112] Dag.Put This commit was moved from ipfs/go-ipfs-http-client@3217104469020f0c66d8457842f8221b64289d74 --- client/httpapi/api.go | 2 +- client/httpapi/dag.go | 69 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dag.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3df45f38c24..c2ec1ad6719 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { } func (api *HttpApi) Dag() iface.DagAPI { - return nil + return (*DagAPI)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go new file mode 100644 index 00000000000..c3b29e7ee0b --- /dev/null +++ b/client/httpapi/dag.go @@ -0,0 +1,69 @@ +package httpapi + +import ( + "context" + "fmt" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type DagAPI HttpApi + +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { + options, err := caopts.DagPutOptions(opts...) + if err != nil { + return nil, err + } + + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + + codec, ok := cid.CodecToStr[options.Codec] + if !ok { + return nil, fmt.Errorf("unknowm codec %d", options.MhType) + } + + if options.MhLength != -1 { + return nil, fmt.Errorf("setting hash len is not supported yet") + } + + var out struct{ + Cid cid.Cid + } + err = api.core().request("dht/put"). + Option("hash", mht). + Option("format", codec). + Option("input-enc", options.InputEnc). + FileBody(src). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + return iface.IpldPath(out.Cid), nil +} + +func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { + panic("implement me") +} + +func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { + panic("implement me") +} + +func (api *DagAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 101910f04ed0575be19efaacb44b1bce73717033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:22:10 +0100 Subject: [PATCH 020/112] Object.New This commit was moved from ipfs/go-ipfs-http-client@60321ed42fd8f42e964e37ab5e96839fb8fd77b6 --- client/httpapi/api.go | 2 +- client/httpapi/ipldnode.go | 8 ++++ client/httpapi/object.go | 82 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/object.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c2ec1ad6719..cf9a633bc8b 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -154,7 +154,7 @@ func (api *HttpApi) Pin() iface.PinAPI { } func (api *HttpApi) Object() iface.ObjectAPI { - return nil + return (*ObjectAPI)(api) } func (api *HttpApi) Dht() iface.DhtAPI { diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index b8e6fba0180..1f584ac0700 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -20,6 +20,14 @@ type ipldNode struct { api *HttpApi } +func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { + return &ipldNode{ + ctx: ctx, + path: p, + api: a, + } +} + func (n *ipldNode) RawData() []byte { r, err := n.api.Block().Get(n.ctx, n.path) if err != nil { diff --git a/client/httpapi/object.go b/client/httpapi/object.go new file mode 100644 index 00000000000..9c1063af993 --- /dev/null +++ b/client/httpapi/object.go @@ -0,0 +1,82 @@ +package httpapi + +import ( + "context" + "github.com/ipfs/go-cid" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" +) + +type ObjectAPI HttpApi + +type objectOut struct { + Hash string +} + +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { + options, err := caopts.ObjectNewOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/new", options.Type).Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil +} + +func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { + panic("implement me") +} + +func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { + panic("implement me") +} + +func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { + panic("implement me") +} + +func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { + panic("implement me") +} + +func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { + panic("implement me") +} + +func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { + panic("implement me") +} + +func (api *ObjectAPI) core() *HttpApi { + return (*HttpApi)(api) +} From 31a4c3754b87e68f60525533ba9c6bb9c0b1021e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 09:47:32 +0100 Subject: [PATCH 021/112] Fix Dag.Put This commit was moved from ipfs/go-ipfs-http-client@5b2c99abdedaf464f061511bc7b24b1ade66546f --- client/httpapi/dag.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index c3b29e7ee0b..152d2c6b0a7 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -3,30 +3,25 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-cid" "io" + "math" "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" ) type DagAPI HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (iface.ResolvedPath, error) { +func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { options, err := caopts.DagPutOptions(opts...) if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) - } - codec, ok := cid.CodecToStr[options.Codec] if !ok { return nil, fmt.Errorf("unknowm codec %d", options.MhType) @@ -39,12 +34,19 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...options.DagPu var out struct{ Cid cid.Cid } - err = api.core().request("dht/put"). - Option("hash", mht). + req := api.core().request("dag/put"). Option("format", codec). - Option("input-enc", options.InputEnc). - FileBody(src). - Exec(ctx, &out) + Option("input-enc", options.InputEnc) + + if options.MhType != math.MaxUint64 { + mht, ok := mh.Codes[options.MhType] + if !ok { + return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + } + req.Option("hash", mht) + } + + err = req.FileBody(src).Exec(ctx, &out) if err != nil { return nil, err } @@ -56,7 +58,7 @@ func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error panic("implement me") } -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...options.DagTreeOption) ([]iface.Path, error) { +func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { panic("implement me") } From c213e2654235ffbdb6ce2f3a7df3c027ba53e02f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 9 Jan 2019 21:55:45 +0100 Subject: [PATCH 022/112] Unixfs.Add progress events This commit was moved from ipfs/go-ipfs-http-client@2f3a77b686ee8c4194ee6c21cbdecd57f2081519 --- client/httpapi/api.go | 4 +-- client/httpapi/api_test.go | 6 ++--- client/httpapi/apifile.go | 24 +++++++++--------- client/httpapi/block.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/ipldnode.go | 4 +-- client/httpapi/key.go | 5 ++-- client/httpapi/object.go | 2 +- client/httpapi/pin.go | 3 +-- client/httpapi/unixfs.go | 51 ++++++++++++++++++++++++++++++++++---- 10 files changed, 71 insertions(+), 32 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index cf9a633bc8b..c686a8d0dac 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -99,8 +99,8 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } return &HttpApi{ - url: url, - httpcli: c, + url: url, + httpcli: c, applyGlobal: func(*RequestBuilder) {}, } } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 67a2dbdce53..b822378f73a 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -53,7 +53,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n - 1), "0"} + connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} if err := c.Run(connectArgs); err != nil { return nil, err } @@ -95,8 +95,8 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) } c := &gohttp.Client{ Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, DisableCompression: true, }, } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 541189b609e..d9da2397598 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -22,12 +22,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } } - var stat struct{ + var stat struct { Hash string Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()). Exec(ctx, &stat) + err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -43,12 +43,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) } type apiFile struct { - ctx context.Context + ctx context.Context core *HttpApi size int64 path iface.Path - r io.ReadCloser + r io.ReadCloser at int64 } @@ -96,7 +96,7 @@ func (f *apiFile) Size() (int64, error) { func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { f := &apiFile{ - ctx: ctx, + ctx: ctx, core: api.core(), size: size, path: p, @@ -106,14 +106,14 @@ func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (fi } type apiIter struct { - ctx context.Context + ctx context.Context core *UnixfsAPI err error - dec *json.Decoder + dec *json.Decoder curFile files.Node - cur lsLink + cur lsLink } func (it *apiIter) Err() error { @@ -179,7 +179,7 @@ func (it *apiIter) Node() files.Node { } type apiDir struct { - ctx context.Context + ctx context.Context core *UnixfsAPI size int64 path iface.Path @@ -197,9 +197,9 @@ func (d *apiDir) Size() (int64, error) { func (d *apiDir) Entries() files.DirIterator { return &apiIter{ - ctx: d.ctx, + ctx: d.ctx, core: d.core, - dec: d.dec, + dec: d.dec, } } @@ -216,7 +216,7 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (fil } d := &apiDir{ - ctx: ctx, + ctx: ctx, core: api, size: size, path: p, diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 185aa0a42be..f933d3c4afa 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -16,7 +16,7 @@ import ( type BlockAPI HttpApi type blockStat struct { - Key string + Key string BSize int `json:"Size"` } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 152d2c6b0a7..20c233196d8 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -31,7 +31,7 @@ func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPut return nil, fmt.Errorf("setting hash len is not supported yet") } - var out struct{ + var out struct { Cid cid.Cid } req := api.core().request("dag/put"). diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 1f584ac0700..8e028e1a7ed 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -22,9 +22,9 @@ type ipldNode struct { func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ - ctx: ctx, + ctx: ctx, path: p, - api: a, + api: a, } } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 87b573f98a5..3653526ebef 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -13,7 +13,7 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` - Id string + Id string } func (k *keyOutput) Name() string { @@ -35,7 +35,6 @@ func (k *keyOutput) valid() error { return err } - func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { options, err := caopts.KeyGenerateOptions(opts...) if err != nil { @@ -65,7 +64,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { - var id struct{ID string} + var id struct{ ID string } if err := api.core().request("id").Exec(ctx, &id); err != nil { return nil, err } diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 9c1063af993..6259473c7ce 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,7 +14,7 @@ import ( type ObjectAPI HttpApi type objectOut struct { - Hash string + Hash string } func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 02dcc422247..ea34dc5409d 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -20,7 +20,7 @@ type pinRefKeyList struct { type pin struct { path iface.ResolvedPath - typ string + typ string } func (p *pin) Path() iface.ResolvedPath { @@ -31,7 +31,6 @@ func (p *pin) Type() string { return p.typ } - func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { panic("implement me") } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 77daf7b9b7e..7e63d241f87 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -2,9 +2,11 @@ package httpapi import ( "context" + "encoding/json" "fmt" "github.com/ipfs/go-cid" "github.com/pkg/errors" + "io" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -39,7 +41,6 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). - //Option("", options.Events). Option("fscache", options.FsCache). Option("hidden", options.Hidden). Option("inline", options.Inline). @@ -47,11 +48,10 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). - //Option("", options.Progress). Option("silent", options.Silent). Option("stdin-name", options.StdinName). Option("wrap-with-directory", options.Wrap). - Option("quieter", true) // TODO: rm after event impl + Option("progress", options.Progress) if options.RawLeavesSet { req.Option("raw-leaves", options.RawLeaves) @@ -73,9 +73,50 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix } var out addEvent - if err := req.Exec(ctx, &out); err != nil { //TODO: ndjson events + resp, err := req.Send(ctx) + if err != nil { return nil, err } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Output.Close() + dec := json.NewDecoder(resp.Output) +loop: + for { + var evt addEvent + switch err := dec.Decode(&evt); err { + case nil: + case io.EOF: + break loop + default: + return nil, err + } + out = evt + + if options.Events != nil { + ifevt := &iface.AddEvent{ + Name: out.Name, + Size: out.Size, + Bytes: out.Bytes, + } + + if out.Hash != "" { + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + ifevt.Path = iface.IpfsPath(c) + } + + select { + case options.Events <- ifevt: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } c, err := cid.Parse(out.Hash) if err != nil { @@ -120,7 +161,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, err links[i] = &format.Link{ Name: l.Name, Size: l.Size, - Cid: c, + Cid: c, } } return links, nil From 266c2f92c50df9b50bbc5aa093c793bb5d1a2265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 14:33:03 +0100 Subject: [PATCH 023/112] Implement Key API This commit was moved from ipfs/go-ipfs-http-client@281b2bf65b19b17e4f0c047ac8f9901306bf171f --- client/httpapi/key.go | 57 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 3653526ebef..417415ddbf4 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -2,6 +2,7 @@ package httpapi import ( "context" + "errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -49,18 +50,47 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { - panic("implement me") + options, err := caopts.KeyRenameOptions(opts...) + if err != nil { + return nil, false, err + } + + var out struct{ + Was string + Now string + Id string + Overwrite bool + } + err = api.core().request("key/rename", oldName, newName). + Option("force", options.Force). + Exec(ctx, &out) + if err != nil { + return nil, false, err + } + + id := &keyOutput{JName: out.Now, Id: out.Id} + return id, out.Overwrite, id.valid() } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { - panic("implement me") + var out struct{ Keys []*keyOutput } + if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]iface.Key, len(out.Keys)) + for i, k := range out.Keys { + if err := k.valid(); err != nil { + return nil, err + } + res[i] = k + } + + return res, nil } func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { @@ -70,14 +100,19 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { } out := keyOutput{JName: "self", Id: id.ID} - if err := out.valid(); err != nil { - return nil, err - } - return &out, nil + return &out, out.valid() } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { - panic("implement me") + var out struct{ Keys []keyOutput } + if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + return nil, err + } + if len(out.Keys) != 1 { + return nil, errors.New("got unexpected number of keys back") + } + + return &out.Keys[0], out.Keys[0].valid() } func (api *KeyAPI) core() *HttpApi { From 9d647d011efa551bae28c50cafb402a35d33b04a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 15:02:15 +0100 Subject: [PATCH 024/112] Implement Pin API This commit was moved from ipfs/go-ipfs-http-client@5bb7a581323aad4dd765a0916f6552cd5902d1e2 --- client/httpapi/pin.go | 92 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ea34dc5409d..34cafad71b6 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "github.com/ipfs/go-cid" + "github.com/pkg/errors" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" @@ -31,8 +33,14 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(context.Context, iface.Path, ...caopts.PinAddOption) error { - panic("implement me") +func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { + options, err := caopts.PinAddOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/add", p.String()). + Option("recursive", options.Recursive).Exec(ctx, nil) } func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { @@ -65,11 +73,85 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { - panic("implement me") + options, err := caopts.PinUpdateOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/update"). + Option("unpin", options.Unpin).Exec(ctx, nil) +} + +type pinVerifyRes struct { + Cid string + JOk bool `json:"Ok"` + JBadNodes []*badNode `json:"BadNodes,omitempty"` +} + +func (r *pinVerifyRes) Ok() bool { + return r.JOk +} + +func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { + out := make([]iface.BadPinNode, len(r.JBadNodes)) + for i, n := range r.JBadNodes { + out[i] = n + } + return out +} + +type badNode struct { + Cid string + JErr string `json:"Err"` } -func (api *PinAPI) Verify(context.Context) (<-chan iface.PinStatus, error) { - panic("implement me") +func (n *badNode) Path() iface.ResolvedPath { + c, err := cid.Parse(n.Cid) + if err != nil { + return nil // todo: handle this better + } + return iface.IpldPath(c) +} + +func (n *badNode) Err() error { + if n.JErr != "" { + return errors.New(n.JErr) + } + if _, err := cid.Parse(n.Cid); err != nil { + return err + } + return nil +} + +func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { + resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan iface.PinStatus) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + for { + var out pinVerifyRes + if err := dec.Decode(&out); err != nil { + return // todo: handle non io.EOF somehow + } + + select { + case res <- &out: + case <-ctx.Done(): + return + } + } + }() + + return res, nil } func (api *PinAPI) core() *HttpApi { From eaa19388d4d86eebf09d93a3a637e0690df722bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 16:31:07 +0100 Subject: [PATCH 025/112] Implement Object API This commit was moved from ipfs/go-ipfs-http-client@38149e46c8c3a1544dc3e3108ef72ffbc885e410 --- client/httpapi/block.go | 2 +- client/httpapi/ipldnode.go | 7 +- client/httpapi/key.go | 8 +- client/httpapi/object.go | 216 +++++++++++++++++++++++++++++++++---- client/httpapi/pin.go | 6 +- 5 files changed, 208 insertions(+), 31 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f933d3c4afa..f8334a10980 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -67,7 +67,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(context.Background()) + resp, err := api.core().request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 8e028e1a7ed..2294a95b42b 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -3,7 +3,6 @@ package httpapi import ( "context" "errors" - "fmt" "io/ioutil" "strconv" @@ -20,11 +19,11 @@ type ipldNode struct { api *HttpApi } -func (a *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { +func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { return &ipldNode{ ctx: ctx, path: p, - api: a, + api: api, } } @@ -47,7 +46,7 @@ func (n *ipldNode) Cid() cid.Cid { } func (n *ipldNode) String() string { - return fmt.Sprintf("[Block %s]", n.Cid()) + return n.Cid().String() } func (n *ipldNode) Loggable() map[string]interface{} { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 417415ddbf4..5087b6d9989 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -59,10 +59,10 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o return nil, false, err } - var out struct{ - Was string - Now string - Id string + var out struct { + Was string + Now string + Id string Overwrite bool } err = api.core().request("key/rename", oldName, newName). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6259473c7ce..4bc78799166 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -1,14 +1,17 @@ package httpapi import ( + "bytes" "context" - "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" ) type ObjectAPI HttpApi @@ -37,44 +40,219 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil } -func (api *ObjectAPI) Put(context.Context, io.Reader, ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { + options, err := caopts.ObjectPutOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/put"). + Option("inputenc", options.InputEnc). + Option("datafieldenc", options.DataType). + Option("pin", options.Pin). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(context.Context, iface.Path) (format.Node, error) { - panic("implement me") +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { + r, err := api.core().Block().Get(ctx, p) + if err != nil { + return nil, err + } + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(context.Context, iface.Path) (io.Reader, error) { - panic("implement me") +func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { + resp, err := api.core().request("object/data", p.String()).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + //TODO: make Data return ReadCloser to avoid copying + defer resp.Close() + b := new(bytes.Buffer) + if _, err := io.Copy(b, resp.Output); err != nil { + return nil, err + } + + return b, nil } -func (api *ObjectAPI) Links(context.Context, iface.Path) ([]*format.Link, error) { - panic("implement me") +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { + var out struct { + Links []struct { + Name string + Hash string + Size uint64 + } + } + if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]*format.Link, len(out.Links)) + for i, l := range out.Links { + c, err := cid.Parse(l.Hash) + if err != nil { + return nil, err + } + + res[i] = &format.Link{ + Cid: c, + Name: l.Name, + Size: l.Size, + } + } + + return res, nil } -func (api *ObjectAPI) Stat(context.Context, iface.Path) (*iface.ObjectStat, error) { - panic("implement me") +func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { + var out struct { + Hash string + NumLinks int + BlockSize int + LinksSize int + DataSize int + CumulativeSize int + } + if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return &iface.ObjectStat{ + Cid: c, + NumLinks: out.NumLinks, + BlockSize: out.BlockSize, + LinksSize: out.LinksSize, + DataSize: out.DataSize, + CumulativeSize: out.CumulativeSize, + }, nil } func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { - panic("implement me") + options, err := caopts.ObjectAddLinkOptions(opts...) + if err != nil { + return nil, err + } + + var out objectOut + err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + Option("create", options.Create). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { - panic("implement me") + var out objectOut + err := api.core().request("object/patch/rm-link", base.String(), link). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil +} + +func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/append-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { + var out objectOut + err := api.core().request("object/patch/set-data", p.String()). + FileBody(r). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + c, err := cid.Parse(out.Hash) + if err != nil { + return nil, err + } + + return iface.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(context.Context, iface.Path, io.Reader) (iface.ResolvedPath, error) { - panic("implement me") +type change struct { + Type iface.ChangeType + Path string + Before cid.Cid + After cid.Cid } -func (api *ObjectAPI) Diff(context.Context, iface.Path, iface.Path) ([]iface.ObjectChange, error) { - panic("implement me") +func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { + var out struct { + Changes []change + } + if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + return nil, err + } + res := make([]iface.ObjectChange, len(out.Changes)) + for i, ch := range out.Changes { + res[i] = iface.ObjectChange{ + Type: ch.Type, + Path: ch.Path, + } + if ch.Before != cid.Undef { + res[i].Before = iface.IpfsPath(ch.Before) + } + if ch.After != cid.Undef { + res[i].After = iface.IpfsPath(ch.After) + } + } + return res, nil } func (api *ObjectAPI) core() *HttpApi { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 34cafad71b6..1624b786758 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -83,8 +83,8 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` + Cid string + JOk bool `json:"Ok"` JBadNodes []*badNode `json:"BadNodes,omitempty"` } @@ -101,7 +101,7 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { } type badNode struct { - Cid string + Cid string JErr string `json:"Err"` } From ed9f2dd091693e914e9de072b2c23c2f991210a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:01:33 +0100 Subject: [PATCH 026/112] Implement DHT Api This commit was moved from ipfs/go-ipfs-http-client@c77355067a22300a8abe6b93b354de0ae0a3b548 --- client/httpapi/api.go | 2 +- client/httpapi/dht.go | 98 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/dht.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c686a8d0dac..208b958e931 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -158,7 +158,7 @@ func (api *HttpApi) Object() iface.ObjectAPI { } func (api *HttpApi) Dht() iface.DhtAPI { - return nil + return (*DhtAPI)(api) } func (api *HttpApi) Swarm() iface.SwarmAPI { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go new file mode 100644 index 00000000000..17d3b0500c2 --- /dev/null +++ b/client/httpapi/dht.go @@ -0,0 +1,98 @@ +package httpapi + +import ( + "context" + "encoding/json" + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + notif "github.com/libp2p/go-libp2p-routing/notifications" +) + +type DhtAPI HttpApi + +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + if err != nil { + return peerstore.PeerInfo{}, err + } + if resp.Error != nil { + return peerstore.PeerInfo{}, resp.Error + } + defer resp.Close() + dec := json.NewDecoder(resp.Output) + for { + if err := dec.Decode(&out); err != nil { + return peerstore.PeerInfo{}, err + } + if out.Type == notif.FinalPeer { + return out.Responses[0], nil + } + } +} + +func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { + options, err := caopts.DhtFindProvidersOptions(opts...) + if err != nil { + return nil, err + } + resp, err := api.core().request("dht/findprovs", p.String()). + Option("num-providers", options.NumProviders). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + res := make(chan peerstore.PeerInfo) + + go func() { + defer resp.Close() + defer close(res) + dec := json.NewDecoder(resp.Output) + + for { + var out struct { + Type notif.QueryEventType + Responses []peerstore.PeerInfo + } + + if err := dec.Decode(&out); err != nil { + return // todo: handle this somehow + } + if out.Type == notif.Provider { + for _, pi := range out.Responses { + select { + case res <- pi: + case <-ctx.Done(): + return + } + } + } + } + }() + + return res, nil +} + +func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { + options, err := caopts.DhtProvideOptions(opts...) + if err != nil { + return err + } + + return api.core().request("dht/provide", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) +} + +func (api *DhtAPI) core() *HttpApi { + return (*HttpApi)(api) +} From cae0ff2379f29a205426f7a8199c398885f33897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 17:36:35 +0100 Subject: [PATCH 027/112] Implement Swarm Api This commit was moved from ipfs/go-ipfs-http-client@01105690d2da6e3f3581fa364c6284248a251a35 --- client/httpapi/api.go | 2 +- client/httpapi/swarm.go | 176 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 client/httpapi/swarm.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 208b958e931..a5b56eff634 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -162,7 +162,7 @@ func (api *HttpApi) Dht() iface.DhtAPI { } func (api *HttpApi) Swarm() iface.SwarmAPI { - return nil + return (*SwarmAPI)(api) } func (api *HttpApi) PubSub() iface.PubSubAPI { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go new file mode 100644 index 00000000000..cf0ee2a9b36 --- /dev/null +++ b/client/httpapi/swarm.go @@ -0,0 +1,176 @@ +package httpapi + +import ( + "context" + "github.com/libp2p/go-libp2p-protocol" + "time" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + + inet "github.com/libp2p/go-libp2p-net" + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + "github.com/multiformats/go-multiaddr" +) + +type SwarmAPI HttpApi + +func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + saddrs := make([]string, len(pi.Addrs)) + for i, addr := range pi.Addrs { + saddrs[i] = addr.String() + } + + return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) +} + +func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { + return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) +} + +type streamInfo struct { + Protocol string +} + +type connInfo struct { + Addr string + Peer string + JLatency time.Duration `json:"Latency"` + Muxer string + JDirection inet.Direction `json:"Direction"` + JStreams []streamInfo `json:"Streams"` +} + +func (c *connInfo) valid() error { + _, err := multiaddr.NewMultiaddr(c.Addr) + if err != nil { + return err + } + + _, err = peer.IDB58Decode(c.Peer) + return err +} + +func (c *connInfo) ID() peer.ID { + id, _ := peer.IDB58Decode(c.Peer) + return id +} + +func (c *connInfo) Address() multiaddr.Multiaddr { + a, _ := multiaddr.NewMultiaddr(c.Addr) + return a +} + +func (c *connInfo) Direction() inet.Direction { + return c.JDirection +} + +func (c *connInfo) Latency() (time.Duration, error) { + return c.JLatency, nil +} + +func (c *connInfo) Streams() ([]protocol.ID, error) { + res := make([]protocol.ID, len(c.JStreams)) + for i, stream := range c.JStreams { + res[i] = protocol.ID(stream.Protocol) + } + return res, nil +} + +func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { + var out struct { + Peers []*connInfo + } + + err := api.core().request("swarm/peers"). + Option("streams", true). + Option("latency", true). + Exec(ctx, &out) + if err != nil { + return nil, err + } + + res := make([]iface.ConnectionInfo, len(out.Peers)) + for i, conn := range out.Peers { + if err := conn.valid(); err != nil { + return nil, err + } + res[i] = conn + } + + return res, nil +} + +func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Multiaddr, error) { + var out struct { + Addrs map[string][]string + } + if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + return nil, err + } + res := map[peer.ID][]multiaddr.Multiaddr{} + for spid, saddrs := range out.Addrs { + addrs := make([]multiaddr.Multiaddr, len(saddrs)) + + for i, addr := range saddrs { + a, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + addrs[i] = a + } + + pid, err := peer.IDB58Decode(spid) + if err != nil { + return nil, err + } + + res[pid] = addrs + } + + return res, nil +} + +func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, error) { + var out struct { + Strings []string + } + + if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]multiaddr.Multiaddr, len(out.Strings)) + for i, addr := range out.Strings { + ma, err := multiaddr.NewMultiaddr(addr) + if err != nil { + return nil, err + } + res[i] = ma + } + return res, nil +} + +func (api *SwarmAPI) core() *HttpApi { + return (*HttpApi)(api) +} From c2e0872f6dfba6f6d3f74a4b9eb755053a3f483e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:19:50 +0100 Subject: [PATCH 028/112] Implement PubSub Api This commit was moved from ipfs/go-ipfs-http-client@7abddda1d33595dffb0e0da782b52660147f6855 --- client/httpapi/api.go | 2 +- client/httpapi/api_test.go | 2 +- client/httpapi/pubsub.go | 129 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/pubsub.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index a5b56eff634..3a8e0e0031f 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -166,5 +166,5 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { } func (api *HttpApi) PubSub() iface.PubSubAPI { - return nil + return (*PubsubAPI)(api) } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index b822378f73a..1acece88c3c 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,7 +47,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--offline=" + strconv.FormatBool(n == 1)} + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go new file mode 100644 index 00000000000..3c5e3c50d60 --- /dev/null +++ b/client/httpapi/pubsub.go @@ -0,0 +1,129 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "io" + + "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/libp2p/go-libp2p-peer" +) + +type PubsubAPI HttpApi + +func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + return nil, err + } + + return out.Strings, nil +} + +func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { + options, err := caopts.PubSubPeersOptions(opts...) + if err != nil { + return nil, err + } + + var out struct { + Strings []string + } + + if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + return nil, err + } + + res := make([]peer.ID, len(out.Strings)) + for i, sid := range out.Strings { + id, err := peer.IDB58Decode(sid) + if err != nil { + return nil, err + } + res[i] = id + } + return res, nil +} + +func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { + return api.core().request("pubsub/pub", topic). + FileBody(bytes.NewReader(message)). + Exec(ctx, nil) +} + +type pubsubSub struct { + io.Closer + dec *json.Decoder +} + +type pubsubMessage struct { + JFrom []byte `json:"from,omitempty"` + JData []byte `json:"data,omitempty"` + JSeqno []byte `json:"seqno,omitempty"` + JTopicIDs []string `json:"topicIDs,omitempty"` +} + +func (msg *pubsubMessage) valid() error { + _, err := peer.IDFromBytes(msg.JFrom) + return err +} + +func (msg *pubsubMessage) From() peer.ID { + id, _ := peer.IDFromBytes(msg.JFrom) + return id +} + +func (msg *pubsubMessage) Data() []byte { + return msg.JData +} + +func (msg *pubsubMessage) Seq() []byte { + return msg.JSeqno +} + +func (msg *pubsubMessage) Topics() []string { + return msg.JTopicIDs +} + +func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { + // TODO: handle ctx + + var msg pubsubMessage + if err := s.dec.Decode(&msg); err != nil { + return nil, err + } + return &msg, msg.valid() +} + +func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + options, err := caopts.PubSubSubscribeOptions(opts...) + if err != nil { + return nil, err + } + + resp, err := api.core().request("pubsub/sub", topic). + Option("discover", options.Discover). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + return &pubsubSub{ + Closer: resp, + dec: json.NewDecoder(resp.Output), + }, nil +} + +func (api *PubsubAPI) core() *HttpApi { + return (*HttpApi)(api) +} + From 9b24cf0aafa94a7b4cb4401b6fbb9c959bde09b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 15 Jan 2019 18:49:11 +0100 Subject: [PATCH 029/112] Use cids in DHT calls This commit was moved from ipfs/go-ipfs-http-client@163b25f8b88eb98ddae50978440e3940070fabc0 --- client/httpapi/dht.go | 20 ++++++++++++++++++-- client/httpapi/pubsub.go | 3 +-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 17d3b0500c2..8fedabbf83d 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -42,7 +42,13 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err != nil { return nil, err } - resp, err := api.core().request("dht/findprovs", p.String()). + + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + resp, err := api.core().request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -60,6 +66,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop for { var out struct { + Extra string Type notif.QueryEventType Responses []peerstore.PeerInfo } @@ -67,6 +74,10 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } + if out.Type == notif.QueryError { + return // usually a 'not found' error + // todo: handle other errors + } if out.Type == notif.Provider { for _, pi := range out.Responses { select { @@ -88,7 +99,12 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", p.String()). + rp, err := api.core().ResolvePath(ctx, p) + if err != nil { + return err + } + + return api.core().request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 3c5e3c50d60..fb9bb74601c 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -119,11 +119,10 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return &pubsubSub{ Closer: resp, - dec: json.NewDecoder(resp.Output), + dec: json.NewDecoder(resp.Output), }, nil } func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } - From b7e258cc1015e518198c50b02fcc6ebdd8767375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Jan 2019 21:31:09 +0100 Subject: [PATCH 030/112] Reimplement DAG as DAGService This commit was moved from ipfs/go-ipfs-http-client@f34a5f6d2569bb21f022d3231e811b3db6262ca8 --- client/httpapi/api.go | 6 ++- client/httpapi/apifile.go | 5 ++ client/httpapi/dag.go | 103 +++++++++++++++++++++++++------------- 3 files changed, 77 insertions(+), 37 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 3a8e0e0031f..7d4fbd0e081 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -10,6 +10,8 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -137,8 +139,8 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() iface.DagAPI { - return (*DagAPI)(api) +func (api *HttpApi) Dag() format.DAGService { + return (*HttpDagServ)(api) } func (api *HttpApi) Name() iface.NameAPI { diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index d9da2397598..99ae72059c8 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -125,6 +125,11 @@ func (it *apiIter) Name() string { } func (it *apiIter) Next() bool { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + var out lsOutput if err := it.dec.Decode(&out); err != nil { if err != io.EOF { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 20c233196d8..de2934c2f11 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -1,71 +1,104 @@ package httpapi import ( + "bytes" "context" "fmt" - "io" - "math" + "io/ioutil" + "sync" "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - mh "github.com/multiformats/go-multihash" ) -type DagAPI HttpApi +type HttpDagServ HttpApi -func (api *DagAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.DagPutOption) (iface.ResolvedPath, error) { - options, err := caopts.DagPutOptions(opts...) +func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { + r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) if err != nil { return nil, err } - codec, ok := cid.CodecToStr[options.Codec] - if !ok { - return nil, fmt.Errorf("unknowm codec %d", options.MhType) + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err } - if options.MhLength != -1 { - return nil, fmt.Errorf("setting hash len is not supported yet") + blk, err := blocks.NewBlockWithCid(data, c) + if err != nil { + return nil, err } - var out struct { - Cid cid.Cid + return format.DefaultBlockDecoder.Decode(blk) +} + +func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { + out := make(chan *format.NodeOption) + wg := sync.WaitGroup{} + wg.Add(len(cids)) + + for _, c := range cids { + // TODO: Consider limiting concurrency of this somehow + go func() { + defer wg.Done() + n, err := api.Get(ctx, c) + + select { + case out <- &format.NodeOption{Node: n, Err: err}: + case <-ctx.Done(): + } + }() } - req := api.core().request("dag/put"). - Option("format", codec). - Option("input-enc", options.InputEnc) - - if options.MhType != math.MaxUint64 { - mht, ok := mh.Codes[options.MhType] - if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) - } - req.Option("hash", mht) + return out +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + c := nd.Cid() + prefix := c.Prefix() + format := cid.CodecToStr[prefix.Codec] + if prefix.Version == 0 { + format = "v0" } - err = req.FileBody(src).Exec(ctx, &out) + stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), + options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) if err != nil { - return nil, err + return err } - - return iface.IpldPath(out.Cid), nil + if !stat.Path().Cid().Equals(c) { + return fmt.Errorf("cids didn't match - local %s, remote %s", c.String(), stat.Path().Cid().String()) + } + return nil } -func (api *DagAPI) Get(ctx context.Context, path iface.Path) (format.Node, error) { - panic("implement me") +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + for _, nd := range nds { + // TODO: optimize + if err := api.Add(ctx, nd); err != nil { + return err + } + } + return nil } -func (api *DagAPI) Tree(ctx context.Context, path iface.Path, opts ...caopts.DagTreeOption) ([]iface.Path, error) { - panic("implement me") +func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { + return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? } -func (api *DagAPI) Batch(ctx context.Context) iface.DagBatch { - panic("implement me") +func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { + for _, c := range cids { + // TODO: optimize + if err := api.Remove(ctx, c); err != nil { + return err + } + } + return nil } -func (api *DagAPI) core() *HttpApi { +func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From c3f2970f8a7302344154e6d7fe74485f0dd272c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:42:32 +0100 Subject: [PATCH 031/112] block: Pin option This commit was moved from ipfs/go-ipfs-http-client@def66919dfbee162595852334468719f57158121 --- client/httpapi/block.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index f8334a10980..24df9629ed8 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -53,6 +53,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). + Option("pin", options.Pin). FileBody(r) var out blockStat From af2edd12eb44f477ef2e0dc22ae31a75df645b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:00 +0100 Subject: [PATCH 032/112] tests: enable filestore This commit was moved from ipfs/go-ipfs-http-client@904e8eeeb1b677d458c819122d91264040120d3b --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 1acece88c3c..bf1d6b9b04b 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -47,6 +47,11 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) return nil, err } + filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} + if err := c.Run(filestoreArgs); err != nil { + return nil, err + } + startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} if err := c.Run(startArgs); err != nil { return nil, err From 75cf2be1026a7f206e61d164342dae4807d36bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:43:26 +0100 Subject: [PATCH 033/112] apifile: Implement Seek This commit was moved from ipfs/go-ipfs-http-client@69cc3e8106552605ecfdfa5c6f352996dc6f9497 --- client/httpapi/apifile.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 99ae72059c8..ec916d8e299 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,13 +6,15 @@ import ( "fmt" "github.com/ipfs/go-cid" "io" + "io/ioutil" "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs-files" - unixfspb "github.com/ipfs/go-unixfs/pb" ) +const forwardSeekLimit = 1 << 14 //16k + func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error @@ -80,7 +82,24 @@ func (f *apiFile) Read(p []byte) (int, error) { } func (f *apiFile) Seek(offset int64, whence int) (int64, error) { - panic("implement me") //TODO + switch whence { + case io.SeekEnd: + offset = f.size + offset + case io.SeekCurrent: + offset = f.at + offset + } + if f.at == offset { //noop + return offset, nil + } + + if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + + f.at += r + return f.at, err + } + f.at = offset + return f.at, f.reset() } func (f *apiFile) Close() error { @@ -156,17 +175,17 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case unixfspb.Data_HAMTShard: + case iface.THAMTShard: fallthrough - case unixfspb.Data_Metadata: + case iface.TMetadata: fallthrough - case unixfspb.Data_Directory: + case iface.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case unixfspb.Data_File: + case iface.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err From bb83ccb15feb4854c7bdb4821fbcf90b89f36163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:44:48 +0100 Subject: [PATCH 034/112] response: read trailing error headers This commit was moved from ipfs/go-ipfs-http-client@83dfd84ba8bf2341478bb98d437ae1f14fe29e18 --- client/httpapi/response.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f6e7f3ab709..745c9a2a99d 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -2,6 +2,7 @@ package httpapi import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -13,6 +14,24 @@ import ( files "github.com/ipfs/go-ipfs-files" ) +type trailerReader struct { + resp *http.Response +} + +func (r *trailerReader) Read(b []byte) (int, error) { + n, err := r.resp.Body.Read(b) + if err != nil { + if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + err = errors.New(e) + } + } + return n, err +} + +func (r *trailerReader) Close() error { + return r.resp.Body.Close() +} + type Response struct { Output io.ReadCloser Error *Error @@ -56,9 +75,6 @@ type Error struct { func (e *Error) Error() string { var out string - if e.Command != "" { - out = e.Command + ": " - } if e.Code != 0 { out = fmt.Sprintf("%s%d: ", out, e.Code) } @@ -93,7 +109,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.Output = resp.Body + nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ Command: r.Command, From 88139ddc50e111e39721990bb86bb1c018c488b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:45:24 +0100 Subject: [PATCH 035/112] dag: Interface updates This commit was moved from ipfs/go-ipfs-http-client@7bea2efb45cba072b4e24c20b46a4372be3a716d --- client/httpapi/api.go | 3 +-- client/httpapi/dag.go | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 7d4fbd0e081..2c7a97c9926 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -11,7 +11,6 @@ import ( "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipld-format" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -139,7 +138,7 @@ func (api *HttpApi) Block() iface.BlockAPI { return (*BlockAPI)(api) } -func (api *HttpApi) Dag() format.DAGService { +func (api *HttpApi) Dag() iface.APIDagService { return (*HttpDagServ)(api) } diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index de2934c2f11..eacf631b92c 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -15,7 +15,9 @@ import ( "github.com/ipfs/go-ipld-format" ) -type HttpDagServ HttpApi +type httpNodeAdder HttpApi +type HttpDagServ httpNodeAdder +type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) @@ -56,7 +58,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for return out } -func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { +func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() format := cid.CodecToStr[prefix.Codec] @@ -65,7 +67,9 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), - options.Block.Hash(prefix.MhType, prefix.MhLength), options.Block.Format(format)) + options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.Format(format), + options.Block.Pin(pin)) if err != nil { return err } @@ -75,16 +79,36 @@ func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { return nil } -func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { +func (api *httpNodeAdder) addMany(ctx context.Context, nds []format.Node, pin bool) error { for _, nd := range nds { // TODO: optimize - if err := api.Add(ctx, nd); err != nil { + if err := api.add(ctx, nd, pin); err != nil { return err } } return nil } +func (api *HttpDagServ) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, false) +} + +func (api *HttpDagServ) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, false) +} + +func (api *pinningHttpNodeAdder) Add(ctx context.Context, nd format.Node) error { + return (*httpNodeAdder)(api).add(ctx, nd, true) +} + +func (api *pinningHttpNodeAdder) AddMany(ctx context.Context, nds []format.Node) error { + return (*httpNodeAdder)(api).addMany(ctx, nds, true) +} + +func (api *HttpDagServ) Pinning() format.NodeAdder { + return (*pinningHttpNodeAdder)(api) +} + func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? } @@ -99,6 +123,10 @@ func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { return nil } +func (api *httpNodeAdder) core() *HttpApi { + return (*HttpApi)(api) +} + func (api *HttpDagServ) core() *HttpApi { return (*HttpApi)(api) } From 93bfcf91cf979b267758c04aa14c6435a687fc36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:46:00 +0100 Subject: [PATCH 036/112] unixfs: updated ls This commit was moved from ipfs/go-ipfs-http-client@bb8d9d1a60e4161c4cf3a33c468746050f267a12 --- client/httpapi/unixfs.go | 94 ++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 7e63d241f87..67d05309b29 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,7 +13,6 @@ import ( "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" - unixfspb "github.com/ipfs/go-unixfs/pb" mh "github.com/multiformats/go-multihash" ) @@ -129,7 +128,7 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type unixfspb.Data_DataType + Type iface.FileType } type lsObject struct { @@ -141,30 +140,87 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path) ([]*format.Link, error) { - var out lsOutput - err := api.core().request("ls", p.String()).Exec(ctx, &out) +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { + options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err } - if len(out.Objects) != 1 { - return nil, errors.New("unexpected objects len") + resp, err := api.core().request("ls", p.String()). + Option("resolve-type", options.ResolveChildren). + Option("size", options.ResolveChildren). + Option("stream", true). + Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error } - links := make([]*format.Link, len(out.Objects[0].Links)) - for i, l := range out.Objects[0].Links { - c, err := cid.Parse(l.Hash) - if err != nil { - return nil, err - } - links[i] = &format.Link{ - Name: l.Name, - Size: l.Size, - Cid: c, + dec := json.NewDecoder(resp.Output) + out := make(chan iface.LsLink) + + go func() { + defer resp.Close() + defer close(out) + + for { + var link lsOutput + if err := dec.Decode(&link); err != nil { + if err == io.EOF { + return + } + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case <-ctx.Done(): + } + return + } + + if len(link.Objects[0].Links) != 1 { + select { + case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case <-ctx.Done(): + } + return + } + + l0 := link.Objects[0].Links[0] + + c, err := cid.Decode(l0.Hash) + if err != nil { + select { + case out <- iface.LsLink{Err: err}: + case <-ctx.Done(): + } + return + } + + select { + case out <- iface.LsLink{ + Link: &format.Link{ + Cid: c, + Name: l0.Name, + Size: l0.Size, + }, + Size: l0.Size, + Type: l0.Type, + }: + case <-ctx.Done(): + } } - } - return links, nil + }() + + return out, nil } func (api *UnixfsAPI) core() *HttpApi { From 1bc854bf2169f82d2275232e123adae721b8dc0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Feb 2019 19:53:02 +0100 Subject: [PATCH 037/112] pin: handle Rm options This commit was moved from ipfs/go-ipfs-http-client@e85e856ea2841cd3e0cf4cb7b87441539af42bbe --- client/httpapi/pin.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 1624b786758..237b3232976 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -68,8 +68,15 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path) error { - return api.core().request("pin/rm", p.String()).Exec(ctx, nil) +func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { + options, err := caopts.PinRmOptions(opts...) + if err != nil { + return err + } + + return api.core().request("pin/rm", p.String()). + Option("recursive", options.Recursive). + Exec(ctx, nil) } func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { From c543354b1772323cb2cfbf41c06522297bf27667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Feb 2019 22:36:30 +0100 Subject: [PATCH 038/112] Check for redirects This commit was moved from ipfs/go-ipfs-http-client@19c65db4f0fd1549fc6f224efd868d4f00da997d --- client/httpapi/api.go | 18 +++++++++++++----- client/httpapi/requestbuilder.go | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 2c7a97c9926..da0836d4a6f 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -2,6 +2,7 @@ package httpapi import ( "errors" + "fmt" "io/ioutil" gohttp "net/http" "os" @@ -27,7 +28,7 @@ var ErrNotImplemented = errors.New("not implemented") type HttpApi struct { url string - httpcli *gohttp.Client + httpcli gohttp.Client applyGlobal func(*RequestBuilder) } @@ -50,8 +51,8 @@ func NewPathApi(p string) iface.CoreAPI { return NewApi(a) } -func ApiAddr(p string) ma.Multiaddr { - baseDir, err := homedir.Expand(p) +func ApiAddr(ipfspath string) ma.Multiaddr { + baseDir, err := homedir.Expand(ipfspath) if err != nil { return nil } @@ -99,11 +100,18 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { } } - return &HttpApi{ + api := &HttpApi{ url: url, - httpcli: c, + httpcli: *c, applyGlobal: func(*RequestBuilder) {}, } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 831e6d71c0b..628ad03cdf3 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -92,7 +92,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { req.Opts = r.opts req.Headers = r.headers req.Body = r.body - return req.Send(r.shell.httpcli) + return req.Send(&r.shell.httpcli) } // Exec sends the request a request and decodes the response. From ef5bf40df1d0d790aeb213147c53412145894bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 11 Feb 2019 18:40:38 +0100 Subject: [PATCH 039/112] Update imports to use extracted interface This commit was moved from ipfs/go-ipfs-http-client@fad467bc437a1282ed3813d828c6f8ec5b7b2d7a --- client/httpapi/api.go | 5 ++--- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 9 ++++----- client/httpapi/block.go | 4 ++-- client/httpapi/dag.go | 5 ++--- client/httpapi/dht.go | 4 ++-- client/httpapi/ipldnode.go | 3 +-- client/httpapi/key.go | 5 ++--- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 5 ++--- client/httpapi/path.go | 3 +-- client/httpapi/pin.go | 6 +++--- client/httpapi/pubsub.go | 5 ++--- client/httpapi/swarm.go | 5 ++--- client/httpapi/unixfs.go | 11 +++++------ 15 files changed, 36 insertions(+), 47 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index da0836d4a6f..698e36524e7 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -9,9 +9,8 @@ import ( "path" "strings" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + iface "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" homedir "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index bf1d6b9b04b..5621ab87f0f 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,10 +10,9 @@ import ( "strconv" "testing" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-ipfs/core/coreapi/interface/tests" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec916d8e299..1e5f61a9a7b 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -4,13 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/ipfs/go-cid" "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -92,8 +91,8 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { return offset, nil } - if f.at < offset && offset - f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset - f.at) + if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip + r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 24df9629ed8..45c73472c59 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -8,8 +8,8 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index eacf631b92c..d0059698ab6 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -7,12 +7,11 @@ import ( "io/ioutil" "sync" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/options" ) type httpNodeAdder HttpApi diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 8fedabbf83d..dc7dd6beaf6 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -3,9 +3,9 @@ package httpapi import ( "context" "encoding/json" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go index 2294a95b42b..43fa5d50de9 100644 --- a/client/httpapi/ipldnode.go +++ b/client/httpapi/ipldnode.go @@ -6,11 +6,10 @@ import ( "io/ioutil" "strconv" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) type ipldNode struct { diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 5087b6d9989..dc2adf8b7fe 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,8 @@ import ( "context" "errors" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index fbc440b9666..d760e6188ae 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -3,10 +3,10 @@ package httpapi import ( "context" "fmt" - "github.com/ipfs/go-ipfs/namesys/opts" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/options/namesys" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 4bc78799166..3b648d82bc0 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,12 +6,11 @@ import ( "io" "io/ioutil" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 5701326fc56..68dc2633dd5 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,11 +3,10 @@ package httpapi import ( "context" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core" ) func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 237b3232976..4c4e5713ce6 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,11 +3,11 @@ package httpapi import ( "context" "encoding/json" + "github.com/ipfs/go-cid" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/pkg/errors" - - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" ) type PinAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index fb9bb74601c..3345097806d 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,9 +6,8 @@ import ( "encoding/json" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-peer" ) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index cf0ee2a9b36..d179b6540fe 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -2,14 +2,13 @@ package httpapi import ( "context" - "github.com/libp2p/go-libp2p-protocol" "time" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - + "github.com/ipfs/interface-go-ipfs-core" inet "github.com/libp2p/go-libp2p-net" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" + "github.com/libp2p/go-libp2p-protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 67d05309b29..1f340b6573a 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -3,16 +3,15 @@ package httpapi import ( "context" "encoding/json" + "errors" "fmt" - "github.com/ipfs/go-cid" - "github.com/pkg/errors" "io" - "github.com/ipfs/go-ipfs/core/coreapi/interface" - caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipld-format" + "github.com/ipfs/interface-go-ipfs-core" + caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -208,7 +207,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf select { case out <- iface.LsLink{ Link: &format.Link{ - Cid: c, + Cid: c, Name: l0.Name, Size: l0.Size, }, From 0813d808b54341414441a56e5cd3a7722a516a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:29:31 +0100 Subject: [PATCH 040/112] Fix govet warning in Dag This commit was moved from ipfs/go-ipfs-http-client@62552b33959401c7a9d9463dffba433e522b2f15 --- client/httpapi/dag.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index d0059698ab6..3f54ced34f0 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -44,7 +44,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for for _, c := range cids { // TODO: Consider limiting concurrency of this somehow - go func() { + go func(c cid.Cid) { defer wg.Done() n, err := api.Get(ctx, c) @@ -52,7 +52,7 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for case out <- &format.NodeOption{Node: n, Err: err}: case <-ctx.Done(): } - }() + }(c) } return out } From abc30e384fafe157490df0dae80a22ddeb6255a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 12 Feb 2019 12:48:04 +0100 Subject: [PATCH 041/112] Register iptb plugin once This commit was moved from ipfs/go-ipfs-http-client@d6c8cbd5e643d2131477120a76159074d1a07cd3 --- client/httpapi/api_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5621ab87f0f..fa38866e1a2 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -19,9 +19,7 @@ import ( "github.com/ipfs/iptb/testbed/interfaces" ) -type NodeProvider struct{} - -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", NewNode: local.NewNode, @@ -31,8 +29,13 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) BuiltIn: true, }, false) if err != nil { - return nil, err + panic(err) } +} + +type NodeProvider struct{} + +func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { From 9a6ee6f5e153b96afda0f4ca1ee893cf28aa4f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 17:30:04 +0100 Subject: [PATCH 042/112] Improve test node spawning This commit was moved from ipfs/go-ipfs-http-client@6bb2a287a6f5090792e07f3e3a74871c459b395f --- client/httpapi/api_test.go | 198 ++++++++++++++++++++++++------------- 1 file changed, 131 insertions(+), 67 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fa38866e1a2..af2180384d6 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,23 +2,23 @@ package httpapi import ( "context" - "fmt" "io/ioutil" gohttp "net/http" "os" - "path" "strconv" + "sync" "testing" "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" - "github.com/ipfs/iptb/cli" "github.com/ipfs/iptb/testbed" "github.com/ipfs/iptb/testbed/interfaces" + ma "github.com/multiformats/go-multiaddr" ) +const parallelSpeculativeNodes = 15 // 15 seems to work best + func init() { _, err := testbed.RegisterPlugin(testbed.IptbPlugin{ From: "", @@ -33,99 +33,163 @@ func init() { } } -type NodeProvider struct{} +type NodeProvider struct { + simple <-chan func(context.Context) ([]iface.CoreAPI, error) +} + +func newNodeProvider(ctx context.Context) *NodeProvider { + simpleNodes := make(chan func(context.Context) ([]iface.CoreAPI, error), parallelSpeculativeNodes) + + np := &NodeProvider{ + simple: simpleNodes, + } + + // start basic nodes speculatively in parallel + for i := 0; i < parallelSpeculativeNodes; i++ { + go func() { + for { + ctx, cancel := context.WithCancel(ctx) + + snd, err := np.makeAPISwarm(ctx, false, 1) + + res := func(ctx context.Context) ([]iface.CoreAPI, error) { + if err != nil { + return nil, err + } + + go func() { + <-ctx.Done() + cancel() + }() + + return snd, nil + } + + select { + case simpleNodes <- res: + case <-ctx.Done(): + return + } + } + }() + } + + return np +} + +func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { + if !fullIdentity && n == 1 { + return (<-np.simple)(ctx) + } + return np.makeAPISwarm(ctx, fullIdentity, n) +} -func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { +func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { dir, err := ioutil.TempDir("", "httpapi-tb-") if err != nil { return nil, err } - c := cli.NewCli() //TODO: is there a better way? + tb := testbed.NewTestbed(dir) - initArgs := []string{"iptb", "--IPTB_ROOT", dir, "auto", "-type", "localipfs", "-count", strconv.FormatInt(int64(n), 10)} - if err := c.Run(initArgs); err != nil { + specs, err := testbed.BuildSpecs(tb.Dir(), n, "localipfs", nil) + if err != nil { return nil, err } - filestoreArgs := []string{"iptb", "--IPTB_ROOT", dir, "run", fmt.Sprintf("[0-%d]", n-1), "--", "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"} - if err := c.Run(filestoreArgs); err != nil { + if err := testbed.WriteNodeSpecs(tb.Dir(), specs); err != nil { return nil, err } - startArgs := []string{"iptb", "--IPTB_ROOT", dir, "start", "-wait", "--", "--enable-pubsub-experiment", "--offline=" + strconv.FormatBool(n == 1)} - if err := c.Run(startArgs); err != nil { + nodes, err := tb.Nodes() + if err != nil { return nil, err } - if n > 1 { - connectArgs := []string{"iptb", "--IPTB_ROOT", dir, "connect", fmt.Sprintf("[1-%d]", n-1), "0"} - if err := c.Run(connectArgs); err != nil { - return nil, err - } + apis := make([]iface.CoreAPI, n) + + wg := sync.WaitGroup{} + zero := sync.WaitGroup{} + + wg.Add(len(nodes)) + zero.Add(1) + + for i, nd := range nodes { + go func(i int, nd testbedi.Core) { + defer wg.Done() + + if _, err := nd.Init(ctx, "--empty-repo"); err != nil { + panic(err) + } + + if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { + panic(err) + } + + if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { + panic(err) + } + + if i > 0 { + zero.Wait() + if err := nd.Connect(ctx, nodes[0]); err != nil { + panic(err) + } + } else { + zero.Done() + } + + addr, err := nd.APIAddr() + if err != nil { + panic(err) + } + + maddr, err := ma.NewMultiaddr(addr) + if err != nil { + panic(err) + } + + c := &gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + DisableCompression: true, + }, + } + apis[i] = NewApiWithClient(maddr, c) + + // empty node is pinned even with --empty-repo, we don't want that + emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + if err != nil { + panic(err) + } + if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { + panic(err) + } + }(i, nd) } + wg.Wait() + go func() { <-ctx.Done() defer os.Remove(dir) defer func() { - _ = c.Run([]string{"iptb", "--IPTB_ROOT", dir, "stop"}) + for _, nd := range nodes { + _ = nd.Stop(context.Background()) + } }() }() - apis := make([]iface.CoreAPI, n) - - for i := range apis { - tb := testbed.NewTestbed(path.Join(dir, "testbeds", "default")) - - node, err := tb.Node(i) - if err != nil { - return nil, err - } - - attrNode, ok := node.(testbedi.Attribute) - if !ok { - return nil, fmt.Errorf("node does not implement attributes") - } - - pth, err := attrNode.Attr("path") - if err != nil { - return nil, err - } - - a := ApiAddr(pth) - if a == nil { - return nil, fmt.Errorf("nil addr for node") - } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - DisableCompression: true, - }, - } - apis[i] = NewApiWithClient(a, c) - - // node cleanup - // TODO: pass --empty-repo somehow (how?) - pins, err := apis[i].Pin().Ls(ctx, caopts.Pin.Type.Recursive()) - if err != nil { - return nil, err - } - for _, pin := range pins { //TODO: parallel - if err := apis[i].Pin().Rm(ctx, pin.Path()); err != nil { - return nil, err - } - } - - } - return apis, nil } func TestHttpApi(t *testing.T) { - tests.TestApi(&NodeProvider{})(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tests.TestApi(newNodeProvider(ctx))(t) } From f7dd0c690960ea79a7aad7f40b11362512d173bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:12:02 +0100 Subject: [PATCH 043/112] Improve apifile error messages This commit was moved from ipfs/go-ipfs-http-client@3393b8379021695126da83afdf3bf54eaff828f4 --- client/httpapi/apifile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1e5f61a9a7b..8647606844b 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -157,12 +157,12 @@ func (it *apiIter) Next() bool { } if len(out.Objects) != 1 { - it.err = fmt.Errorf("len(out.Objects) != 1 (is %d)", len(out.Objects)) + it.err = fmt.Errorf("ls returned more objects than expected (%d)", len(out.Objects)) return false } if len(out.Objects[0].Links) != 1 { - it.err = fmt.Errorf("len(out.Objects[0].Links) != 1 (is %d)", len(out.Objects[0].Links)) + it.err = fmt.Errorf("ls returned more links than expected (%d)", len(out.Objects[0].Links)) return false } From e1b14d78c72edf0a782e362afb0e818e032c31cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 18:45:19 +0100 Subject: [PATCH 044/112] Implement missing methods This commit was moved from ipfs/go-ipfs-http-client@b76413dfe55fd33d39a680458282d85565cc1c69 --- client/httpapi/api.go | 3 -- client/httpapi/name.go | 63 +++++++++++++++++++++++++++++++++++++++--- client/httpapi/path.go | 9 ++++-- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 698e36524e7..e106d8d87a6 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,7 +1,6 @@ package httpapi import ( - "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -23,8 +22,6 @@ const ( EnvDir = "IPFS_PATH" ) -var ErrNotImplemented = errors.New("not implemented") - type HttpApi struct { url string httpcli gohttp.Client diff --git a/client/httpapi/name.go b/client/httpapi/name.go index d760e6188ae..58fb59249b9 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -2,7 +2,9 @@ package httpapi import ( "context" + "encoding/json" "fmt" + "io" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -38,11 +40,11 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na req := api.core().request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). - Option("lifetime", options.ValidTime.String()). + Option("lifetime", options.ValidTime). Option("resolve", false) if options.TTL != nil { - req.Option("ttl", options.TTL.String()) + req.Option("ttl", options.TTL) } var out ipnsEntry @@ -57,7 +59,60 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { - return nil, ErrNotImplemented + options, err := caopts.NameResolveOptions(opts...) + if err != nil { + return nil, err + } + + ropts := nsopts.ProcessOpts(options.ResolveOpts) + if ropts.Depth != nsopts.DefaultDepthLimit && ropts.Depth != 1 { + return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) + } + + req := api.core().request("name/resolve", name). + Option("nocache", !options.Cache). + Option("recursive", ropts.Depth != 1). + Option("dht-record-count", ropts.DhtRecordCount). + Option("dht-timeout", ropts.DhtTimeout). + Option("stream", true) + resp, err := req.Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + res := make(chan iface.IpnsResult) + + go func() { + defer close(res) + defer resp.Close() + + dec := json.NewDecoder(resp.Output) + + for { + var out struct{ Path string } + err := dec.Decode(&out) + if err == io.EOF { + return + } + var ires iface.IpnsResult + if err == nil { + ires.Path, err = iface.ParsePath(out.Path) + } + + select { + case res <- ires: + case <-ctx.Done(): + } + if err != nil { + return + } + } + }() + + return res, nil } func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { @@ -75,7 +130,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). - Option("dht-timeout", ropts.DhtTimeout.String()) + Option("dht-timeout", ropts.DhtTimeout) var out struct{ Path string } if err := req.Exec(ctx, &out); err != nil { diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 68dc2633dd5..8c819121ad1 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -42,6 +42,11 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(context.Context, iface.Path) (ipld.Node, error) { - return nil, ErrNotImplemented +func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { + rp, err := api.ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + return api.Dag().Get(ctx, rp.Cid()) } From 42273cab06348b19a813dee77f7cbe5ed7cc78d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:05:17 +0100 Subject: [PATCH 045/112] Don't use valid() pattern This commit was moved from ipfs/go-ipfs-http-client@4d07c48f98b0af0b18c6c13f80f04c966fc43bdc --- client/httpapi/block.go | 20 +++++++------------- client/httpapi/key.go | 28 ++++++++++++++++------------ client/httpapi/name.go | 14 ++++---------- client/httpapi/pubsub.go | 12 +++++------- 4 files changed, 32 insertions(+), 42 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 45c73472c59..fd4d9bab971 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -18,24 +18,16 @@ type BlockAPI HttpApi type blockStat struct { Key string BSize int `json:"Size"` + + cid cid.Cid } func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) valid() (iface.ResolvedPath, error) { - c, err := cid.Parse(s.Key) - if err != nil { - return nil, err - } - - return iface.IpldPath(c), nil -} - func (s *blockStat) Path() iface.ResolvedPath { - p, _ := s.valid() - return p + return iface.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +52,8 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } @@ -118,7 +111,8 @@ func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, e if err != nil { return nil, err } - if _, err := out.valid(); err != nil { + out.cid, err = cid.Parse(out.Key) + if err != nil { return nil, err } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index dc2adf8b7fe..a16c30d8ef8 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -14,6 +14,8 @@ type KeyAPI HttpApi type keyOutput struct { JName string `json:"Name"` Id string + + pid peer.ID } func (k *keyOutput) Name() string { @@ -26,13 +28,7 @@ func (k *keyOutput) Path() iface.Path { } func (k *keyOutput) ID() peer.ID { - p, _ := peer.IDB58Decode(k.Id) - return p -} - -func (k *keyOutput) valid() error { - _, err := peer.IDB58Decode(k.Id) - return err + return k.pid } func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.KeyGenerateOption) (iface.Key, error) { @@ -49,7 +45,8 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, opts ...caopts.KeyRenameOption) (iface.Key, bool, error) { @@ -72,7 +69,8 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - return id, out.Overwrite, id.valid() + id.pid, err = peer.IDB58Decode(id.Id) + return id, out.Overwrite, err } func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { @@ -83,7 +81,9 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { - if err := k.valid(); err != nil { + var err error + k.pid, err = peer.IDB58Decode(k.Id) + if err != nil { return nil, err } res[i] = k @@ -98,8 +98,10 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { return nil, err } + var err error out := keyOutput{JName: "self", Id: id.ID} - return &out, out.valid() + out.pid, err = peer.IDB58Decode(out.Id) + return &out, err } func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { @@ -111,7 +113,9 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { return nil, errors.New("got unexpected number of keys back") } - return &out.Keys[0], out.Keys[0].valid() + var err error + out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + return &out.Keys[0], err } func (api *KeyAPI) core() *HttpApi { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 58fb59249b9..b848aa81978 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -16,10 +16,8 @@ type NameAPI HttpApi type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` -} -func (e *ipnsEntry) valid() (iface.Path, error) { - return iface.ParsePath(e.JValue) + path iface.Path } func (e *ipnsEntry) Name() string { @@ -27,8 +25,7 @@ func (e *ipnsEntry) Name() string { } func (e *ipnsEntry) Value() iface.Path { - p, _ := e.valid() - return p + return e.path } func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { @@ -51,11 +48,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - if _, err := out.valid(); err != nil { - return nil, err - } - - return &out, nil + out.path, err = iface.ParsePath(out.JValue) + return &out, err } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 3345097806d..e27cb565557 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -66,16 +66,12 @@ type pubsubMessage struct { JData []byte `json:"data,omitempty"` JSeqno []byte `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` -} -func (msg *pubsubMessage) valid() error { - _, err := peer.IDFromBytes(msg.JFrom) - return err + from peer.ID } func (msg *pubsubMessage) From() peer.ID { - id, _ := peer.IDFromBytes(msg.JFrom) - return id + return msg.from } func (msg *pubsubMessage) Data() []byte { @@ -97,7 +93,9 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if err := s.dec.Decode(&msg); err != nil { return nil, err } - return &msg, msg.valid() + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { From 93f684617a3adbd7cba751ee421e32b75cd3dafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:07:45 +0100 Subject: [PATCH 046/112] dag: remove unused waitgroup This commit was moved from ipfs/go-ipfs-http-client@934fc60a7c41a14b679fd89fec3b349650a76c81 --- client/httpapi/dag.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 3f54ced34f0..a613d18dec5 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -39,13 +39,10 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { out := make(chan *format.NodeOption) - wg := sync.WaitGroup{} - wg.Add(len(cids)) for _, c := range cids { // TODO: Consider limiting concurrency of this somehow go func(c cid.Cid) { - defer wg.Done() n, err := api.Get(ctx, c) select { From cc964b4ab8ea5d059be2c05e64ecdca851dcd8b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:15:21 +0100 Subject: [PATCH 047/112] Simplify Object.New, remove ipldnode.go This commit was moved from ipfs/go-ipfs-http-client@0752a6ee63a2ec42aabb6c9acac6ef6c4a94af38 --- client/httpapi/dag.go | 1 - client/httpapi/ipldnode.go | 113 ------------------------------------- client/httpapi/object.go | 34 +++++------ 3 files changed, 18 insertions(+), 130 deletions(-) delete mode 100644 client/httpapi/ipldnode.go diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index a613d18dec5..669b5f893ad 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io/ioutil" - "sync" "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" diff --git a/client/httpapi/ipldnode.go b/client/httpapi/ipldnode.go deleted file mode 100644 index 43fa5d50de9..00000000000 --- a/client/httpapi/ipldnode.go +++ /dev/null @@ -1,113 +0,0 @@ -package httpapi - -import ( - "context" - "errors" - "io/ioutil" - "strconv" - - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" -) - -type ipldNode struct { - ctx context.Context //TODO: should we re-consider adding ctx to ipld interfaces? - path iface.ResolvedPath - api *HttpApi -} - -func (api *HttpApi) nodeFromPath(ctx context.Context, p iface.ResolvedPath) ipld.Node { - return &ipldNode{ - ctx: ctx, - path: p, - api: api, - } -} - -func (n *ipldNode) RawData() []byte { - r, err := n.api.Block().Get(n.ctx, n.path) - if err != nil { - panic(err) // TODO: eww, should we add errors too / better ideas? - } - - b, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - - return b -} - -func (n *ipldNode) Cid() cid.Cid { - return n.path.Cid() -} - -func (n *ipldNode) String() string { - return n.Cid().String() -} - -func (n *ipldNode) Loggable() map[string]interface{} { - return nil //TODO: we can't really do better here, can we? -} - -// TODO: should we use 'full'/real ipld codecs for this? js-ipfs-api does that. -// We can also give people a choice -func (n *ipldNode) Resolve(path []string) (interface{}, []string, error) { - p := ipfspath.Join([]string{n.path.String(), ipfspath.Join(path)}) - - var out interface{} - n.api.request("dag/get", p).Exec(n.ctx, &out) - - // TODO: this is more than likely wrong, fix if we decide to stick with this 'http-ipld-node' hack - for len(path) > 0 { - switch o := out.(type) { - case map[string]interface{}: - v, ok := o[path[0]] - if !ok { - // TODO: ipld links - return nil, nil, errors.New("no element under this path") - } - out = v - case []interface{}: - n, err := strconv.ParseUint(path[0], 10, 32) - if err != nil { - return nil, nil, err - } - if len(o) < int(n) { - return nil, nil, errors.New("no element under this path") - } - out = o[n] - } - path = path[1:] - } - - return out, path, nil -} - -func (n *ipldNode) Tree(path string, depth int) []string { - panic("implement me") -} - -func (n *ipldNode) ResolveLink(path []string) (*ipld.Link, []string, error) { - panic("implement me") -} - -func (n *ipldNode) Copy() ipld.Node { - panic("implement me") -} - -func (n *ipldNode) Links() []*ipld.Link { - panic("implement me") -} - -func (n *ipldNode) Stat() (*ipld.NodeStat, error) { - panic("implement me") -} - -func (n *ipldNode) Size() (uint64, error) { - panic("implement me") -} - -var _ ipld.Node = &ipldNode{} diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 3b648d82bc0..5a06f74d99a 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -3,12 +3,15 @@ package httpapi import ( "bytes" "context" + "fmt" "io" "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + dag "github.com/ipfs/go-merkledag" + ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" ) @@ -19,24 +22,23 @@ type objectOut struct { Hash string } -func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (format.Node, error) { +func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) (ipld.Node, error) { options, err := caopts.ObjectNewOptions(opts...) if err != nil { return nil, err } - var out objectOut - err = api.core().request("object/new", options.Type).Exec(ctx, &out) - if err != nil { - return nil, err - } - - c, err := cid.Parse(out.Hash) - if err != nil { - return nil, err + var n ipld.Node + switch options.Type { + case "empty": + n = new(dag.ProtoNode) + case "unixfs-dir": + n = ft.EmptyDirNode() + default: + return nil, fmt.Errorf("unknown object type: %s", options.Type) } - return api.core().nodeFromPath(ctx, iface.IpfsPath(c)), nil + return n, nil } func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { @@ -64,7 +66,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return iface.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (format.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -96,7 +98,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -107,14 +109,14 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*format.Link, if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } - res := make([]*format.Link, len(out.Links)) + res := make([]*ipld.Link, len(out.Links)) for i, l := range out.Links { c, err := cid.Parse(l.Hash) if err != nil { return nil, err } - res[i] = &format.Link{ + res[i] = &ipld.Link{ Cid: c, Name: l.Name, Size: l.Size, From a23da822de54922faf1f82a86c25660e8cca8bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Feb 2019 19:35:32 +0100 Subject: [PATCH 048/112] swarm: attach peerid if needed This commit was moved from ipfs/go-ipfs-http-client@aa88b18ac497400d16b36af6489af50a003c298c --- client/httpapi/swarm.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index d179b6540fe..0bb36aca3fa 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -17,6 +17,13 @@ type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { + if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + addr = addr.Encapsulate(pidma) + } saddrs[i] = addr.String() } From f005b8dc5666916ff2a0ff1eef58b7795670ec73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 16:39:00 +0100 Subject: [PATCH 049/112] pin: verify: parse bad node cids early This commit was moved from ipfs/go-ipfs-http-client@7032dfc9b016f2c584e2102e80fb60c3ecf573ea --- client/httpapi/pin.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 4c4e5713ce6..b11efd35c5d 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -110,23 +110,18 @@ func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { type badNode struct { Cid string JErr string `json:"Err"` + + cid cid.Cid } func (n *badNode) Path() iface.ResolvedPath { - c, err := cid.Parse(n.Cid) - if err != nil { - return nil // todo: handle this better - } - return iface.IpldPath(c) + return iface.IpldPath(n.cid) } func (n *badNode) Err() error { if n.JErr != "" { return errors.New(n.JErr) } - if _, err := cid.Parse(n.Cid); err != nil { - return err - } return nil } @@ -150,6 +145,13 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { return // todo: handle non io.EOF somehow } + for i, n := range out.JBadNodes { + out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + if err != nil { + return + } + } + select { case res <- &out: case <-ctx.Done(): From 4a6d36d98b6527e9c13918ac901c4cbc68630b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:02:13 +0100 Subject: [PATCH 050/112] pubsub: handle ctx This commit was moved from ipfs/go-ipfs-http-client@d451a4943c5311d7e32fcf86f56bdc1eda977fc7 --- client/httpapi/pubsub.go | 70 ++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index e27cb565557..edc1a9709ef 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -57,8 +57,10 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) } type pubsubSub struct { - io.Closer - dec *json.Decoder + messages chan pubsubMessage + + done chan struct{} + rcloser io.Closer } type pubsubMessage struct { @@ -68,6 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID + err error } func (msg *pubsubMessage) From() peer.ID { @@ -87,15 +90,20 @@ func (msg *pubsubMessage) Topics() []string { } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { - // TODO: handle ctx - - var msg pubsubMessage - if err := s.dec.Decode(&msg); err != nil { - return nil, err + select { + case msg, ok := <-s.messages: + if !ok { + return nil, io.EOF + } + if msg.err != nil { + return nil, msg.err + } + var err error + msg.from, err = peer.IDFromBytes(msg.JFrom) + return &msg, err + case <-ctx.Done(): + return nil, ctx.Err() } - var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { @@ -114,10 +122,44 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, resp.Error } - return &pubsubSub{ - Closer: resp, - dec: json.NewDecoder(resp.Output), - }, nil + sub := &pubsubSub{ + messages: make(chan pubsubMessage), + done: make(chan struct{}), + } + + dec := json.NewDecoder(resp.Output) + + go func() { + defer close(sub.messages) + + for { + var msg pubsubMessage + if err := dec.Decode(&msg); err != nil { + if err == io.EOF { + return + } + msg.err = err + } + + select { + case sub.messages <- msg: + case <-sub.done: + return + case <-ctx.Done(): + return + } + } + }() + + return sub, nil +} + +func (s*pubsubSub) Close() error { + if s.done != nil { + close(s.done) + s.done = nil + } + return s.rcloser.Close() } func (api *PubsubAPI) core() *HttpApi { From 7229dbbf73fd925da390fc5db5a25caf398b1fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:04:33 +0100 Subject: [PATCH 051/112] don't read all and then throw away the buffer This commit was moved from ipfs/go-ipfs-http-client@5c96c2954a5d45dd907dca9942d2773d76b8a71b --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 745c9a2a99d..b9e83eb3d49 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -139,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - ioutil.ReadAll(resp.Body) + io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } From b6ace8dd401321b842c5f0054d9ed025fd19cd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:12:37 +0100 Subject: [PATCH 052/112] response: handle late errors This commit was moved from ipfs/go-ipfs-http-client@139e9e5ff1b52a92c169d28bf56ff58e36b49ca3 --- client/httpapi/requestbuilder.go | 4 ++-- client/httpapi/response.go | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 628ad03cdf3..af43ce236ba 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -103,11 +103,11 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { } if res == nil { - httpRes.Close() + lateErr := httpRes.Close() if httpRes.Error != nil { return httpRes.Error } - return nil + return lateErr } return httpRes.Decode(res) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index b9e83eb3d49..f773130e83c 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -39,9 +39,14 @@ type Response struct { func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - //ioutil.ReadAll(r.Output) // TODO: might not be a good idea in case there is a lot of data - return r.Output.Close() + + // always drain output (response body) //TODO: make optional for things like cat + _, err1 := io.Copy(ioutil.Discard, r.Output) + err2 := r.Output.Close() + if err1 != nil { + return err1 + } + return err2 } return nil } From 7be8d01ee732be71f08f4482e0d5c5b61731ee81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:27:12 +0100 Subject: [PATCH 053/112] response: option to disable output draining This commit was moved from ipfs/go-ipfs-http-client@9f3d9635fa4e977b6ec202c9e4f78ca1c64fc0dd --- client/httpapi/api.go | 1 + client/httpapi/apifile.go | 2 +- client/httpapi/pubsub.go | 4 ++-- client/httpapi/request.go | 2 ++ client/httpapi/requestbuilder.go | 7 +++++++ client/httpapi/response.go | 10 ++++++++-- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e106d8d87a6..f74300216e3 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -131,6 +131,7 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, + drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 8647606844b..e3cb85ea460 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,7 +57,7 @@ func (f *apiFile) reset() error { if f.r != nil { f.r.Close() } - req := f.core.request("cat", f.path.String()) + req := f.core.request("cat", f.path.String()).NoDrain() if f.at != 0 { req.Option("offset", f.at) } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index edc1a9709ef..b3e45ed36c2 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -113,8 +113,8 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover). - Send(ctx) + Option("discover", options.Discover).NoDrain().Send(ctx) + if err != nil { return nil, err } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac6780..18cfb7fd0c5 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,6 +13,7 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string + DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -30,5 +31,6 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), + DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236ba..8b040522e2f 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,6 +19,7 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader + drainOut bool shell *HttpApi } @@ -84,6 +85,12 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } +// NoDrain disables output draining in response closer +func (r *RequestBuilder) NoDrain() *RequestBuilder { + r.drainOut = false + return r +} + // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index f773130e83c..cd3cc2b71e7 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -35,13 +35,18 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error + + drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) //TODO: make optional for things like cat - _, err1 := io.Copy(ioutil.Discard, r.Output) + // always drain output (response body) + var err1 error + if r.drainOutput { + _, err1 = io.Copy(ioutil.Discard, r.Output) + } err2 := r.Output.Close() if err1 != nil { return err1 @@ -114,6 +119,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) + nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ From 3088776f8150fe6967a245cf0360f5219e2dc9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:30:02 +0100 Subject: [PATCH 054/112] swarm: always append peer IDs This commit was moved from ipfs/go-ipfs-http-client@49267901c71f77f098deaf8966aa2dcecf22c2df --- client/httpapi/swarm.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0bb36aca3fa..13a814bc413 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -15,16 +15,14 @@ import ( type SwarmAPI HttpApi func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { + pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) + if err != nil { + return err + } + saddrs := make([]string, len(pi.Addrs)) for i, addr := range pi.Addrs { - if _, err := addr.ValueForProtocol(multiaddr.P_P2P); err == multiaddr.ErrProtocolNotFound { - pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) - if err != nil { - return err - } - addr = addr.Encapsulate(pidma) - } - saddrs[i] = addr.String() + saddrs[i] = addr.Encapsulate(pidma).String() } return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) From 745bf92506bd488bde4f695c4d644e7d3b58f0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Feb 2019 17:32:40 +0100 Subject: [PATCH 055/112] gofmt This commit was moved from ipfs/go-ipfs-http-client@f3c2c350861470c05617cf74e7b78d06782bb706 --- client/httpapi/api.go | 6 +++--- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f74300216e3..6dbfe3b6a81 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -128,9 +128,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, drainOut: true, } } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index b3e45ed36c2..49c58ab8822 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -59,7 +59,7 @@ func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) type pubsubSub struct { messages chan pubsubMessage - done chan struct{} + done chan struct{} rcloser io.Closer } @@ -70,7 +70,7 @@ type pubsubMessage struct { JTopicIDs []string `json:"topicIDs,omitempty"` from peer.ID - err error + err error } func (msg *pubsubMessage) From() peer.ID { @@ -124,7 +124,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), - done: make(chan struct{}), + done: make(chan struct{}), } dec := json.NewDecoder(resp.Output) @@ -154,7 +154,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return sub, nil } -func (s*pubsubSub) Close() error { +func (s *pubsubSub) Close() error { if s.done != nil { close(s.done) s.done = nil diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 18cfb7fd0c5..d65d638350e 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string DrainOut bool // if set, resp.Close will read all remaining data } @@ -26,11 +26,11 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 8b040522e2f..c5bc44124f2 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader drainOut bool shell *HttpApi From 27aa13fe44561e8d9469ce537e64baa60d17a705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 21:53:17 +0100 Subject: [PATCH 056/112] return errors from constructor methods This commit was moved from ipfs/go-ipfs-http-client@dbee4e27aaf9ef8e3fa7e25ac1f4f28df8a31e0d --- client/httpapi/api.go | 39 ++++++++++++++++---------------------- client/httpapi/api_test.go | 5 ++++- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6dbfe3b6a81..e4603bd380d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -29,8 +29,7 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } -//TODO: Return errors here -func NewLocalApi() iface.CoreAPI { +func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -39,40 +38,34 @@ func NewLocalApi() iface.CoreAPI { return NewPathApi(baseDir) } -func NewPathApi(p string) iface.CoreAPI { - a := ApiAddr(p) - if a == nil { - return nil +func NewPathApi(p string) (iface.CoreAPI, error) { + a, err := ApiAddr(p) + if err != nil { + if err == os.ErrNotExist { + err = nil + } + return nil, err } return NewApi(a) } -func ApiAddr(ipfspath string) ma.Multiaddr { +func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { - return nil + return nil, err } apiFile := path.Join(baseDir, DefaultApiFile) - if _, err := os.Stat(apiFile); err != nil { - return nil - } - api, err := ioutil.ReadFile(apiFile) if err != nil { - return nil - } - - maddr, err := ma.NewMultiaddr(strings.TrimSpace(string(api))) - if err != nil { - return nil + return nil, err } - return maddr + return ma.NewMultiaddr(strings.TrimSpace(string(api))) } -func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? +func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ Proxy: gohttp.ProxyFromEnvironment, @@ -83,10 +76,10 @@ func NewApi(a ma.Multiaddr) *HttpApi { // TODO: should be MAddr? return NewApiWithClient(a, c) } -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { +func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { - return nil // TODO: return that error + return nil, err } if a, err := ma.NewMultiaddr(url); err == nil { @@ -107,7 +100,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) *HttpApi { return fmt.Errorf("unexpected redirect") } - return api + return api, nil } func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index af2180384d6..2f6d47c58ed 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -157,7 +157,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) DisableCompression: true, }, } - apis[i] = NewApiWithClient(maddr, c) + apis[i], err = NewApiWithClient(maddr, c) + if err != nil { + panic(err) + } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") From fc299e7e849d46e364204234f9ea3d3aa52e42cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:19:56 +0100 Subject: [PATCH 057/112] pin verify: use temporary struct This commit was moved from ipfs/go-ipfs-http-client@e400fa3b380b589c2ae385a66a007d685147e4c7 --- client/httpapi/pin.go | 57 +++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index b11efd35c5d..f667a54875c 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -90,27 +90,20 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o } type pinVerifyRes struct { - Cid string - JOk bool `json:"Ok"` - JBadNodes []*badNode `json:"BadNodes,omitempty"` + ok bool + badNodes []iface.BadPinNode } func (r *pinVerifyRes) Ok() bool { - return r.JOk + return r.ok } func (r *pinVerifyRes) BadNodes() []iface.BadPinNode { - out := make([]iface.BadPinNode, len(r.JBadNodes)) - for i, n := range r.JBadNodes { - out[i] = n - } - return out + return r.badNodes } type badNode struct { - Cid string - JErr string `json:"Err"` - + err error cid cid.Cid } @@ -119,10 +112,7 @@ func (n *badNode) Path() iface.ResolvedPath { } func (n *badNode) Err() error { - if n.JErr != "" { - return errors.New(n.JErr) - } - return nil + return n.err } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { @@ -140,20 +130,45 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { defer close(res) dec := json.NewDecoder(resp.Output) for { - var out pinVerifyRes + var out struct { + Cid string + Ok bool + + BadNodes []struct{ + Cid string + Err string + } + } if err := dec.Decode(&out); err != nil { return // todo: handle non io.EOF somehow } - for i, n := range out.JBadNodes { - out.JBadNodes[i].cid, err = cid.Decode(n.Cid) + badNodes := make([]iface.BadPinNode, len(out.BadNodes)) + for i, n := range out.BadNodes { + c, err := cid.Decode(n.Cid) if err != nil { - return + badNodes[i] = &badNode{ + cid: c, + err: err, + } + continue + } + + if n.Err != "" { + err = errors.New(n.Err) + } + badNodes[i] = &badNode{ + cid: c, + err: err, } } select { - case res <- &out: + case res <- &pinVerifyRes{ + ok: out.Ok, + + badNodes: badNodes, + }: case <-ctx.Done(): return } From a058e7d31a32d1bca98e41886a310149b872af84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:27:48 +0100 Subject: [PATCH 058/112] response: Document zero-result Decode behaviour This commit was moved from ipfs/go-ipfs-http-client@902bc5ef8c1b9595e97728dca3da0d0da124f142 --- client/httpapi/response.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index cd3cc2b71e7..26f549c9af7 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -71,6 +71,9 @@ func (r *Response) Decode(dec interface{}) error { } n++ } + + // Decode expects at least one result. For calls where zero results are valid, + // use Send and construct json Decoder manually. if n > 0 && err == io.EOF { err = nil } From db1af499c9d661a70ac6fcce9cc7347b934385e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 22:33:57 +0100 Subject: [PATCH 059/112] use mime.ParseMediaType for Content-Type response parsing This commit was moved from ipfs/go-ipfs-http-client@217d1945237321c12f96abef82707217a30813b0 --- client/httpapi/response.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 26f549c9af7..708d2714fb7 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,14 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" + "mime" "net/http" "net/url" "os" - "strings" - - files "github.com/ipfs/go-ipfs-files" ) type trailerReader struct { @@ -116,9 +115,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType := resp.Header.Get("Content-Type") - parts := strings.Split(contentType, ";") - contentType = parts[0] + + contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return nil, err + } nresp := new(Response) From f34788e6ee61aad0ee1970a4f3d004bdebdbcd27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:11:08 +0100 Subject: [PATCH 060/112] cleanup Swarm.Peers This commit was moved from ipfs/go-ipfs-http-client@fd7858dc57b86761599b0e844f3536f2d4a0d953 --- client/httpapi/api.go | 1 - client/httpapi/apifile.go | 14 +++--- client/httpapi/pubsub.go | 9 ++-- client/httpapi/request.go | 2 - client/httpapi/requestbuilder.go | 7 --- client/httpapi/response.go | 19 +++++--- client/httpapi/swarm.go | 83 +++++++++++++++++--------------- 7 files changed, 69 insertions(+), 66 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index e4603bd380d..c5a706d0574 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -124,7 +124,6 @@ func (api *HttpApi) request(command string, args ...string) *RequestBuilder { command: command, args: args, shell: api, - drainOut: true, } } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e3cb85ea460..a8eb0de1af2 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -49,15 +49,15 @@ type apiFile struct { size int64 path iface.Path - r io.ReadCloser + r *Response at int64 } func (f *apiFile) reset() error { if f.r != nil { - f.r.Close() + f.r.Cancel() } - req := f.core.request("cat", f.path.String()).NoDrain() + req := f.core.request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -68,12 +68,12 @@ func (f *apiFile) reset() error { if resp.Error != nil { return resp.Error } - f.r = resp.Output + f.r = resp return nil } func (f *apiFile) Read(p []byte) (int, error) { - n, err := f.r.Read(p) + n, err := f.r.Output.Read(p) if n > 0 { f.at += int64(n) } @@ -92,7 +92,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r, offset-f.at) + r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err @@ -103,7 +103,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { func (f *apiFile) Close() error { if f.r != nil { - return f.r.Close() + return f.r.Cancel() } return nil } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 49c58ab8822..2ac04b53ce3 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -60,7 +60,7 @@ type pubsubSub struct { messages chan pubsubMessage done chan struct{} - rcloser io.Closer + rcloser func() error } type pubsubMessage struct { @@ -113,7 +113,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt } resp, err := api.core().request("pubsub/sub", topic). - Option("discover", options.Discover).NoDrain().Send(ctx) + Option("discover", options.Discover).Send(ctx) if err != nil { return nil, err @@ -125,6 +125,9 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt sub := &pubsubSub{ messages: make(chan pubsubMessage), done: make(chan struct{}), + rcloser: func() error { + return resp.Cancel() + }, } dec := json.NewDecoder(resp.Output) @@ -159,7 +162,7 @@ func (s *pubsubSub) Close() error { close(s.done) s.done = nil } - return s.rcloser.Close() + return s.rcloser() } func (api *PubsubAPI) core() *HttpApi { diff --git a/client/httpapi/request.go b/client/httpapi/request.go index d65d638350e..f6f7ee486ef 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -13,7 +13,6 @@ type Request struct { Opts map[string]string Body io.Reader Headers map[string]string - DrainOut bool // if set, resp.Close will read all remaining data } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -31,6 +30,5 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque Args: args, Opts: opts, Headers: make(map[string]string), - DrainOut: true, } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index c5bc44124f2..ba407217f21 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -19,7 +19,6 @@ type RequestBuilder struct { opts map[string]string headers map[string]string body io.Reader - drainOut bool shell *HttpApi } @@ -85,12 +84,6 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { return r } -// NoDrain disables output draining in response closer -func (r *RequestBuilder) NoDrain() *RequestBuilder { - r.drainOut = false - return r -} - // Send sends the request and return the response. func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 708d2714fb7..c9f178f9ce8 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -34,18 +34,13 @@ func (r *trailerReader) Close() error { type Response struct { Output io.ReadCloser Error *Error - - drainOutput bool } func (r *Response) Close() error { if r.Output != nil { - // always drain output (response body) - var err1 error - if r.drainOutput { - _, err1 = io.Copy(ioutil.Discard, r.Output) - } + // drain output (response body) + _, err1 := io.Copy(ioutil.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -55,6 +50,15 @@ func (r *Response) Close() error { return nil } +// Cancel aborts running request (without draining request body) +func (r *Response) Cancel() error { + if r.Output != nil { + return r.Output.Close() + } + + return nil +} + func (r *Response) Decode(dec interface{}) error { defer r.Close() if r.Error != nil { @@ -123,7 +127,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp := new(Response) - nresp.drainOutput = r.DrainOut nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { e := &Error{ diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 13a814bc413..8827119178a 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -32,74 +32,81 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) } -type streamInfo struct { - Protocol string -} - type connInfo struct { - Addr string - Peer string - JLatency time.Duration `json:"Latency"` - Muxer string - JDirection inet.Direction `json:"Direction"` - JStreams []streamInfo `json:"Streams"` -} - -func (c *connInfo) valid() error { - _, err := multiaddr.NewMultiaddr(c.Addr) - if err != nil { - return err - } - - _, err = peer.IDB58Decode(c.Peer) - return err + addr multiaddr.Multiaddr + peer peer.ID + latency time.Duration + muxer string + direction inet.Direction + streams []protocol.ID } func (c *connInfo) ID() peer.ID { - id, _ := peer.IDB58Decode(c.Peer) - return id + return c.peer } func (c *connInfo) Address() multiaddr.Multiaddr { - a, _ := multiaddr.NewMultiaddr(c.Addr) - return a + return c.addr } func (c *connInfo) Direction() inet.Direction { - return c.JDirection + return c.direction } func (c *connInfo) Latency() (time.Duration, error) { - return c.JLatency, nil + return c.latency, nil } func (c *connInfo) Streams() ([]protocol.ID, error) { - res := make([]protocol.ID, len(c.JStreams)) - for i, stream := range c.JStreams { - res[i] = protocol.ID(stream.Protocol) - } - return res, nil + return c.streams, nil } func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { - var out struct { - Peers []*connInfo + var resp struct { + Peers []struct{ + Addr string + Peer string + Latency time.Duration + Muxer string + Direction inet.Direction + Streams []struct { + Protocol string + } + } } err := api.core().request("swarm/peers"). Option("streams", true). Option("latency", true). - Exec(ctx, &out) + Exec(ctx, &resp) if err != nil { return nil, err } - res := make([]iface.ConnectionInfo, len(out.Peers)) - for i, conn := range out.Peers { - if err := conn.valid(); err != nil { + res := make([]iface.ConnectionInfo, len(resp.Peers)) + for i, conn := range resp.Peers { + out := &connInfo{ + latency: conn.Latency, + muxer: conn.Muxer, + direction: conn.Direction, + } + + out.peer, err = peer.IDB58Decode(conn.Peer) + if err != nil { return nil, err } - res[i] = conn + + out.addr, err = multiaddr.NewMultiaddr(conn.Addr) + if err != nil { + return nil, err + } + + out.streams = make([]protocol.ID, len(conn.Streams)) + for i, p := range conn.Streams { + out.streams[i] = protocol.ID(p.Protocol) + } + + res[i] = out } return res, nil From aa4d6e16f13bd753b77d0a8fc2174e41af89310e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:26:34 +0100 Subject: [PATCH 061/112] Add some docs to constructors This commit was moved from ipfs/go-ipfs-http-client@e34cd600e690bfa9bd20d6089b9335ac6090d4dd --- client/httpapi/api.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c5a706d0574..9e8d5518817 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -22,6 +22,11 @@ const ( EnvDir = "IPFS_PATH" ) +// HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using +// IPFS HTTP API. +// +// For interface docs see +// https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string httpcli gohttp.Client @@ -29,6 +34,11 @@ type HttpApi struct { applyGlobal func(*RequestBuilder) } +// NewLocalApi tries to construct new HttpApi instance communicating with local +// IPFS daemon +// +// Daemon api address is pulled from the $IPFS_PATH/api file. +// If $IPFS_PATH env var is not present, it defaults to ~/.ipfs func NewLocalApi() (iface.CoreAPI, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { @@ -38,8 +48,10 @@ func NewLocalApi() (iface.CoreAPI, error) { return NewPathApi(baseDir) } -func NewPathApi(p string) (iface.CoreAPI, error) { - a, err := ApiAddr(p) +// NewPathApi constructs new HttpApi by pulling api address from specified +// ipfspath. Api file should be located at $ipfspath/api +func NewPathApi(ipfspath string) (iface.CoreAPI, error) { + a, err := ApiAddr(ipfspath) if err != nil { if err == os.ErrNotExist { err = nil @@ -49,6 +61,7 @@ func NewPathApi(p string) (iface.CoreAPI, error) { return NewApi(a) } +// ApiAddr reads api file in specified ipfs path func ApiAddr(ipfspath string) (ma.Multiaddr, error) { baseDir, err := homedir.Expand(ipfspath) if err != nil { @@ -65,6 +78,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return ma.NewMultiaddr(strings.TrimSpace(string(api))) } +// NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { c := &gohttp.Client{ Transport: &gohttp.Transport{ @@ -76,6 +90,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { return NewApiWithClient(a, c) } +// NewApiWithClient constructs HttpApi with specified endpoint and custom http client func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { From f6a3a4fb2d715ffc0394408287b23eae0d322631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 19 Feb 2019 23:28:39 +0100 Subject: [PATCH 062/112] gofmt This commit was moved from ipfs/go-ipfs-http-client@65cd935e13a2ea05befbc69d72d64a89056952ea --- client/httpapi/api.go | 6 +++--- client/httpapi/pin.go | 4 ++-- client/httpapi/request.go | 22 +++++++++++----------- client/httpapi/requestbuilder.go | 10 +++++----- client/httpapi/response.go | 1 - client/httpapi/swarm.go | 18 +++++++++--------- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 9e8d5518817..b67fb0b483a 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -136,9 +136,9 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) func (api *HttpApi) request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ - command: command, - args: args, - shell: api, + command: command, + args: args, + shell: api, } } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f667a54875c..0111d626a09 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -132,9 +132,9 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { for { var out struct { Cid string - Ok bool + Ok bool - BadNodes []struct{ + BadNodes []struct { Cid string Err string } diff --git a/client/httpapi/request.go b/client/httpapi/request.go index f6f7ee486ef..58c61ac6780 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,12 +7,12 @@ import ( ) type Request struct { - ApiBase string - Command string - Args []string - Opts map[string]string - Body io.Reader - Headers map[string]string + ApiBase string + Command string + Args []string + Opts map[string]string + Body io.Reader + Headers map[string]string } func NewRequest(ctx context.Context, url, command string, args ...string) *Request { @@ -25,10 +25,10 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ - ApiBase: url + "/api/v0", - Command: command, - Args: args, - Opts: opts, - Headers: make(map[string]string), + ApiBase: url + "/api/v0", + Command: command, + Args: args, + Opts: opts, + Headers: make(map[string]string), } } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index ba407217f21..af43ce236ba 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -14,11 +14,11 @@ import ( // RequestBuilder is an IPFS commands request builder. type RequestBuilder struct { - command string - args []string - opts map[string]string - headers map[string]string - body io.Reader + command string + args []string + opts map[string]string + headers map[string]string + body io.Reader shell *HttpApi } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index c9f178f9ce8..84f28214dcf 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -119,7 +119,6 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } - contentType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil { return nil, err diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 8827119178a..0814debee3c 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -33,10 +33,10 @@ func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) e } type connInfo struct { - addr multiaddr.Multiaddr - peer peer.ID + addr multiaddr.Multiaddr + peer peer.ID latency time.Duration - muxer string + muxer string direction inet.Direction streams []protocol.ID } @@ -63,11 +63,11 @@ func (c *connInfo) Streams() ([]protocol.ID, error) { func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) { var resp struct { - Peers []struct{ - Addr string - Peer string + Peers []struct { + Addr string + Peer string Latency time.Duration - Muxer string + Muxer string Direction inet.Direction Streams []struct { Protocol string @@ -86,8 +86,8 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { out := &connInfo{ - latency: conn.Latency, - muxer: conn.Muxer, + latency: conn.Latency, + muxer: conn.Muxer, direction: conn.Direction, } From ad844e3d0b0221333c04896ed80f11afed301495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 16:50:38 +0100 Subject: [PATCH 063/112] response: simplify Decode This commit was moved from ipfs/go-ipfs-http-client@adbfda4c1c6ee037df3f91ca73f60de1f4620d67 --- client/httpapi/api.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 20 +++----------------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b67fb0b483a..17a289cc597 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -53,7 +53,7 @@ func NewLocalApi() (iface.CoreAPI, error) { func NewPathApi(ipfspath string) (iface.CoreAPI, error) { a, err := ApiAddr(ipfspath) if err != nil { - if err == os.ErrNotExist { + if os.IsNotExist(err) { err = nil } return nil, err diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index af43ce236ba..2ffed7a0ae7 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -110,5 +110,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return lateErr } - return httpRes.Decode(res) + return httpRes.decode(res) } diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 84f28214dcf..794cd14704f 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -59,28 +59,14 @@ func (r *Response) Cancel() error { return nil } -func (r *Response) Decode(dec interface{}) error { +// Decode reads request body and decodes it as json +func (r *Response) decode(dec interface{}) error { defer r.Close() if r.Error != nil { return r.Error } - n := 0 - var err error - for { - err = json.NewDecoder(r.Output).Decode(dec) - if err != nil { - break - } - n++ - } - - // Decode expects at least one result. For calls where zero results are valid, - // use Send and construct json Decoder manually. - if n > 0 && err == io.EOF { - err = nil - } - return err + return json.NewDecoder(r.Output).Decode(dec) } type Error struct { From 19d91fbb259220e91c8887ee8a09218865418720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Feb 2019 22:08:02 +0100 Subject: [PATCH 064/112] response: pass close error in decode This commit was moved from ipfs/go-ipfs-http-client@4a87232ecaa96246f038d15eb7ee6094216a201e --- client/httpapi/response.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 794cd14704f..624a488d7d7 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -61,12 +61,17 @@ func (r *Response) Cancel() error { // Decode reads request body and decodes it as json func (r *Response) decode(dec interface{}) error { - defer r.Close() if r.Error != nil { return r.Error } - return json.NewDecoder(r.Output).Decode(dec) + err := json.NewDecoder(r.Output).Decode(dec) + err2 := r.Close() + if err != nil { + return err + } + + return err2 } type Error struct { From cc9968d5dda71c21b3c90eb376458cacadb59e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:39:12 +0100 Subject: [PATCH 065/112] request: fix Content-Disposition header in Send This commit was moved from ipfs/go-ipfs-http-client@b7db17c63b69dc4fbb726dc720e9c594fa9ed495 --- client/httpapi/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 624a488d7d7..339c7365861 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -102,7 +102,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { if fr, ok := r.Body.(*files.MultiFileReader); ok { req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary()) - req.Header.Set("Content-Disposition", "form-data: name=\"files\"") + req.Header.Set("Content-Disposition", "form-data; name=\"files\"") } resp, err := c.Do(req) From 47b820150bddb9868274ed2c6faf79c6b8cbe369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Feb 2019 14:46:18 +0100 Subject: [PATCH 066/112] test: don't panic on errors in async node construction This commit was moved from ipfs/go-ipfs-http-client@95ce0f3949da47b2db92508b8c7b0a00f502682b --- client/httpapi/api_test.go | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2f6d47c58ed..df45c15af50 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -114,27 +114,32 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) wg.Add(len(nodes)) zero.Add(1) + errs := make(chan error, len(nodes)) for i, nd := range nodes { go func(i int, nd testbedi.Core) { defer wg.Done() if _, err := nd.Init(ctx, "--empty-repo"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.RunCmd(ctx, nil, "ipfs", "config", "--json", "Experimental.FilestoreEnabled", "true"); err != nil { - panic(err) + errs <- err + return } if _, err := nd.Start(ctx, true, "--enable-pubsub-experiment", "--offline="+strconv.FormatBool(n == 1)); err != nil { - panic(err) + errs <- err + return } if i > 0 { zero.Wait() if err := nd.Connect(ctx, nodes[0]); err != nil { - panic(err) + errs <- err + return } } else { zero.Done() @@ -142,12 +147,14 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) addr, err := nd.APIAddr() if err != nil { - panic(err) + errs <- err + return } maddr, err := ma.NewMultiaddr(addr) if err != nil { - panic(err) + errs <- err + return } c := &gohttp.Client{ @@ -159,16 +166,19 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } apis[i], err = NewApiWithClient(maddr, c) if err != nil { - panic(err) + errs <- err + return } // empty node is pinned even with --empty-repo, we don't want that emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err != nil { - panic(err) + errs <- err + return } if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { - panic(err) + errs <- err + return } }(i, nd) } @@ -187,7 +197,12 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) }() }() - return apis, nil + select { + case err = <-errs: + default: + } + + return apis, err } func TestHttpApi(t *testing.T) { From 0e800c3d8431fdcdc22ee2a2ac3337cb009553c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 18:15:37 +0000 Subject: [PATCH 067/112] update interface-go-ipfs-core and handle breaking changes * No more Hidden, StdinName, and Wrap options. * LsLink -> DirEntry with file types that don't expose internals. This commit was moved from ipfs/go-ipfs-http-client@5a7161eeab7127da4348dda6a6dc16f07b051f36 --- client/httpapi/apifile.go | 13 ++++----- client/httpapi/unixfs.go | 58 ++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index a8eb0de1af2..e0c0c9e1b97 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -8,8 +8,9 @@ import ( "io/ioutil" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + iface "github.com/ipfs/interface-go-ipfs-core" ) const forwardSeekLimit = 1 << 14 //16k @@ -174,17 +175,13 @@ func (it *apiIter) Next() bool { } switch it.cur.Type { - case iface.THAMTShard: - fallthrough - case iface.TMetadata: - fallthrough - case iface.TDirectory: + case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } - case iface.TFile: + case unixfs.TFile: it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 1f340b6573a..4f717b1291b 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,9 +8,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-files" - "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" + files "github.com/ipfs/go-ipfs-files" + unixfs "github.com/ipfs/go-unixfs" + unixfs_pb "github.com/ipfs/go-unixfs/pb" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" mh "github.com/multiformats/go-multihash" ) @@ -40,15 +41,12 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). Option("fscache", options.FsCache). - Option("hidden", options.Hidden). Option("inline", options.Inline). Option("inline-limit", options.InlineLimit). Option("nocopy", options.NoCopy). Option("only-hash", options.OnlyHash). Option("pin", options.Pin). Option("silent", options.Silent). - Option("stdin-name", options.StdinName). - Option("wrap-with-directory", options.Wrap). Option("progress", options.Progress) if options.RawLeavesSet { @@ -62,13 +60,8 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix req.Option("trickle", true) } - switch c := f.(type) { - case files.Directory: - req.Body(files.NewMultiFileReader(c, false)) - case files.File: - d := files.NewMapDirectory(map[string]files.Node{"": c}) // unwrapped on the other side - req.Body(files.NewMultiFileReader(d, false)) - } + d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side + req.Body(files.NewMultiFileReader(d, false)) var out addEvent resp, err := req.Send(ctx) @@ -127,7 +120,8 @@ loop: type lsLink struct { Name, Hash string Size uint64 - Type iface.FileType + Type unixfs_pb.Data_DataType + Target string } type lsObject struct { @@ -139,7 +133,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.LsLink, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err @@ -158,7 +152,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf } dec := json.NewDecoder(resp.Output) - out := make(chan iface.LsLink) + out := make(chan iface.DirEntry) go func() { defer resp.Close() @@ -171,7 +165,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return } select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return @@ -179,7 +173,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Objects len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Objects len")}: case <-ctx.Done(): } return @@ -187,7 +181,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf if len(link.Objects[0].Links) != 1 { select { - case out <- iface.LsLink{Err: errors.New("unexpected Links len")}: + case out <- iface.DirEntry{Err: errors.New("unexpected Links len")}: case <-ctx.Done(): } return @@ -198,21 +192,29 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf c, err := cid.Decode(l0.Hash) if err != nil { select { - case out <- iface.LsLink{Err: err}: + case out <- iface.DirEntry{Err: err}: case <-ctx.Done(): } return } + var ftype iface.FileType + switch l0.Type { + case unixfs.TRaw, unixfs.TFile: + ftype = iface.TFile + case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: + ftype = iface.TDirectory + case unixfs.TSymlink: + ftype = iface.TSymlink + } + select { - case out <- iface.LsLink{ - Link: &format.Link{ - Cid: c, - Name: l0.Name, - Size: l0.Size, - }, - Size: l0.Size, - Type: l0.Type, + case out <- iface.DirEntry{ + Name: l0.Name, + Cid: c, + Size: l0.Size, + Type: ftype, + Target: l0.Target, }: case <-ctx.Done(): } From e76ecd30384799dfe388671ebd153e88903d9388 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 16:54:05 +0000 Subject: [PATCH 068/112] nit: return concrete types This is generally considered the "better" way to do things in go. It also allows us to expose functions like `request` at some point without changing the API interface itself. This commit was moved from ipfs/go-ipfs-http-client@524e5d82e6bcd2c4956151a4811addad656d7bf0 --- client/httpapi/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 17a289cc597..ef6eff3e823 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -39,7 +39,7 @@ type HttpApi struct { // // Daemon api address is pulled from the $IPFS_PATH/api file. // If $IPFS_PATH env var is not present, it defaults to ~/.ipfs -func NewLocalApi() (iface.CoreAPI, error) { +func NewLocalApi() (*HttpApi, error) { baseDir := os.Getenv(EnvDir) if baseDir == "" { baseDir = DefaultPathRoot @@ -50,7 +50,7 @@ func NewLocalApi() (iface.CoreAPI, error) { // NewPathApi constructs new HttpApi by pulling api address from specified // ipfspath. Api file should be located at $ipfspath/api -func NewPathApi(ipfspath string) (iface.CoreAPI, error) { +func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { From c6f704b453cd72734ca03793cd370021f4a39db9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:02:04 +0000 Subject: [PATCH 069/112] return an error when the API file isn't found fixes #7 This commit was moved from ipfs/go-ipfs-http-client@897d1b1bcdbf9f6616991366e66ac55e18a99a9c --- client/httpapi/api.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ef6eff3e823..6c41ef3ab8f 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -1,6 +1,7 @@ package httpapi import ( + "errors" "fmt" "io/ioutil" gohttp "net/http" @@ -22,6 +23,9 @@ const ( EnvDir = "IPFS_PATH" ) +// ErrApiNotFound if we fail to find a running daemon. +var ErrApiNotFound = errors.New("ipfs api address could not be found") + // HttpApi implements github.com/ipfs/interface-go-ipfs-core/CoreAPI using // IPFS HTTP API. // @@ -54,7 +58,7 @@ func NewPathApi(ipfspath string) (*HttpApi, error) { a, err := ApiAddr(ipfspath) if err != nil { if os.IsNotExist(err) { - err = nil + err = ErrApiNotFound } return nil, err } From 40f40b5a6593af189b81f43e393316b381bd541d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:04:33 +0000 Subject: [PATCH 070/112] fix golang-ci lint nits This commit was moved from ipfs/go-ipfs-http-client@ccf20b0ef5f440aca754f94325fca7202275ba8b --- client/httpapi/apifile.go | 2 +- client/httpapi/response.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index e0c0c9e1b97..1fd751ede7b 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -56,7 +56,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { - f.r.Cancel() + _ = f.r.Cancel() } req := f.core.request("cat", f.path.String()) if f.at != 0 { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c7365861..ed303388607 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -147,8 +147,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() + _, _ = io.Copy(ioutil.Discard, resp.Body) + _ = resp.Body.Close() } return nresp, nil From c791d73355e33139e2face5fc96884c5444f96c7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 26 Mar 2019 17:05:02 +0000 Subject: [PATCH 071/112] avoid canceling multiple times This commit was moved from ipfs/go-ipfs-http-client@fe16408225d4c9b688528f67ffabe08198b53ab5 --- client/httpapi/apifile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 1fd751ede7b..9e6df95c24b 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -57,6 +57,7 @@ type apiFile struct { func (f *apiFile) reset() error { if f.r != nil { _ = f.r.Cancel() + f.r = nil } req := f.core.request("cat", f.path.String()) if f.at != 0 { From 1c07bed90465210c27dba3cda850913cf397b5c5 Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Mon, 1 Apr 2019 17:13:39 -0700 Subject: [PATCH 072/112] Export (*HttpApi).request to enable building custom requests This commit was moved from ipfs/go-ipfs-http-client@8c9ed7dbc284b4d112f6805914a43e63f234df78 --- client/httpapi/api.go | 2 +- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 8 ++++---- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 10 +++++----- client/httpapi/name.go | 6 +++--- client/httpapi/object.go | 18 +++++++++--------- client/httpapi/path.go | 2 +- client/httpapi/pin.go | 10 +++++----- client/httpapi/pubsub.go | 8 ++++---- client/httpapi/swarm.go | 12 ++++++------ client/httpapi/unixfs.go | 4 ++-- 12 files changed, 46 insertions(+), 46 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 6c41ef3ab8f..dad205fd09d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -138,7 +138,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { return &RequestBuilder{ command: command, args: args, diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 9e6df95c24b..6e3433a51bf 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -29,7 +29,7 @@ func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) Type string Size int64 // unixfs size } - err := api.core().request("files/stat", p.String()).Exec(ctx, &stat) + err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat) if err != nil { return nil, err } @@ -59,7 +59,7 @@ func (f *apiFile) reset() error { _ = f.r.Cancel() f.r = nil } - req := f.core.request("cat", f.path.String()) + req := f.core.Request("cat", f.path.String()) if f.at != 0 { req.Option("offset", f.at) } @@ -225,7 +225,7 @@ func (d *apiDir) Entries() files.DirIterator { } func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index fd4d9bab971..3389f4c317e 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -41,7 +41,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("block/put"). + req := api.core().Request("block/put"). Option("mhtype", mht). Option("mhlen", options.MhLength). Option("format", options.Codec). @@ -61,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP } func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("block/get", p.String()).Send(ctx) + resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR Error string `json:",omitempty"` }{} - req := api.core().request("block/rm"). + req := api.core().Request("block/rm"). Option("force", options.Force). Arguments(p.String()) @@ -107,7 +107,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { var out blockStat - err := api.core().request("block/stat", p.String()).Exec(ctx, &out) + err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { return nil, err } diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index dc7dd6beaf6..f7417cb7fc3 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -18,7 +18,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, Type notif.QueryEventType Responses []peerstore.PeerInfo } - resp, err := api.core().request("dht/findpeer", p.Pretty()).Send(ctx) + resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { return peerstore.PeerInfo{}, err } @@ -48,7 +48,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return nil, err } - resp, err := api.core().request("dht/findprovs", rp.Cid().String()). + resp, err := api.core().Request("dht/findprovs", rp.Cid().String()). Option("num-providers", options.NumProviders). Send(ctx) if err != nil { @@ -104,7 +104,7 @@ func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.Dht return err } - return api.core().request("dht/provide", rp.Cid().String()). + return api.core().Request("dht/provide", rp.Cid().String()). Option("recursive", options.Recursive). Exec(ctx, nil) } diff --git a/client/httpapi/key.go b/client/httpapi/key.go index a16c30d8ef8..1571a3350bf 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -38,7 +38,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key } var out keyOutput - err = api.core().request("key/gen", name). + err = api.core().Request("key/gen", name). Option("type", options.Algorithm). Option("size", options.Size). Exec(ctx, &out) @@ -61,7 +61,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o Id string Overwrite bool } - err = api.core().request("key/rename", oldName, newName). + err = api.core().Request("key/rename", oldName, newName). Option("force", options.Force). Exec(ctx, &out) if err != nil { @@ -75,7 +75,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { var out struct{ Keys []*keyOutput } - if err := api.core().request("key/list").Exec(ctx, &out); err != nil { + if err := api.core().Request("key/list").Exec(ctx, &out); err != nil { return nil, err } @@ -94,7 +94,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var id struct{ ID string } - if err := api.core().request("id").Exec(ctx, &id); err != nil { + if err := api.core().Request("id").Exec(ctx, &id); err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { var out struct{ Keys []keyOutput } - if err := api.core().request("key/rm", name).Exec(ctx, &out); err != nil { + if err := api.core().Request("key/rm", name).Exec(ctx, &out); err != nil { return nil, err } if len(out.Keys) != 1 { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b848aa81978..0641d1a22d6 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -34,7 +34,7 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na return nil, err } - req := api.core().request("name/publish", p.String()). + req := api.core().Request("name/publish", p.String()). Option("key", options.Key). Option("allow-offline", options.AllowOffline). Option("lifetime", options.ValidTime). @@ -63,7 +63,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). @@ -120,7 +120,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, fmt.Errorf("Name.Resolve: depth other than 1 or %d not supported", nsopts.DefaultDepthLimit) } - req := api.core().request("name/resolve", name). + req := api.core().Request("name/resolve", name). Option("nocache", !options.Cache). Option("recursive", ropts.Depth != 1). Option("dht-record-count", ropts.DhtRecordCount). diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 5a06f74d99a..a1ee486b32c 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -48,7 +48,7 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec } var out objectOut - err = api.core().request("object/put"). + err = api.core().Request("object/put"). Option("inputenc", options.InputEnc). Option("datafieldenc", options.DataType). Option("pin", options.Pin). @@ -80,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) } func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { - resp, err := api.core().request("object/data", p.String()).Send(ctx) + resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er Size uint64 } } - if err := api.core().request("object/links", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/links", p.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]*ipld.Link, len(out.Links)) @@ -135,7 +135,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat DataSize int CumulativeSize int } - if err := api.core().request("object/stat", p.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/stat", p.String()).Exec(ctx, &out); err != nil { return nil, err } @@ -161,7 +161,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, } var out objectOut - err = api.core().request("object/patch/add-link", base.String(), name, child.String()). + err = api.core().Request("object/patch/add-link", base.String(), name, child.String()). Option("create", options.Create). Exec(ctx, &out) if err != nil { @@ -178,7 +178,7 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/rm-link", base.String(), link). + err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) if err != nil { return nil, err @@ -194,7 +194,7 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/append-data", p.String()). + err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -211,7 +211,7 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { var out objectOut - err := api.core().request("object/patch/set-data", p.String()). + err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). Exec(ctx, &out) if err != nil { @@ -237,7 +237,7 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i var out struct { Changes []change } - if err := api.core().request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { + if err := api.core().Request("object/diff", a.String(), b.String()).Exec(ctx, &out); err != nil { return nil, err } res := make([]iface.ObjectChange, len(out.Changes)) diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 8c819121ad1..639ca2f8e20 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -24,7 +24,7 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res } } - if err := api.request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 0111d626a09..f779640f174 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -39,7 +39,7 @@ func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddO return err } - return api.core().request("pin/add", p.String()). + return api.core().Request("pin/add", p.String()). Option("recursive", options.Recursive).Exec(ctx, nil) } @@ -50,7 +50,7 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. } var out pinRefKeyList - err = api.core().request("pin/ls"). + err = api.core().Request("pin/ls"). Option("type", options.Type).Exec(ctx, &out) if err != nil { return nil, err @@ -74,7 +74,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt return err } - return api.core().request("pin/rm", p.String()). + return api.core().Request("pin/rm", p.String()). Option("recursive", options.Recursive). Exec(ctx, nil) } @@ -85,7 +85,7 @@ func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, o return err } - return api.core().request("pin/update"). + return api.core().Request("pin/update"). Option("unpin", options.Unpin).Exec(ctx, nil) } @@ -116,7 +116,7 @@ func (n *badNode) Err() error { } func (api *PinAPI) Verify(ctx context.Context) (<-chan iface.PinStatus, error) { - resp, err := api.core().request("pin/verify").Option("verbose", true).Send(ctx) + resp, err := api.core().Request("pin/verify").Option("verbose", true).Send(ctx) if err != nil { return nil, err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 2ac04b53ce3..380b933dc95 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -18,7 +18,7 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { Strings []string } - if err := api.core().request("pubsub/ls").Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } @@ -35,7 +35,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +51,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", topic). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -112,7 +112,7 @@ func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopt return nil, err } - resp, err := api.core().request("pubsub/sub", topic). + resp, err := api.core().Request("pubsub/sub", topic). Option("discover", options.Discover).Send(ctx) if err != nil { diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0814debee3c..0e47df2ab36 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -25,11 +25,11 @@ func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { saddrs[i] = addr.Encapsulate(pidma).String() } - return api.core().request("swarm/connect", saddrs...).Exec(ctx, nil) + return api.core().Request("swarm/connect", saddrs...).Exec(ctx, nil) } func (api *SwarmAPI) Disconnect(ctx context.Context, addr multiaddr.Multiaddr) error { - return api.core().request("swarm/disconnect", addr.String()).Exec(ctx, nil) + return api.core().Request("swarm/disconnect", addr.String()).Exec(ctx, nil) } type connInfo struct { @@ -75,7 +75,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) } } - err := api.core().request("swarm/peers"). + err := api.core().Request("swarm/peers"). Option("streams", true). Option("latency", true). Exec(ctx, &resp) @@ -116,7 +116,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu var out struct { Addrs map[string][]string } - if err := api.core().request("swarm/addrs").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs").Exec(ctx, &out); err != nil { return nil, err } res := map[peer.ID][]multiaddr.Multiaddr{} @@ -147,7 +147,7 @@ func (api *SwarmAPI) LocalAddrs(ctx context.Context) ([]multiaddr.Multiaddr, err Strings []string } - if err := api.core().request("swarm/addrs/local").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/local").Exec(ctx, &out); err != nil { return nil, err } @@ -167,7 +167,7 @@ func (api *SwarmAPI) ListenAddrs(ctx context.Context) ([]multiaddr.Multiaddr, er Strings []string } - if err := api.core().request("swarm/addrs/listen").Exec(ctx, &out); err != nil { + if err := api.core().Request("swarm/addrs/listen").Exec(ctx, &out); err != nil { return nil, err } diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 4f717b1291b..84609194bff 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -36,7 +36,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix return nil, fmt.Errorf("unknowm mhType %d", options.MhType) } - req := api.core().request("add"). + req := api.core().Request("add"). Option("hash", mht). Option("chunker", options.Chunker). Option("cid-version", options.CidVersion). @@ -139,7 +139,7 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.Unixf return nil, err } - resp, err := api.core().request("ls", p.String()). + resp, err := api.core().Request("ls", p.String()). Option("resolve-type", options.ResolveChildren). Option("size", options.ResolveChildren). Option("stream", true). From 70158edd5950671a26631788d15204350d174f52 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 4 Apr 2019 00:36:32 -0700 Subject: [PATCH 073/112] feed through context see https://github.com/ipfs/go-ipfs-api/pull/173 all credit to @optman This commit was moved from ipfs/go-ipfs-http-client@7b75a8c577988e4728474a02144bf5bf83260527 --- client/httpapi/request.go | 2 ++ client/httpapi/response.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client/httpapi/request.go b/client/httpapi/request.go index 58c61ac6780..dd5293b7a6e 100644 --- a/client/httpapi/request.go +++ b/client/httpapi/request.go @@ -7,6 +7,7 @@ import ( ) type Request struct { + Ctx context.Context ApiBase string Command string Args []string @@ -25,6 +26,7 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque "stream-channels": "true", } return &Request{ + Ctx: ctx, ApiBase: url + "/api/v0", Command: command, Args: args, diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 339c7365861..29ae38c69ee 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -95,6 +95,8 @@ func (r *Request) Send(c *http.Client) (*Response, error) { return nil, err } + req = req.WithContext(r.Ctx) + // Add any headers that were supplied via the RequestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) From 3b02ea46462d1f87a64e3fe23f73ee0f186b40d5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 9 Apr 2019 19:53:07 -0700 Subject: [PATCH 074/112] api: add authenticated transport, and direct connection This commit was moved from ipfs/go-ipfs-http-client@d04afa02b059d2c6a631f23a6923eb57ecb21015 --- client/httpapi/api.go | 36 ++++++++++++++++++++++++++++++++++++ client/httpapi/api_test.go | 25 +++++++++++++++++++++++-- client/httpapi/transport.go | 21 +++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index dad205fd09d..c0eacd7cbfd 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -122,6 +122,42 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } +// NewDirectAPIClient is used to instantiate a HttpApi client +// that connects to an endpoint which leverages additional http paths. +// +// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 +// you should use NewDirectAPIClient. +func NewDirectAPIClient(url string) (*HttpApi, error) { + api := &HttpApi{ + url: url, + httpcli: gohttp.Client{ + Transport: &gohttp.Transport{ + Proxy: gohttp.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }, + applyGlobal: func(*RequestBuilder) {}, + } + + // We don't support redirects. + api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + return fmt.Errorf("unexpected redirect") + } + + return api, nil +} + +// WithAuthorization is used to wrap an instance of HttpApi +// with an authenticated transport, such as JWT +func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { + return &HttpApi{ + url: api.url, + httpcli: gohttp.Client{ + Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), + }, + } +} + func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af50..f1b1bf26942 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -9,11 +9,11 @@ import ( "sync" "testing" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" - "github.com/ipfs/iptb/testbed/interfaces" + testbedi "github.com/ipfs/iptb/testbed/interfaces" ma "github.com/multiformats/go-multiaddr" ) @@ -211,3 +211,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } + +func TestDirectAPI(t *testing.T) { + type args struct { + url, header, value string + } + tests := []struct { + name string + args args + }{ + {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + api, err := NewDirectAPIClient(tt.args.url) + if err != nil { + t.Fatal(err) + } + api = api.WithAuthorization(tt.args.header, tt.args.value) + }) + } +} diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go new file mode 100644 index 00000000000..3fb0e6e00a2 --- /dev/null +++ b/client/httpapi/transport.go @@ -0,0 +1,21 @@ +package httpapi + +import "net/http" + +type transport struct { + header, value string + httptr http.RoundTripper +} + +func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { + return &transport{ + header: header, + value: value, + httptr: tr, + } +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set(t.header, t.value) + return t.httptr.RoundTrip(req) +} From 3b67615201cd38aceb6de61e87da1552be7689fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 26 Apr 2019 20:30:55 +0200 Subject: [PATCH 075/112] gomod: update iface with path refactor This commit was moved from ipfs/go-ipfs-http-client@0bd65a67ee2994966538fae6d034ddb9fac466e6 --- client/httpapi/api_test.go | 7 ++----- client/httpapi/apifile.go | 16 ++++++++-------- client/httpapi/block.go | 11 ++++++----- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 6 +++--- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 17 +++++++++-------- client/httpapi/object.go | 35 ++++++++++++++++++----------------- client/httpapi/path.go | 18 +++++++++--------- client/httpapi/pin.go | 17 +++++++++-------- client/httpapi/unixfs.go | 9 +++++---- 11 files changed, 75 insertions(+), 73 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index df45c15af50..b1bae5565e9 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" @@ -171,11 +172,7 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } // empty node is pinned even with --empty-repo, we don't want that - emptyNode, err := iface.ParsePath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - errs <- err - return - } + emptyNode := path.New("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err := apis[i].Pin().Rm(ctx, emptyNode); err != nil { errs <- err return diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 6e3433a51bf..afd9934e61f 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -10,12 +10,12 @@ import ( "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" unixfs "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k -func (api *UnixfsAPI) Get(ctx context.Context, p iface.Path) (files.Node, error) { +func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) { if p.Mutable() { // use resolved path in case we are dealing with IPNS / MFS var err error p, err = api.core().ResolvePath(ctx, p) @@ -48,7 +48,7 @@ type apiFile struct { ctx context.Context core *HttpApi size int64 - path iface.Path + path path.Path r *Response at int64 @@ -114,7 +114,7 @@ func (f *apiFile) Size() (int64, error) { return f.size, nil } -func (api *UnixfsAPI) getFile(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) { f := &apiFile{ ctx: ctx, core: api.core(), @@ -177,13 +177,13 @@ func (it *apiIter) Next() bool { switch it.cur.Type { case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory: - it.curFile, err = it.core.getDir(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getDir(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false } case unixfs.TFile: - it.curFile, err = it.core.getFile(it.ctx, iface.IpfsPath(c), int64(it.cur.Size)) + it.curFile, err = it.core.getFile(it.ctx, path.IpfsPath(c), int64(it.cur.Size)) if err != nil { it.err = err return false @@ -203,7 +203,7 @@ type apiDir struct { ctx context.Context core *UnixfsAPI size int64 - path iface.Path + path path.Path dec *json.Decoder } @@ -224,7 +224,7 @@ func (d *apiDir) Entries() files.DirIterator { } } -func (api *UnixfsAPI) getDir(ctx context.Context, p iface.Path, size int64) (files.Node, error) { +func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) { resp, err := api.core().Request("ls", p.String()). Option("resolve-size", true). Option("stream", true).Send(ctx) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 3389f4c317e..640f186f504 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -26,8 +27,8 @@ func (s *blockStat) Size() int { return s.BSize } -func (s *blockStat) Path() iface.ResolvedPath { - return iface.IpldPath(s.cid) +func (s *blockStat) Path() path.Resolved { + return path.IpldPath(s.cid) } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { @@ -60,7 +61,7 @@ func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockP return &out, nil } -func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("block/get", p.String()).Send(ctx) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *BlockAPI) Get(ctx context.Context, p iface.Path) (io.Reader, error) { return b, nil } -func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockRmOption) error { +func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRmOption) error { options, err := caopts.BlockRmOptions(opts...) if err != nil { return err @@ -105,7 +106,7 @@ func (api *BlockAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.BlockR return nil } -func (api *BlockAPI) Stat(ctx context.Context, p iface.Path) (iface.BlockStat, error) { +func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 669b5f893ad..15d9a9c62ed 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -9,8 +9,8 @@ import ( "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-ipld-format" - "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type httpNodeAdder HttpApi @@ -18,7 +18,7 @@ type HttpDagServ httpNodeAdder type pinningHttpNodeAdder httpNodeAdder func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) { - r, err := api.core().Block().Get(ctx, iface.IpldPath(c)) + r, err := api.core().Block().Get(ctx, path.IpldPath(c)) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (api *HttpDagServ) Pinning() format.NodeAdder { } func (api *HttpDagServ) Remove(ctx context.Context, c cid.Cid) error { - return api.core().Block().Rm(ctx, iface.IpldPath(c)) //TODO: should we force rm? + return api.core().Block().Rm(ctx, path.IpldPath(c)) //TODO: should we force rm? } func (api *HttpDagServ) RemoveMany(ctx context.Context, cids []cid.Cid) error { diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index f7417cb7fc3..e12caa1c534 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" notif "github.com/libp2p/go-libp2p-routing/notifications" @@ -37,7 +37,7 @@ func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, } } -func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -93,7 +93,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p iface.Path, opts ...caop return res, nil } -func (api *DhtAPI) Provide(ctx context.Context, p iface.Path, opts ...caopts.DhtProvideOption) error { +func (api *DhtAPI) Provide(ctx context.Context, p path.Path, opts ...caopts.DhtProvideOption) error { options, err := caopts.DhtProvideOptions(opts...) if err != nil { return err diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 1571a3350bf..9a47e2ed010 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-peer" ) @@ -22,9 +23,8 @@ func (k *keyOutput) Name() string { return k.JName } -func (k *keyOutput) Path() iface.Path { - p, _ := iface.ParsePath("/ipns/" + k.Id) - return p +func (k *keyOutput) Path() path.Path { + return path.New("/ipns/" + k.Id) } func (k *keyOutput) ID() peer.ID { diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 0641d1a22d6..47227a2ab89 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/options/namesys" + "github.com/ipfs/interface-go-ipfs-core/path" ) type NameAPI HttpApi @@ -17,18 +18,18 @@ type ipnsEntry struct { JName string `json:"Name"` JValue string `json:"Value"` - path iface.Path + path path.Path } func (e *ipnsEntry) Name() string { return e.JName } -func (e *ipnsEntry) Value() iface.Path { +func (e *ipnsEntry) Value() path.Path { return e.path } -func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { +func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (iface.IpnsEntry, error) { options, err := caopts.NamePublishOptions(opts...) if err != nil { return nil, err @@ -48,8 +49,8 @@ func (api *NameAPI) Publish(ctx context.Context, p iface.Path, opts ...caopts.Na if err := req.Exec(ctx, &out); err != nil { return nil, err } - out.path, err = iface.ParsePath(out.JValue) - return &out, err + out.path = path.New(out.JValue) + return &out, out.path.IsValid() } func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.NameResolveOption) (<-chan iface.IpnsResult, error) { @@ -93,7 +94,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name } var ires iface.IpnsResult if err == nil { - ires.Path, err = iface.ParsePath(out.Path) + ires.Path = path.New(out.Path) } select { @@ -109,7 +110,7 @@ func (api *NameAPI) Search(ctx context.Context, name string, opts ...caopts.Name return res, nil } -func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (iface.Path, error) { +func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.NameResolveOption) (path.Path, error) { options, err := caopts.NameResolveOptions(opts...) if err != nil { return nil, err @@ -131,7 +132,7 @@ func (api *NameAPI) Resolve(ctx context.Context, name string, opts ...caopts.Nam return nil, err } - return iface.ParsePath(out.Path) + return path.New(out.Path), nil } func (api *NameAPI) core() *HttpApi { diff --git a/client/httpapi/object.go b/client/httpapi/object.go index a1ee486b32c..280b2490f93 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -14,6 +14,7 @@ import ( ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi @@ -41,7 +42,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( return n, nil } -func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.ObjectPutOption) (path.Resolved, error) { options, err := caopts.ObjectPutOptions(opts...) if err != nil { return nil, err @@ -63,10 +64,10 @@ func (api *ObjectAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.Objec return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { r, err := api.core().Block().Get(ctx, p) if err != nil { return nil, err @@ -79,7 +80,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p iface.Path) (ipld.Node, error) return merkledag.DecodeProtobuf(b) } -func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) { +func (api *ObjectAPI) Data(ctx context.Context, p path.Path) (io.Reader, error) { resp, err := api.core().Request("object/data", p.String()).Send(ctx) if err != nil { return nil, err @@ -98,7 +99,7 @@ func (api *ObjectAPI) Data(ctx context.Context, p iface.Path) (io.Reader, error) return b, nil } -func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, error) { +func (api *ObjectAPI) Links(ctx context.Context, p path.Path) ([]*ipld.Link, error) { var out struct { Links []struct { Name string @@ -126,7 +127,7 @@ func (api *ObjectAPI) Links(ctx context.Context, p iface.Path) ([]*ipld.Link, er return res, nil } -func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat, error) { +func (api *ObjectAPI) Stat(ctx context.Context, p path.Path) (*iface.ObjectStat, error) { var out struct { Hash string NumLinks int @@ -154,7 +155,7 @@ func (api *ObjectAPI) Stat(ctx context.Context, p iface.Path) (*iface.ObjectStat }, nil } -func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, child iface.Path, opts ...caopts.ObjectAddLinkOption) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...caopts.ObjectAddLinkOption) (path.Resolved, error) { options, err := caopts.ObjectAddLinkOptions(opts...) if err != nil { return nil, err @@ -173,10 +174,10 @@ func (api *ObjectAPI) AddLink(ctx context.Context, base iface.Path, name string, return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) (iface.ResolvedPath, error) { +func (api *ObjectAPI) RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/rm-link", base.String(), link). Exec(ctx, &out) @@ -189,10 +190,10 @@ func (api *ObjectAPI) RmLink(ctx context.Context, base iface.Path, link string) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) AppendData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/append-data", p.String()). FileBody(r). @@ -206,10 +207,10 @@ func (api *ObjectAPI) AppendData(ctx context.Context, p iface.Path, r io.Reader) return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } -func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (iface.ResolvedPath, error) { +func (api *ObjectAPI) SetData(ctx context.Context, p path.Path, r io.Reader) (path.Resolved, error) { var out objectOut err := api.core().Request("object/patch/set-data", p.String()). FileBody(r). @@ -223,7 +224,7 @@ func (api *ObjectAPI) SetData(ctx context.Context, p iface.Path, r io.Reader) (i return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type change struct { @@ -233,7 +234,7 @@ type change struct { After cid.Cid } -func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]iface.ObjectChange, error) { +func (api *ObjectAPI) Diff(ctx context.Context, a path.Path, b path.Path) ([]iface.ObjectChange, error) { var out struct { Changes []change } @@ -247,10 +248,10 @@ func (api *ObjectAPI) Diff(ctx context.Context, a iface.Path, b iface.Path) ([]i Path: ch.Path, } if ch.Before != cid.Undef { - res[i].Before = iface.IpfsPath(ch.Before) + res[i].Before = path.IpfsPath(ch.Before) } if ch.After != cid.Undef { - res[i].After = iface.IpfsPath(ch.After) + res[i].After = path.IpfsPath(ch.After) } } return res, nil diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 639ca2f8e20..619dd0e0d2c 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -6,10 +6,10 @@ import ( cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core" + "github.com/ipfs/interface-go-ipfs-core/path" ) -func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.ResolvedPath, error) { +func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { var out struct { Cid cid.Cid RemPath string @@ -18,31 +18,31 @@ func (api *HttpApi) ResolvePath(ctx context.Context, path iface.Path) (iface.Res //TODO: this is hacky, fixing https://github.com/ipfs/go-ipfs/issues/5703 would help var err error - if path.Namespace() == "ipns" { - if path, err = api.Name().Resolve(ctx, path.String()); err != nil { + if p.Namespace() == "ipns" { + if p, err = api.Name().Resolve(ctx, p.String()); err != nil { return nil, err } } - if err := api.Request("dag/resolve", path.String()).Exec(ctx, &out); err != nil { + if err := api.Request("dag/resolve", p.String()).Exec(ctx, &out); err != nil { return nil, err } // TODO: - ipath, err := ipfspath.FromSegments("/"+path.Namespace()+"/", out.Cid.String(), out.RemPath) + ipath, err := ipfspath.FromSegments("/"+p.Namespace()+"/", out.Cid.String(), out.RemPath) if err != nil { return nil, err } - root, err := cid.Parse(ipfspath.Path(path.String()).Segments()[1]) + root, err := cid.Parse(ipfspath.Path(p.String()).Segments()[1]) if err != nil { return nil, err } - return iface.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil + return path.NewResolvedPath(ipath, out.Cid, root, out.RemPath), nil } -func (api *HttpApi) ResolveNode(ctx context.Context, p iface.Path) (ipld.Node, error) { +func (api *HttpApi) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) { rp, err := api.ResolvePath(ctx, p) if err != nil { return nil, err diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index f779640f174..ec0fc91a8c2 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) @@ -21,11 +22,11 @@ type pinRefKeyList struct { } type pin struct { - path iface.ResolvedPath + path path.Resolved typ string } -func (p *pin) Path() iface.ResolvedPath { +func (p *pin) Path() path.Resolved { return p.path } @@ -33,7 +34,7 @@ func (p *pin) Type() string { return p.typ } -func (api *PinAPI) Add(ctx context.Context, p iface.Path, opts ...caopts.PinAddOption) error { +func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOption) error { options, err := caopts.PinAddOptions(opts...) if err != nil { return err @@ -62,13 +63,13 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. if err != nil { return nil, err } - pins = append(pins, &pin{typ: p.Type, path: iface.IpldPath(c)}) + pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) } return pins, nil } -func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOption) error { +func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { options, err := caopts.PinRmOptions(opts...) if err != nil { return err @@ -79,7 +80,7 @@ func (api *PinAPI) Rm(ctx context.Context, p iface.Path, opts ...caopts.PinRmOpt Exec(ctx, nil) } -func (api *PinAPI) Update(ctx context.Context, from iface.Path, to iface.Path, opts ...caopts.PinUpdateOption) error { +func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opts ...caopts.PinUpdateOption) error { options, err := caopts.PinUpdateOptions(opts...) if err != nil { return err @@ -107,8 +108,8 @@ type badNode struct { cid cid.Cid } -func (n *badNode) Path() iface.ResolvedPath { - return iface.IpldPath(n.cid) +func (n *badNode) Path() path.Resolved { + return path.IpldPath(n.cid) } func (n *badNode) Err() error { diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 84609194bff..5e27fb0367f 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -13,6 +13,7 @@ import ( unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) @@ -25,7 +26,7 @@ type addEvent struct { type UnixfsAPI HttpApi -func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (iface.ResolvedPath, error) { +func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.UnixfsAddOption) (path.Resolved, error) { options, _, err := caopts.UnixfsAddOptions(opts...) if err != nil { return nil, err @@ -98,7 +99,7 @@ loop: return nil, err } - ifevt.Path = iface.IpfsPath(c) + ifevt.Path = path.IpfsPath(c) } select { @@ -114,7 +115,7 @@ loop: return nil, err } - return iface.IpfsPath(c), nil + return path.IpfsPath(c), nil } type lsLink struct { @@ -133,7 +134,7 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p iface.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { options, err := caopts.UnixfsLsOptions(opts...) if err != nil { return nil, err From 11a990c7b12db59d9d03cbd9b28c2b8c75abfa26 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:13:59 -0700 Subject: [PATCH 076/112] api: expose headers via HttpApi, copy headers during request build This commit was moved from ipfs/go-ipfs-http-client@033befdef3f06a4a80fb9cf7c3afecc1a383700c --- client/httpapi/api.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index c0eacd7cbfd..712801d5323 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" gohttp "net/http" "os" "path" @@ -32,9 +33,9 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // For interface docs see // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { - url string - httpcli gohttp.Client - + url string + httpcli gohttp.Client + Headers http.Header applyGlobal func(*RequestBuilder) } @@ -175,10 +176,17 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { + var headers map[string]string + if api.Headers != nil { + for k := range api.Headers { + headers[k] = api.Headers.Get(k) + } + } return &RequestBuilder{ command: command, args: args, shell: api, + headers: headers, } } From d8c286bb15e628dbe7ddd2370c77ef02a5dc16b2 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:15 -0700 Subject: [PATCH 077/112] api: remove WithAuthorization, DirectAPI client, instantiate header map This commit was moved from ipfs/go-ipfs-http-client@9c7d495b03db4d130a8fd87217093beb33e6fdb7 --- client/httpapi/api.go | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 712801d5323..ffcc85f2b51 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -112,6 +112,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -123,20 +124,11 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return api, nil } -// NewDirectAPIClient is used to instantiate a HttpApi client -// that connects to an endpoint which leverages additional http paths. -// -// If you need to connect to a IPFS HTTP API located at https://foo.bar/baz/api/v0 -// you should use NewDirectAPIClient. -func NewDirectAPIClient(url string) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { api := &HttpApi{ - url: url, - httpcli: gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, - DisableKeepAlives: true, - }, - }, + url: url, + httpcli: *c, + Headers: make(map[string][]string), applyGlobal: func(*RequestBuilder) {}, } @@ -144,21 +136,9 @@ func NewDirectAPIClient(url string) (*HttpApi, error) { api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { return fmt.Errorf("unexpected redirect") } - return api, nil } -// WithAuthorization is used to wrap an instance of HttpApi -// with an authenticated transport, such as JWT -func (api *HttpApi) WithAuthorization(header, value string) *HttpApi { - return &HttpApi{ - url: api.url, - httpcli: gohttp.Client{ - Transport: newAuthenticatedTransport(api.httpcli.Transport, header, value), - }, - } -} - func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) { options, err := caopts.ApiOptions(opts...) if err != nil { @@ -176,7 +156,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { - var headers map[string]string + headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) From 89600e62ce160cf5f3aa2a752b3806a273d1d142 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 13:41:37 -0700 Subject: [PATCH 078/112] api: add wip api test This commit was moved from ipfs/go-ipfs-http-client@2b48dd36ad170149bd4c62a4194587bf7c704f7e --- client/httpapi/api_test.go | 62 ++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index f1b1bf26942..7b519ce7dd6 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -3,11 +3,13 @@ package httpapi import ( "context" "io/ioutil" + "net/http" gohttp "net/http" "os" "strconv" "sync" "testing" + "time" iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +214,51 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func TestDirectAPI(t *testing.T) { - type args struct { - url, header, value string +func Test_NewURLApiWithClient(t *testing.T) { + t.Skip() + var ( + url = "127.0.0.1:65501" + headerToTest = "Test-Header" + expectedHeaderValue = "thisisaheadertest" + ) + server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) + if err != nil { + t.Fatal(err) } - tests := []struct { - name string - args args - }{ - {"Success", args{"http://127.0.0.1:5001/foo/bar/api/v0", "Authorization", "Bearer token"}}, + defer server.Close() + go func() { + server.ListenAndServe() + }() + time.Sleep(time.Second * 2) + api, err := NewURLApiWithClient(url, &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DisableKeepAlives: true, + }, + }) + if err != nil { + t.Fatal(err) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - api, err := NewDirectAPIClient(tt.args.url) - if err != nil { - t.Fatal(err) - } - api = api.WithAuthorization(tt.args.header, tt.args.value) - }) + api.Headers.Set(headerToTest, expectedHeaderValue) + if _, err := api.Pin().Ls(context.Background()); err != nil { + t.Fatal(err) } } + +/// testHTTPServer spins up a test go http server +// used to check headers +func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { + r := http.NewServeMux() + r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }) + return &http.Server{ + Handler: r, + Addr: url, + }, nil +} From efe3b6054e8f62191e74e6664adec06c306c06aa Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:00:48 -0700 Subject: [PATCH 079/112] api: call NewURLApiWithClient from NewApiWithClient This commit was moved from ipfs/go-ipfs-http-client@aea6890f36001f15d24a4b373979bb1b9bf81483 --- client/httpapi/api.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ffcc85f2b51..eb99230eb1e 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -109,19 +109,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { } } - api := &HttpApi{ - url: url, - httpcli: *c, - Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, - } - - // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { - return fmt.Errorf("unexpected redirect") - } - - return api, nil + return NewURLApiWithClient(url, c) } func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { From fd5010bd2a852ea8fa3a2b08e199312fb3b71097 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 16:13:47 -0700 Subject: [PATCH 080/112] api: cleanup header and NewURLApiWithClient test This commit was moved from ipfs/go-ipfs-http-client@b8b55cb89a5dd830915f83e43e8afa7a01597905 --- client/httpapi/api_test.go | 46 +++++++++++++------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 52fa67d0e2c..fadd9663e9d 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -5,13 +5,14 @@ import ( "io/ioutil" "net/http" gohttp "net/http" + "net/http/httptest" "os" "strconv" "sync" "testing" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/interface-go-ipfs-core/tests" @@ -212,23 +213,24 @@ func TestHttpApi(t *testing.T) { tests.TestApi(newNodeProvider(ctx))(t) } -func Test_NewURLApiWithClient(t *testing.T) { - t.Skip() +func Test_NewURLApiWithClient_With_Headers(t *testing.T) { var ( - url = "127.0.0.1:65501" headerToTest = "Test-Header" expectedHeaderValue = "thisisaheadertest" ) - server, err := testHTTPServer(url, headerToTest, expectedHeaderValue) - if err != nil { - t.Fatal(err) - } - defer server.Close() - go func() { - server.ListenAndServe() - }() + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + val := r.Header.Get(headerToTest) + if val == expectedHeaderValue { + w.WriteHeader(400) + return + } + w.WriteHeader(200) + }), + ) + defer ts.Close() time.Sleep(time.Second * 2) - api, err := NewURLApiWithClient(url, &http.Client{ + api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, @@ -242,21 +244,3 @@ func Test_NewURLApiWithClient(t *testing.T) { t.Fatal(err) } } - -/// testHTTPServer spins up a test go http server -// used to check headers -func testHTTPServer(url, headerToTest, expectedHeaderValue string) (*http.Server, error) { - r := http.NewServeMux() - r.HandleFunc("/api/v0/pin/ls", func(w http.ResponseWriter, r *http.Request) { - val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { - w.WriteHeader(400) - return - } - w.WriteHeader(200) - }) - return &http.Server{ - Handler: r, - Addr: url, - }, nil -} From daf1b72809a3d1af819d7ff44aabcc6bc6471fe8 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 17:19:35 -0700 Subject: [PATCH 081/112] api: fix failing test This commit was moved from ipfs/go-ipfs-http-client@ea507216d821455c351440ba33775f564ea18ea8 --- client/httpapi/api_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index fadd9663e9d..5f012b47672 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "strconv" + "strings" "sync" "testing" "time" @@ -221,15 +222,14 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { ts := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { val := r.Header.Get(headerToTest) - if val == expectedHeaderValue { + if val != expectedHeaderValue { w.WriteHeader(400) return } - w.WriteHeader(200) + http.ServeContent(w, r, "", time.Now(), strings.NewReader("test")) }), ) defer ts.Close() - time.Sleep(time.Second * 2) api, err := NewURLApiWithClient(ts.URL, &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -240,7 +240,7 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } api.Headers.Set(headerToTest, expectedHeaderValue) - if _, err := api.Pin().Ls(context.Background()); err != nil { + if err := api.Pin().Rm(context.Background(), path.New("/ipfs/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv")); err != nil { t.Fatal(err) } } From 8b9e189169b51c46956a18e08f9dc7bc668401b4 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 May 2019 19:14:15 -0700 Subject: [PATCH 082/112] remove unused transport.go file This commit was moved from ipfs/go-ipfs-http-client@320130421f4727d72d68a3c6a5c0e633e1dabc38 --- client/httpapi/transport.go | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 client/httpapi/transport.go diff --git a/client/httpapi/transport.go b/client/httpapi/transport.go deleted file mode 100644 index 3fb0e6e00a2..00000000000 --- a/client/httpapi/transport.go +++ /dev/null @@ -1,21 +0,0 @@ -package httpapi - -import "net/http" - -type transport struct { - header, value string - httptr http.RoundTripper -} - -func newAuthenticatedTransport(tr http.RoundTripper, header, value string) *transport { - return &transport{ - header: header, - value: value, - httptr: tr, - } -} - -func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header.Set(t.header, t.value) - return t.httptr.RoundTrip(req) -} From 8d953a7359c488cde5299ca704594bd8e1349ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 10 May 2019 15:38:20 +0200 Subject: [PATCH 083/112] Use interface for RequestBuilder This commit was moved from ipfs/go-ipfs-http-client@0534b7ca8319329459c505a546f3025862a735f5 --- client/httpapi/api.go | 10 ++++----- client/httpapi/requestbuilder.go | 36 ++++++++++++++++++++++---------- client/httpapi/response.go | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index eb99230eb1e..5c2eb76d6d0 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -36,7 +36,7 @@ type HttpApi struct { url string httpcli gohttp.Client Headers http.Header - applyGlobal func(*RequestBuilder) + applyGlobal func(*requestBuilder) } // NewLocalApi tries to construct new HttpApi instance communicating with local @@ -117,7 +117,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { url: url, httpcli: *c, Headers: make(map[string][]string), - applyGlobal: func(*RequestBuilder) {}, + applyGlobal: func(*requestBuilder) {}, } // We don't support redirects. @@ -134,7 +134,7 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) } subApi := *api - subApi.applyGlobal = func(req *RequestBuilder) { + subApi.applyGlobal = func(req *requestBuilder) { if options.Offline { req.Option("offline", options.Offline) } @@ -143,14 +143,14 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error) return &subApi, nil } -func (api *HttpApi) Request(command string, args ...string) *RequestBuilder { +func (api *HttpApi) Request(command string, args ...string) RequestBuilder { headers := make(map[string]string) if api.Headers != nil { for k := range api.Headers { headers[k] = api.Headers.Get(k) } } - return &RequestBuilder{ + return &requestBuilder{ command: command, args: args, shell: api, diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 2ffed7a0ae7..7012a8935bf 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -12,8 +12,20 @@ import ( "github.com/ipfs/go-ipfs-files" ) -// RequestBuilder is an IPFS commands request builder. -type RequestBuilder struct { +type RequestBuilder interface { + Arguments(args ...string) RequestBuilder + BodyString(body string) RequestBuilder + BodyBytes(body []byte) RequestBuilder + Body(body io.Reader) RequestBuilder + FileBody(body io.Reader) RequestBuilder + Option(key string, value interface{}) RequestBuilder + Header(name, value string) RequestBuilder + Send(ctx context.Context) (*Response, error) + Exec(ctx context.Context, res interface{}) error +} + +// requestBuilder is an IPFS commands request builder. +type requestBuilder struct { command string args []string opts map[string]string @@ -24,29 +36,29 @@ type RequestBuilder struct { } // Arguments adds the arguments to the args. -func (r *RequestBuilder) Arguments(args ...string) *RequestBuilder { +func (r *requestBuilder) Arguments(args ...string) RequestBuilder { r.args = append(r.args, args...) return r } // BodyString sets the request body to the given string. -func (r *RequestBuilder) BodyString(body string) *RequestBuilder { +func (r *requestBuilder) BodyString(body string) RequestBuilder { return r.Body(strings.NewReader(body)) } // BodyBytes sets the request body to the given buffer. -func (r *RequestBuilder) BodyBytes(body []byte) *RequestBuilder { +func (r *requestBuilder) BodyBytes(body []byte) RequestBuilder { return r.Body(bytes.NewReader(body)) } // Body sets the request body to the given reader. -func (r *RequestBuilder) Body(body io.Reader) *RequestBuilder { +func (r *requestBuilder) Body(body io.Reader) RequestBuilder { r.body = body return r } // FileBody sets the request body to the given reader wrapped into multipartreader. -func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { +func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) @@ -55,7 +67,7 @@ func (r *RequestBuilder) FileBody(body io.Reader) *RequestBuilder { } // Option sets the given option. -func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { +func (r *requestBuilder) Option(key string, value interface{}) RequestBuilder { var s string switch v := value.(type) { case bool: @@ -76,7 +88,7 @@ func (r *RequestBuilder) Option(key string, value interface{}) *RequestBuilder { } // Header sets the given header. -func (r *RequestBuilder) Header(name, value string) *RequestBuilder { +func (r *requestBuilder) Header(name, value string) RequestBuilder { if r.headers == nil { r.headers = make(map[string]string, 1) } @@ -85,7 +97,7 @@ func (r *RequestBuilder) Header(name, value string) *RequestBuilder { } // Send sends the request and return the response. -func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { +func (r *requestBuilder) Send(ctx context.Context) (*Response, error) { r.shell.applyGlobal(r) req := NewRequest(ctx, r.shell.url, r.command, r.args...) @@ -96,7 +108,7 @@ func (r *RequestBuilder) Send(ctx context.Context) (*Response, error) { } // Exec sends the request a request and decodes the response. -func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { +func (r *requestBuilder) Exec(ctx context.Context, res interface{}) error { httpRes, err := r.Send(ctx) if err != nil { return err @@ -112,3 +124,5 @@ func (r *RequestBuilder) Exec(ctx context.Context, res interface{}) error { return httpRes.decode(res) } + +var _ RequestBuilder = &requestBuilder{} diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 2d6af49d2ca..db22a66faf4 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -97,7 +97,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { req = req.WithContext(r.Ctx) - // Add any headers that were supplied via the RequestBuilder. + // Add any headers that were supplied via the requestBuilder. for k, v := range r.Headers { req.Header.Add(k, v) } From 1b81f2ef10c40b5ba5a50ae545a62cc050c11813 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 11 Jun 2019 17:41:49 -0700 Subject: [PATCH 084/112] add extended error handling fixes #19 This commit was moved from ipfs/go-ipfs-http-client@8e3552ac1e6ea18ffd5f49918eef578de2b224fb --- client/httpapi/response.go | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index db22a66faf4..056b2d68d8b 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -13,6 +13,48 @@ import ( "os" ) +// Error codes adapted from go-ipfs-cmds. We should find a better solution. + +// ErrorType signfies a category of errors +type ErrorType uint + +// ErrorTypes convey what category of error ocurred +const ( + // ErrNormal is a normal error. The command failed for some reason that's not a bug. + ErrNormal ErrorType = iota + // ErrClient means the client made an invalid request. + ErrClient + // ErrImplementation means there's a bug in the implementation. + ErrImplementation + // ErrRateLimited is returned when the operation has been rate-limited. + ErrRateLimited + // ErrForbidden is returned when the client doesn't have permission to + // perform the requested operation. + ErrForbidden +) + +func (e ErrorType) Error() string { + return e.String() +} + +func (e ErrorType) String() string { + switch e { + case ErrNormal: + return "command failed" + case ErrClient: + return "invalid argument" + case ErrImplementation: + return "internal error" + case ErrRateLimited: + return "rate limited" + case ErrForbidden: + return "request forbidden" + default: + return "unknown error code" + } + +} + type trailerReader struct { resp *http.Response } @@ -77,7 +119,13 @@ func (r *Response) decode(dec interface{}) error { type Error struct { Command string Message string - Code int + Code ErrorType +} + +// Unwrap returns the base error (an ErrorType). Works with go 1.14 error +// helpers. +func (e *Error) Unwrap() error { + return e.Code } func (e *Error) Error() string { @@ -133,11 +181,23 @@ func (r *Request) Send(c *http.Client) (*Response, error) { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } e.Message = string(out) + + // set special status codes. + switch resp.StatusCode { + case http.StatusNotFound, http.StatusBadRequest: + e.Code = ErrClient + case http.StatusTooManyRequests: + e.Code = ErrRateLimited + case http.StatusForbidden: + e.Code = ErrForbidden + } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err) } default: + // This is a server-side bug (probably). + e.Code = ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 2612353d85385a595e889dfa247363a8a4ab02c3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 13 Jun 2019 12:00:58 -0700 Subject: [PATCH 085/112] use error from go-ipfs-cmds note: this drops the Command field This commit was moved from ipfs/go-ipfs-http-client@523a26f0a853a2d6779a21d7bad239bf2012833b --- client/httpapi/response.go | 79 +++++--------------------------------- 1 file changed, 10 insertions(+), 69 deletions(-) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 056b2d68d8b..95cbf13ec1b 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -4,56 +4,19 @@ import ( "encoding/json" "errors" "fmt" - "github.com/ipfs/go-ipfs-files" "io" "io/ioutil" "mime" "net/http" "net/url" "os" -) -// Error codes adapted from go-ipfs-cmds. We should find a better solution. - -// ErrorType signfies a category of errors -type ErrorType uint - -// ErrorTypes convey what category of error ocurred -const ( - // ErrNormal is a normal error. The command failed for some reason that's not a bug. - ErrNormal ErrorType = iota - // ErrClient means the client made an invalid request. - ErrClient - // ErrImplementation means there's a bug in the implementation. - ErrImplementation - // ErrRateLimited is returned when the operation has been rate-limited. - ErrRateLimited - // ErrForbidden is returned when the client doesn't have permission to - // perform the requested operation. - ErrForbidden + cmds "github.com/ipfs/go-ipfs-cmds" + cmdhttp "github.com/ipfs/go-ipfs-cmds/http" + files "github.com/ipfs/go-ipfs-files" ) -func (e ErrorType) Error() string { - return e.String() -} - -func (e ErrorType) String() string { - switch e { - case ErrNormal: - return "command failed" - case ErrClient: - return "invalid argument" - case ErrImplementation: - return "internal error" - case ErrRateLimited: - return "rate limited" - case ErrForbidden: - return "request forbidden" - default: - return "unknown error code" - } - -} +type Error = cmds.Error type trailerReader struct { resp *http.Response @@ -62,7 +25,7 @@ type trailerReader struct { func (r *trailerReader) Read(b []byte) (int, error) { n, err := r.resp.Body.Read(b) if err != nil { - if e := r.resp.Trailer.Get("X-Stream-Error"); e != "" { + if e := r.resp.Trailer.Get(cmdhttp.StreamErrHeader); e != "" { err = errors.New(e) } } @@ -116,26 +79,6 @@ func (r *Response) decode(dec interface{}) error { return err2 } -type Error struct { - Command string - Message string - Code ErrorType -} - -// Unwrap returns the base error (an ErrorType). Works with go 1.14 error -// helpers. -func (e *Error) Unwrap() error { - return e.Code -} - -func (e *Error) Error() string { - var out string - if e.Code != 0 { - out = fmt.Sprintf("%s%d: ", out, e.Code) - } - return out + e.Message -} - func (r *Request) Send(c *http.Client) (*Response, error) { url := r.getURL() req, err := http.NewRequest("POST", url, r.Body) @@ -169,9 +112,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = &trailerReader{resp} if resp.StatusCode >= http.StatusBadRequest { - e := &Error{ - Command: r.Command, - } + e := new(Error) switch { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" @@ -185,11 +126,11 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // set special status codes. switch resp.StatusCode { case http.StatusNotFound, http.StatusBadRequest: - e.Code = ErrClient + e.Code = cmds.ErrClient case http.StatusTooManyRequests: - e.Code = ErrRateLimited + e.Code = cmds.ErrRateLimited case http.StatusForbidden: - e.Code = ErrForbidden + e.Code = cmds.ErrForbidden } case contentType == "application/json": if err = json.NewDecoder(resp.Body).Decode(e); err != nil { @@ -197,7 +138,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { } default: // This is a server-side bug (probably). - e.Code = ErrImplementation + e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) out, err := ioutil.ReadAll(resp.Body) if err != nil { From 9345cfa7b83d3fb2fb470fa1e77e7af89087452b Mon Sep 17 00:00:00 2001 From: godcong Date: Sun, 21 Jul 2019 12:33:35 +0800 Subject: [PATCH 086/112] update to consolidated libp2p interface package (#21) and fix parsing of connection latencies This commit was moved from ipfs/go-ipfs-http-client@fd5cce4cbc9d95617ded33849c437a492f5e9d96 --- client/httpapi/dht.go | 31 +++++++++++++++---------------- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/swarm.go | 20 ++++++++++---------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index e12caa1c534..0a78514f973 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,38 +6,37 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - notif "github.com/libp2p/go-libp2p-routing/notifications" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/routing" ) type DhtAPI HttpApi -func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peerstore.PeerInfo, error) { +func (api *DhtAPI) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) { var out struct { - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } resp, err := api.core().Request("dht/findpeer", p.Pretty()).Send(ctx) if err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } if resp.Error != nil { - return peerstore.PeerInfo{}, resp.Error + return peer.AddrInfo{}, resp.Error } defer resp.Close() dec := json.NewDecoder(resp.Output) for { if err := dec.Decode(&out); err != nil { - return peerstore.PeerInfo{}, err + return peer.AddrInfo{}, err } - if out.Type == notif.FinalPeer { + if out.Type == routing.FinalPeer { return out.Responses[0], nil } } } -func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peerstore.PeerInfo, error) { +func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopts.DhtFindProvidersOption) (<-chan peer.AddrInfo, error) { options, err := caopts.DhtFindProvidersOptions(opts...) if err != nil { return nil, err @@ -57,7 +56,7 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt if resp.Error != nil { return nil, resp.Error } - res := make(chan peerstore.PeerInfo) + res := make(chan peer.AddrInfo) go func() { defer resp.Close() @@ -67,18 +66,18 @@ func (api *DhtAPI) FindProviders(ctx context.Context, p path.Path, opts ...caopt for { var out struct { Extra string - Type notif.QueryEventType - Responses []peerstore.PeerInfo + Type routing.QueryEventType + Responses []peer.AddrInfo } if err := dec.Decode(&out); err != nil { return // todo: handle this somehow } - if out.Type == notif.QueryError { + if out.Type == routing.QueryError { return // usually a 'not found' error // todo: handle other errors } - if out.Type == notif.Provider { + if out.Type == routing.Provider { for _, pi := range out.Responses { select { case res <- pi: diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 9a47e2ed010..cf22c61089c 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 380b933dc95..81ddb5211c3 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-core/peer" ) type PubsubAPI HttpApi diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 0e47df2ab36..10280c39f08 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,16 +5,15 @@ import ( "time" "github.com/ipfs/interface-go-ipfs-core" - inet "github.com/libp2p/go-libp2p-net" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - "github.com/libp2p/go-libp2p-protocol" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/multiformats/go-multiaddr" ) type SwarmAPI HttpApi -func (api *SwarmAPI) Connect(ctx context.Context, pi peerstore.PeerInfo) error { +func (api *SwarmAPI) Connect(ctx context.Context, pi peer.AddrInfo) error { pidma, err := multiaddr.NewComponent("p2p", pi.ID.Pretty()) if err != nil { return err @@ -37,7 +36,7 @@ type connInfo struct { peer peer.ID latency time.Duration muxer string - direction inet.Direction + direction network.Direction streams []protocol.ID } @@ -49,7 +48,7 @@ func (c *connInfo) Address() multiaddr.Multiaddr { return c.addr } -func (c *connInfo) Direction() inet.Direction { +func (c *connInfo) Direction() network.Direction { return c.direction } @@ -66,9 +65,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) Peers []struct { Addr string Peer string - Latency time.Duration + Latency string Muxer string - Direction inet.Direction + Direction network.Direction Streams []struct { Protocol string } @@ -85,8 +84,9 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) res := make([]iface.ConnectionInfo, len(resp.Peers)) for i, conn := range resp.Peers { + latency, _ := time.ParseDuration(conn.Latency) out := &connInfo{ - latency: conn.Latency, + latency: latency, muxer: conn.Muxer, direction: conn.Direction, } From be8f80237d4ba8ed8d2ef278180b2b3d446c60f0 Mon Sep 17 00:00:00 2001 From: godcong Date: Tue, 13 Aug 2019 07:16:47 +0800 Subject: [PATCH 087/112] fix path miss in windows/add a new api create function (#23) * fix path was not always correct in windows * add new api create function for if the port was different with api file. * Update api.go remove no need commit This commit was moved from ipfs/go-ipfs-http-client@8c87debb1233f2af7a0cd102a934b630d8a12147 --- client/httpapi/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 5c2eb76d6d0..ec16c4b2da8 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -7,12 +7,12 @@ import ( "net/http" gohttp "net/http" "os" - "path" + "path/filepath" "strings" iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - homedir "github.com/mitchellh/go-homedir" + "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) @@ -73,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { return nil, err } - apiFile := path.Join(baseDir, DefaultApiFile) + apiFile := filepath.Join(baseDir, DefaultApiFile) api, err := ioutil.ReadFile(apiFile) if err != nil { From b6b2ae355bf86c4859f234240b44b35656685ece Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 30 Aug 2019 18:44:24 -0700 Subject: [PATCH 088/112] doc: mark this package experimental instead of WIP (#30) We're at a point where this package works pretty well and users should consider using it. This commit was moved from ipfs/go-ipfs-http-client@3e8506bbecd9b77ae30d04c7b997d9d612654beb --- client/httpapi/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index dd09142e32b..e5ab1c61227 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -8,7 +8,10 @@ > IPFS CoreAPI implementation using HTTP API -This project is WIP, use https://github.com/ipfs/go-ipfs-api for now +This package is experimental and subject to change. If you need to depend on +something less likely to change, please use +[go-ipfs-api](https://github.com/ipfs/go-ipfs-api). If you'd like the latest and +greatest features, please use _this_ package. ## Documentation From d48f8fce64d6496a18b442c74021829554151823 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 10 Sep 2019 18:07:48 -0700 Subject: [PATCH 089/112] file: implement ReadAt This commit was moved from ipfs/go-ipfs-http-client@88a9b615205816ea20ea38fbee170b3256e51b94 --- client/httpapi/apifile.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index afd9934e61f..ec2f7588d59 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -82,6 +82,25 @@ func (f *apiFile) Read(p []byte) (int, error) { return n, err } +func (f *apiFile) ReadAt(p []byte, off int64) (int, error) { + // Always make a new request. This method should be parallel-safe. + resp, err := f.core.Request("cat", f.path.String()). + Option("offset", off).Option("length", len(p)).Send(f.ctx) + if err != nil { + return 0, err + } + if resp.Error != nil { + return 0, resp.Error + } + defer resp.Output.Close() + + n, err := io.ReadFull(resp.Output, p) + if err == io.ErrUnexpectedEOF { + err = io.EOF + } + return n, err +} + func (f *apiFile) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekEnd: From 8f9306304efdc60147eccbb49380b83e3c7cb4ea Mon Sep 17 00:00:00 2001 From: Joel Gustafson Date: Sat, 7 Dec 2019 01:08:48 -0500 Subject: [PATCH 090/112] Update pin.go This commit was moved from ipfs/go-ipfs-http-client@31b4e60d961ada25cb7b40317b18011b284436a7 --- client/httpapi/pin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index ec0fc91a8c2..6026c71b38a 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -86,7 +86,7 @@ func (api *PinAPI) Update(ctx context.Context, from path.Path, to path.Path, opt return err } - return api.core().Request("pin/update"). + return api.core().Request("pin/update", from.String(), to.String()). Option("unpin", options.Unpin).Exec(ctx, nil) } From d9004a9061070263dcc01976334836acb9493530 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sun, 3 May 2020 12:04:04 -0700 Subject: [PATCH 091/112] IDB58Decode -> Decode Allow Non-RSA keys to be decoded by the HTTP client This commit was moved from ipfs/go-ipfs-http-client@b0549d86231d69a933b0e4b771c350cf1bf5c27b --- client/httpapi/key.go | 12 ++++++------ client/httpapi/pubsub.go | 4 ++-- client/httpapi/swarm.go | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index cf22c61089c..78b67517c01 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/libp2p/go-libp2p-core/peer" @@ -45,7 +45,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key if err != nil { return nil, err } - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -69,7 +69,7 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o } id := &keyOutput{JName: out.Now, Id: out.Id} - id.pid, err = peer.IDB58Decode(id.Id) + id.pid, err = peer.Decode(id.Id) return id, out.Overwrite, err } @@ -82,7 +82,7 @@ func (api *KeyAPI) List(ctx context.Context) ([]iface.Key, error) { res := make([]iface.Key, len(out.Keys)) for i, k := range out.Keys { var err error - k.pid, err = peer.IDB58Decode(k.Id) + k.pid, err = peer.Decode(k.Id) if err != nil { return nil, err } @@ -100,7 +100,7 @@ func (api *KeyAPI) Self(ctx context.Context) (iface.Key, error) { var err error out := keyOutput{JName: "self", Id: id.ID} - out.pid, err = peer.IDB58Decode(out.Id) + out.pid, err = peer.Decode(out.Id) return &out, err } @@ -114,7 +114,7 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (iface.Key, error) { } var err error - out.Keys[0].pid, err = peer.IDB58Decode(out.Keys[0].Id) + out.Keys[0].pid, err = peer.Decode(out.Keys[0].Id) return &out.Keys[0], err } diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 81ddb5211c3..da7c59ef1d2 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,7 +6,7 @@ import ( "encoding/json" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" ) @@ -41,7 +41,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio res := make([]peer.ID, len(out.Strings)) for i, sid := range out.Strings { - id, err := peer.IDB58Decode(sid) + id, err := peer.Decode(sid) if err != nil { return nil, err } diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 10280c39f08..1cb0d91df7c 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" @@ -91,7 +91,7 @@ func (api *SwarmAPI) Peers(ctx context.Context) ([]iface.ConnectionInfo, error) direction: conn.Direction, } - out.peer, err = peer.IDB58Decode(conn.Peer) + out.peer, err = peer.Decode(conn.Peer) if err != nil { return nil, err } @@ -131,7 +131,7 @@ func (api *SwarmAPI) KnownAddrs(ctx context.Context) (map[peer.ID][]multiaddr.Mu addrs[i] = a } - pid, err := peer.IDB58Decode(spid) + pid, err := peer.Decode(spid) if err != nil { return nil, err } From 7ebef96ba12bfdfc550412748ee16eadcaffd25d Mon Sep 17 00:00:00 2001 From: Bryan Stenson Date: Tue, 12 May 2020 20:09:16 +0000 Subject: [PATCH 092/112] update contributing link This commit was moved from ipfs/go-ipfs-http-client@ef2a556e19e8d9ebafcb24b3574724c6d7718bc8 --- client/httpapi/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index e5ab1c61227..376e6590e85 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -25,7 +25,7 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ### Want to hack on IPFS? -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) ## License From 247cb05fbc4fa3863542ecd9e0e5109c927790c6 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Fri, 31 Jul 2020 10:20:59 -0700 Subject: [PATCH 093/112] Bump interface-go-ipfs-core version (#117) * fix issues #26 Implement an IsPinned function * fix issues #113 Implement interface v0.3.0 * ci: update to use go-ipfs v0.6.0 * ci: bump go version Co-authored-by: godcong Co-authored-by: Adin Schmahmann This commit was moved from ipfs/go-ipfs-http-client@4e8a6c8b981f808062caa4ef7481babc0a39f298 --- client/httpapi/pin.go | 54 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 6026c71b38a..bd3e5423bc3 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -3,9 +3,10 @@ package httpapi import ( "context" "encoding/json" + "strings" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" @@ -24,6 +25,11 @@ type pinRefKeyList struct { type pin struct { path path.Resolved typ string + err error +} + +func (p *pin) Err() error { + return p.err } func (p *pin) Path() path.Resolved { @@ -44,7 +50,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp Option("recursive", options.Recursive).Exec(ctx, nil) } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan iface.Pin, error) { options, err := caopts.PinLsOptions(opts...) if err != nil { return nil, err @@ -57,16 +63,46 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]iface. return nil, err } - pins := make([]iface.Pin, 0, len(out.Keys)) - for hash, p := range out.Keys { - c, err := cid.Parse(hash) - if err != nil { - return nil, err + pins := make(chan iface.Pin) + go func(ch chan<- iface.Pin) { + defer close(ch) + for hash, p := range out.Keys { + c, e := cid.Parse(hash) + if e != nil { + ch <- &pin{typ: p.Type, err: e} + return + } + ch <- &pin{typ: p.Type, path: path.IpldPath(c), err: e} } - pins = append(pins, &pin{typ: p.Type, path: path.IpldPath(c)}) + }(pins) + return pins, nil +} + +// IsPinned returns whether or not the given cid is pinned +// and an explanation of why its pinned +func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { + options, err := caopts.PinIsPinnedOptions(opts...) + if err != nil { + return "", false, err + } + var out pinRefKeyList + err = api.core().Request("pin/ls"). + Option("type", options.WithType). + Option("arg", p.String()). + Exec(ctx, &out) + if err != nil { + // TODO: This error-type discrimination based on sub-string matching is brittle. + // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 + if strings.Index(err.Error(), "is not pinned") != -1 { + return "", false, nil + } + return "", false, err } - return pins, nil + for _, obj := range out.Keys { + return obj.Type, true, nil + } + return "", false, errors.New("http api returned no error and no results") } func (api *PinAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.PinRmOption) error { From ea2a35686db523b544245cbaea5592021d0cd163 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 15 May 2021 18:21:24 -0700 Subject: [PATCH 094/112] fix staticcheck This commit was moved from ipfs/go-ipfs-http-client@4461a0bb952ce0a05a5bab9728cc64b658375068 --- client/httpapi/api.go | 17 ++++++++--------- client/httpapi/api_test.go | 7 +++---- client/httpapi/object.go | 3 +-- client/httpapi/pin.go | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index ec16c4b2da8..f589e4267a4 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "net/http" - gohttp "net/http" "os" "path/filepath" "strings" @@ -14,7 +13,7 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) const ( @@ -34,7 +33,7 @@ var ErrApiNotFound = errors.New("ipfs api address could not be found") // https://godoc.org/github.com/ipfs/interface-go-ipfs-core#CoreAPI type HttpApi struct { url string - httpcli gohttp.Client + httpcli http.Client Headers http.Header applyGlobal func(*requestBuilder) } @@ -85,9 +84,9 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { // NewApi constructs HttpApi with specified endpoint func NewApi(a ma.Multiaddr) (*HttpApi, error) { - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, }, } @@ -96,7 +95,7 @@ func NewApi(a ma.Multiaddr) (*HttpApi, error) { } // NewApiWithClient constructs HttpApi with specified endpoint and custom http client -func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { +func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { _, url, err := manet.DialArgs(a) if err != nil { return nil, err @@ -112,7 +111,7 @@ func NewApiWithClient(a ma.Multiaddr, c *gohttp.Client) (*HttpApi, error) { return NewURLApiWithClient(url, c) } -func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { +func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { api := &HttpApi{ url: url, httpcli: *c, @@ -121,7 +120,7 @@ func NewURLApiWithClient(url string, c *gohttp.Client) (*HttpApi, error) { } // We don't support redirects. - api.httpcli.CheckRedirect = func(_ *gohttp.Request, _ []*gohttp.Request) error { + api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("unexpected redirect") } return api, nil diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5f012b47672..35a57d2c089 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -4,7 +4,6 @@ import ( "context" "io/ioutil" "net/http" - gohttp "net/http" "net/http/httptest" "os" "strconv" @@ -163,9 +162,9 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) return } - c := &gohttp.Client{ - Transport: &gohttp.Transport{ - Proxy: gohttp.ProxyFromEnvironment, + c := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, DisableKeepAlives: true, DisableCompression: true, }, diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 280b2490f93..6ec7f5503cd 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -10,7 +10,6 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" - dag "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" @@ -32,7 +31,7 @@ func (api *ObjectAPI) New(ctx context.Context, opts ...caopts.ObjectNewOption) ( var n ipld.Node switch options.Type { case "empty": - n = new(dag.ProtoNode) + n = new(merkledag.ProtoNode) case "unixfs-dir": n = ft.EmptyDirNode() default: diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index bd3e5423bc3..13de2d3893d 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -93,7 +93,7 @@ func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.Pin if err != nil { // TODO: This error-type discrimination based on sub-string matching is brittle. // It is addressed by this open issue: https://github.com/ipfs/go-ipfs/issues/7563 - if strings.Index(err.Error(), "is not pinned") != -1 { + if strings.Contains(err.Error(), "is not pinned") { return "", false, nil } return "", false, err From 5322cf89e45197afb74cc36104ee581ca179c1d6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 29 Nov 2021 22:24:27 +0100 Subject: [PATCH 095/112] feat: pubsub http rpc with multibase (#151) * feat: pubsub http rpc with multibase This updates HTTP RPC wire format to one from https://github.com/ipfs/go-ipfs/pull/8183 * chore: use updated go-ipfs * chore: switch ci to go-ipfs master This commit was moved from ipfs/go-ipfs-http-client@c832fc0ce1d8235c830fecd0dd44dda04a9fba37 --- client/httpapi/README.md | 6 ++-- client/httpapi/pubsub.go | 78 +++++++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index 376e6590e85..d3e14ac3f1b 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -1,8 +1,8 @@ # go-ipfs-http-api -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](https://protocol.ai) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.io/) +[![](https://img.shields.io/badge/matrix-%23ipfs-blue.svg?style=flat-square)](https://app.element.io/#/room/#ipfs:matrix.org) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-http-api?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-http-api) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index da7c59ef1d2..72f5923761f 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -9,6 +9,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/libp2p/go-libp2p-core/peer" + mbase "github.com/multiformats/go-multibase" ) type PubsubAPI HttpApi @@ -21,8 +22,15 @@ func (api *PubsubAPI) Ls(ctx context.Context) ([]string, error) { if err := api.core().Request("pubsub/ls").Exec(ctx, &out); err != nil { return nil, err } - - return out.Strings, nil + topics := make([]string, len(out.Strings)) + for n, mb := range out.Strings { + _, topic, err := mbase.Decode(mb) + if err != nil { + return nil, err + } + topics[n] = string(topic) + } + return topics, nil } func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOption) ([]peer.ID, error) { @@ -35,7 +43,11 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio Strings []string } - if err := api.core().Request("pubsub/peers", options.Topic).Exec(ctx, &out); err != nil { + var optionalTopic string + if len(options.Topic) > 0 { + optionalTopic = toMultibase([]byte(options.Topic)) + } + if err := api.core().Request("pubsub/peers", optionalTopic).Exec(ctx, &out); err != nil { return nil, err } @@ -51,7 +63,7 @@ func (api *PubsubAPI) Peers(ctx context.Context, opts ...caopts.PubSubPeersOptio } func (api *PubsubAPI) Publish(ctx context.Context, topic string, message []byte) error { - return api.core().Request("pubsub/pub", topic). + return api.core().Request("pubsub/pub", toMultibase([]byte(topic))). FileBody(bytes.NewReader(message)). Exec(ctx, nil) } @@ -64,13 +76,18 @@ type pubsubSub struct { } type pubsubMessage struct { - JFrom []byte `json:"from,omitempty"` - JData []byte `json:"data,omitempty"` - JSeqno []byte `json:"seqno,omitempty"` + JFrom string `json:"from,omitempty"` + JData string `json:"data,omitempty"` + JSeqno string `json:"seqno,omitempty"` JTopicIDs []string `json:"topicIDs,omitempty"` - from peer.ID - err error + // real values after unpacking from text/multibase envelopes + from peer.ID + data []byte + seqno []byte + topics []string + + err error } func (msg *pubsubMessage) From() peer.ID { @@ -78,15 +95,17 @@ func (msg *pubsubMessage) From() peer.ID { } func (msg *pubsubMessage) Data() []byte { - return msg.JData + return msg.data } func (msg *pubsubMessage) Seq() []byte { - return msg.JSeqno + return msg.seqno } +// TODO: do we want to keep this interface as []string, +// or change to more correct [][]byte? func (msg *pubsubMessage) Topics() []string { - return msg.JTopicIDs + return msg.topics } func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { @@ -98,22 +117,41 @@ func (s *pubsubSub) Next(ctx context.Context) (iface.PubSubMessage, error) { if msg.err != nil { return nil, msg.err } + // unpack values from text/multibase envelopes var err error - msg.from, err = peer.IDFromBytes(msg.JFrom) - return &msg, err + msg.from, err = peer.Decode(msg.JFrom) + if err != nil { + return nil, err + } + _, msg.data, err = mbase.Decode(msg.JData) + if err != nil { + return nil, err + } + _, msg.seqno, err = mbase.Decode(msg.JSeqno) + if err != nil { + return nil, err + } + for _, mbt := range msg.JTopicIDs { + _, topic, err := mbase.Decode(mbt) + if err != nil { + return nil, err + } + msg.topics = append(msg.topics, string(topic)) + } + return &msg, nil case <-ctx.Done(): return nil, ctx.Err() } } func (api *PubsubAPI) Subscribe(ctx context.Context, topic string, opts ...caopts.PubSubSubscribeOption) (iface.PubSubSubscription, error) { + /* right now we have no options (discover got deprecated) options, err := caopts.PubSubSubscribeOptions(opts...) if err != nil { return nil, err } - - resp, err := api.core().Request("pubsub/sub", topic). - Option("discover", options.Discover).Send(ctx) + */ + resp, err := api.core().Request("pubsub/sub", toMultibase([]byte(topic))).Send(ctx) if err != nil { return nil, err @@ -168,3 +206,9 @@ func (s *pubsubSub) Close() error { func (api *PubsubAPI) core() *HttpApi { return (*HttpApi)(api) } + +// Encodes bytes into URL-safe multibase that can be sent over HTTP RPC (URL or body) +func toMultibase(data []byte) string { + mb, _ := mbase.Encode(mbase.Base64url, data) + return mb +} From b52c80ca4da5730f47674946025c9490fcd50e1f Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 14 Jan 2022 16:16:26 +0100 Subject: [PATCH 096/112] skip TestHttpApi on Windows This commit was moved from ipfs/go-ipfs-http-client@a41c7956c0da2a4c51e15794b22dea09395cc7ef --- client/httpapi/api_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 35a57d2c089..d5f27f0ec9d 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "os" + "runtime" "strconv" "strings" "sync" @@ -207,6 +208,10 @@ func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) } func TestHttpApi(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping due to #142") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() From b85aa9639f861d659b49e0396b409398462226a2 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 27 Mar 2022 14:11:47 +0200 Subject: [PATCH 097/112] fix: make Block().* return correct ABI based ipld.ErrNotFound errors This commit was moved from ipfs/go-ipfs-http-client@4f5f8e9b144d1f38e0fc37c5080845611dcbcf1e --- client/httpapi/abyfy_errors.go | 104 ++++++++++++++++++++++++++++ client/httpapi/abyfy_errors_test.go | 60 ++++++++++++++++ client/httpapi/block.go | 11 +-- 3 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 client/httpapi/abyfy_errors.go create mode 100644 client/httpapi/abyfy_errors_test.go diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/abyfy_errors.go new file mode 100644 index 00000000000..e4f4afb7cb3 --- /dev/null +++ b/client/httpapi/abyfy_errors.go @@ -0,0 +1,104 @@ +package httpapi + +import ( + "errors" + "strings" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +type prePostWrappedNotFoundError struct { + pre string + post string + + wrapped ipld.ErrNotFound +} + +func (e prePostWrappedNotFoundError) String() string { + return e.Error() +} + +func (e prePostWrappedNotFoundError) Error() string { + return e.pre + e.wrapped.Error() + e.post +} + +func (e prePostWrappedNotFoundError) Unwrap() error { + return e.wrapped +} + +func abyfyIpldNotFoundFallbackToMSG(msg string) error { + err, handled := abyfyIpldNotFound(msg) + if handled { + return err + } + + return errors.New(msg) +} + +func abyfyIpldNotFoundFallbackToError(msg error) error { + err, handled := abyfyIpldNotFound(msg.Error()) + if handled { + return err + } + + return msg +} + +// This file handle parsing and returning the correct ABI based errors from error messages +//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type +func abyfyIpldNotFound(msg string) (error, bool) { + if msg == "" { + return nil, true // Fast path + } + + // The patern we search for is: + // node not found (fallback) + // or + // CID not found (here we parse the CID) + notFoundIndex := strings.LastIndex(msg, " not found") + + if notFoundIndex == -1 { + // Unknown, ot found not found + return nil, false + } + + preNotFound := msg[:notFoundIndex] + + var c cid.Cid + var preIndex int + if strings.HasSuffix(preNotFound, "node") { + // Fallback case + c = cid.Undef + preIndex = notFoundIndex - len("node") + } else { + // Assume that CIDs does not include whitespace to pull out the CID + preIndex = strings.LastIndexByte(preNotFound, ' ') + // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space + preIndex++ + var err error + c, err = cid.Decode(preNotFound[preIndex:]) + if err != nil { + // Unknown + return nil, false + } + } + + postIndex := notFoundIndex + len(" not found") + + err := ipld.ErrNotFound{Cid: c} + + pre := msg[:preIndex] + post := msg[postIndex:] + + if len(pre) > 0 || len(post) > 0 { + // We have some text to wrap arround the ErrNotFound one + return prePostWrappedNotFoundError{ + pre: pre, + post: post, + wrapped: err, + }, true + } + + return err, true +} diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/abyfy_errors_test.go new file mode 100644 index 00000000000..fe1cf0707bf --- /dev/null +++ b/client/httpapi/abyfy_errors_test.go @@ -0,0 +1,60 @@ +package httpapi + +import ( + "errors" + "fmt" + "testing" + + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} + +func doAbyfyIpldNotFoundTest(t *testing.T, original error) { + originalMsg := original.Error() + + rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + + rebuiltMsg := rebuilt.Error() + + if originalMsg != rebuiltMsg { + t.Errorf("expected message to be %q; got %q", originalMsg, rebuiltMsg) + } + + originalNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(original) + if originalNotFound != rebuiltNotFound { + t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + } +} + +func TestAbyfyIpldNotFound(t *testing.T) { + if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { + t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + for _, err := range [...]error{ + errors.New("file not found"), + errors.New(" not found"), + errors.New("Bad_CID not found"), + errors.New("network connection timeout"), + ipld.ErrNotFound{Cid: cid.Undef}, + ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, + ipld.ErrNotFound{Cid: cid.NewCidV1(cid.Raw, randomSha256MH)}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doAbyfyIpldNotFoundTest(t, err) + } + } +} diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 640f186f504..e78ba228175 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -3,7 +3,6 @@ package httpapi import ( "bytes" "context" - "errors" "fmt" "io" @@ -67,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, resp.Error + return nil, abyfyIpldNotFoundFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -99,18 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - if removedBlock.Error != "" { - return errors.New(removedBlock.Error) - } - - return nil + return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, err + return nil, abyfyIpldNotFoundFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { From 5a8e2b77c0e4361a0dd6e57a7f4129adba2f4d8a Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 30 Mar 2022 04:32:12 +0200 Subject: [PATCH 098/112] chore: rename abyfyIpldErrNotFound to parseIPLDErrNotFound This commit was moved from ipfs/go-ipfs-http-client@7aa002992970058e37bb82f9837a442227562d33 --- client/httpapi/block.go | 6 +++--- client/httpapi/{abyfy_errors.go => errors.go} | 10 +++++----- .../httpapi/{abyfy_errors_test.go => errors_test.go} | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) rename client/httpapi/{abyfy_errors.go => errors.go} (88%) rename client/httpapi/{abyfy_errors_test.go => errors_test.go} (84%) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index e78ba228175..0ee838e8326 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, abyfyIpldNotFoundFallbackToError(resp.Error) + return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return abyfyIpldNotFoundFallbackToMSG(removedBlock.Error) + return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, abyfyIpldNotFoundFallbackToError(err) + return nil, parseIPLDNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/abyfy_errors.go b/client/httpapi/errors.go similarity index 88% rename from client/httpapi/abyfy_errors.go rename to client/httpapi/errors.go index e4f4afb7cb3..a527e545232 100644 --- a/client/httpapi/abyfy_errors.go +++ b/client/httpapi/errors.go @@ -27,8 +27,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func abyfyIpldNotFoundFallbackToMSG(msg string) error { - err, handled := abyfyIpldNotFound(msg) +func parseIPLDNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseIPLDNotFound(msg) if handled { return err } @@ -36,8 +36,8 @@ func abyfyIpldNotFoundFallbackToMSG(msg string) error { return errors.New(msg) } -func abyfyIpldNotFoundFallbackToError(msg error) error { - err, handled := abyfyIpldNotFound(msg.Error()) +func parseIPLDNotFoundWithFallbackToError(msg error) error { + err, handled := parseIPLDNotFound(msg.Error()) if handled { return err } @@ -47,7 +47,7 @@ func abyfyIpldNotFoundFallbackToError(msg error) error { // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func abyfyIpldNotFound(msg string) (error, bool) { +func parseIPLDNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } diff --git a/client/httpapi/abyfy_errors_test.go b/client/httpapi/errors_test.go similarity index 84% rename from client/httpapi/abyfy_errors_test.go rename to client/httpapi/errors_test.go index fe1cf0707bf..08def204dc8 100644 --- a/client/httpapi/abyfy_errors_test.go +++ b/client/httpapi/errors_test.go @@ -12,10 +12,10 @@ import ( var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1, 0xc9, 0x96, 0xad, 0xee, 0xd, 0x26, 0x84, 0x2, 0xc9, 0xc9, 0x5c, 0xf9, 0x5c, 0x4d, 0x9b, 0xc3, 0x3f, 0xfb, 0x4a, 0xd8, 0xaf, 0x28, 0x6b, 0xca, 0x1a, 0xf2} -func doAbyfyIpldNotFoundTest(t *testing.T, original error) { +func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := abyfyIpldNotFoundFallbackToMSG(originalMsg) + rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -30,8 +30,8 @@ func doAbyfyIpldNotFoundTest(t *testing.T, original error) { } } -func TestAbyfyIpldNotFound(t *testing.T) { - if err := abyfyIpldNotFoundFallbackToMSG(""); err != nil { +func TestParseIPLDNotFound(t *testing.T) { + if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -54,7 +54,7 @@ func TestAbyfyIpldNotFound(t *testing.T) { err = fmt.Errorf(wrap, err) } - doAbyfyIpldNotFoundTest(t, err) + doParseIpldNotFoundTest(t, err) } } } From a2a60768ea7e5d5dad1ad1f0dcf8811cca24d3eb Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:36:11 +0200 Subject: [PATCH 099/112] fix: parseIPLDErrNotFound test This commit was moved from ipfs/go-ipfs-http-client@2e09c4b3ab580f71c652c82926ba7b1580a2b495 --- client/httpapi/errors_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 08def204dc8..502c10966df 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -24,9 +24,9 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } originalNotFound := ipld.IsNotFound(original) - rebuiltNotFound := ipld.IsNotFound(original) + rebuiltNotFound := ipld.IsNotFound(rebuilt) if originalNotFound != rebuiltNotFound { - t.Errorf("expected Ipld.IsNotFound to be %t; got %t", originalNotFound, rebuiltNotFound) + t.Errorf("for %q expected Ipld.IsNotFound to be %t; got %t", originalMsg, originalNotFound, rebuiltNotFound) } } From ddd36645b25c97651deb81c092c7669efabd959e Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Mar 2022 23:43:27 +0200 Subject: [PATCH 100/112] feat: update the error parsing for go-ipld-format to v0.4.0 This commit was moved from ipfs/go-ipfs-http-client@296534fd1647a1ec9cae08314daf41ec96d55763 --- client/httpapi/errors.go | 54 +++++++++++++++++++++-------------- client/httpapi/errors_test.go | 14 +++++++-- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index a527e545232..f42d9257523 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -45,6 +45,18 @@ func parseIPLDNotFoundWithFallbackToError(msg error) error { return msg } +// Use a string to move it into RODATA +// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) +const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + +func notAsciiLetterOrDigits(r rune) bool { + if r > 'z' { + return true + } + + return notAsciiLetterOrDigitsLUT[r] > 0 +} + // This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseIPLDNotFound(msg string) (error, bool) { @@ -53,46 +65,46 @@ func parseIPLDNotFound(msg string) (error, bool) { } // The patern we search for is: - // node not found (fallback) - // or - // CID not found (here we parse the CID) - notFoundIndex := strings.LastIndex(msg, " not found") + const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ + // We try to parse the CID, if it's invalid we give up and return a simple text error. + // We also accept "node" in place of the CID because that means it's an Undefined CID. - if notFoundIndex == -1 { - // Unknown, ot found not found + keyIndex := strings.Index(msg, ipldErrNotFoundKey) + + if keyIndex < 0 { // Unknown error return nil, false } - preNotFound := msg[:notFoundIndex] + cidStart := keyIndex + len(ipldErrNotFoundKey) + msgPostKey := msg[cidStart:] var c cid.Cid - var preIndex int - if strings.HasSuffix(preNotFound, "node") { + var postIndex int + if strings.HasPrefix(msgPostKey, "node") { // Fallback case c = cid.Undef - preIndex = notFoundIndex - len("node") + postIndex = len("node") } else { - // Assume that CIDs does not include whitespace to pull out the CID - preIndex = strings.LastIndexByte(preNotFound, ' ') - // + 1 is to normalise not founds to zeros and point to the start of the CID, not the previous space - preIndex++ + // Assume that CIDs only contain a-zA-Z0-9 characters. + // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. + postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + if postIndex < 0 { + postIndex = len(msgPostKey) + } + var err error - c, err = cid.Decode(preNotFound[preIndex:]) + c, err = cid.Decode(msgPostKey[:postIndex]) if err != nil { // Unknown return nil, false } } - postIndex := notFoundIndex + len(" not found") - err := ipld.ErrNotFound{Cid: c} - - pre := msg[:preIndex] - post := msg[postIndex:] + pre := msg[:keyIndex] + post := msgPostKey[postIndex:] if len(pre) > 0 || len(post) > 0 { - // We have some text to wrap arround the ErrNotFound one return prePostWrappedNotFoundError{ pre: pre, post: post, diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 502c10966df..1b7de879826 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,6 +3,7 @@ package httpapi import ( "errors" "fmt" + "strings" "testing" "github.com/ipfs/go-cid" @@ -42,9 +43,8 @@ func TestParseIPLDNotFound(t *testing.T) { "%w is wrong", } { for _, err := range [...]error{ - errors.New("file not found"), - errors.New(" not found"), - errors.New("Bad_CID not found"), + errors.New("ipld: could not find "), + errors.New("ipld: could not find Bad_CID"), errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, @@ -58,3 +58,11 @@ func TestParseIPLDNotFound(t *testing.T) { } } } + +func TestNotAsciiLetterOrDigits(t *testing.T) { + for i := rune(0); i <= 256; i++ { + if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { + t.Errorf("%q is incorrectly identified", i) + } + } +} From 775bcb7f09de0d13a64b013ccaefa253e44c3403 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Fri, 1 Apr 2022 00:23:55 +0200 Subject: [PATCH 101/112] feat: add blockstore: block not found matching too This commit was moved from ipfs/go-ipfs-http-client@a3354f062c97bcb6412e74ecfb24e84208edc479 --- client/httpapi/block.go | 6 ++-- client/httpapi/errors.go | 56 +++++++++++++++++++++++++++++++---- client/httpapi/errors_test.go | 25 ++++++++++++++-- 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 0ee838e8326..c074f79407e 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -66,7 +66,7 @@ func (api *BlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) { return nil, err } if resp.Error != nil { - return nil, parseIPLDNotFoundWithFallbackToError(resp.Error) + return nil, parseErrNotFoundWithFallbackToError(resp.Error) } //TODO: make get return ReadCloser to avoid copying @@ -98,14 +98,14 @@ func (api *BlockAPI) Rm(ctx context.Context, p path.Path, opts ...caopts.BlockRm return err } - return parseIPLDNotFoundWithFallbackToMSG(removedBlock.Error) + return parseErrNotFoundWithFallbackToMSG(removedBlock.Error) } func (api *BlockAPI) Stat(ctx context.Context, p path.Path) (iface.BlockStat, error) { var out blockStat err := api.core().Request("block/stat", p.String()).Exec(ctx, &out) if err != nil { - return nil, parseIPLDNotFoundWithFallbackToError(err) + return nil, parseErrNotFoundWithFallbackToError(err) } out.cid, err = cid.Parse(out.Key) if err != nil { diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f42d9257523..b2a6f86de27 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -8,6 +8,8 @@ import ( ipld "github.com/ipfs/go-ipld-format" ) +// This file handle parsing and returning the correct ABI based errors from error messages + type prePostWrappedNotFoundError struct { pre string post string @@ -27,8 +29,8 @@ func (e prePostWrappedNotFoundError) Unwrap() error { return e.wrapped } -func parseIPLDNotFoundWithFallbackToMSG(msg string) error { - err, handled := parseIPLDNotFound(msg) +func parseErrNotFoundWithFallbackToMSG(msg string) error { + err, handled := parseErrNotFound(msg) if handled { return err } @@ -36,8 +38,8 @@ func parseIPLDNotFoundWithFallbackToMSG(msg string) error { return errors.New(msg) } -func parseIPLDNotFoundWithFallbackToError(msg error) error { - err, handled := parseIPLDNotFound(msg.Error()) +func parseErrNotFoundWithFallbackToError(msg error) error { + err, handled := parseErrNotFound(msg.Error()) if handled { return err } @@ -57,13 +59,25 @@ func notAsciiLetterOrDigits(r rune) bool { return notAsciiLetterOrDigitsLUT[r] > 0 } -// This file handle parsing and returning the correct ABI based errors from error messages //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type -func parseIPLDNotFound(msg string) (error, bool) { +func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path } + if err, handled := parseIPLDErrNotFound(msg); handled { + return err, true + } + + if err, handled := parseBlockstoreNotFound(msg); handled { + return err, true + } + + return nil, false +} + +//lint:ignore ST1008 using error as values +func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ // We try to parse the CID, if it's invalid we give up and return a simple text error. @@ -114,3 +128,33 @@ func parseIPLDNotFound(msg string) (error, bool) { return err, true } + +// This is a simple error type that just return msg as Error(). +// But that also match ipld.ErrNotFound when called with Is(err). +// That is needed to keep compatiblity with code that use string.Contains(err.Error(), "blockstore: block not found") +// and code using ipld.ErrNotFound +type blockstoreNotFoundMatchingIPLDErrNotFound struct { + msg string +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) String() string { + return e.Error() +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Error() string { + return e.msg +} + +func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { + _, ok := err.(ipld.ErrNotFound) + return ok +} + +//lint:ignore ST1008 using error as values +func parseBlockstoreNotFound(msg string) (error, bool) { + if !strings.Contains(msg, "blockstore: block not found") { + return nil, false + } + + return blockstoreNotFoundMatchingIPLDErrNotFound{msg: msg}, true +} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 1b7de879826..09437bfbfcc 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -16,7 +16,7 @@ var randomSha256MH = mh.Multihash{0x12, 0x20, 0x88, 0x82, 0x73, 0x37, 0x7c, 0xc1 func doParseIpldNotFoundTest(t *testing.T, original error) { originalMsg := original.Error() - rebuilt := parseIPLDNotFoundWithFallbackToMSG(originalMsg) + rebuilt := parseErrNotFoundWithFallbackToMSG(originalMsg) rebuiltMsg := rebuilt.Error() @@ -32,7 +32,7 @@ func doParseIpldNotFoundTest(t *testing.T, original error) { } func TestParseIPLDNotFound(t *testing.T) { - if err := parseIPLDNotFoundWithFallbackToMSG(""); err != nil { + if err := parseErrNotFoundWithFallbackToMSG(""); err != nil { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } @@ -59,6 +59,27 @@ func TestParseIPLDNotFound(t *testing.T) { } } +func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { + if !ipld.IsNotFound(blockstoreNotFoundMatchingIPLDErrNotFound{}) { + t.Fatalf("expected blockstoreNotFoundMatchingIPLDErrNotFound to match ipld.IsNotFound; got false") + } + + for _, wrap := range [...]string{ + "", + "merkledag: %w", + "testing: %w the test", + "%w is wrong", + } { + var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + + if wrap != "" { + err = fmt.Errorf(wrap, err) + } + + doParseIpldNotFoundTest(t, err) + } +} + func TestNotAsciiLetterOrDigits(t *testing.T) { for i := rune(0); i <= 256; i++ { if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { From 54f6e90870c61a686f57b0bd0f1ad4923615897b Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 03:25:39 +0200 Subject: [PATCH 102/112] fix: change CID breaking logic when parsing ipld.ErrNotFound This commit was moved from ipfs/go-ipfs-http-client@a3b49352bfd1b885567018092a06f303de18b7aa --- client/httpapi/errors.go | 24 +++++++++--------------- client/httpapi/errors_test.go | 18 +++++++----------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index b2a6f86de27..f1a57a04960 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -47,18 +47,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -// Use a string to move it into RODATA -// print("".join("\\x01" if chr(i) not in string.ascii_letters + string.digits else "\\x00" for i in range(ord('z')+1))) -const notAsciiLetterOrDigitsLUT = "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - -func notAsciiLetterOrDigits(r rune) bool { - if r > 'z' { - return true - } - - return notAsciiLetterOrDigitsLUT[r] > 0 -} - //lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { @@ -76,6 +64,12 @@ func parseErrNotFound(msg string) (error, bool) { return nil, false } +// Assume CIDs break on: +// - Whitespaces: " \t\n\r\v\f" +// - Semicolon: ";" this is to parse ipld.ErrNotFound wrapped in multierr +// - Double Quotes: "\"" this is for parsing %q and %#v formating +const cidBreakSet = " \t\n\r\v\f;\"" + //lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: @@ -99,9 +93,9 @@ func parseIPLDErrNotFound(msg string) (error, bool) { c = cid.Undef postIndex = len("node") } else { - // Assume that CIDs only contain a-zA-Z0-9 characters. - // This is true because go-ipld-format use go-cid#Cid.String which use base{3{2,6},58}. - postIndex = strings.IndexFunc(msgPostKey, notAsciiLetterOrDigits) + postIndex = strings.IndexFunc(msgPostKey, func(r rune) bool { + return strings.ContainsAny(string(r), cidBreakSet) + }) if postIndex < 0 { postIndex = len(msgPostKey) } diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 09437bfbfcc..86f91cdf5f7 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "strings" "testing" "github.com/ipfs/go-cid" @@ -36,12 +35,17 @@ func TestParseIPLDNotFound(t *testing.T) { t.Errorf("expected empty string to give no error; got %T %q", err, err.Error()) } - for _, wrap := range [...]string{ + cidBreaks := make([]string, len(cidBreakSet)) + for i, v := range cidBreakSet { + cidBreaks[i] = "%w" + string(v) + } + + for _, wrap := range append(cidBreaks, "", "merkledag: %w", "testing: %w the test", "%w is wrong", - } { + ) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), @@ -79,11 +83,3 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { doParseIpldNotFoundTest(t, err) } } - -func TestNotAsciiLetterOrDigits(t *testing.T) { - for i := rune(0); i <= 256; i++ { - if notAsciiLetterOrDigits(i) != !strings.ContainsAny(string(i), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") { - t.Errorf("%q is incorrectly identified", i) - } - } -} From f2f2109bc1e8f5b9c636597eb7478e0a4ddca427 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sat, 2 Apr 2022 04:24:12 +0200 Subject: [PATCH 103/112] test: add a false case test to blockstore parsing This commit was moved from ipfs/go-ipfs-http-client@75f597aa16c512ec02f549836e998f1fc29a3846 --- client/httpapi/errors_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 86f91cdf5f7..7709fc9c53a 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -74,12 +74,15 @@ func TestBlockstoreNotFoundMatchingIPLDErrNotFound(t *testing.T) { "testing: %w the test", "%w is wrong", } { - var err error = blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"} + for _, err := range [...]error{ + errors.New("network connection timeout"), + blockstoreNotFoundMatchingIPLDErrNotFound{"blockstore: block not found"}, + } { + if wrap != "" { + err = fmt.Errorf(wrap, err) + } - if wrap != "" { - err = fmt.Errorf(wrap, err) + doParseIpldNotFoundTest(t, err) } - - doParseIpldNotFoundTest(t, err) } } From 60a548b621ff229a648a268c4d06f6e206ad6099 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 5 Apr 2022 20:11:06 +0200 Subject: [PATCH 104/112] correctness: only match CIDs matching go-cid.Cid.String output This commit was moved from ipfs/go-ipfs-http-client@34cc489461985cf4139819e589fdd09bdc034c24 --- client/httpapi/errors.go | 19 +++++++++++++++++-- client/httpapi/errors_test.go | 7 +++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index f1a57a04960..1ccf6182c7b 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -3,9 +3,11 @@ package httpapi import ( "errors" "strings" + "unicode/utf8" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" ) // This file handle parsing and returning the correct ABI based errors from error messages @@ -97,15 +99,28 @@ func parseIPLDErrNotFound(msg string) (error, bool) { return strings.ContainsAny(string(r), cidBreakSet) }) if postIndex < 0 { + // no breakage meaning the string look like this something + "ipld: could not find bafy" postIndex = len(msgPostKey) } + cidStr := msgPostKey[:postIndex] + var err error - c, err = cid.Decode(msgPostKey[:postIndex]) + c, err = cid.Decode(cidStr) if err != nil { - // Unknown + // failed to decode CID give up return nil, false } + + // check that the CID is either a CIDv0 or a base32 multibase + // because that what ipld.ErrNotFound.Error() -> cid.Cid.String() do currently + if c.Version() != 0 { + baseRune, _ := utf8.DecodeRuneInString(cidStr) + if baseRune == utf8.RuneError || baseRune != mbase.Base32 { + // not a multibase we expect, give up + return nil, false + } + } } err := ipld.ErrNotFound{Cid: c} diff --git a/client/httpapi/errors_test.go b/client/httpapi/errors_test.go index 7709fc9c53a..c8b98d08eb2 100644 --- a/client/httpapi/errors_test.go +++ b/client/httpapi/errors_test.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + mbase "github.com/multiformats/go-multibase" mh "github.com/multiformats/go-multihash" ) @@ -40,6 +41,11 @@ func TestParseIPLDNotFound(t *testing.T) { cidBreaks[i] = "%w" + string(v) } + base58BTCEncoder, err := mbase.NewEncoder(mbase.Base58BTC) + if err != nil { + t.Fatalf("expected to find Base58BTC encoder; got error %q", err.Error()) + } + for _, wrap := range append(cidBreaks, "", "merkledag: %w", @@ -49,6 +55,7 @@ func TestParseIPLDNotFound(t *testing.T) { for _, err := range [...]error{ errors.New("ipld: could not find "), errors.New("ipld: could not find Bad_CID"), + errors.New("ipld: could not find " + cid.NewCidV1(cid.Raw, randomSha256MH).Encode(base58BTCEncoder)), // Test that we only accept CIDv0 and base32 CIDs errors.New("network connection timeout"), ipld.ErrNotFound{Cid: cid.Undef}, ipld.ErrNotFound{Cid: cid.NewCidV0(randomSha256MH)}, From fd209019f27c7a16f94bfc41ebc7b815daee9903 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 21 Apr 2022 22:18:19 +0200 Subject: [PATCH 105/112] fix: interop with 'block put' from go-ipfs 0.13 (#158) * chore: interop with go-ipfs 0.13 Applies necessary changes to ensure 'block/put' works and is backward-compatible. Context: https://github.com/ipfs/go-ipfs/pull/8568 * chore: 0.3.1 bumping as patch because we bumped to 0.3.0 recently, as part of other (unreleased) go-ipfs 0.13 work This commit was moved from ipfs/go-ipfs-http-client@ecf364c9898d78034e3232574aae70d1bf85357c --- client/httpapi/block.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/client/httpapi/block.go b/client/httpapi/block.go index c074f79407e..7b4cf0edab5 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -7,9 +7,10 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) @@ -31,20 +32,33 @@ func (s *blockStat) Path() path.Resolved { } func (api *BlockAPI) Put(ctx context.Context, r io.Reader, opts ...caopts.BlockPutOption) (iface.BlockStat, error) { - options, _, err := caopts.BlockPutOptions(opts...) + options, err := caopts.BlockPutOptions(opts...) + px := options.CidPrefix if err != nil { return nil, err } - mht, ok := mh.Codes[options.MhType] + mht, ok := mh.Codes[px.MhType] if !ok { - return nil, fmt.Errorf("unknowm mhType %d", options.MhType) + return nil, fmt.Errorf("unknowm mhType %d", px.MhType) + } + + var cidOptKey, cidOptVal string + switch { + case px.Version == 0 && px.Codec == cid.DagProtobuf: + // ensure legacy --format=v0 passes as BlockPutOption still works + cidOptKey = "format" + cidOptVal = "v0" + default: + // pass codec as string + cidOptKey = "cid-codec" + cidOptVal = mc.Code(px.Codec).String() } req := api.core().Request("block/put"). Option("mhtype", mht). - Option("mhlen", options.MhLength). - Option("format", options.Codec). + Option("mhlen", px.MhLength). + Option(cidOptKey, cidOptVal). Option("pin", options.Pin). FileBody(r) From 1cdb9adf898a8a2b9d2cd0d3fe8ba6467a822166 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 23 Jun 2022 22:39:01 +0200 Subject: [PATCH 106/112] fix: interop with go-ipfs 0.13 (#160) This ensures cid-codec introduced in https://github.com/ipfs/go-ipfs/pull/8568 gets correctly passed (+ we maintain backward-compatibility with CIDv0) This commit was moved from ipfs/go-ipfs-http-client@9c9f43fd9ca36c7ebdc664bb6936a6fe9881255b --- client/httpapi/dag.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 15d9a9c62ed..879c1e49932 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,12 @@ import ( "fmt" "io/ioutil" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" + multicodec "github.com/multiformats/go-multicodec" ) type httpNodeAdder HttpApi @@ -56,13 +57,21 @@ func (api *HttpDagServ) GetMany(ctx context.Context, cids []cid.Cid) <-chan *for func (api *httpNodeAdder) add(ctx context.Context, nd format.Node, pin bool) error { c := nd.Cid() prefix := c.Prefix() - format := cid.CodecToStr[prefix.Codec] + + // preserve 'cid-codec' when sent over HTTP + cidCodec := multicodec.Code(prefix.Codec).String() + + // 'format' got replaced by 'cid-codec' in https://github.com/ipfs/interface-go-ipfs-core/pull/80 + // but we still support it here for backward-compatibility with use of CIDv0 + format := "" if prefix.Version == 0 { + cidCodec = "" format = "v0" } stat, err := api.core().Block().Put(ctx, bytes.NewReader(nd.RawData()), options.Block.Hash(prefix.MhType, prefix.MhLength), + options.Block.CidCodec(cidCodec), options.Block.Format(format), options.Block.Pin(pin)) if err != nil { From 0fff1d5d8e4cb2debaff1ee4e85b1266907101af Mon Sep 17 00:00:00 2001 From: web3-bot Date: Fri, 23 Sep 2022 07:43:33 +0000 Subject: [PATCH 107/112] stop using the deprecated io/ioutil package This commit was moved from ipfs/go-ipfs-http-client@026ba730a1fe9a4a25a22449bec8d69c2262ccfc --- client/httpapi/api.go | 3 +-- client/httpapi/api_test.go | 3 +-- client/httpapi/apifile.go | 3 +-- client/httpapi/dag.go | 4 ++-- client/httpapi/name.go | 4 ++-- client/httpapi/object.go | 5 ++--- client/httpapi/requestbuilder.go | 5 ++--- client/httpapi/response.go | 9 ++++----- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index f589e4267a4..97440a72436 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -3,7 +3,6 @@ package httpapi import ( "errors" "fmt" - "io/ioutil" "net/http" "os" "path/filepath" @@ -74,7 +73,7 @@ func ApiAddr(ipfspath string) (ma.Multiaddr, error) { apiFile := filepath.Join(baseDir, DefaultApiFile) - api, err := ioutil.ReadFile(apiFile) + api, err := os.ReadFile(apiFile) if err != nil { return nil, err } diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index d5f27f0ec9d..5960ea2c027 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -2,7 +2,6 @@ package httpapi import ( "context" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -92,7 +91,7 @@ func (np *NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n i func (NodeProvider) makeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]iface.CoreAPI, error) { - dir, err := ioutil.TempDir("", "httpapi-tb-") + dir, err := os.MkdirTemp("", "httpapi-tb-") if err != nil { return nil, err } diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index ec2f7588d59..c4884b924f0 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" @@ -113,7 +112,7 @@ func (f *apiFile) Seek(offset int64, whence int) (int64, error) { } if f.at < offset && offset-f.at < forwardSeekLimit { //forward skip - r, err := io.CopyN(ioutil.Discard, f.r.Output, offset-f.at) + r, err := io.CopyN(io.Discard, f.r.Output, offset-f.at) f.at += r return f.at, err diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 879c1e49932..f32c67c42ef 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "io" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -24,7 +24,7 @@ func (api *HttpDagServ) Get(ctx context.Context, c cid.Cid) (format.Node, error) return nil, err } - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/name.go b/client/httpapi/name.go index 47227a2ab89..b6a7836033a 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/options/namesys" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 6ec7f5503cd..894369223fc 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -5,13 +5,12 @@ import ( "context" "fmt" "io" - "io/ioutil" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" ft "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" ) @@ -71,7 +70,7 @@ func (api *ObjectAPI) Get(ctx context.Context, p path.Path) (ipld.Node, error) { if err != nil { return nil, err } - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 7012a8935bf..039bca03641 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -5,11 +5,10 @@ import ( "context" "fmt" "io" - "io/ioutil" "strconv" "strings" - "github.com/ipfs/go-ipfs-files" + files "github.com/ipfs/go-ipfs-files" ) type RequestBuilder interface { @@ -59,7 +58,7 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder { // FileBody sets the request body to the given reader wrapped into multipartreader. func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder { - pr, _ := files.NewReaderPathFile("/dev/stdin", ioutil.NopCloser(body), nil) + pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil) d := files.NewMapDirectory(map[string]files.Node{"": pr}) r.body = files.NewMultiFileReader(d, false) diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 95cbf13ec1b..8a491ab7316 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime" "net/http" "net/url" @@ -45,7 +44,7 @@ func (r *Response) Close() error { if r.Output != nil { // drain output (response body) - _, err1 := io.Copy(ioutil.Discard, r.Output) + _, err1 := io.Copy(io.Discard, r.Output) err2 := r.Output.Close() if err1 != nil { return err1 @@ -117,7 +116,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { case resp.StatusCode == http.StatusNotFound: e.Message = "command not found" case contentType == "text/plain": - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err) } @@ -140,7 +139,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { // This is a server-side bug (probably). e.Code = cmds.ErrImplementation fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType) - out, err := ioutil.ReadAll(resp.Body) + out, err := io.ReadAll(resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err) } @@ -150,7 +149,7 @@ func (r *Request) Send(c *http.Client) (*Response, error) { nresp.Output = nil // drain body and close - _, _ = io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() } From 1193b2978f7859ed8e1df4be50e980ff1bde7015 Mon Sep 17 00:00:00 2001 From: galargh Date: Sat, 1 Oct 2022 17:37:23 +0200 Subject: [PATCH 108/112] chore: remove unused linter directives This commit was moved from ipfs/go-ipfs-http-client@c4fc1a7740bfaad2abcba2563d569422e525324c --- client/httpapi/errors.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/httpapi/errors.go b/client/httpapi/errors.go index 1ccf6182c7b..59e4ad70543 100644 --- a/client/httpapi/errors.go +++ b/client/httpapi/errors.go @@ -49,7 +49,6 @@ func parseErrNotFoundWithFallbackToError(msg error) error { return msg } -//lint:ignore ST1008 this function is not using the error as a mean to return failure but it massages it to return the correct type func parseErrNotFound(msg string) (error, bool) { if msg == "" { return nil, true // Fast path @@ -72,7 +71,6 @@ func parseErrNotFound(msg string) (error, bool) { // - Double Quotes: "\"" this is for parsing %q and %#v formating const cidBreakSet = " \t\n\r\v\f;\"" -//lint:ignore ST1008 using error as values func parseIPLDErrNotFound(msg string) (error, bool) { // The patern we search for is: const ipldErrNotFoundKey = "ipld: could not find " /*CID*/ @@ -159,7 +157,6 @@ func (e blockstoreNotFoundMatchingIPLDErrNotFound) Is(err error) bool { return ok } -//lint:ignore ST1008 using error as values func parseBlockstoreNotFound(msg string) (error, bool) { if !strings.Contains(msg, "blockstore: block not found") { return nil, false From 75b6f45f765237d611168a5e1c09915f380e7bd7 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 6 Feb 2023 15:32:57 +0100 Subject: [PATCH 109/112] chore: bumps for Kubo 0.18 and Routing API with stub for Put This commit was moved from ipfs/go-ipfs-http-client@c076c3cb71d08fb9448ae9bf6916dab137322722 --- client/httpapi/api.go | 4 +++ client/httpapi/apifile.go | 2 +- client/httpapi/dag.go | 2 +- client/httpapi/dht.go | 4 +-- client/httpapi/key.go | 2 +- client/httpapi/pubsub.go | 2 +- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/routing.go | 55 ++++++++++++++++++++++++++++++++ client/httpapi/swarm.go | 6 ++-- client/httpapi/unixfs.go | 2 +- 11 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 client/httpapi/routing.go diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 97440a72436..10df94603f7 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -195,3 +195,7 @@ func (api *HttpApi) Swarm() iface.SwarmAPI { func (api *HttpApi) PubSub() iface.PubSubAPI { return (*PubsubAPI)(api) } + +func (api *HttpApi) Routing() iface.RoutingAPI { + return (*RoutingAPI)(api) +} diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index c4884b924f0..80fd13cd4fe 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -7,7 +7,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" "github.com/ipfs/interface-go-ipfs-core/path" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index f32c67c42ef..2fbfea1ff29 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index 0a78514f973..ebecfcaeb3a 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -6,8 +6,8 @@ import ( caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" ) type DhtAPI HttpApi diff --git a/client/httpapi/key.go b/client/httpapi/key.go index 78b67517c01..b785bbf453d 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -7,7 +7,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" ) type KeyAPI HttpApi diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index 72f5923761f..ec45d1a51be 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -8,7 +8,7 @@ import ( iface "github.com/ipfs/interface-go-ipfs-core" caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index 039bca03641..bc6f4c8a94f 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 8a491ab7316..99932ca0da7 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -12,7 +12,7 @@ import ( cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/routing.go b/client/httpapi/routing.go new file mode 100644 index 00000000000..18f5332766f --- /dev/null +++ b/client/httpapi/routing.go @@ -0,0 +1,55 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + + "github.com/libp2p/go-libp2p/core/routing" +) + +type RoutingAPI HttpApi + +func (api *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) { + resp, err := api.core().Request("routing/get", key).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + defer resp.Close() + + var out routing.QueryEvent + + dec := json.NewDecoder(resp.Output) + if err := dec.Decode(&out); err != nil { + return nil, err + } + + res, err := base64.StdEncoding.DecodeString(out.Extra) + if err != nil { + return nil, err + } + + return res, nil +} + +func (api *RoutingAPI) Put(ctx context.Context, key string, value []byte) error { + resp, err := api.core().Request("routing/put", key). + FileBody(bytes.NewReader(value)). + Send(ctx) + + if err != nil { + return err + } + if resp.Error != nil { + return resp.Error + } + return nil +} + +func (api *RoutingAPI) core() *HttpApi { + return (*HttpApi)(api) +} diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index 1cb0d91df7c..c5dfbe56452 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -5,9 +5,9 @@ import ( "time" iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" "github.com/multiformats/go-multiaddr" ) diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 5e27fb0367f..38f23d5b9b4 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -8,7 +8,7 @@ import ( "io" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-libipfs/files" unixfs "github.com/ipfs/go-unixfs" unixfs_pb "github.com/ipfs/go-unixfs/pb" iface "github.com/ipfs/interface-go-ipfs-core" From 5c21cf0ce95d7c2710d5bb5dd58e2871c0fbc7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= <92101826+PedrobyJoao@users.noreply.github.com> Date: Tue, 28 Mar 2023 07:38:24 -0300 Subject: [PATCH 110/112] docs: adding example of connection and pinning to README (#173) Co-authored-by: JPexplorer This commit was moved from ipfs/go-ipfs-http-client@b0de2b028e6ef03d4dba3ed105641ad299d4f7be --- client/httpapi/README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/client/httpapi/README.md b/client/httpapi/README.md index d3e14ac3f1b..da25f24feb0 100644 --- a/client/httpapi/README.md +++ b/client/httpapi/README.md @@ -17,6 +17,41 @@ greatest features, please use _this_ package. https://godoc.org/github.com/ipfs/go-ipfs-http-api +### Example + +Pin file on your local IPFS node based on its CID: + +```go +package main + +import ( + "context" + "fmt" + + ipfsClient "github.com/ipfs/go-ipfs-http-client" + path "github.com/ipfs/interface-go-ipfs-core/path" +) + +func main() { + // "Connect" to local node + node, err := ipfsClient.NewLocalApi() + if err != nil { + fmt.Printf(err) + return + } + // Pin a given file by its CID + ctx := context.Background() + cid := "bafkreidtuosuw37f5xmn65b3ksdiikajy7pwjjslzj2lxxz2vc4wdy3zku" + p := path.New(cid) + err = node.Pin().Add(ctx, p) + if err != nil { + fmt.Printf(err) + return + } + return +} +``` + ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-http-api/issues)! From 8bebfc00b7192de63b79082bd2666d8d1bf76126 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 16 Apr 2023 12:31:09 +0200 Subject: [PATCH 111/112] chore: migrate from go-libipfs to boxo This commit was moved from ipfs/go-ipfs-http-client@ae996cbe5a91708d245100267c45094ab52c42ad --- client/httpapi/api.go | 4 ++-- client/httpapi/api_test.go | 7 +++---- client/httpapi/apifile.go | 6 +++--- client/httpapi/block.go | 6 +++--- client/httpapi/dag.go | 6 +++--- client/httpapi/dht.go | 4 ++-- client/httpapi/key.go | 6 +++--- client/httpapi/name.go | 8 ++++---- client/httpapi/object.go | 10 +++++----- client/httpapi/path.go | 4 ++-- client/httpapi/pin.go | 6 +++--- client/httpapi/pubsub.go | 4 ++-- client/httpapi/requestbuilder.go | 2 +- client/httpapi/response.go | 2 +- client/httpapi/swarm.go | 2 +- client/httpapi/unixfs.go | 12 ++++++------ 16 files changed, 44 insertions(+), 45 deletions(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index 10df94603f7..b7cfb2b520d 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strings" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/mitchellh/go-homedir" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 5960ea2c027..2832d722d37 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -12,10 +12,9 @@ import ( "testing" "time" - iface "github.com/ipfs/interface-go-ipfs-core" - "github.com/ipfs/interface-go-ipfs-core/path" - - "github.com/ipfs/interface-go-ipfs-core/tests" + iface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/coreiface/tests" local "github.com/ipfs/iptb-plugins/local" "github.com/ipfs/iptb/testbed" testbedi "github.com/ipfs/iptb/testbed/interfaces" diff --git a/client/httpapi/apifile.go b/client/httpapi/apifile.go index 80fd13cd4fe..25fd7c3b371 100644 --- a/client/httpapi/apifile.go +++ b/client/httpapi/apifile.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - "github.com/ipfs/interface-go-ipfs-core/path" ) const forwardSeekLimit = 1 << 14 //16k diff --git a/client/httpapi/block.go b/client/httpapi/block.go index 7b4cf0edab5..2a794c26f52 100644 --- a/client/httpapi/block.go +++ b/client/httpapi/block.go @@ -6,10 +6,10 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) diff --git a/client/httpapi/dag.go b/client/httpapi/dag.go index 2fbfea1ff29..795e1d78d9e 100644 --- a/client/httpapi/dag.go +++ b/client/httpapi/dag.go @@ -6,11 +6,11 @@ import ( "fmt" "io" + "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" - "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" multicodec "github.com/multiformats/go-multicodec" ) diff --git a/client/httpapi/dht.go b/client/httpapi/dht.go index ebecfcaeb3a..a2910fef6d3 100644 --- a/client/httpapi/dht.go +++ b/client/httpapi/dht.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" ) diff --git a/client/httpapi/key.go b/client/httpapi/key.go index b785bbf453d..434e98fe521 100644 --- a/client/httpapi/key.go +++ b/client/httpapi/key.go @@ -4,9 +4,9 @@ import ( "context" "errors" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/libp2p/go-libp2p/core/peer" ) diff --git a/client/httpapi/name.go b/client/httpapi/name.go index b6a7836033a..f82f69f3ac4 100644 --- a/client/httpapi/name.go +++ b/client/httpapi/name.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - "github.com/ipfs/interface-go-ipfs-core/path" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + nsopts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/coreiface/path" ) type NameAPI HttpApi diff --git a/client/httpapi/object.go b/client/httpapi/object.go index 894369223fc..4e3b9ef6be1 100644 --- a/client/httpapi/object.go +++ b/client/httpapi/object.go @@ -6,13 +6,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipld/merkledag" + ft "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" - ft "github.com/ipfs/go-unixfs" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" ) type ObjectAPI HttpApi diff --git a/client/httpapi/path.go b/client/httpapi/path.go index 619dd0e0d2c..d69d425ab66 100644 --- a/client/httpapi/path.go +++ b/client/httpapi/path.go @@ -3,10 +3,10 @@ package httpapi import ( "context" + "github.com/ipfs/boxo/coreiface/path" + ipfspath "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/interface-go-ipfs-core/path" ) func (api *HttpApi) ResolvePath(ctx context.Context, p path.Path) (path.Resolved, error) { diff --git a/client/httpapi/pin.go b/client/httpapi/pin.go index 13de2d3893d..30a3d7b7a90 100644 --- a/client/httpapi/pin.go +++ b/client/httpapi/pin.go @@ -5,10 +5,10 @@ import ( "encoding/json" "strings" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" "github.com/pkg/errors" ) diff --git a/client/httpapi/pubsub.go b/client/httpapi/pubsub.go index ec45d1a51be..28f1ef8e609 100644 --- a/client/httpapi/pubsub.go +++ b/client/httpapi/pubsub.go @@ -6,8 +6,8 @@ import ( "encoding/json" "io" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" "github.com/libp2p/go-libp2p/core/peer" mbase "github.com/multiformats/go-multibase" ) diff --git a/client/httpapi/requestbuilder.go b/client/httpapi/requestbuilder.go index bc6f4c8a94f..476aed7866d 100644 --- a/client/httpapi/requestbuilder.go +++ b/client/httpapi/requestbuilder.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/boxo/files" ) type RequestBuilder interface { diff --git a/client/httpapi/response.go b/client/httpapi/response.go index 99932ca0da7..189b4367161 100644 --- a/client/httpapi/response.go +++ b/client/httpapi/response.go @@ -10,9 +10,9 @@ import ( "net/url" "os" + "github.com/ipfs/boxo/files" cmds "github.com/ipfs/go-ipfs-cmds" cmdhttp "github.com/ipfs/go-ipfs-cmds/http" - "github.com/ipfs/go-libipfs/files" ) type Error = cmds.Error diff --git a/client/httpapi/swarm.go b/client/httpapi/swarm.go index c5dfbe56452..9b073078da0 100644 --- a/client/httpapi/swarm.go +++ b/client/httpapi/swarm.go @@ -4,7 +4,7 @@ import ( "context" "time" - iface "github.com/ipfs/interface-go-ipfs-core" + iface "github.com/ipfs/boxo/coreiface" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" diff --git a/client/httpapi/unixfs.go b/client/httpapi/unixfs.go index 38f23d5b9b4..b9c34c59ffc 100644 --- a/client/httpapi/unixfs.go +++ b/client/httpapi/unixfs.go @@ -7,13 +7,13 @@ import ( "fmt" "io" + iface "github.com/ipfs/boxo/coreiface" + caopts "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/files" + unixfs "github.com/ipfs/boxo/ipld/unixfs" + unixfs_pb "github.com/ipfs/boxo/ipld/unixfs/pb" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/files" - unixfs "github.com/ipfs/go-unixfs" - unixfs_pb "github.com/ipfs/go-unixfs/pb" - iface "github.com/ipfs/interface-go-ipfs-core" - caopts "github.com/ipfs/interface-go-ipfs-core/options" - "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" ) From 7d8262307cb40c1f21fd79fe52cdfd36a7dcb5cd Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 15 May 2023 17:18:51 +0100 Subject: [PATCH 112/112] fix: use https URI when multiaddress specifies tls (#177) Currently any clients created through `NewApiWithClient` will make a HTTP request to the api, even if the multiaddress specifies TLS or (the deprecated multiaddr option) https. This commit addresses this by having NewApiWithClient iterate the available protocols for the multiaddress, specifying the URL proto as https if it finds TLS or HTTPS is specified. The default continues to be http for those multiaddresses that do not specify these options. Should resolve #176 This commit was moved from ipfs/go-ipfs-http-client@7e1de1f7ccfa8cd1f616e94664886fe04d70806e --- client/httpapi/api.go | 16 +++++++++++++++- client/httpapi/api_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/client/httpapi/api.go b/client/httpapi/api.go index b7cfb2b520d..5584de85d99 100644 --- a/client/httpapi/api.go +++ b/client/httpapi/api.go @@ -107,7 +107,21 @@ func NewApiWithClient(a ma.Multiaddr, c *http.Client) (*HttpApi, error) { } } - return NewURLApiWithClient(url, c) + proto := "http://" + + // By default, DialArgs is going to provide details suitable for connecting + // a socket to, but not really suitable for making an informed choice of http + // protocol. For multiaddresses specifying tls and/or https we want to make + // a https request instead of a http request. + protocols := a.Protocols() + for _, p := range protocols { + if p.Code == ma.P_HTTPS || p.Code == ma.P_TLS { + proto = "https://" + break + } + } + + return NewURLApiWithClient(proto+url, c) } func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) { diff --git a/client/httpapi/api_test.go b/client/httpapi/api_test.go index 2832d722d37..a6d684b0d07 100644 --- a/client/httpapi/api_test.go +++ b/client/httpapi/api_test.go @@ -246,3 +246,31 @@ func Test_NewURLApiWithClient_With_Headers(t *testing.T) { t.Fatal(err) } } + +func Test_NewURLApiWithClient_HTTP_Variant(t *testing.T) { + testcases := []struct { + address string + expected string + }{ + {address: "/ip4/127.0.0.1/tcp/80", expected: "http://127.0.0.1:80"}, + {address: "/ip4/127.0.0.1/tcp/443/tls", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/https", expected: "https://127.0.0.1:443"}, + {address: "/ip4/127.0.0.1/tcp/443/tls/http", expected: "https://127.0.0.1:443"}, + } + + for _, tc := range testcases { + address, err := ma.NewMultiaddr(tc.address) + if err != nil { + t.Fatal(err) + } + + api, err := NewApiWithClient(address, &http.Client{}) + if err != nil { + t.Fatal(err) + } + + if api.url != tc.expected { + t.Errorf("Expected = %s; got %s", tc.expected, api.url) + } + } +}