in path)
+func HasDNSLinkOrigin(gwURL string, path string) bool {
+ if gwURL != "" {
+ fqdn := stripPort(strings.TrimPrefix(gwURL, "//"))
+ return strings.HasPrefix(path, "/ipns/"+fqdn)
+ }
+ return false
+}
+
+func stripPort(hostname string) string {
+ host, _, err := net.SplitHostPort(hostname)
+ if err == nil {
+ return host
+ }
+ return hostname
+}
diff --git a/gateway/assets/build.sh b/gateway/assets/build.sh
new file mode 100755
index 000000000..531bbfc02
--- /dev/null
+++ b/gateway/assets/build.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -euo pipefail
+
+function build() {
+ rm -f $1
+ sed '/{{ .Path }}
+
+
+
+
+
+
+
+
+
+
+ Preview as JSON (application/json)
+
+
+
+
+ Or download as:
+
+
+
+
+
+
+
{{ .Path }}
+
+
+
+
+
+
+
+
+ {{ if .BackLink }}
+
+
+
+
+
+ ..
+
+
+ {{ end }}
+ {{ range .Listing }}
+
+
+
+
+
+ {{ .Name }}
+
+
+ {{ if .Hash }}
+
+ {{ .ShortHash }}
+
+ {{ end }}
+
+ {{ .Size }}
+
+ {{ end }}
+
+
+
{{ .Path }}
+
+
+
+
+
+
+
+
+
+ Preview as JSON (application/json)
+
+
+
+
+ Or download as:
+
+
+
+
+
+
+
{{ .Path }}
+
+
+
+
+
+
+
+ {{ if .BackLink }}
+
+
+
+
+
+ ..
+
+
+ {{ end }}
+ {{ range .Listing }}
+
+
+
+
+
+ {{ .Name }}
+
+
+ {{ if .Hash }}
+
+ {{ .ShortHash }}
+
+ {{ end }}
+
+ {{ .Size }}
+
+ {{ end }}
+
+
+
Test paths: DAG , Directory .`
+ _, _ = w.Write([]byte(html))
+ default:
+ http.Redirect(w, r, "/", http.StatusSeeOther)
+ }
+ })
+
+ if _, err := os.Stat(directoryTemplateFile); err != nil {
+ wd, _ := os.Getwd()
+ fmt.Printf("could not open template file %q, relative to %q: %s\n", directoryTemplateFile, wd, err)
+ os.Exit(1)
+ }
+
+ if _, err := os.Stat(dagTemplateFile); err != nil {
+ wd, _ := os.Getwd()
+ fmt.Printf("could not open template file %q, relative to %q: %s\n", dagTemplateFile, wd, err)
+ os.Exit(1)
+ }
+
+ fmt.Printf("listening on localhost:3000\n")
+ _ = http.ListenAndServe("localhost:3000", mux)
+}
+
+// Copied from ../assets.go
+type DagTemplateData struct {
+ Path string
+ CID string
+ CodecName string
+ CodecHex string
+}
+
+type DirectoryTemplateData struct {
+ GatewayURL string
+ DNSLink bool
+ Listing []DirectoryItem
+ Size string
+ Path string
+ Breadcrumbs []Breadcrumb
+ BackLink string
+ Hash string
+}
+
+type DirectoryItem struct {
+ Size string
+ Name string
+ Path string
+ Hash string
+ ShortHash string
+}
+
+type Breadcrumb struct {
+ Name string
+ Path string
+}
diff --git a/gateway/errors.go b/gateway/errors.go
new file mode 100644
index 000000000..191426742
--- /dev/null
+++ b/gateway/errors.go
@@ -0,0 +1,181 @@
+package gateway
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/ipfs/boxo/path/resolver"
+ "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+var (
+ ErrInternalServerError = NewErrorResponseForCode(http.StatusInternalServerError)
+ ErrGatewayTimeout = NewErrorResponseForCode(http.StatusGatewayTimeout)
+ ErrBadGateway = NewErrorResponseForCode(http.StatusBadGateway)
+ ErrServiceUnavailable = NewErrorResponseForCode(http.StatusServiceUnavailable)
+ ErrTooManyRequests = NewErrorResponseForCode(http.StatusTooManyRequests)
+)
+
+type ErrorRetryAfter struct {
+ Err error
+ RetryAfter time.Duration
+}
+
+// NewErrorWithRetryAfter wraps any error in RetryAfter hint that
+// gets passed to HTTP clients in Retry-After HTTP header.
+func NewErrorRetryAfter(err error, retryAfter time.Duration) *ErrorRetryAfter {
+ if err == nil {
+ err = ErrServiceUnavailable
+ }
+ if retryAfter < 0 {
+ retryAfter = 0
+ }
+ return &ErrorRetryAfter{
+ RetryAfter: retryAfter,
+ Err: err,
+ }
+}
+
+func (e *ErrorRetryAfter) Error() string {
+ var text string
+ if e.Err != nil {
+ text = e.Err.Error()
+ }
+ if e.RetryAfter != 0 {
+ text += fmt.Sprintf(", retry after %s", e.Humanized())
+ }
+ return text
+}
+
+func (e *ErrorRetryAfter) Unwrap() error {
+ return e.Err
+}
+
+func (e *ErrorRetryAfter) Is(err error) bool {
+ switch err.(type) {
+ case *ErrorRetryAfter:
+ return true
+ default:
+ return false
+ }
+}
+
+func (e *ErrorRetryAfter) RoundSeconds() time.Duration {
+ return e.RetryAfter.Round(time.Second)
+}
+
+func (e *ErrorRetryAfter) Humanized() string {
+ return e.RoundSeconds().String()
+}
+
+// HTTPHeaderValue returns the Retry-After header value as a string, representing the number
+// of seconds to wait before making a new request, rounded to the nearest second.
+// This function follows the Retry-After header definition as specified in RFC 9110.
+func (e *ErrorRetryAfter) HTTPHeaderValue() string {
+ return strconv.Itoa(int(e.RoundSeconds().Seconds()))
+}
+
+// Custom type for collecting error details to be handled by `webError`. When an error
+// of this type is returned to the gateway handler, the StatusCode will be used for
+// the response status.
+type ErrorResponse struct {
+ StatusCode int
+ Err error
+}
+
+func NewErrorResponseForCode(statusCode int) *ErrorResponse {
+ return NewErrorResponse(errors.New(http.StatusText(statusCode)), statusCode)
+}
+
+func NewErrorResponse(err error, statusCode int) *ErrorResponse {
+ return &ErrorResponse{
+ Err: err,
+ StatusCode: statusCode,
+ }
+}
+
+func (e *ErrorResponse) Is(err error) bool {
+ switch err.(type) {
+ case *ErrorResponse:
+ return true
+ default:
+ return false
+ }
+}
+
+func (e *ErrorResponse) Error() string {
+ var text string
+ if e.Err != nil {
+ text = e.Err.Error()
+ }
+ return text
+}
+
+func (e *ErrorResponse) Unwrap() error {
+ return e.Err
+}
+
+func webError(w http.ResponseWriter, err error, defaultCode int) {
+ code := defaultCode
+
+ // Pass Retry-After hint to the client
+ var era *ErrorRetryAfter
+ if errors.As(err, &era) {
+ if era.RetryAfter > 0 {
+ w.Header().Set("Retry-After", era.HTTPHeaderValue())
+ // Adjust defaultCode if needed
+ if code != http.StatusTooManyRequests && code != http.StatusServiceUnavailable {
+ code = http.StatusTooManyRequests
+ }
+ }
+ err = era.Unwrap()
+ }
+
+ // Handle status code
+ switch {
+ case errors.Is(err, &cid.ErrInvalidCid{}):
+ code = http.StatusBadRequest
+ case isErrNotFound(err):
+ code = http.StatusNotFound
+ case errors.Is(err, context.DeadlineExceeded):
+ code = http.StatusGatewayTimeout
+ }
+
+ // Handle explicit code in ErrorResponse
+ var gwErr *ErrorResponse
+ if errors.As(err, &gwErr) {
+ code = gwErr.StatusCode
+ }
+
+ http.Error(w, err.Error(), code)
+}
+
+func isErrNotFound(err error) bool {
+ if ipld.IsNotFound(err) {
+ return true
+ }
+
+ // Checks if err is a resolver.ErrNoLink. resolver.ErrNoLink does not implement
+ // the .Is interface and cannot be directly compared to. Therefore, errors.Is
+ // always returns false with it.
+ for {
+ _, ok := err.(resolver.ErrNoLink)
+ if ok {
+ return true
+ }
+
+ err = errors.Unwrap(err)
+ if err == nil {
+ return false
+ }
+ }
+}
+
+func webRequestError(w http.ResponseWriter, err *ErrorResponse) {
+ webError(w, err.Err, err.StatusCode)
+}
diff --git a/gateway/errors_test.go b/gateway/errors_test.go
new file mode 100644
index 000000000..05e6ca887
--- /dev/null
+++ b/gateway/errors_test.go
@@ -0,0 +1,65 @@
+package gateway
+
+import (
+ "errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestErrRetryAfterIs(t *testing.T) {
+ var err error
+
+ err = NewErrorRetryAfter(errors.New("test"), 10*time.Second)
+ assert.True(t, errors.Is(err, &ErrorRetryAfter{}), "pointer to error must be error")
+
+ err = fmt.Errorf("wrapped: %w", err)
+ assert.True(t, errors.Is(err, &ErrorRetryAfter{}), "wrapped pointer to error must be error")
+}
+
+func TestErrRetryAfterAs(t *testing.T) {
+ var (
+ err error
+ errRA *ErrorRetryAfter
+ )
+
+ err = NewErrorRetryAfter(errors.New("test"), 25*time.Second)
+ assert.True(t, errors.As(err, &errRA), "pointer to error must be error")
+ assert.EqualValues(t, errRA.RetryAfter, 25*time.Second)
+
+ err = fmt.Errorf("wrapped: %w", err)
+ assert.True(t, errors.As(err, &errRA), "wrapped pointer to error must be error")
+ assert.EqualValues(t, errRA.RetryAfter, 25*time.Second)
+}
+
+func TestWebError(t *testing.T) {
+ t.Parallel()
+
+ t.Run("429 Too Many Requests", func(t *testing.T) {
+ err := fmt.Errorf("wrapped for testing: %w", NewErrorRetryAfter(ErrTooManyRequests, 0))
+ w := httptest.NewRecorder()
+ webError(w, err, http.StatusInternalServerError)
+ assert.Equal(t, http.StatusTooManyRequests, w.Result().StatusCode)
+ assert.Zero(t, len(w.Result().Header.Values("Retry-After")))
+ })
+
+ t.Run("429 Too Many Requests with Retry-After header", func(t *testing.T) {
+ err := NewErrorRetryAfter(ErrTooManyRequests, 25*time.Second)
+ w := httptest.NewRecorder()
+ webError(w, err, http.StatusInternalServerError)
+ assert.Equal(t, http.StatusTooManyRequests, w.Result().StatusCode)
+ assert.Equal(t, "25", w.Result().Header.Get("Retry-After"))
+ })
+
+ t.Run("503 Service Unavailable with Retry-After header", func(t *testing.T) {
+ err := NewErrorRetryAfter(ErrServiceUnavailable, 50*time.Second)
+ w := httptest.NewRecorder()
+ webError(w, err, http.StatusInternalServerError)
+ assert.Equal(t, http.StatusServiceUnavailable, w.Result().StatusCode)
+ assert.Equal(t, "50", w.Result().Header.Get("Retry-After"))
+ })
+}
diff --git a/gateway/gateway.go b/gateway/gateway.go
new file mode 100644
index 000000000..f32e0646f
--- /dev/null
+++ b/gateway/gateway.go
@@ -0,0 +1,120 @@
+package gateway
+
+import (
+ "context"
+ "net/http"
+ "sort"
+
+ "github.com/ipfs/boxo/blocks"
+ iface "github.com/ipfs/boxo/coreiface"
+ "github.com/ipfs/boxo/coreiface/path"
+ "github.com/ipfs/boxo/files"
+ cid "github.com/ipfs/go-cid"
+)
+
+// Config is the configuration used when creating a new gateway handler.
+type Config struct {
+ Headers map[string][]string
+}
+
+// API defines the minimal set of API services required for a gateway handler.
+type API interface {
+ // GetUnixFsNode returns a read-only handle to a file tree referenced by a path.
+ GetUnixFsNode(context.Context, path.Resolved) (files.Node, error)
+
+ // LsUnixFsDir returns the list of links in a directory.
+ LsUnixFsDir(context.Context, path.Resolved) (<-chan iface.DirEntry, error)
+
+ // GetBlock return a block from a certain CID.
+ GetBlock(context.Context, cid.Cid) (blocks.Block, error)
+
+ // GetIPNSRecord retrieves the best IPNS record for a given CID (libp2p-key)
+ // from the routing system.
+ GetIPNSRecord(context.Context, cid.Cid) ([]byte, error)
+
+ // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN.
+ // Unlike ResolvePath, it does not perform recursive resolution. It only
+ // checks for the existence of a DNSLink TXT record with path starting with
+ // /ipfs/ or /ipns/ and returns the path as-is.
+ GetDNSLinkRecord(context.Context, string) (path.Path, error)
+
+ // IsCached returns whether or not the path exists locally.
+ IsCached(context.Context, path.Path) bool
+
+ // ResolvePath resolves the path using UnixFS resolver. If the path does not
+ // exist due to a missing link, it should return an error of type:
+ // https://pkg.go.dev/github.com/ipfs/go-path@v0.3.0/resolver#ErrNoLink
+ ResolvePath(context.Context, path.Path) (path.Resolved, error)
+}
+
+// A helper function to clean up a set of headers:
+// 1. Canonicalizes.
+// 2. Deduplicates.
+// 3. Sorts.
+func cleanHeaderSet(headers []string) []string {
+ // Deduplicate and canonicalize.
+ m := make(map[string]struct{}, len(headers))
+ for _, h := range headers {
+ m[http.CanonicalHeaderKey(h)] = struct{}{}
+ }
+ result := make([]string, 0, len(m))
+ for k := range m {
+ result = append(result, k)
+ }
+
+ // Sort
+ sort.Strings(result)
+ return result
+}
+
+// AddAccessControlHeaders adds default headers used for controlling
+// cross-origin requests. This function adds several values to the
+// Access-Control-Allow-Headers and Access-Control-Expose-Headers entries.
+// If the Access-Control-Allow-Origin entry is missing a value of '*' is
+// added, indicating that browsers should allow requesting code from any
+// origin to access the resource.
+// If the Access-Control-Allow-Methods entry is missing a value of 'GET' is
+// added, indicating that browsers may use the GET method when issuing cross
+// origin requests.
+func AddAccessControlHeaders(headers map[string][]string) {
+ // Hard-coded headers.
+ const ACAHeadersName = "Access-Control-Allow-Headers"
+ const ACEHeadersName = "Access-Control-Expose-Headers"
+ const ACAOriginName = "Access-Control-Allow-Origin"
+ const ACAMethodsName = "Access-Control-Allow-Methods"
+
+ if _, ok := headers[ACAOriginName]; !ok {
+ // Default to *all*
+ headers[ACAOriginName] = []string{"*"}
+ }
+ if _, ok := headers[ACAMethodsName]; !ok {
+ // Default to GET
+ headers[ACAMethodsName] = []string{http.MethodGet}
+ }
+
+ headers[ACAHeadersName] = cleanHeaderSet(
+ append([]string{
+ "Content-Type",
+ "User-Agent",
+ "Range",
+ "X-Requested-With",
+ }, headers[ACAHeadersName]...))
+
+ headers[ACEHeadersName] = cleanHeaderSet(
+ append([]string{
+ "Content-Length",
+ "Content-Range",
+ "X-Chunked-Output",
+ "X-Stream-Output",
+ "X-Ipfs-Path",
+ "X-Ipfs-Roots",
+ }, headers[ACEHeadersName]...))
+}
+
+type RequestContextKey string
+
+const (
+ DNSLinkHostnameKey RequestContextKey = "dnslink-hostname"
+ GatewayHostnameKey RequestContextKey = "gw-hostname"
+ ContentPathKey RequestContextKey = "content-path"
+)
diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go
new file mode 100644
index 000000000..b78f95e45
--- /dev/null
+++ b/gateway/gateway_test.go
@@ -0,0 +1,637 @@
+package gateway
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ gopath "path"
+ "regexp"
+ "strings"
+ "testing"
+
+ "github.com/ipfs/boxo/blocks"
+ "github.com/ipfs/boxo/blockservice"
+ blockstore "github.com/ipfs/boxo/blockstore"
+ iface "github.com/ipfs/boxo/coreiface"
+ nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
+ ipath "github.com/ipfs/boxo/coreiface/path"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
+ "github.com/ipfs/boxo/files"
+ carblockstore "github.com/ipfs/boxo/ipld/car/v2/blockstore"
+ "github.com/ipfs/boxo/ipld/merkledag"
+ "github.com/ipfs/boxo/namesys"
+ "github.com/ipfs/boxo/namesys/resolve"
+ path "github.com/ipfs/boxo/path"
+ "github.com/ipfs/boxo/path/resolver"
+ "github.com/ipfs/boxo/unixfs"
+ ufile "github.com/ipfs/boxo/unixfs/file"
+ uio "github.com/ipfs/boxo/unixfs/io"
+ "github.com/ipfs/go-cid"
+ format "github.com/ipfs/go-ipld-format"
+ "github.com/ipfs/go-unixfsnode"
+ dagpb "github.com/ipld/go-codec-dagpb"
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/node/basicnode"
+ "github.com/ipld/go-ipld-prime/schema"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/stretchr/testify/assert"
+)
+
+type mockNamesys map[string]path.Path
+
+func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.ResolveOpt) (value path.Path, err error) {
+ cfg := nsopts.DefaultResolveOpts()
+ for _, o := range opts {
+ o(&cfg)
+ }
+ depth := cfg.Depth
+ if depth == nsopts.UnlimitedDepth {
+ // max uint
+ depth = ^uint(0)
+ }
+ for strings.HasPrefix(name, "/ipns/") {
+ if depth == 0 {
+ return value, namesys.ErrResolveRecursion
+ }
+ depth--
+
+ var ok bool
+ value, ok = m[name]
+ if !ok {
+ return "", namesys.ErrResolveFailed
+ }
+ name = value.String()
+ }
+ return value, nil
+}
+
+func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...nsopts.ResolveOpt) <-chan namesys.Result {
+ out := make(chan namesys.Result, 1)
+ v, err := m.Resolve(ctx, name, opts...)
+ out <- namesys.Result{Path: v, Err: err}
+ close(out)
+ return out
+}
+
+func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...nsopts.PublishOption) error {
+ return errors.New("not implemented for mockNamesys")
+}
+
+func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) {
+ return nil, false
+}
+
+type mockAPI struct {
+ blockStore blockstore.Blockstore
+ blockService blockservice.BlockService
+ dagService format.DAGService
+ resolver resolver.Resolver
+ namesys mockNamesys
+}
+
+func newMockAPI(t *testing.T) (*mockAPI, cid.Cid) {
+ r, err := os.Open("./testdata/fixtures.car")
+ assert.Nil(t, err)
+
+ blockStore, err := carblockstore.NewReadOnly(r, nil)
+ assert.Nil(t, err)
+
+ t.Cleanup(func() {
+ blockStore.Close()
+ r.Close()
+ })
+
+ cids, err := blockStore.Roots()
+ assert.Nil(t, err)
+ assert.Len(t, cids, 1)
+
+ blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
+ dagService := merkledag.NewDAGService(blockService)
+
+ fetcherConfig := bsfetcher.NewFetcherConfig(blockService)
+ fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+ if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
+ return tlnkNd.LinkTargetNodePrototype(), nil
+ }
+ return basicnode.Prototype.Any, nil
+ })
+ fetcher := fetcherConfig.WithReifier(unixfsnode.Reify)
+ resolver := resolver.NewBasicResolver(fetcher)
+
+ return &mockAPI{
+ blockStore: blockService.Blockstore(),
+ blockService: blockService,
+ dagService: dagService,
+ resolver: resolver,
+ namesys: mockNamesys{},
+ }, cids[0]
+}
+
+func (api *mockAPI) GetUnixFsNode(ctx context.Context, p ipath.Resolved) (files.Node, error) {
+ nd, err := api.resolveNode(ctx, p)
+ if err != nil {
+ return nil, err
+ }
+
+ return ufile.NewUnixfsFile(ctx, api.dagService, nd)
+}
+
+func (api *mockAPI) LsUnixFsDir(ctx context.Context, p ipath.Resolved) (<-chan iface.DirEntry, error) {
+ node, err := api.resolveNode(ctx, p)
+ if err != nil {
+ return nil, err
+ }
+
+ dir, err := uio.NewDirectoryFromNode(api.dagService, node)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make(chan iface.DirEntry, uio.DefaultShardWidth)
+
+ go func() {
+ defer close(out)
+ for l := range dir.EnumLinksAsync(ctx) {
+ select {
+ case out <- api.processLink(ctx, l):
+ case <-ctx.Done():
+ return
+ }
+ }
+ }()
+
+ return out, nil
+}
+
+func (api *mockAPI) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+ return api.blockService.GetBlock(ctx, c)
+}
+
+func (api *mockAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
+ return nil, routing.ErrNotSupported
+}
+
+func (api *mockAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) {
+ if api.namesys != nil {
+ p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
+ if err == namesys.ErrResolveRecursion {
+ err = nil
+ }
+ return ipath.New(p.String()), err
+ }
+
+ return nil, errors.New("not implemented")
+}
+
+func (api *mockAPI) IsCached(ctx context.Context, p ipath.Path) bool {
+ rp, err := api.ResolvePath(ctx, p)
+ if err != nil {
+ return false
+ }
+
+ has, _ := api.blockStore.Has(ctx, rp.Cid())
+ return has
+}
+
+func (api *mockAPI) ResolvePath(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) {
+ if _, ok := ip.(ipath.Resolved); ok {
+ return ip.(ipath.Resolved), nil
+ }
+
+ err := ip.IsValid()
+ if err != nil {
+ return nil, err
+ }
+
+ p := path.Path(ip.String())
+ if p.Segments()[0] == "ipns" {
+ p, err = resolve.ResolveIPNS(ctx, api.namesys, p)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ if p.Segments()[0] != "ipfs" {
+ return nil, fmt.Errorf("unsupported path namespace: %s", ip.Namespace())
+ }
+
+ node, rest, err := api.resolver.ResolveToLastNode(ctx, p)
+ if err != nil {
+ return nil, err
+ }
+
+ root, err := cid.Parse(p.Segments()[1])
+ if err != nil {
+ return nil, err
+ }
+
+ return ipath.NewResolvedPath(p, node, root, gopath.Join(rest...)), nil
+}
+
+func (api *mockAPI) resolveNode(ctx context.Context, p ipath.Path) (format.Node, error) {
+ rp, err := api.ResolvePath(ctx, p)
+ if err != nil {
+ return nil, err
+ }
+
+ node, err := api.dagService.Get(ctx, rp.Cid())
+ if err != nil {
+ return nil, fmt.Errorf("get node: %w", err)
+ }
+ return node, nil
+}
+
+func (api *mockAPI) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry {
+ if result.Err != nil {
+ return iface.DirEntry{Err: result.Err}
+ }
+
+ link := iface.DirEntry{
+ Name: result.Link.Name,
+ Cid: result.Link.Cid,
+ }
+
+ switch link.Cid.Type() {
+ case cid.Raw:
+ link.Type = iface.TFile
+ link.Size = result.Link.Size
+ case cid.DagProtobuf:
+ link.Size = result.Link.Size
+ }
+
+ return link
+}
+
+func doWithoutRedirect(req *http.Request) (*http.Response, error) {
+ tag := "without-redirect"
+ c := &http.Client{
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ return errors.New(tag)
+ },
+ }
+ res, err := c.Do(req)
+ if err != nil && !strings.Contains(err.Error(), tag) {
+ return nil, err
+ }
+ return res, nil
+}
+
+func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mockAPI, cid.Cid) {
+ api, root := newMockAPI(t)
+ ts := newTestServer(t, api)
+ return ts, api, root
+}
+
+func newTestServer(t *testing.T, api API) *httptest.Server {
+ config := Config{Headers: map[string][]string{}}
+ AddAccessControlHeaders(config.Headers)
+
+ handler := NewHandler(config, api)
+ mux := http.NewServeMux()
+ mux.Handle("/ipfs/", handler)
+ mux.Handle("/ipns/", handler)
+ handler = WithHostname(mux, api, map[string]*Specification{}, false)
+
+ ts := httptest.NewServer(handler)
+ t.Cleanup(func() { ts.Close() })
+
+ return ts
+}
+
+func matchPathOrBreadcrumbs(s string, expected string) bool {
+ matched, _ := regexp.MatchString("Index of(\n|\r\n)[\t ]*"+regexp.QuoteMeta(expected), s)
+ return matched
+}
+
+func TestGatewayGet(t *testing.T) {
+ ts, api, root := newTestServerAndNode(t, nil)
+ t.Logf("test server url: %s", ts.URL)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name(), "fnord"))
+ assert.Nil(t, err)
+
+ api.namesys["/ipns/example.com"] = path.FromCid(k.Cid())
+ api.namesys["/ipns/working.example.com"] = path.FromString(k.String())
+ api.namesys["/ipns/double.example.com"] = path.FromString("/ipns/working.example.com")
+ api.namesys["/ipns/triple.example.com"] = path.FromString("/ipns/double.example.com")
+ api.namesys["/ipns/broken.example.com"] = path.FromString("/ipns/" + k.Cid().String())
+ // We picked .man because:
+ // 1. It's a valid TLD.
+ // 2. Go treats it as the file extension for "man" files (even though
+ // nobody actually *uses* this extension, AFAIK).
+ //
+ // Unfortunately, this may not work on all platforms as file type
+ // detection is platform dependent.
+ api.namesys["/ipns/example.man"] = path.FromString(k.String())
+
+ t.Log(ts.URL)
+ for _, test := range []struct {
+ host string
+ path string
+ status int
+ text string
+ }{
+ {"127.0.0.1:8080", "/", http.StatusNotFound, "404 page not found\n"},
+ {"127.0.0.1:8080", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"},
+ {"127.0.0.1:8080", "/ipfs/this-is-not-a-cid", http.StatusBadRequest, "failed to resolve /ipfs/this-is-not-a-cid: invalid path \"/ipfs/this-is-not-a-cid\": invalid CID: invalid cid: illegal base32 data at input byte 3\n"},
+ {"127.0.0.1:8080", k.String(), http.StatusOK, "fnord"},
+ {"127.0.0.1:8080", "/ipns/nxdomain.example.com", http.StatusInternalServerError, "failed to resolve /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"},
+ {"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusInternalServerError, "failed to resolve /ipns/\\r\\n\\r\\nhello: " + namesys.ErrResolveFailed.Error() + "\n"},
+ {"127.0.0.1:8080", "/ipns/k51qzi5uqu5djucgtwlxrbfiyfez1nb0ct58q5s4owg6se02evza05dfgi6tw5", http.StatusInternalServerError, "failed to resolve /ipns/k51qzi5uqu5djucgtwlxrbfiyfez1nb0ct58q5s4owg6se02evza05dfgi6tw5: " + namesys.ErrResolveFailed.Error() + "\n"},
+ {"127.0.0.1:8080", "/ipns/example.com", http.StatusOK, "fnord"},
+ {"example.com", "/", http.StatusOK, "fnord"},
+
+ {"working.example.com", "/", http.StatusOK, "fnord"},
+ {"double.example.com", "/", http.StatusOK, "fnord"},
+ {"triple.example.com", "/", http.StatusOK, "fnord"},
+ {"working.example.com", k.String(), http.StatusNotFound, "failed to resolve /ipns/working.example.com" + k.String() + ": no link named \"ipfs\" under " + k.Cid().String() + "\n"},
+ {"broken.example.com", "/", http.StatusInternalServerError, "failed to resolve /ipns/broken.example.com/: " + namesys.ErrResolveFailed.Error() + "\n"},
+ {"broken.example.com", k.String(), http.StatusInternalServerError, "failed to resolve /ipns/broken.example.com" + k.String() + ": " + namesys.ErrResolveFailed.Error() + "\n"},
+ // This test case ensures we don't treat the TLD as a file extension.
+ {"example.man", "/", http.StatusOK, "fnord"},
+ } {
+ testName := "http://" + test.host + test.path
+ t.Run(testName, func(t *testing.T) {
+ var c http.Client
+ r, err := http.NewRequest(http.MethodGet, ts.URL+test.path, nil)
+ assert.Nil(t, err)
+ r.Host = test.host
+ resp, err := c.Do(r)
+ assert.Nil(t, err)
+ defer resp.Body.Close()
+ assert.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+ body, err := io.ReadAll(resp.Body)
+ assert.Nil(t, err)
+ assert.Equal(t, test.status, resp.StatusCode, "body", body)
+ assert.Equal(t, test.text, string(body))
+ })
+ }
+}
+
+func TestUriQueryRedirect(t *testing.T) {
+ ts, _, _ := newTestServerAndNode(t, mockNamesys{})
+
+ cid := "QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR"
+ for _, test := range []struct {
+ path string
+ status int
+ location string
+ }{
+ // - Browsers will send original URI in URL-escaped form
+ // - We expect query parameters to be persisted
+ // - We drop fragments, as those should not be sent by a browser
+ {"/ipfs/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"},
+ {"/ipfs/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"},
+ {"/ipfs/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid},
+ {"/ipfs?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/?uri=ipfs://" + cid},
+ {"/ipfs/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid},
+ {"/ipns/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"},
+ {"/ipns/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"},
+ {"/ipns?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/?uri=ipns://" + cid},
+ {"/ipns/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid},
+ {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid},
+ {"/ipfs/?uri=unsupported://" + cid, http.StatusBadRequest, ""},
+ {"/ipfs/?uri=invaliduri", http.StatusBadRequest, ""},
+ {"/ipfs/?uri=" + cid, http.StatusBadRequest, ""},
+ } {
+ testName := ts.URL + test.path
+ t.Run(testName, func(t *testing.T) {
+ r, err := http.NewRequest(http.MethodGet, ts.URL+test.path, nil)
+ assert.Nil(t, err)
+ resp, err := doWithoutRedirect(r)
+ assert.Nil(t, err)
+ defer resp.Body.Close()
+ assert.Equal(t, test.status, resp.StatusCode)
+ assert.Equal(t, test.location, resp.Header.Get("Location"))
+ })
+ }
+}
+
+func TestIPNSHostnameRedirect(t *testing.T) {
+ ts, api, root := newTestServerAndNode(t, nil)
+ t.Logf("test server url: %s", ts.URL)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name()))
+ assert.Nil(t, err)
+
+ t.Logf("k: %s\n", k)
+ api.namesys["/ipns/example.net"] = path.FromString(k.String())
+
+ // make request to directory containing index.html
+ req, err := http.NewRequest(http.MethodGet, ts.URL+"/foo", nil)
+ assert.Nil(t, err)
+ req.Host = "example.net"
+
+ res, err := doWithoutRedirect(req)
+ assert.Nil(t, err)
+
+ // expect 301 redirect to same path, but with trailing slash
+ assert.Equal(t, http.StatusMovedPermanently, res.StatusCode)
+ hdr := res.Header["Location"]
+ assert.Positive(t, len(hdr), "location header not present")
+ assert.Equal(t, hdr[0], "/foo/")
+
+ // make request with prefix to directory containing index.html
+ req, err = http.NewRequest(http.MethodGet, ts.URL+"/foo", nil)
+ assert.Nil(t, err)
+ req.Host = "example.net"
+
+ res, err = doWithoutRedirect(req)
+ assert.Nil(t, err)
+ // expect 301 redirect to same path, but with prefix and trailing slash
+ assert.Equal(t, http.StatusMovedPermanently, res.StatusCode)
+
+ hdr = res.Header["Location"]
+ assert.Positive(t, len(hdr), "location header not present")
+ assert.Equal(t, hdr[0], "/foo/")
+
+ // make sure /version isn't exposed
+ req, err = http.NewRequest(http.MethodGet, ts.URL+"/version", nil)
+ assert.Nil(t, err)
+ req.Host = "example.net"
+
+ res, err = doWithoutRedirect(req)
+ assert.Nil(t, err)
+ assert.Equal(t, http.StatusNotFound, res.StatusCode)
+}
+
+// Test directory listing on DNSLink website
+// (scenario when Host header is the same as URL hostname)
+// This is basic regression test: additional end-to-end tests
+// can be found in test/sharness/t0115-gateway-dir-listing.sh
+func TestIPNSHostnameBacklinks(t *testing.T) {
+ ts, api, root := newTestServerAndNode(t, nil)
+ t.Logf("test server url: %s", ts.URL)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name()))
+ assert.Nil(t, err)
+
+ // create /ipns/example.net/foo/
+ k2, err := api.ResolvePath(ctx, ipath.Join(k, "foo? #<'"))
+ assert.Nil(t, err)
+
+ k3, err := api.ResolvePath(ctx, ipath.Join(k, "foo? #<'/bar"))
+ assert.Nil(t, err)
+
+ t.Logf("k: %s\n", k)
+ api.namesys["/ipns/example.net"] = path.FromString(k.String())
+
+ // make request to directory listing
+ req, err := http.NewRequest(http.MethodGet, ts.URL+"/foo%3F%20%23%3C%27/", nil)
+ assert.Nil(t, err)
+ req.Host = "example.net"
+
+ res, err := doWithoutRedirect(req)
+ assert.Nil(t, err)
+
+ // expect correct links
+ body, err := io.ReadAll(res.Body)
+ assert.Nil(t, err)
+ s := string(body)
+ t.Logf("body: %s\n", string(body))
+
+ assert.True(t, matchPathOrBreadcrumbs(s, "/ipns/example.net /foo? #<' "), "expected a path in directory listing")
+ // https://github.com/ipfs/dir-index-html/issues/42
+ assert.Contains(t, s, "", "expected backlink in directory listing")
+ assert.Contains(t, s, " ", "expected file in directory listing")
+ assert.Contains(t, s, s, k2.Cid().String(), "expected hash in directory listing")
+
+ // make request to directory listing at root
+ req, err = http.NewRequest(http.MethodGet, ts.URL, nil)
+ assert.Nil(t, err)
+ req.Host = "example.net"
+
+ res, err = doWithoutRedirect(req)
+ assert.Nil(t, err)
+
+ // expect correct backlinks at root
+ body, err = io.ReadAll(res.Body)
+ assert.Nil(t, err)
+
+ s = string(body)
+ t.Logf("body: %s\n", string(body))
+
+ assert.True(t, matchPathOrBreadcrumbs(s, "/"), "expected a path in directory listing")
+ assert.NotContains(t, s, " ", "expected no backlink in directory listing of the root CID")
+ assert.Contains(t, s, " ", "expected file in directory listing")
+ // https://github.com/ipfs/dir-index-html/issues/42
+ assert.Contains(t, s, " example.net /foo? #<' /bar "), "expected a path in directory listing")
+ assert.Contains(t, s, "", "expected backlink in directory listing")
+ assert.Contains(t, s, " ", "expected file in directory listing")
+ assert.Contains(t, s, k3.Cid().String(), "expected hash in directory listing")
+}
+
+func TestPretty404(t *testing.T) {
+ ts, api, root := newTestServerAndNode(t, nil)
+ t.Logf("test server url: %s", ts.URL)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ k, err := api.ResolvePath(ctx, ipath.Join(ipath.IpfsPath(root), t.Name()))
+ assert.Nil(t, err)
+
+ host := "example.net"
+ api.namesys["/ipns/"+host] = path.FromString(k.String())
+
+ for _, test := range []struct {
+ path string
+ accept string
+ status int
+ text string
+ }{
+ {"/ipfs-404.html", "text/html", http.StatusOK, "Custom 404"},
+ {"/nope", "text/html", http.StatusNotFound, "Custom 404"},
+ {"/nope", "text/*", http.StatusNotFound, "Custom 404"},
+ {"/nope", "*/*", http.StatusNotFound, "Custom 404"},
+ {"/nope", "application/json", http.StatusNotFound, fmt.Sprintf("failed to resolve /ipns/example.net/nope: no link named \"nope\" under %s\n", k.Cid().String())},
+ {"/deeper/nope", "text/html", http.StatusNotFound, "Deep custom 404"},
+ {"/deeper/", "text/html", http.StatusOK, ""},
+ {"/deeper", "text/html", http.StatusOK, ""},
+ {"/nope/nope", "text/html", http.StatusNotFound, "Custom 404"},
+ } {
+ testName := fmt.Sprintf("%s %s", test.path, test.accept)
+ t.Run(testName, func(t *testing.T) {
+ var c http.Client
+ req, err := http.NewRequest("GET", ts.URL+test.path, nil)
+ assert.Nil(t, err)
+ req.Header.Add("Accept", test.accept)
+ req.Host = host
+ resp, err := c.Do(req)
+ assert.Nil(t, err)
+ defer resp.Body.Close()
+ assert.Equal(t, test.status, resp.StatusCode)
+ body, err := io.ReadAll(resp.Body)
+ assert.Nil(t, err)
+ if test.text != "" {
+ assert.Equal(t, test.text, string(body))
+ }
+ })
+ }
+}
+
+func TestCacheControlImmutable(t *testing.T) {
+ ts, _, root := newTestServerAndNode(t, nil)
+ t.Logf("test server url: %s", ts.URL)
+
+ req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/", nil)
+ assert.Nil(t, err)
+
+ res, err := doWithoutRedirect(req)
+ assert.Nil(t, err)
+
+ // check the immutable tag isn't set
+ hdrs, ok := res.Header["Cache-Control"]
+ if ok {
+ for _, hdr := range hdrs {
+ assert.NotContains(t, hdr, "immutable", "unexpected Cache-Control: immutable on directory listing")
+ }
+ }
+}
+
+func TestGoGetSupport(t *testing.T) {
+ ts, _, root := newTestServerAndNode(t, nil)
+ t.Logf("test server url: %s", ts.URL)
+
+ // mimic go-get
+ req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+root.String()+"?go-get=1", nil)
+ assert.Nil(t, err)
+
+ res, err := doWithoutRedirect(req)
+ assert.Nil(t, err)
+ assert.Equal(t, http.StatusOK, res.StatusCode)
+}
diff --git a/gateway/handler.go b/gateway/handler.go
new file mode 100644
index 000000000..02b907f6f
--- /dev/null
+++ b/gateway/handler.go
@@ -0,0 +1,880 @@
+package gateway
+
+import (
+ "context"
+ "fmt"
+ "html/template"
+ "io"
+ "mime"
+ "net/http"
+ "net/textproto"
+ "net/url"
+ gopath "path"
+ "regexp"
+ "runtime/debug"
+ "strings"
+ "time"
+
+ coreiface "github.com/ipfs/boxo/coreiface"
+ ipath "github.com/ipfs/boxo/coreiface/path"
+ cid "github.com/ipfs/go-cid"
+ logging "github.com/ipfs/go-log"
+ mc "github.com/multiformats/go-multicodec"
+ prometheus "github.com/prometheus/client_golang/prometheus"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+ "go.uber.org/zap"
+)
+
+var log = logging.Logger("core/server")
+
+const (
+ ipfsPathPrefix = "/ipfs/"
+ ipnsPathPrefix = "/ipns/"
+ immutableCacheControl = "public, max-age=29030400, immutable"
+)
+
+var (
+ onlyASCII = regexp.MustCompile("[[:^ascii:]]")
+ noModtime = time.Unix(0, 0) // disables Last-Modified header if passed as modtime
+)
+
+// HTML-based redirect for errors which can be recovered from, but we want
+// to provide hint to people that they should fix things on their end.
+var redirectTemplate = template.Must(template.New("redirect").Parse(`
+
+
+ {{.ErrorMsg}} (if a redirect does not happen in 10 seconds, use "{{.SuggestedPath}}" instead)
+
+`))
+
+type redirectTemplateData struct {
+ RedirectURL string
+ SuggestedPath string
+ ErrorMsg string
+}
+
+// handler is a HTTP handler that serves IPFS objects (accessible by default at /ipfs/)
+// (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link)
+type handler struct {
+ config Config
+ api API
+
+ // generic metrics
+ firstContentBlockGetMetric *prometheus.HistogramVec
+ unixfsGetMetric *prometheus.SummaryVec // deprecated, use firstContentBlockGetMetric
+
+ // response type metrics
+ getMetric *prometheus.HistogramVec
+ unixfsFileGetMetric *prometheus.HistogramVec
+ unixfsDirIndexGetMetric *prometheus.HistogramVec
+ unixfsGenDirListingGetMetric *prometheus.HistogramVec
+ carStreamGetMetric *prometheus.HistogramVec
+ rawBlockGetMetric *prometheus.HistogramVec
+ tarStreamGetMetric *prometheus.HistogramVec
+ jsoncborDocumentGetMetric *prometheus.HistogramVec
+ ipnsRecordGetMetric *prometheus.HistogramVec
+}
+
+// StatusResponseWriter enables us to override HTTP Status Code passed to
+// WriteHeader function inside of http.ServeContent. Decision is based on
+// presence of HTTP Headers such as Location.
+type statusResponseWriter struct {
+ http.ResponseWriter
+}
+
+func (sw *statusResponseWriter) WriteHeader(code int) {
+ // Check if we need to adjust Status Code to account for scheduled redirect
+ // This enables us to return payload along with HTTP 301
+ // for subdomain redirect in web browsers while also returning body for cli
+ // tools which do not follow redirects by default (curl, wget).
+ redirect := sw.ResponseWriter.Header().Get("Location")
+ if redirect != "" && code == http.StatusOK {
+ code = http.StatusMovedPermanently
+ log.Debugw("subdomain redirect", "location", redirect, "status", code)
+ }
+ sw.ResponseWriter.WriteHeader(code)
+}
+
+// ServeContent replies to the request using the content in the provided ReadSeeker
+// and returns the status code written and any error encountered during a write.
+// It wraps http.ServeContent which takes care of If-None-Match+Etag,
+// Content-Length and range requests.
+func ServeContent(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) (int, bool, error) {
+ ew := &errRecordingResponseWriter{ResponseWriter: w}
+ http.ServeContent(ew, req, name, modtime, content)
+
+ // When we calculate some metrics we want a flag that lets us to ignore
+ // errors and 304 Not Modified, and only care when requested data
+ // was sent in full.
+ dataSent := ew.code/100 == 2 && ew.err == nil
+
+ return ew.code, dataSent, ew.err
+}
+
+// errRecordingResponseWriter wraps a ResponseWriter to record the status code and any write error.
+type errRecordingResponseWriter struct {
+ http.ResponseWriter
+ code int
+ err error
+}
+
+func (w *errRecordingResponseWriter) WriteHeader(code int) {
+ if w.code == 0 {
+ w.code = code
+ }
+ w.ResponseWriter.WriteHeader(code)
+}
+
+func (w *errRecordingResponseWriter) Write(p []byte) (int, error) {
+ n, err := w.ResponseWriter.Write(p)
+ if err != nil && w.err == nil {
+ w.err = err
+ }
+ return n, err
+}
+
+// ReadFrom exposes errRecordingResponseWriter's underlying ResponseWriter to io.Copy
+// to allow optimized methods to be taken advantage of.
+func (w *errRecordingResponseWriter) ReadFrom(r io.Reader) (n int64, err error) {
+ n, err = io.Copy(w.ResponseWriter, r)
+ if err != nil && w.err == nil {
+ w.err = err
+ }
+ return n, err
+}
+
+func newSummaryMetric(name string, help string) *prometheus.SummaryVec {
+ summaryMetric := prometheus.NewSummaryVec(
+ prometheus.SummaryOpts{
+ Namespace: "ipfs",
+ Subsystem: "http",
+ Name: name,
+ Help: help,
+ },
+ []string{"gateway"},
+ )
+ if err := prometheus.Register(summaryMetric); err != nil {
+ if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+ summaryMetric = are.ExistingCollector.(*prometheus.SummaryVec)
+ } else {
+ log.Errorf("failed to register ipfs_http_%s: %v", name, err)
+ }
+ }
+ return summaryMetric
+}
+
+func newHistogramMetric(name string, help string) *prometheus.HistogramVec {
+ // We can add buckets as a parameter in the future, but for now using static defaults
+ // suggested in https://github.com/ipfs/kubo/issues/8441
+ defaultBuckets := []float64{0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60}
+ histogramMetric := prometheus.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Namespace: "ipfs",
+ Subsystem: "http",
+ Name: name,
+ Help: help,
+ Buckets: defaultBuckets,
+ },
+ []string{"gateway"},
+ )
+ if err := prometheus.Register(histogramMetric); err != nil {
+ if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+ histogramMetric = are.ExistingCollector.(*prometheus.HistogramVec)
+ } else {
+ log.Errorf("failed to register ipfs_http_%s: %v", name, err)
+ }
+ }
+ return histogramMetric
+}
+
+// NewHandler returns an http.Handler that can act as a gateway to IPFS content
+// offlineApi is a version of the API that should not make network requests for missing data
+func NewHandler(c Config, api API) http.Handler {
+ return newHandler(c, api)
+}
+
+func newHandler(c Config, api API) *handler {
+ i := &handler{
+ config: c,
+ api: api,
+ // Improved Metrics
+ // ----------------------------
+ // Time till the first content block (bar in /ipfs/cid/foo/bar)
+ // (format-agnostic, across all response types)
+ firstContentBlockGetMetric: newHistogramMetric(
+ "gw_first_content_block_get_latency_seconds",
+ "The time till the first content block is received on GET from the gateway.",
+ ),
+
+ // Response-type specific metrics
+ // ----------------------------
+ // Generic: time it takes to execute a successful gateway request (all request types)
+ getMetric: newHistogramMetric(
+ "gw_get_duration_seconds",
+ "The time to GET a successful response to a request (all content types).",
+ ),
+ // UnixFS: time it takes to return a file
+ unixfsFileGetMetric: newHistogramMetric(
+ "gw_unixfs_file_get_duration_seconds",
+ "The time to serve an entire UnixFS file from the gateway.",
+ ),
+ // UnixFS: time it takes to find and serve an index.html file on behalf of a directory.
+ unixfsDirIndexGetMetric: newHistogramMetric(
+ "gw_unixfs_dir_indexhtml_get_duration_seconds",
+ "The time to serve an index.html file on behalf of a directory from the gateway. This is a subset of gw_unixfs_file_get_duration_seconds.",
+ ),
+ // UnixFS: time it takes to generate static HTML with directory listing
+ unixfsGenDirListingGetMetric: newHistogramMetric(
+ "gw_unixfs_gen_dir_listing_get_duration_seconds",
+ "The time to serve a generated UnixFS HTML directory listing from the gateway.",
+ ),
+ // CAR: time it takes to return requested CAR stream
+ carStreamGetMetric: newHistogramMetric(
+ "gw_car_stream_get_duration_seconds",
+ "The time to GET an entire CAR stream from the gateway.",
+ ),
+ // Block: time it takes to return requested Block
+ rawBlockGetMetric: newHistogramMetric(
+ "gw_raw_block_get_duration_seconds",
+ "The time to GET an entire raw Block from the gateway.",
+ ),
+ // TAR: time it takes to return requested TAR stream
+ tarStreamGetMetric: newHistogramMetric(
+ "gw_tar_stream_get_duration_seconds",
+ "The time to GET an entire TAR stream from the gateway.",
+ ),
+ // JSON/CBOR: time it takes to return requested DAG-JSON/-CBOR document
+ jsoncborDocumentGetMetric: newHistogramMetric(
+ "gw_jsoncbor_get_duration_seconds",
+ "The time to GET an entire DAG-JSON/CBOR block from the gateway.",
+ ),
+ // IPNS Record: time it takes to return IPNS record
+ ipnsRecordGetMetric: newHistogramMetric(
+ "gw_ipns_record_get_duration_seconds",
+ "The time to GET an entire IPNS Record from the gateway.",
+ ),
+
+ // Legacy Metrics
+ // ----------------------------
+ unixfsGetMetric: newSummaryMetric( // TODO: remove?
+ // (deprecated, use firstContentBlockGetMetric instead)
+ "unixfs_get_latency_seconds",
+ "DEPRECATED: does not do what you think, use gw_first_content_block_get_latency_seconds instead.",
+ ),
+ }
+ return i
+}
+
+func (i *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ defer panicHandler(w)
+
+ // the hour is a hard fallback, we don't expect it to happen, but just in case
+ ctx, cancel := context.WithTimeout(r.Context(), time.Hour)
+ defer cancel()
+ r = r.WithContext(ctx)
+
+ switch r.Method {
+ case http.MethodGet, http.MethodHead:
+ i.getOrHeadHandler(w, r)
+ return
+ case http.MethodOptions:
+ i.optionsHandler(w, r)
+ return
+ }
+
+ w.Header().Add("Allow", http.MethodGet)
+ w.Header().Add("Allow", http.MethodHead)
+ w.Header().Add("Allow", http.MethodOptions)
+
+ errmsg := "Method " + r.Method + " not allowed: read only access"
+ http.Error(w, errmsg, http.StatusMethodNotAllowed)
+}
+
+func (i *handler) optionsHandler(w http.ResponseWriter, r *http.Request) {
+ /*
+ OPTIONS is a noop request that is used by the browsers to check
+ if server accepts cross-site XMLHttpRequest (indicated by the presence of CORS headers)
+ https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS#Preflighted_requests
+ */
+ i.addUserHeaders(w) // return all custom headers (including CORS ones, if set)
+}
+
+func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
+ begin := time.Now()
+
+ logger := log.With("from", r.RequestURI)
+ logger.Debug("http request received")
+
+ if err := handleUnsupportedHeaders(r); err != nil {
+ webRequestError(w, err)
+ return
+ }
+
+ if requestHandled := handleProtocolHandlerRedirect(w, r, logger); requestHandled {
+ return
+ }
+
+ if err := handleServiceWorkerRegistration(r); err != nil {
+ webRequestError(w, err)
+ return
+ }
+
+ contentPath := ipath.New(r.URL.Path)
+ ctx := context.WithValue(r.Context(), ContentPathKey, contentPath)
+ r = r.WithContext(ctx)
+
+ if requestHandled := i.handleOnlyIfCached(w, r, contentPath, logger); requestHandled {
+ return
+ }
+
+ if requestHandled := handleSuperfluousNamespace(w, r, contentPath); requestHandled {
+ return
+ }
+
+ // Detect when explicit Accept header or ?format parameter are present
+ responseFormat, formatParams, err := customResponseFormat(r)
+ if err != nil {
+ webError(w, fmt.Errorf("error while processing the Accept header: %w", err), http.StatusBadRequest)
+ return
+ }
+ trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat))
+
+ resolvedPath, contentPath, ok := i.handlePathResolution(w, r, responseFormat, contentPath, logger)
+ if !ok {
+ return
+ }
+ trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String()))
+
+ // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified
+ if inm := r.Header.Get("If-None-Match"); inm != "" {
+ pathCid := resolvedPath.Cid()
+ // need to check against both File and Dir Etag variants
+ // because this inexpensive check happens before we do any I/O
+ cidEtag := getEtag(r, pathCid)
+ dirEtag := getDirListingEtag(pathCid)
+ if etagMatch(inm, cidEtag, dirEtag) {
+ // Finish early if client already has a matching Etag
+ w.WriteHeader(http.StatusNotModified)
+ return
+ }
+ }
+
+ if err := i.handleGettingFirstBlock(r, begin, contentPath, resolvedPath); err != nil {
+ webRequestError(w, err)
+ return
+ }
+
+ if err := i.setCommonHeaders(w, r, contentPath); err != nil {
+ webRequestError(w, err)
+ return
+ }
+
+ var success bool
+
+ // Support custom response formats passed via ?format or Accept HTTP header
+ switch responseFormat {
+ case "", "application/json", "application/cbor":
+ switch mc.Code(resolvedPath.Cid().Prefix().Codec) {
+ case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor:
+ logger.Debugw("serving codec", "path", contentPath)
+ success = i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat)
+ default:
+ logger.Debugw("serving unixfs", "path", contentPath)
+ success = i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
+ }
+ case "application/vnd.ipld.raw":
+ logger.Debugw("serving raw block", "path", contentPath)
+ success = i.serveRawBlock(r.Context(), w, r, resolvedPath, contentPath, begin)
+ case "application/vnd.ipld.car":
+ logger.Debugw("serving car stream", "path", contentPath)
+ carVersion := formatParams["version"]
+ success = i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin)
+ case "application/x-tar":
+ logger.Debugw("serving tar file", "path", contentPath)
+ success = i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
+ case "application/vnd.ipld.dag-json", "application/vnd.ipld.dag-cbor":
+ logger.Debugw("serving codec", "path", contentPath)
+ success = i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat)
+ case "application/vnd.ipfs.ipns-record":
+ logger.Debugw("serving ipns record", "path", contentPath)
+ success = i.serveIpnsRecord(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
+ default: // catch-all for unsuported application/vnd.*
+ err := fmt.Errorf("unsupported format %q", responseFormat)
+ webError(w, err, http.StatusBadRequest)
+ return
+ }
+
+ if success {
+ i.getMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds())
+ }
+}
+
+func (i *handler) addUserHeaders(w http.ResponseWriter) {
+ for k, v := range i.config.Headers {
+ w.Header()[k] = v
+ }
+}
+
+func panicHandler(w http.ResponseWriter) {
+ if r := recover(); r != nil {
+ log.Error("A panic occurred in the gateway handler!")
+ log.Error(r)
+ debug.PrintStack()
+ w.WriteHeader(http.StatusInternalServerError)
+ }
+}
+
+func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, fileCid cid.Cid) (modtime time.Time) {
+ // Set Etag to based on CID (override whatever was set before)
+ w.Header().Set("Etag", getEtag(r, fileCid))
+
+ // Set Cache-Control and Last-Modified based on contentPath properties
+ if contentPath.Mutable() {
+ // mutable namespaces such as /ipns/ can't be cached forever
+
+ /* For now we set Last-Modified to Now() to leverage caching heuristics built into modern browsers:
+ * https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768
+ * but we should not set it to fake values and use Cache-Control based on TTL instead */
+ modtime = time.Now()
+
+ // TODO: set Cache-Control based on TTL of IPNS/DNSLink: https://github.com/ipfs/kubo/issues/1818#issuecomment-1015849462
+ // TODO: set Last-Modified based on /ipns/ publishing timestamp?
+ } else {
+ // immutable! CACHE ALL THE THINGS, FOREVER! wolololol
+ w.Header().Set("Cache-Control", immutableCacheControl)
+
+ // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control)
+ modtime = noModtime
+
+ // TODO: set Last-Modified? - TBD - /ipfs/ modification metadata is present in unixfs 1.5 https://github.com/ipfs/kubo/issues/6920?
+ }
+
+ return modtime
+}
+
+// Set Content-Disposition if filename URL query param is present, return preferred filename
+func addContentDispositionHeader(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) string {
+ /* This logic enables:
+ * - creation of HTML links that trigger "Save As.." dialog instead of being rendered by the browser
+ * - overriding the filename used when saving subresource assets on HTML page
+ * - providing a default filename for HTTP clients when downloading direct /ipfs/CID without any subpath
+ */
+
+ // URL param ?filename=cat.jpg triggers Content-Disposition: [..] filename
+ // which impacts default name used in "Save As.." dialog
+ name := getFilename(contentPath)
+ urlFilename := r.URL.Query().Get("filename")
+ if urlFilename != "" {
+ disposition := "inline"
+ // URL param ?download=true triggers Content-Disposition: [..] attachment
+ // which skips rendering and forces "Save As.." dialog in browsers
+ if r.URL.Query().Get("download") == "true" {
+ disposition = "attachment"
+ }
+ setContentDispositionHeader(w, urlFilename, disposition)
+ name = urlFilename
+ }
+ return name
+}
+
+// Set Content-Disposition to arbitrary filename and disposition
+func setContentDispositionHeader(w http.ResponseWriter, filename string, disposition string) {
+ utf8Name := url.PathEscape(filename)
+ asciiName := url.PathEscape(onlyASCII.ReplaceAllLiteralString(filename, "_"))
+ w.Header().Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, asciiName, utf8Name))
+}
+
+// Set X-Ipfs-Roots with logical CID array for efficient HTTP cache invalidation.
+func (i *handler) buildIpfsRootsHeader(contentPath string, r *http.Request) (string, error) {
+ /*
+ These are logical roots where each CID represent one path segment
+ and resolves to either a directory or the root block of a file.
+ The main purpose of this header is allow HTTP caches to do smarter decisions
+ around cache invalidation (eg. keep specific subdirectory/file if it did not change)
+
+ A good example is Wikipedia, which is HAMT-sharded, but we only care about
+ logical roots that represent each segment of the human-readable content
+ path:
+
+ Given contentPath = /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey
+ rootCidList is a generated by doing `ipfs resolve -r` on each sub path:
+ /ipns/en.wikipedia-on-ipfs.org → bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze
+ /ipns/en.wikipedia-on-ipfs.org/wiki/ → bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4
+ /ipns/en.wikipedia-on-ipfs.org/wiki/Block_of_Wikipedia_in_Turkey → bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma
+
+ The result is an ordered array of values:
+ X-Ipfs-Roots: bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze,bafybeihn2f7lhumh4grizksi2fl233cyszqadkn424ptjajfenykpsaiw4,bafkreibn6euazfvoghepcm4efzqx5l3hieof2frhp254hio5y7n3hv5rma
+
+ Note that while the top one will change every time any article is changed,
+ the last root (responsible for specific article) may not change at all.
+ */
+ var sp strings.Builder
+ var pathRoots []string
+ pathSegments := strings.Split(contentPath[6:], "/")
+ sp.WriteString(contentPath[:5]) // /ipfs or /ipns
+ for _, root := range pathSegments {
+ if root == "" {
+ continue
+ }
+ sp.WriteString("/")
+ sp.WriteString(root)
+ resolvedSubPath, err := i.api.ResolvePath(r.Context(), ipath.New(sp.String()))
+ if err != nil {
+ return "", err
+ }
+ pathRoots = append(pathRoots, resolvedSubPath.Cid().String())
+ }
+ rootCidList := strings.Join(pathRoots, ",") // convention from rfc2616#sec4.2
+ return rootCidList, nil
+}
+
+func getFilename(contentPath ipath.Path) string {
+ s := contentPath.String()
+ if (strings.HasPrefix(s, ipfsPathPrefix) || strings.HasPrefix(s, ipnsPathPrefix)) && strings.Count(gopath.Clean(s), "/") <= 2 {
+ // Don't want to treat ipfs.io in /ipns/ipfs.io as a filename.
+ return ""
+ }
+ return gopath.Base(s)
+}
+
+// etagMatch evaluates if we can respond with HTTP 304 Not Modified
+// It supports multiple weak and strong etags passed in If-None-Matc stringh
+// including the wildcard one.
+func etagMatch(ifNoneMatchHeader string, cidEtag string, dirEtag string) bool {
+ buf := ifNoneMatchHeader
+ for {
+ buf = textproto.TrimString(buf)
+ if len(buf) == 0 {
+ break
+ }
+ if buf[0] == ',' {
+ buf = buf[1:]
+ continue
+ }
+ // If-None-Match: * should match against any etag
+ if buf[0] == '*' {
+ return true
+ }
+ etag, remain := scanETag(buf)
+ if etag == "" {
+ break
+ }
+ // Check for match both strong and weak etags
+ if etagWeakMatch(etag, cidEtag) || etagWeakMatch(etag, dirEtag) {
+ return true
+ }
+ buf = remain
+ }
+ return false
+}
+
+// scanETag determines if a syntactically valid ETag is present at s. If so,
+// the ETag and remaining text after consuming ETag is returned. Otherwise,
+// it returns "", "".
+// (This is the same logic as one executed inside of http.ServeContent)
+func scanETag(s string) (etag string, remain string) {
+ s = textproto.TrimString(s)
+ start := 0
+ if strings.HasPrefix(s, "W/") {
+ start = 2
+ }
+ if len(s[start:]) < 2 || s[start] != '"' {
+ return "", ""
+ }
+ // ETag is either W/"text" or "text".
+ // See RFC 7232 2.3.
+ for i := start + 1; i < len(s); i++ {
+ c := s[i]
+ switch {
+ // Character values allowed in ETags.
+ case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
+ case c == '"':
+ return s[:i+1], s[i+1:]
+ default:
+ return "", ""
+ }
+ }
+ return "", ""
+}
+
+// etagWeakMatch reports whether a and b match using weak ETag comparison.
+func etagWeakMatch(a, b string) bool {
+ return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
+}
+
+// generate Etag value based on HTTP request and CID
+func getEtag(r *http.Request, cid cid.Cid) string {
+ prefix := `"`
+ suffix := `"`
+ responseFormat, _, err := customResponseFormat(r)
+ if err == nil && responseFormat != "" {
+ // application/vnd.ipld.foo → foo
+ // application/x-bar → x-bar
+ shortFormat := responseFormat[strings.LastIndexAny(responseFormat, "/.")+1:]
+ // Etag: "cid.shortFmt" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
+ suffix = `.` + shortFormat + suffix
+ }
+ // TODO: include selector suffix when https://github.com/ipfs/kubo/issues/8769 lands
+ return prefix + cid.String() + suffix
+}
+
+// return explicit response format if specified in request as query parameter or via Accept HTTP header
+func customResponseFormat(r *http.Request) (mediaType string, params map[string]string, err error) {
+ if formatParam := r.URL.Query().Get("format"); formatParam != "" {
+ // translate query param to a content type
+ switch formatParam {
+ case "raw":
+ return "application/vnd.ipld.raw", nil, nil
+ case "car":
+ return "application/vnd.ipld.car", nil, nil
+ case "tar":
+ return "application/x-tar", nil, nil
+ case "json":
+ return "application/json", nil, nil
+ case "cbor":
+ return "application/cbor", nil, nil
+ case "dag-json":
+ return "application/vnd.ipld.dag-json", nil, nil
+ case "dag-cbor":
+ return "application/vnd.ipld.dag-cbor", nil, nil
+ case "ipns-record":
+ return "application/vnd.ipfs.ipns-record", nil, nil
+ }
+ }
+ // Browsers and other user agents will send Accept header with generic types like:
+ // Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8
+ // We only care about explicit, vendor-specific content-types and respond to the first match (in order).
+ // TODO: make this RFC compliant and respect weights (eg. return CAR for Accept:application/vnd.ipld.dag-json;q=0.1,application/vnd.ipld.car;q=0.2)
+ for _, header := range r.Header.Values("Accept") {
+ for _, value := range strings.Split(header, ",") {
+ accept := strings.TrimSpace(value)
+ // respond to the very first matching content type
+ if strings.HasPrefix(accept, "application/vnd.ipld") ||
+ strings.HasPrefix(accept, "application/x-tar") ||
+ strings.HasPrefix(accept, "application/json") ||
+ strings.HasPrefix(accept, "application/cbor") ||
+ strings.HasPrefix(accept, "application/vnd.ipfs") {
+ mediatype, params, err := mime.ParseMediaType(accept)
+ if err != nil {
+ return "", nil, err
+ }
+ return mediatype, params, nil
+ }
+ }
+ }
+ // If none of special-cased content types is found, return empty string
+ // to indicate default, implicit UnixFS response should be prepared
+ return "", nil, nil
+}
+
+// returns unquoted path with all special characters revealed as \u codes
+func debugStr(path string) string {
+ q := fmt.Sprintf("%+q", path)
+ if len(q) >= 3 {
+ q = q[1 : len(q)-1]
+ }
+ return q
+}
+
+// Resolve the provided contentPath including any special handling related to
+// the requested responseFormat. Returned ok flag indicates if gateway handler
+// should continue processing the request.
+func (i *handler) handlePathResolution(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath ipath.Path, logger *zap.SugaredLogger) (resolvedPath ipath.Resolved, newContentPath ipath.Path, ok bool) {
+ // Attempt to resolve the provided path.
+ resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
+
+ switch err {
+ case nil:
+ return resolvedPath, contentPath, true
+ case coreiface.ErrOffline:
+ err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err)
+ webError(w, err, http.StatusServiceUnavailable)
+ return nil, nil, false
+ default:
+ // The path can't be resolved.
+ if isUnixfsResponseFormat(responseFormat) {
+ // If we have origin isolation (subdomain gw, DNSLink website),
+ // and response type is UnixFS (default for website hosting)
+ // check for presence of _redirects file and apply rules defined there.
+ // See: https://github.com/ipfs/specs/pull/290
+ if hasOriginIsolation(r) {
+ resolvedPath, newContentPath, ok, hadMatchingRule := i.serveRedirectsIfPresent(w, r, resolvedPath, contentPath, logger)
+ if hadMatchingRule {
+ logger.Debugw("applied a rule from _redirects file")
+ return resolvedPath, newContentPath, ok
+ }
+ }
+
+ // if Accept is text/html, see if ipfs-404.html is present
+ // This logic isn't documented and will likely be removed at some point.
+ // Any 404 logic in _redirects above will have already run by this time, so it's really an extra fall back
+ if i.serveLegacy404IfPresent(w, r, contentPath) {
+ logger.Debugw("served legacy 404")
+ return nil, nil, false
+ }
+ }
+
+ err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err)
+ webError(w, err, http.StatusInternalServerError)
+ return nil, nil, false
+ }
+}
+
+// Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore.
+// https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#cache-control-request-header
+func (i *handler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) (requestHandled bool) {
+ if r.Header.Get("Cache-Control") == "only-if-cached" {
+ if !i.api.IsCached(r.Context(), contentPath) {
+ if r.Method == http.MethodHead {
+ w.WriteHeader(http.StatusPreconditionFailed)
+ return true
+ }
+ errMsg := fmt.Sprintf("%q not in local datastore", contentPath.String())
+ http.Error(w, errMsg, http.StatusPreconditionFailed)
+ return true
+ }
+ if r.Method == http.MethodHead {
+ w.WriteHeader(http.StatusOK)
+ return true
+ }
+ }
+ return false
+}
+
+func handleUnsupportedHeaders(r *http.Request) (err *ErrorResponse) {
+ // X-Ipfs-Gateway-Prefix was removed (https://github.com/ipfs/kubo/issues/7702)
+ // TODO: remove this after go-ipfs 0.13 ships
+ if prfx := r.Header.Get("X-Ipfs-Gateway-Prefix"); prfx != "" {
+ err := fmt.Errorf("unsupported HTTP header: X-Ipfs-Gateway-Prefix support was removed: https://github.com/ipfs/kubo/issues/7702")
+ return NewErrorResponse(err, http.StatusBadRequest)
+ }
+ return nil
+}
+
+// ?uri query param support for requests produced by web browsers
+// via navigator.registerProtocolHandler Web API
+// https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler
+// TLDR: redirect /ipfs/?uri=ipfs%3A%2F%2Fcid%3Fquery%3Dval to /ipfs/cid?query=val
+func handleProtocolHandlerRedirect(w http.ResponseWriter, r *http.Request, logger *zap.SugaredLogger) (requestHandled bool) {
+ if uriParam := r.URL.Query().Get("uri"); uriParam != "" {
+ u, err := url.Parse(uriParam)
+ if err != nil {
+ webError(w, fmt.Errorf("failed to parse uri query parameter: %w", err), http.StatusBadRequest)
+ return true
+ }
+ if u.Scheme != "ipfs" && u.Scheme != "ipns" {
+ webError(w, fmt.Errorf("uri query parameter scheme must be ipfs or ipns: %w", err), http.StatusBadRequest)
+ return true
+ }
+ path := u.Path
+ if u.RawQuery != "" { // preserve query if present
+ path = path + "?" + u.RawQuery
+ }
+
+ redirectURL := gopath.Join("/", u.Scheme, u.Host, path)
+ logger.Debugw("uri param, redirect", "to", redirectURL, "status", http.StatusMovedPermanently)
+ http.Redirect(w, r, redirectURL, http.StatusMovedPermanently)
+ return true
+ }
+
+ return false
+}
+
+// Disallow Service Worker registration on namespace roots
+// https://github.com/ipfs/kubo/issues/4025
+func handleServiceWorkerRegistration(r *http.Request) (err *ErrorResponse) {
+ if r.Header.Get("Service-Worker") == "script" {
+ matched, _ := regexp.MatchString(`^/ip[fn]s/[^/]+$`, r.URL.Path)
+ if matched {
+ err := fmt.Errorf("registration is not allowed for this scope")
+ return NewErrorResponse(fmt.Errorf("navigator.serviceWorker: %w", err), http.StatusBadRequest)
+ }
+ }
+
+ return nil
+}
+
+// Attempt to fix redundant /ipfs/ namespace as long as resulting
+// 'intended' path is valid. This is in case gremlins were tickled
+// wrong way and user ended up at /ipfs/ipfs/{cid} or /ipfs/ipns/{id}
+// like in bafybeien3m7mdn6imm425vc2s22erzyhbvk5n3ofzgikkhmdkh5cuqbpbq :^))
+func handleSuperfluousNamespace(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) (requestHandled bool) {
+ // If the path is valid, there's nothing to do
+ if pathErr := contentPath.IsValid(); pathErr == nil {
+ return false
+ }
+
+ // If there's no superflous namespace, there's nothing to do
+ if !(strings.HasPrefix(r.URL.Path, "/ipfs/ipfs/") || strings.HasPrefix(r.URL.Path, "/ipfs/ipns/")) {
+ return false
+ }
+
+ // Attempt to fix the superflous namespace
+ intendedPath := ipath.New(strings.TrimPrefix(r.URL.Path, "/ipfs"))
+ if err := intendedPath.IsValid(); err != nil {
+ webError(w, fmt.Errorf("invalid ipfs path: %w", err), http.StatusBadRequest)
+ return true
+ }
+ intendedURL := intendedPath.String()
+ if r.URL.RawQuery != "" {
+ // we render HTML, so ensure query entries are properly escaped
+ q, _ := url.ParseQuery(r.URL.RawQuery)
+ intendedURL = intendedURL + "?" + q.Encode()
+ }
+ // return HTTP 400 (Bad Request) with HTML error page that:
+ // - points at correct canonical path via Apache-2.0 Software License
+
+```
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+```
+MIT Software License
+
+```
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+```
+ + ).
+ if int64(dataOffset) < PragmaSize+HeaderSize {
+ return n, fmt.Errorf("invalid data payload offset: %v", dataOffset)
+ }
+ // Assert the data size validity.
+ // It must be larger than zero.
+ // Technically, it should be at least 11 bytes (i.e. a valid CARv1 header with no roots) but
+ // we let further parsing of the header to signal invalid data payload header.
+ if int64(dataSize) <= 0 {
+ return n, fmt.Errorf("invalid data payload size: %v", dataSize)
+ }
+ // Assert the index offset validity.
+ if int64(indexOffset) < 0 {
+ return n, fmt.Errorf("invalid index offset: %v", indexOffset)
+ }
+ h.DataOffset = dataOffset
+ h.DataSize = dataSize
+ h.IndexOffset = indexOffset
+ return n, nil
+}
diff --git a/ipld/car/v2/car_test.go b/ipld/car/v2/car_test.go
new file mode 100644
index 000000000..c5d35063c
--- /dev/null
+++ b/ipld/car/v2/car_test.go
@@ -0,0 +1,247 @@
+package car_test
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCarV2PragmaLength(t *testing.T) {
+ tests := []struct {
+ name string
+ want interface{}
+ got interface{}
+ }{
+ {
+ "ActualSizeShouldBe11",
+ 11,
+ len(carv2.Pragma),
+ },
+ {
+ "ShouldStartWithVarint(10)",
+ carv2.Pragma[0],
+ 10,
+ },
+ }
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ assert.EqualValues(t, tt.want, tt.got, "CarV2Pragma got = %v, want %v", tt.got, tt.want)
+ })
+ }
+}
+
+func TestCarV2PragmaIsValidCarV1Header(t *testing.T) {
+ v1h, err := carv1.ReadHeader(bytes.NewReader(carv2.Pragma), carv1.DefaultMaxAllowedHeaderSize)
+ assert.NoError(t, err, "cannot decode pragma as CBOR with CARv1 header structure")
+ assert.Equal(t, &carv1.CarHeader{
+ Roots: nil,
+ Version: 2,
+ }, v1h, "CARv2 pragma must be a valid CARv1 header")
+}
+
+func TestHeader_WriteTo(t *testing.T) {
+ tests := []struct {
+ name string
+ target carv2.Header
+ wantWrite []byte
+ wantErr bool
+ }{
+ {
+ "HeaderWithEmptyCharacteristicsIsWrittenAsExpected",
+ carv2.Header{
+ Characteristics: carv2.Characteristics{},
+ DataOffset: 99,
+ },
+ []byte{
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x63, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ },
+ false,
+ },
+ {
+ "NonEmptyHeaderIsWrittenAsExpected",
+ carv2.Header{
+ Characteristics: carv2.Characteristics{
+ Hi: 1001, Lo: 1002,
+ },
+ DataOffset: 99,
+ DataSize: 100,
+ IndexOffset: 101,
+ },
+ []byte{
+ 0xe9, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0xea, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x63, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x64, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x65, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ },
+ false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ buf := &bytes.Buffer{}
+ written, err := tt.target.WriteTo(buf)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("WriteTo() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ gotWrite := buf.Bytes()
+ assert.Equal(t, tt.wantWrite, gotWrite, "Header.WriteTo() gotWrite = %v, wantWrite %v", gotWrite, tt.wantWrite)
+ assert.EqualValues(t, carv2.HeaderSize, uint64(len(gotWrite)), "WriteTo() CARv2 header length must always be %v bytes long", carv2.HeaderSize)
+ assert.EqualValues(t, carv2.HeaderSize, uint64(written), "WriteTo() CARv2 header byte count must always be %v bytes long", carv2.HeaderSize)
+ })
+ }
+}
+
+func TestHeader_ReadFrom(t *testing.T) {
+ tests := []struct {
+ name string
+ target []byte
+ wantHeader carv2.Header
+ wantErr bool
+ }{
+ {
+ "HeaderWithEmptyCharacteristicsIsWrittenAsExpected",
+ []byte{
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x63, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x64, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ },
+ carv2.Header{
+ Characteristics: carv2.Characteristics{},
+ DataOffset: 99,
+ DataSize: 100,
+ },
+ false,
+ },
+ {
+ "NonEmptyHeaderIsWrittenAsExpected",
+
+ []byte{
+ 0xe9, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0xea, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x63, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x64, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x65, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ },
+ carv2.Header{
+ Characteristics: carv2.Characteristics{
+ Hi: 1001, Lo: 1002,
+ },
+ DataOffset: 99,
+ DataSize: 100,
+ IndexOffset: 101,
+ },
+ false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotHeader := carv2.Header{}
+ gotRead, err := gotHeader.ReadFrom(bytes.NewReader(tt.target))
+ assert.NoError(t, err)
+ assert.Equal(t, int64(carv2.HeaderSize), gotRead)
+ assert.Equal(t, tt.wantHeader, gotHeader)
+ })
+ }
+}
+
+func TestHeader_WithPadding(t *testing.T) {
+ tests := []struct {
+ name string
+ subject carv2.Header
+ wantCarV1Offset uint64
+ wantIndexOffset uint64
+ }{
+ {
+ "WhenNoPaddingOffsetsAreSumOfSizes",
+ carv2.NewHeader(123),
+ carv2.PragmaSize + carv2.HeaderSize,
+ carv2.PragmaSize + carv2.HeaderSize + 123,
+ },
+ {
+ "WhenOnlyPaddingCarV1BothOffsetsShift",
+ carv2.NewHeader(123).WithDataPadding(3),
+ carv2.PragmaSize + carv2.HeaderSize + 3,
+ carv2.PragmaSize + carv2.HeaderSize + 3 + 123,
+ },
+ {
+ "WhenPaddingBothCarV1AndIndexBothOffsetsShiftWithAdditionalIndexShift",
+ carv2.NewHeader(123).WithDataPadding(3).WithIndexPadding(7),
+ carv2.PragmaSize + carv2.HeaderSize + 3,
+ carv2.PragmaSize + carv2.HeaderSize + 3 + 123 + 7,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.EqualValues(t, tt.wantCarV1Offset, tt.subject.DataOffset)
+ assert.EqualValues(t, tt.wantIndexOffset, tt.subject.IndexOffset)
+ })
+ }
+}
+
+func TestNewHeaderHasExpectedValues(t *testing.T) {
+ wantCarV1Len := uint64(1413)
+ want := carv2.Header{
+ Characteristics: carv2.Characteristics{},
+ DataOffset: carv2.PragmaSize + carv2.HeaderSize,
+ DataSize: wantCarV1Len,
+ IndexOffset: carv2.PragmaSize + carv2.HeaderSize + wantCarV1Len,
+ }
+ got := carv2.NewHeader(wantCarV1Len)
+ assert.Equal(t, want, got, "NewHeader got = %v, want = %v", got, want)
+}
+
+func TestCharacteristics_StoreIdentityCIDs(t *testing.T) {
+ subject := carv2.Characteristics{}
+ require.False(t, subject.IsFullyIndexed())
+
+ subject.SetFullyIndexed(true)
+ require.True(t, subject.IsFullyIndexed())
+
+ var buf bytes.Buffer
+ written, err := subject.WriteTo(&buf)
+ require.NoError(t, err)
+ require.Equal(t, int64(16), written)
+ require.Equal(t, []byte{
+ 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ }, buf.Bytes())
+
+ var decodedSubject carv2.Characteristics
+ read, err := decodedSubject.ReadFrom(&buf)
+ require.NoError(t, err)
+ require.Equal(t, int64(16), read)
+ require.True(t, decodedSubject.IsFullyIndexed())
+
+ buf.Reset()
+ subject.SetFullyIndexed(false)
+ require.False(t, subject.IsFullyIndexed())
+
+ written, err = subject.WriteTo(&buf)
+ require.NoError(t, err)
+ require.Equal(t, int64(16), written)
+ require.Equal(t, []byte{
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+ }, buf.Bytes())
+
+ var decodedSubjectAgain carv2.Characteristics
+ read, err = decodedSubjectAgain.ReadFrom(&buf)
+ require.NoError(t, err)
+ require.Equal(t, int64(16), read)
+ require.False(t, decodedSubjectAgain.IsFullyIndexed())
+}
diff --git a/ipld/car/v2/doc.go b/ipld/car/v2/doc.go
new file mode 100644
index 000000000..b12266649
--- /dev/null
+++ b/ipld/car/v2/doc.go
@@ -0,0 +1,8 @@
+// Package car allows inspecting and reading CAR files,
+// described at https://ipld.io/specs/transport/car/.
+// The entire library is geared towards the CARv2 spec,
+// but many of the APIs consuming CAR files also accept CARv1.
+//
+// The blockstore sub-package contains an implementation of the
+// go-ipfs-blockstore interface.
+package car
diff --git a/ipld/car/v2/errors.go b/ipld/car/v2/errors.go
new file mode 100644
index 000000000..ee89e0b25
--- /dev/null
+++ b/ipld/car/v2/errors.go
@@ -0,0 +1,18 @@
+package car
+
+import (
+ "fmt"
+)
+
+var _ (error) = (*ErrCidTooLarge)(nil)
+
+// ErrCidTooLarge signals that a CID is too large to include in CARv2 index.
+// See: MaxIndexCidSize.
+type ErrCidTooLarge struct {
+ MaxSize uint64
+ CurrentSize uint64
+}
+
+func (e *ErrCidTooLarge) Error() string {
+ return fmt.Sprintf("cid size is larger than max allowed (%d > %d)", e.CurrentSize, e.MaxSize)
+}
diff --git a/ipld/car/v2/errors_test.go b/ipld/car/v2/errors_test.go
new file mode 100644
index 000000000..56e2c7a09
--- /dev/null
+++ b/ipld/car/v2/errors_test.go
@@ -0,0 +1,12 @@
+package car
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNewErrCidTooLarge_ErrorContainsSizes(t *testing.T) {
+ subject := &ErrCidTooLarge{MaxSize: 1413, CurrentSize: 1414}
+ require.EqualError(t, subject, "cid size is larger than max allowed (1414 > 1413)")
+}
diff --git a/ipld/car/v2/example_test.go b/ipld/car/v2/example_test.go
new file mode 100644
index 000000000..8b9288ebb
--- /dev/null
+++ b/ipld/car/v2/example_test.go
@@ -0,0 +1,133 @@
+package car_test
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/blockstore"
+)
+
+func ExampleWrapV1File() {
+ // We have a sample CARv1 file.
+ // Wrap it as-is in a CARv2, with an index.
+ // Writing the result to testdata allows reusing that file in other tests,
+ // and also helps ensure that the result is deterministic.
+ src := "testdata/sample-v1.car"
+ tdir, err := os.MkdirTemp(os.TempDir(), "example-*")
+ if err != nil {
+ panic(err)
+ }
+ dst := filepath.Join(tdir, "wrapped-v2.car")
+ if err := carv2.WrapV1File(src, dst); err != nil {
+ panic(err)
+ }
+
+ // Open our new CARv2 file and show some info about it.
+ cr, err := carv2.OpenReader(dst)
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ if err := cr.Close(); err != nil {
+ panic(err)
+ }
+ }()
+
+ roots, err := cr.Roots()
+ if err != nil {
+ panic(err)
+ }
+ fmt.Println("Roots:", roots)
+ fmt.Println("Has index:", cr.Header.HasIndex())
+
+ // Verify that the CARv1 remains exactly the same.
+ orig, err := os.ReadFile(src)
+ if err != nil {
+ panic(err)
+ }
+ dr, err := cr.DataReader()
+ if err != nil {
+ panic(err)
+ }
+ inner, err := io.ReadAll(dr)
+ if err != nil {
+ panic(err)
+ }
+ fmt.Println("Inner CARv1 is exactly the same:", bytes.Equal(orig, inner))
+
+ // Verify that the CARv2 works well with its index.
+ bs, err := blockstore.OpenReadOnly(dst)
+ if err != nil {
+ panic(err)
+ }
+ fmt.Println(bs.Get(context.TODO(), roots[0]))
+
+ // Output:
+ // Roots: [bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy]
+ // Has index: true
+ // Inner CARv1 is exactly the same: true
+ // [Block bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy]
+}
+
+// ExampleNewBlockReader instantiates a new BlockReader for a CARv1 file and its wrapped CARv2
+// version. For each file, it prints the version, the root CIDs and the first five block CIDs.
+// Note, the roots and first five block CIDs are identical in both files since both represent the
+// same root CIDs and data blocks.
+func ExampleNewBlockReader() {
+ for _, path := range []string{
+ "testdata/sample-v1.car",
+ "testdata/sample-wrapped-v2.car",
+ } {
+ fmt.Println("File:", path)
+ f, err := os.Open(path)
+ if err != nil {
+ panic(err)
+ }
+ br, err := carv2.NewBlockReader(f)
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ if err := f.Close(); err != nil {
+ panic(err)
+ }
+ }()
+ fmt.Println("Version:", br.Version)
+ fmt.Println("Roots:", br.Roots)
+ fmt.Println("First 5 block CIDs:")
+ for i := 0; i < 5; i++ {
+ bl, err := br.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("\t%v\n", bl.Cid())
+ }
+ }
+ // Output:
+ // File: testdata/sample-v1.car
+ // Version: 1
+ // Roots: [bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy]
+ // First 5 block CIDs:
+ // bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy
+ // bafy2bzaceaycv7jhaegckatnncu5yugzkrnzeqsppzegufr35lroxxnsnpspu
+ // bafy2bzaceb62wdepofqu34afqhbcn4a7jziwblt2ih5hhqqm6zitd3qpzhdp4
+ // bafy2bzaceb3utcspm5jqcdqpih3ztbaztv7yunzkiyfq7up7xmokpxemwgu5u
+ // bafy2bzacedjwekyjresrwjqj4n2r5bnuuu3klncgjo2r3slsp6wgqb37sz4ck
+ // File: testdata/sample-wrapped-v2.car
+ // Version: 2
+ // Roots: [bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy]
+ // First 5 block CIDs:
+ // bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy
+ // bafy2bzaceaycv7jhaegckatnncu5yugzkrnzeqsppzegufr35lroxxnsnpspu
+ // bafy2bzaceb62wdepofqu34afqhbcn4a7jziwblt2ih5hhqqm6zitd3qpzhdp4
+ // bafy2bzaceb3utcspm5jqcdqpih3ztbaztv7yunzkiyfq7up7xmokpxemwgu5u
+ // bafy2bzacedjwekyjresrwjqj4n2r5bnuuu3klncgjo2r3slsp6wgqb37sz4ck
+}
diff --git a/ipld/car/v2/fuzz_test.go b/ipld/car/v2/fuzz_test.go
new file mode 100644
index 000000000..30ec4ff43
--- /dev/null
+++ b/ipld/car/v2/fuzz_test.go
@@ -0,0 +1,141 @@
+//go:build go1.18
+
+package car_test
+
+import (
+ "bytes"
+ "encoding/hex"
+ "io"
+ "os"
+ "path/filepath"
+ "testing"
+
+ car "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+)
+
+// v1FixtureStr is a clean carv1 single-block, single-root CAR
+const v1FixtureStr = "3aa265726f6f747381d82a58250001711220151fe9e73c6267a7060c6f6c4cca943c236f4b196723489608edb42a8b8fa80b6776657273696f6e012c01711220151fe9e73c6267a7060c6f6c4cca943c236f4b196723489608edb42a8b8fa80ba165646f646779f5"
+
+func seedWithCarFiles(f *testing.F) {
+ fixture, err := hex.DecodeString(v1FixtureStr)
+ if err != nil {
+ f.Fatal(err)
+ }
+ f.Add(fixture)
+ files, err := filepath.Glob("testdata/*.car")
+ if err != nil {
+ f.Fatal(err)
+ }
+ for _, fname := range files {
+ func() {
+ file, err := os.Open(fname)
+ if err != nil {
+ f.Fatal(err)
+ }
+ defer file.Close()
+ data, err := io.ReadAll(file)
+ if err != nil {
+ f.Fatal(err)
+ }
+ f.Add(data)
+ }()
+ }
+}
+
+func FuzzBlockReader(f *testing.F) {
+ seedWithCarFiles(f)
+
+ f.Fuzz(func(t *testing.T, data []byte) {
+ r, err := car.NewBlockReader(bytes.NewReader(data))
+ if err != nil {
+ return
+ }
+
+ for {
+ _, err = r.Next()
+ if err == io.EOF {
+ return
+ }
+ }
+ })
+}
+
+func FuzzReader(f *testing.F) {
+ seedWithCarFiles(f)
+
+ f.Fuzz(func(t *testing.T, data []byte) {
+ subject, err := car.NewReader(bytes.NewReader(data))
+ if err != nil {
+ return
+ }
+
+ subject.Roots()
+ _, err = subject.IndexReader()
+ if err != nil {
+ return
+ }
+ dr, err := subject.DataReader()
+ if err != nil {
+ return
+ }
+ car.GenerateIndex(dr)
+ })
+}
+
+func FuzzInspect(f *testing.F) {
+ seedWithCarFiles(f)
+
+ f.Fuzz(func(t *testing.T, data []byte) {
+ reader, err := car.NewReader(bytes.NewReader(data))
+ if err != nil {
+ return
+ }
+
+ // Do differential fuzzing between Inspect and the normal parser
+ _, inspectErr := reader.Inspect(true)
+ if inspectErr == nil {
+ return
+ }
+
+ reader, err = car.NewReader(bytes.NewReader(data))
+ if err != nil {
+ t.Fatal("second NewReader on same data failed", err.Error())
+ }
+ i, err := reader.IndexReader()
+ if err != nil {
+ return
+ }
+ // FIXME: Once indexes are safe to parse, do not skip .car with index in the differential fuzzing.
+ if i == nil {
+ return
+ }
+ dr, err := reader.DataReader()
+ if err != nil {
+ return
+ }
+
+ _, err = carv1.ReadHeader(dr, carv1.DefaultMaxAllowedHeaderSize)
+ if err != nil {
+ return
+ }
+
+ blocks, err := car.NewBlockReader(dr)
+ if err != nil {
+ return
+ }
+
+ for {
+ _, err := blocks.Next()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ // caught error as expected
+ return
+ }
+ }
+
+ t.Fatal("Inspect found error but we red this file correctly:", inspectErr.Error())
+ })
+}
diff --git a/ipld/car/v2/index/doc.go b/ipld/car/v2/index/doc.go
new file mode 100644
index 000000000..b8062cc18
--- /dev/null
+++ b/ipld/car/v2/index/doc.go
@@ -0,0 +1,6 @@
+// package index provides indexing functionality for CARv1 data payload represented as a mapping of
+// CID to offset. This can then be used to implement random access over a CARv1.
+//
+// Index can be written or read using the following static functions: index.WriteTo and
+// index.ReadFrom.
+package index
diff --git a/ipld/car/v2/index/errors.go b/ipld/car/v2/index/errors.go
new file mode 100644
index 000000000..1a10a9846
--- /dev/null
+++ b/ipld/car/v2/index/errors.go
@@ -0,0 +1,6 @@
+package index
+
+import "errors"
+
+// ErrNotFound signals a record is not found in the index.
+var ErrNotFound = errors.New("not found")
diff --git a/ipld/car/v2/index/example_test.go b/ipld/car/v2/index/example_test.go
new file mode 100644
index 000000000..bef2e2e01
--- /dev/null
+++ b/ipld/car/v2/index/example_test.go
@@ -0,0 +1,113 @@
+package index_test
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "reflect"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+)
+
+// ExampleReadFrom unmarshalls an index from an indexed CARv2 file, and for each root CID prints the
+// offset at which its corresponding block starts relative to the wrapped CARv1 data payload.
+func ExampleReadFrom() {
+ // Open the CARv2 file
+ cr, err := carv2.OpenReader("../testdata/sample-wrapped-v2.car")
+ if err != nil {
+ panic(err)
+ }
+ defer cr.Close()
+
+ // Get root CIDs in the CARv1 file.
+ roots, err := cr.Roots()
+ if err != nil {
+ panic(err)
+ }
+
+ // Read and unmarshall index within CARv2 file.
+ ir, err := cr.IndexReader()
+ if err != nil {
+ panic(err)
+ }
+ idx, err := index.ReadFrom(ir)
+ if err != nil {
+ panic(err)
+ }
+
+ // For each root CID print the offset relative to CARv1 data payload.
+ for _, r := range roots {
+ offset, err := index.GetFirst(idx, r)
+ if err != nil {
+ panic(err)
+ }
+ fmt.Printf("Frame with CID %v starts at offset %v relative to CARv1 data payload.\n", r, offset)
+ }
+
+ // Output:
+ // Frame with CID bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy starts at offset 61 relative to CARv1 data payload.
+}
+
+// ExampleWriteTo unmarshalls an index from an indexed CARv2 file, and stores it as a separate
+// file on disk.
+func ExampleWriteTo() {
+ // Open the CARv2 file
+ src := "../testdata/sample-wrapped-v2.car"
+ cr, err := carv2.OpenReader(src)
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ if err := cr.Close(); err != nil {
+ panic(err)
+ }
+ }()
+
+ // Read and unmarshall index within CARv2 file.
+ ir, err := cr.IndexReader()
+ if err != nil {
+ panic(err)
+ }
+ idx, err := index.ReadFrom(ir)
+ if err != nil {
+ panic(err)
+ }
+
+ // Store the index alone onto destination file.
+ f, err := os.CreateTemp(os.TempDir(), "example-index-*.carindex")
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ if err := f.Close(); err != nil {
+ panic(err)
+ }
+ }()
+ _, err = index.WriteTo(idx, f)
+ if err != nil {
+ panic(err)
+ }
+
+ // Seek to the beginning of tile to read it back.
+ _, err = f.Seek(0, io.SeekStart)
+ if err != nil {
+ panic(err)
+ }
+
+ // Read and unmarshall the destination file as a separate index instance.
+ reReadIdx, err := index.ReadFrom(f)
+ if err != nil {
+ panic(err)
+ }
+
+ // Expect indices to be equal.
+ if reflect.DeepEqual(idx, reReadIdx) {
+ fmt.Printf("Saved index file matches the index embedded in CARv2 at %v.\n", src)
+ } else {
+ panic("expected to get the same index as the CARv2 file")
+ }
+
+ // Output:
+ // Saved index file matches the index embedded in CARv2 at ../testdata/sample-wrapped-v2.car.
+}
diff --git a/ipld/car/v2/index/index.go b/ipld/car/v2/index/index.go
new file mode 100644
index 000000000..9a5e0d084
--- /dev/null
+++ b/ipld/car/v2/index/index.go
@@ -0,0 +1,160 @@
+package index
+
+import (
+ "encoding/binary"
+ "fmt"
+ "io"
+
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/go-cid"
+
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-multihash"
+ "github.com/multiformats/go-varint"
+)
+
+// CarIndexNone is a sentinel value used as a multicodec code for the index indicating no index.
+const CarIndexNone = 0x300000
+
+type (
+ // Record is a pre-processed record of a car item and location.
+ Record struct {
+ cid.Cid
+ Offset uint64
+ }
+
+ // Index provides an interface for looking up byte offset of a given CID.
+ //
+ // Note that each indexing mechanism is free to match CIDs however it
+ // sees fit. For example, multicodec.CarIndexSorted only indexes
+ // multihash digests, meaning that Get and GetAll will find matching
+ // blocks even if the CID's encoding multicodec differs. Other index
+ // implementations might index the entire CID, the entire multihash, or
+ // just part of a multihash's digest.
+ //
+ // See: multicodec.CarIndexSorted, multicodec.CarMultihashIndexSorted
+ Index interface {
+ // Codec provides the multicodec code that the index implements.
+ //
+ // Note that this may return a reserved code if the index
+ // implementation is not defined in a spec.
+ Codec() multicodec.Code
+
+ // Marshal encodes the index in serial form.
+ Marshal(w io.Writer) (uint64, error)
+
+ // Unmarshal decodes the index from its serial form.
+ // Note, this function will copy the entire index into memory.
+ //
+ // Do not unmarshal index from untrusted CARv2 files. Instead, the index should be
+ // regenerated from the CARv2 data payload.
+ Unmarshal(r io.Reader) error
+
+ // Load inserts a number of records into the index.
+ // Note that Index will load all given records. Any filtering of the records such as
+ // exclusion of CIDs with multihash.IDENTITY code must occur prior to calling this function.
+ //
+ // Further, the actual information extracted and indexed from the given records entirely
+ // depends on the concrete index implementation.
+ // For example, some index implementations may only store partial multihashes.
+ Load([]Record) error
+
+ // GetAll looks up all blocks matching a given CID,
+ // calling a function for each one of their offsets.
+ //
+ // GetAll stops if the given function returns false,
+ // or there are no more offsets; whichever happens first.
+ //
+ // If no error occurred and the CID isn't indexed,
+ // meaning that no callbacks happen,
+ // ErrNotFound is returned.
+ GetAll(cid.Cid, func(uint64) bool) error
+ }
+
+ // IterableIndex is an index which support iterating over it's elements
+ IterableIndex interface {
+ Index
+
+ // ForEach takes a callback function that will be called
+ // on each entry in the index. The arguments to the callback are
+ // the multihash of the element, and the offset in the car file
+ // where the element appears.
+ //
+ // If the callback returns a non-nil error, the iteration is aborted,
+ // and the ForEach function returns the error to the user.
+ //
+ // An index may contain multiple offsets corresponding to the same multihash, e.g. via duplicate blocks.
+ // In such cases, the given function may be called multiple times with the same multihash but different offset.
+ //
+ // The order of calls to the given function is deterministic, but entirely index-specific.
+ ForEach(func(multihash.Multihash, uint64) error) error
+ }
+)
+
+// GetFirst is a wrapper over Index.GetAll, returning the offset for the first
+// matching indexed CID.
+func GetFirst(idx Index, key cid.Cid) (uint64, error) {
+ var firstOffset uint64
+ err := idx.GetAll(key, func(offset uint64) bool {
+ firstOffset = offset
+ return false
+ })
+ return firstOffset, err
+}
+
+// New constructs a new index corresponding to the given CAR index codec.
+func New(codec multicodec.Code) (Index, error) {
+ switch codec {
+ case multicodec.CarIndexSorted:
+ return newSorted(), nil
+ case multicodec.CarMultihashIndexSorted:
+ return NewMultihashSorted(), nil
+ default:
+ return nil, fmt.Errorf("unknwon index codec: %v", codec)
+ }
+}
+
+// WriteTo writes the given idx into w.
+// The written bytes include the index encoding.
+// This can then be read back using index.ReadFrom
+func WriteTo(idx Index, w io.Writer) (uint64, error) {
+ buf := make([]byte, binary.MaxVarintLen64)
+ b := varint.PutUvarint(buf, uint64(idx.Codec()))
+ n, err := w.Write(buf[:b])
+ if err != nil {
+ return uint64(n), err
+ }
+
+ l, err := idx.Marshal(w)
+ return uint64(n) + l, err
+}
+
+// ReadFrom reads index from r.
+// The reader decodes the index by reading the first byte to interpret the encoding.
+// Returns error if the encoding is not known.
+//
+// Attempting to read index data from untrusted sources is not recommended.
+// Instead, the index should be regenerated from the CARv2 data payload.
+func ReadFrom(r io.Reader) (Index, error) {
+ codec, err := ReadCodec(r)
+ if err != nil {
+ return nil, err
+ }
+ idx, err := New(codec)
+ if err != nil {
+ return nil, err
+ }
+ if err := idx.Unmarshal(r); err != nil {
+ return nil, err
+ }
+ return idx, nil
+}
+
+// ReadCodec reads the codec of the index by decoding the first varint read from r.
+func ReadCodec(r io.Reader) (multicodec.Code, error) {
+ code, err := varint.ReadUvarint(internalio.ToByteReader(r))
+ if err != nil {
+ return 0, err
+ }
+ return multicodec.Code(code), nil
+}
diff --git a/ipld/car/v2/index/index_test.go b/ipld/car/v2/index/index_test.go
new file mode 100644
index 000000000..6dc2a5efd
--- /dev/null
+++ b/ipld/car/v2/index/index_test.go
@@ -0,0 +1,168 @@
+package index
+
+import (
+ "bytes"
+ "io"
+ "os"
+ "path/filepath"
+ "testing"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1/util"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-varint"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNew(t *testing.T) {
+ tests := []struct {
+ name string
+ codec multicodec.Code
+ want Index
+ wantErr bool
+ }{
+ {
+ name: "CarSortedIndexCodecIsConstructed",
+ codec: multicodec.CarIndexSorted,
+ want: newSorted(),
+ },
+ {
+ name: "ValidMultiCodecButUnknwonToIndexIsError",
+ codec: multicodec.Cidv1,
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := New(tt.codec)
+ if tt.wantErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, tt.want, got)
+ }
+ })
+ }
+}
+
+func TestReadFrom(t *testing.T) {
+ idxf, err := os.Open("../testdata/sample-index.carindex")
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, idxf.Close()) })
+
+ subject, err := ReadFrom(idxf)
+ require.NoError(t, err)
+
+ _, err = idxf.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+
+ idxf2, err := os.Open("../testdata/sample-multihash-index-sorted.carindex")
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, idxf2.Close()) })
+
+ subjectInAltFormat, err := ReadFrom(idxf)
+ require.NoError(t, err)
+
+ crf, err := os.Open("../testdata/sample-v1.car")
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, crf.Close()) })
+ cr, err := carv1.NewCarReader(crf)
+ require.NoError(t, err)
+
+ for {
+ wantBlock, err := cr.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+
+ wantCid := wantBlock.Cid()
+ // Get offset from the index for a CID and assert it exists
+ gotOffset, err := GetFirst(subject, wantCid)
+ require.NoError(t, err)
+ require.NotZero(t, gotOffset)
+
+ // Get offset from the index in alternative format for a CID and assert it exists
+ gotOffset2, err := GetFirst(subjectInAltFormat, wantCid)
+ require.NoError(t, err)
+ require.NotZero(t, gotOffset2)
+
+ // Seek to the offset on CARv1 file
+ _, err = crf.Seek(int64(gotOffset), io.SeekStart)
+ require.NoError(t, err)
+
+ // Read the fame at offset and assert the frame corresponds to the expected block.
+ gotCid, gotData, err := util.ReadNode(crf, false, carv1.DefaultMaxAllowedSectionSize)
+ require.NoError(t, err)
+ gotBlock, err := blocks.NewBlockWithCid(gotData, gotCid)
+ require.NoError(t, err)
+ require.Equal(t, wantBlock, gotBlock)
+ }
+}
+
+func TestWriteTo(t *testing.T) {
+ // Read sample index on file
+ idxf, err := os.Open("../testdata/sample-multihash-index-sorted.carindex")
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, idxf.Close()) })
+
+ // Unmarshall to get expected index
+ wantIdx, err := ReadFrom(idxf)
+ require.NoError(t, err)
+
+ // Write the same index out
+ dest := filepath.Join(t.TempDir(), "index-write-to-test.carindex")
+ destF, err := os.Create(dest)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, destF.Close()) })
+ _, err = WriteTo(wantIdx, destF)
+ require.NoError(t, err)
+
+ // Seek to the beginning of the written out file.
+ _, err = destF.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+
+ // Read the written index back
+ gotIdx, err := ReadFrom(destF)
+ require.NoError(t, err)
+
+ // Assert they are equal
+ require.Equal(t, wantIdx, gotIdx)
+}
+
+func TestMarshalledIndexStartsWithCodec(t *testing.T) {
+
+ tests := []struct {
+ path string
+ codec multicodec.Code
+ }{
+ {
+ path: "../testdata/sample-multihash-index-sorted.carindex",
+ codec: multicodec.CarMultihashIndexSorted,
+ },
+ {
+ path: "../testdata/sample-index.carindex",
+ codec: multicodec.CarIndexSorted,
+ },
+ }
+ for _, test := range tests {
+ test := test
+ t.Run(test.codec.String(), func(t *testing.T) {
+ // Read sample index on file
+ idxf, err := os.Open(test.path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, idxf.Close()) })
+
+ // Unmarshall to get expected index
+ wantIdx, err := ReadFrom(idxf)
+ require.NoError(t, err)
+
+ // Assert the first two bytes are the corresponding multicodec code.
+ buf := new(bytes.Buffer)
+ _, err = WriteTo(wantIdx, buf)
+ require.NoError(t, err)
+ require.Equal(t, varint.ToUvarint(uint64(test.codec)), buf.Bytes()[:2])
+ })
+ }
+}
diff --git a/ipld/car/v2/index/indexsorted.go b/ipld/car/v2/index/indexsorted.go
new file mode 100644
index 000000000..93b31e433
--- /dev/null
+++ b/ipld/car/v2/index/indexsorted.go
@@ -0,0 +1,319 @@
+package index
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "sort"
+
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+
+ "github.com/multiformats/go-multicodec"
+
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multihash"
+)
+
+var _ Index = (*multiWidthIndex)(nil)
+
+type (
+ digestRecord struct {
+ digest []byte
+ index uint64
+ }
+ recordSet []digestRecord
+ singleWidthIndex struct {
+ width uint32
+ len uint64 // in struct, len is #items. when marshaled, it's saved as #bytes.
+ index []byte
+ }
+ multiWidthIndex map[uint32]singleWidthIndex
+)
+
+func (d digestRecord) write(buf []byte) {
+ n := copy(buf[:], d.digest)
+ binary.LittleEndian.PutUint64(buf[n:], d.index)
+}
+
+func (r recordSet) Len() int {
+ return len(r)
+}
+
+func (r recordSet) Less(i, j int) bool {
+ return bytes.Compare(r[i].digest, r[j].digest) < 0
+}
+
+func (r recordSet) Swap(i, j int) {
+ r[i], r[j] = r[j], r[i]
+}
+
+func (s *singleWidthIndex) Marshal(w io.Writer) (uint64, error) {
+ l := uint64(0)
+ if err := binary.Write(w, binary.LittleEndian, s.width); err != nil {
+ return 0, err
+ }
+ l += 4
+ if err := binary.Write(w, binary.LittleEndian, int64(len(s.index))); err != nil {
+ return l, err
+ }
+ l += 8
+ n, err := w.Write(s.index)
+ return l + uint64(n), err
+}
+
+func (s *singleWidthIndex) Unmarshal(r io.Reader) error {
+ var width uint32
+ if err := binary.Read(r, binary.LittleEndian, &width); err != nil {
+ if err == io.EOF {
+ return io.ErrUnexpectedEOF
+ }
+ return err
+ }
+ var dataLen uint64
+ if err := binary.Read(r, binary.LittleEndian, &dataLen); err != nil {
+ if err == io.EOF {
+ return io.ErrUnexpectedEOF
+ }
+ return err
+ }
+
+ if err := s.checkUnmarshalLengths(width, dataLen, 0); err != nil {
+ return err
+ }
+
+ buf := make([]byte, dataLen)
+ if _, err := io.ReadFull(r, buf); err != nil {
+ return err
+ }
+ s.index = buf
+ return nil
+}
+
+func (s *singleWidthIndex) checkUnmarshalLengths(width uint32, dataLen, extra uint64) error {
+ if width < 8 {
+ return errors.New("malformed index; width must be at least 8")
+ }
+ const maxWidth = 32 << 20 // 32MiB, to ~match the go-cid maximum
+ if width > maxWidth {
+ return errors.New("index too big; singleWidthIndex width is larger than allowed maximum")
+ }
+ oldDataLen, dataLen := dataLen, dataLen+extra
+ if oldDataLen > dataLen {
+ return errors.New("index too big; singleWidthIndex len is overflowing")
+ }
+ if int64(dataLen) < 0 {
+ return errors.New("index too big; singleWidthIndex len is overflowing int64")
+ }
+ s.width = width
+ s.len = dataLen / uint64(width)
+ return nil
+}
+
+func (s *singleWidthIndex) Less(i int, digest []byte) bool {
+ return bytes.Compare(digest[:], s.index[i*int(s.width):((i+1)*int(s.width)-8)]) <= 0
+}
+
+func (s *singleWidthIndex) GetAll(c cid.Cid, fn func(uint64) bool) error {
+ d, err := multihash.Decode(c.Hash())
+ if err != nil {
+ return err
+ }
+ return s.getAll(d.Digest, fn)
+}
+
+func (s *singleWidthIndex) getAll(d []byte, fn func(uint64) bool) error {
+ idx := sort.Search(int(s.len), func(i int) bool {
+ return s.Less(i, d)
+ })
+
+ var any bool
+ for ; uint64(idx) < s.len; idx++ {
+ digestStart := idx * int(s.width)
+ offsetEnd := (idx + 1) * int(s.width)
+ digestEnd := offsetEnd - 8
+ if bytes.Equal(d[:], s.index[digestStart:digestEnd]) {
+ any = true
+ offset := binary.LittleEndian.Uint64(s.index[digestEnd:offsetEnd])
+ if !fn(offset) {
+ // User signalled to stop searching; therefore, break.
+ break
+ }
+ } else {
+ // No more matches; therefore, break.
+ break
+ }
+ }
+ if !any {
+ return ErrNotFound
+ }
+ return nil
+}
+
+func (s *singleWidthIndex) Load(items []Record) error {
+ m := make(multiWidthIndex)
+ if err := m.Load(items); err != nil {
+ return err
+ }
+ if len(m) != 1 {
+ return fmt.Errorf("unexpected number of cid widths: %d", len(m))
+ }
+ for _, i := range m {
+ s.index = i.index
+ s.len = i.len
+ s.width = i.width
+ return nil
+ }
+ return nil
+}
+
+func (s *singleWidthIndex) forEachDigest(f func(digest []byte, offset uint64) error) error {
+ segmentCount := len(s.index) / int(s.width)
+ for i := 0; i < segmentCount; i++ {
+ digestStart := i * int(s.width)
+ offsetEnd := (i + 1) * int(s.width)
+ digestEnd := offsetEnd - 8
+ digest := s.index[digestStart:digestEnd]
+ offset := binary.LittleEndian.Uint64(s.index[digestEnd:offsetEnd])
+ if err := f(digest, offset); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m *multiWidthIndex) GetAll(c cid.Cid, fn func(uint64) bool) error {
+ d, err := multihash.Decode(c.Hash())
+ if err != nil {
+ return err
+ }
+ if s, ok := (*m)[uint32(len(d.Digest)+8)]; ok {
+ return s.getAll(d.Digest, fn)
+ }
+ return ErrNotFound
+}
+
+func (m *multiWidthIndex) Codec() multicodec.Code {
+ return multicodec.CarIndexSorted
+}
+
+func (m *multiWidthIndex) Marshal(w io.Writer) (uint64, error) {
+ l := uint64(0)
+ if err := binary.Write(w, binary.LittleEndian, int32(len(*m))); err != nil {
+ return l, err
+ }
+ l += 4
+
+ // The widths are unique, but ranging over a map isn't deterministic.
+ // As per the CARv2 spec, we must order buckets by digest length.
+
+ widths := make([]uint32, 0, len(*m))
+ for width := range *m {
+ widths = append(widths, width)
+ }
+ sort.Slice(widths, func(i, j int) bool {
+ return widths[i] < widths[j]
+ })
+
+ for _, width := range widths {
+ bucket := (*m)[width]
+ n, err := bucket.Marshal(w)
+ l += n
+ if err != nil {
+ return l, err
+ }
+ }
+ return l, nil
+}
+
+func (m *multiWidthIndex) Unmarshal(r io.Reader) error {
+ reader := internalio.ToByteReadSeeker(r)
+ var l int32
+ if err := binary.Read(reader, binary.LittleEndian, &l); err != nil {
+ if err == io.EOF {
+ return io.ErrUnexpectedEOF
+ }
+ return err
+ }
+ sum, err := reader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+ if int32(l) < 0 {
+ return errors.New("index too big; multiWidthIndex count is overflowing int32")
+ }
+ for i := 0; i < int(l); i++ {
+ s := singleWidthIndex{}
+ if err := s.Unmarshal(r); err != nil {
+ return err
+ }
+ n, err := reader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+ oldSum := sum
+ sum += n
+ if sum < oldSum {
+ return errors.New("index too big; multiWidthIndex len is overflowing int64")
+ }
+ (*m)[s.width] = s
+ }
+ return nil
+}
+
+func (m *multiWidthIndex) Load(items []Record) error {
+ // Split cids on their digest length
+ idxs := make(map[int][]digestRecord)
+ for _, item := range items {
+ decHash, err := multihash.Decode(item.Hash())
+ if err != nil {
+ return err
+ }
+
+ digest := decHash.Digest
+ idx, ok := idxs[len(digest)]
+ if !ok {
+ idxs[len(digest)] = make([]digestRecord, 0)
+ idx = idxs[len(digest)]
+ }
+ idxs[len(digest)] = append(idx, digestRecord{digest, item.Offset})
+ }
+
+ // Sort each list. then write to compact form.
+ for width, lst := range idxs {
+ sort.Sort(recordSet(lst))
+ rcrdWdth := width + 8
+ compact := make([]byte, rcrdWdth*len(lst))
+ for off, itm := range lst {
+ itm.write(compact[off*rcrdWdth : (off+1)*rcrdWdth])
+ }
+ s := singleWidthIndex{
+ width: uint32(rcrdWdth),
+ len: uint64(len(lst)),
+ index: compact,
+ }
+ (*m)[uint32(width)+8] = s
+ }
+ return nil
+}
+
+func (m *multiWidthIndex) forEachDigest(f func(digest []byte, offset uint64) error) error {
+ sizes := make([]uint32, 0, len(*m))
+ for k := range *m {
+ sizes = append(sizes, k)
+ }
+ sort.Slice(sizes, func(i, j int) bool { return sizes[i] < sizes[j] })
+ for _, s := range sizes {
+ swi := (*m)[s]
+ if err := swi.forEachDigest(f); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func newSorted() Index {
+ m := make(multiWidthIndex)
+ return &m
+}
diff --git a/ipld/car/v2/index/indexsorted_test.go b/ipld/car/v2/index/indexsorted_test.go
new file mode 100644
index 000000000..a0da42354
--- /dev/null
+++ b/ipld/car/v2/index/indexsorted_test.go
@@ -0,0 +1,64 @@
+package index
+
+import (
+ "encoding/binary"
+ "testing"
+
+ "github.com/ipfs/boxo/ipld/merkledag"
+ "github.com/multiformats/go-multicodec"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSortedIndexCodec(t *testing.T) {
+ require.Equal(t, multicodec.CarIndexSorted, newSorted().Codec())
+}
+
+func TestIndexSorted_GetReturnsNotFoundWhenCidDoesNotExist(t *testing.T) {
+ nonExistingKey := merkledag.NewRawNode([]byte("lobstermuncher")).Block.Cid()
+ tests := []struct {
+ name string
+ subject Index
+ }{
+ {
+ "Sorted",
+ newSorted(),
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotOffset, err := GetFirst(tt.subject, nonExistingKey)
+ require.Equal(t, ErrNotFound, err)
+ require.Equal(t, uint64(0), gotOffset)
+ })
+ }
+}
+
+func TestSingleWidthIndex_GetAll(t *testing.T) {
+ l := 4
+ width := 9
+ buf := make([]byte, width*l)
+
+ // Populate the index bytes as total of four records.
+ // The last record should not match the getAll.
+ for i := 0; i < l; i++ {
+ if i < l-1 {
+ buf[i*width] = 1
+ } else {
+ buf[i*width] = 2
+ }
+ binary.LittleEndian.PutUint64(buf[(i*width)+1:(i*width)+width], uint64(14))
+ }
+ subject := &singleWidthIndex{
+ width: 9,
+ len: uint64(l),
+ index: buf,
+ }
+
+ var foundCount int
+ err := subject.getAll([]byte{1}, func(u uint64) bool {
+ foundCount++
+ return true
+ })
+ require.NoError(t, err)
+ require.Equal(t, 3, foundCount)
+}
diff --git a/ipld/car/v2/index/mhindexsorted.go b/ipld/car/v2/index/mhindexsorted.go
new file mode 100644
index 000000000..75ea21ecd
--- /dev/null
+++ b/ipld/car/v2/index/mhindexsorted.go
@@ -0,0 +1,214 @@
+package index
+
+import (
+ "encoding/binary"
+ "errors"
+ "io"
+ "sort"
+
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-multihash"
+)
+
+var (
+ _ Index = (*MultihashIndexSorted)(nil)
+ _ IterableIndex = (*MultihashIndexSorted)(nil)
+)
+
+type (
+ // MultihashIndexSorted maps multihash code (i.e. hashing algorithm) to multiWidthCodedIndex.
+ MultihashIndexSorted map[uint64]*multiWidthCodedIndex
+ // multiWidthCodedIndex stores multihash code for each multiWidthIndex.
+ multiWidthCodedIndex struct {
+ multiWidthIndex
+ code uint64
+ }
+)
+
+func newMultiWidthCodedIndex() *multiWidthCodedIndex {
+ return &multiWidthCodedIndex{
+ multiWidthIndex: make(multiWidthIndex),
+ }
+}
+
+func (m *multiWidthCodedIndex) Marshal(w io.Writer) (uint64, error) {
+ if err := binary.Write(w, binary.LittleEndian, m.code); err != nil {
+ return 8, err
+ }
+ n, err := m.multiWidthIndex.Marshal(w)
+ return 8 + n, err
+}
+
+func (m *multiWidthCodedIndex) Unmarshal(r io.Reader) error {
+ if err := binary.Read(r, binary.LittleEndian, &m.code); err != nil {
+ if err == io.EOF {
+ return io.ErrUnexpectedEOF
+ }
+ return err
+ }
+ return m.multiWidthIndex.Unmarshal(r)
+}
+
+func (m *multiWidthCodedIndex) forEach(f func(mh multihash.Multihash, offset uint64) error) error {
+ return m.multiWidthIndex.forEachDigest(func(digest []byte, offset uint64) error {
+ mh, err := multihash.Encode(digest, m.code)
+ if err != nil {
+ return err
+ }
+ return f(mh, offset)
+ })
+}
+
+func (m *MultihashIndexSorted) Codec() multicodec.Code {
+ return multicodec.CarMultihashIndexSorted
+}
+
+func (m *MultihashIndexSorted) Marshal(w io.Writer) (uint64, error) {
+ if err := binary.Write(w, binary.LittleEndian, int32(len(*m))); err != nil {
+ return 4, err
+ }
+ // The codes are unique, but ranging over a map isn't deterministic.
+ // As per the CARv2 spec, we must order buckets by digest length.
+ // TODO update CARv2 spec to reflect this for the new index type.
+ codes := m.sortedMultihashCodes()
+ l := uint64(4)
+
+ for _, code := range codes {
+ mwci := (*m)[code]
+ n, err := mwci.Marshal(w)
+ l += n
+ if err != nil {
+ return l, err
+ }
+ }
+ return l, nil
+}
+
+func (m *MultihashIndexSorted) sortedMultihashCodes() []uint64 {
+ codes := make([]uint64, 0, len(*m))
+ for code := range *m {
+ codes = append(codes, code)
+ }
+ sort.Slice(codes, func(i, j int) bool {
+ return codes[i] < codes[j]
+ })
+ return codes
+}
+
+func (m *MultihashIndexSorted) Unmarshal(r io.Reader) error {
+ reader := internalio.ToByteReadSeeker(r)
+ var l int32
+ if err := binary.Read(reader, binary.LittleEndian, &l); err != nil {
+ if err == io.EOF {
+ return io.ErrUnexpectedEOF
+ }
+ return err
+ }
+ sum, err := reader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+ if int32(l) < 0 {
+ return errors.New("index too big; MultihashIndexSorted count is overflowing int32")
+ }
+ for i := 0; i < int(l); i++ {
+ mwci := newMultiWidthCodedIndex()
+ if err := mwci.Unmarshal(r); err != nil {
+ return err
+ }
+ n, err := reader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+ oldSum := sum
+ sum += n
+ if sum < oldSum {
+ return errors.New("index too big; MultihashIndexSorted len is overflowing int64")
+ }
+ m.put(mwci)
+ }
+ return nil
+}
+
+func (m *MultihashIndexSorted) put(mwci *multiWidthCodedIndex) {
+ (*m)[mwci.code] = mwci
+}
+
+func (m *MultihashIndexSorted) Load(records []Record) error {
+ // TODO optimize load by avoiding multihash decoding twice.
+ // This implementation decodes multihashes twice: once here to group by code, and once in the
+ // internals of multiWidthIndex to group by digest length. The code can be optimized by
+ // combining the grouping logic into one step where the multihash of a CID is only decoded once.
+ // The optimization would need refactoring of the IndexSorted compaction logic.
+
+ // Group records by multihash code
+ byCode := make(map[uint64][]Record)
+ for _, record := range records {
+ dmh, err := multihash.Decode(record.Hash())
+ if err != nil {
+ return err
+ }
+ code := dmh.Code
+ recsByCode, ok := byCode[code]
+ if !ok {
+ recsByCode = make([]Record, 0)
+ byCode[code] = recsByCode
+ }
+ byCode[code] = append(recsByCode, record)
+ }
+
+ // Load each record group.
+ for code, recsByCode := range byCode {
+ mwci := newMultiWidthCodedIndex()
+ mwci.code = code
+ if err := mwci.Load(recsByCode); err != nil {
+ return err
+ }
+ m.put(mwci)
+ }
+ return nil
+}
+
+func (m *MultihashIndexSorted) GetAll(cid cid.Cid, f func(uint64) bool) error {
+ hash := cid.Hash()
+ dmh, err := multihash.Decode(hash)
+ if err != nil {
+ return err
+ }
+ mwci, err := m.get(dmh)
+ if err != nil {
+ return err
+ }
+ return mwci.GetAll(cid, f)
+}
+
+// ForEach calls f for every multihash and its associated offset stored by this index.
+func (m *MultihashIndexSorted) ForEach(f func(mh multihash.Multihash, offset uint64) error) error {
+ sizes := make([]uint64, 0, len(*m))
+ for k := range *m {
+ sizes = append(sizes, k)
+ }
+ sort.Slice(sizes, func(i, j int) bool { return sizes[i] < sizes[j] })
+ for _, s := range sizes {
+ mwci := (*m)[s]
+ if err := mwci.forEach(f); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m *MultihashIndexSorted) get(dmh *multihash.DecodedMultihash) (*multiWidthCodedIndex, error) {
+ if codedIdx, ok := (*m)[dmh.Code]; ok {
+ return codedIdx, nil
+ }
+ return nil, ErrNotFound
+}
+
+func NewMultihashSorted() *MultihashIndexSorted {
+ index := make(MultihashIndexSorted)
+ return &index
+}
diff --git a/ipld/car/v2/index/mhindexsorted_test.go b/ipld/car/v2/index/mhindexsorted_test.go
new file mode 100644
index 000000000..1b0f2ab86
--- /dev/null
+++ b/ipld/car/v2/index/mhindexsorted_test.go
@@ -0,0 +1,113 @@
+package index_test
+
+import (
+ "bytes"
+ "fmt"
+ "math/rand"
+ "testing"
+
+ "github.com/multiformats/go-multicodec"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multihash"
+ "github.com/stretchr/testify/require"
+)
+
+func TestMutilhashSortedIndex_Codec(t *testing.T) {
+ subject, err := index.New(multicodec.CarMultihashIndexSorted)
+ require.NoError(t, err)
+ require.Equal(t, multicodec.CarMultihashIndexSorted, subject.Codec())
+}
+
+func TestMultiWidthCodedIndex_MarshalUnmarshal(t *testing.T) {
+ rng := rand.New(rand.NewSource(1413))
+ records := generateIndexRecords(t, multihash.SHA2_256, rng)
+
+ // Create a new mh sorted index and load randomly generated records into it.
+ subject, err := index.New(multicodec.CarMultihashIndexSorted)
+ require.NoError(t, err)
+ err = subject.Load(records)
+ require.NoError(t, err)
+
+ // Marshal the index.
+ buf := new(bytes.Buffer)
+ _, err = subject.Marshal(buf)
+ require.NoError(t, err)
+
+ // Unmarshal it back to another instance of mh sorted index.
+ umSubject, err := index.New(multicodec.CarMultihashIndexSorted)
+ require.NoError(t, err)
+ err = umSubject.Unmarshal(buf)
+ require.NoError(t, err)
+
+ // Assert original records are present in both index instances with expected offset.
+ requireContainsAll(t, subject, records)
+ requireContainsAll(t, umSubject, records)
+}
+
+func TestMultiWidthCodedIndex_StableIterate(t *testing.T) {
+ rng := rand.New(rand.NewSource(1414))
+ records := generateIndexRecords(t, multihash.SHA2_256, rng)
+ records = append(records, generateIndexRecords(t, multihash.SHA2_512, rng)...)
+ records = append(records, generateIndexRecords(t, multihash.IDENTITY, rng)...)
+
+ // Create a new mh sorted index and load randomly generated records into it.
+ idx, err := index.New(multicodec.CarMultihashIndexSorted)
+ require.NoError(t, err)
+ err = idx.Load(records)
+ require.NoError(t, err)
+
+ subject, ok := idx.(index.IterableIndex)
+ require.True(t, ok)
+
+ mh := make([]multihash.Multihash, 0, len(records))
+ require.NoError(t, subject.ForEach(func(m multihash.Multihash, _ uint64) error {
+ mh = append(mh, m)
+ return nil
+ }))
+
+ for i := 0; i < 10; i++ {
+ candidate := make([]multihash.Multihash, 0, len(records))
+ require.NoError(t, subject.ForEach(func(m multihash.Multihash, _ uint64) error {
+ candidate = append(candidate, m)
+ return nil
+ }))
+ require.Equal(t, mh, candidate)
+ }
+}
+
+func generateIndexRecords(t *testing.T, hasherCode uint64, rng *rand.Rand) []index.Record {
+ var records []index.Record
+ recordCount := rng.Intn(99) + 1 // Up to 100 records
+ for i := 0; i < recordCount; i++ {
+ records = append(records, index.Record{
+ Cid: generateCidV1(t, hasherCode, rng),
+ Offset: rng.Uint64(),
+ })
+ }
+ return records
+}
+
+func generateCidV1(t *testing.T, hasherCode uint64, rng *rand.Rand) cid.Cid {
+ data := []byte(fmt.Sprintf("🌊d-%d", rng.Uint64()))
+ mh, err := multihash.Sum(data, hasherCode, -1)
+ require.NoError(t, err)
+ return cid.NewCidV1(cid.Raw, mh)
+}
+
+func requireContainsAll(t *testing.T, subject index.Index, nonIdentityRecords []index.Record) {
+ for _, r := range nonIdentityRecords {
+ wantCid := r.Cid
+ wantOffset := r.Offset
+
+ var gotOffsets []uint64
+ err := subject.GetAll(wantCid, func(o uint64) bool {
+ gotOffsets = append(gotOffsets, o)
+ return false
+ })
+ require.NoError(t, err)
+ require.Equal(t, 1, len(gotOffsets))
+ require.Equal(t, wantOffset, gotOffsets[0])
+ }
+}
diff --git a/ipld/car/v2/index_gen.go b/ipld/car/v2/index_gen.go
new file mode 100644
index 000000000..5f1da03e1
--- /dev/null
+++ b/ipld/car/v2/index_gen.go
@@ -0,0 +1,231 @@
+package car
+
+import (
+ "fmt"
+ "io"
+ "os"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multihash"
+ "github.com/multiformats/go-varint"
+)
+
+// GenerateIndex generates index for the given car payload reader.
+// The index can be stored in serialized format using index.WriteTo.
+//
+// Note, the index is re-generated every time even if the payload is in CARv2 format and already has
+// an index. To read existing index when available see ReadOrGenerateIndex.
+// See: LoadIndex.
+func GenerateIndex(v1r io.Reader, opts ...Option) (index.Index, error) {
+ wopts := ApplyOptions(opts...)
+ idx, err := index.New(wopts.IndexCodec)
+ if err != nil {
+ return nil, err
+ }
+ if err := LoadIndex(idx, v1r, opts...); err != nil {
+ return nil, err
+ }
+ return idx, nil
+}
+
+// LoadIndex populates idx with index records generated from r.
+// The r may be in CARv1 or CARv2 format.
+//
+// If the StoreIdentityCIDs option is set when calling LoadIndex, identity
+// CIDs will be included in the index. By default this option is off, and
+// identity CIDs will not be included in the index.
+//
+// Note, the index is re-generated every time even if r is in CARv2 format and already has an index.
+// To read existing index when available see ReadOrGenerateIndex.
+func LoadIndex(idx index.Index, r io.Reader, opts ...Option) error {
+ // Parse Options.
+ o := ApplyOptions(opts...)
+
+ reader := internalio.ToByteReadSeeker(r)
+ pragma, err := carv1.ReadHeader(r, o.MaxAllowedHeaderSize)
+ if err != nil {
+ return fmt.Errorf("error reading car header: %w", err)
+ }
+
+ var dataSize, dataOffset int64
+ switch pragma.Version {
+ case 1:
+ break
+ case 2:
+ // Read V2 header which should appear immediately after pragma according to CARv2 spec.
+ var v2h Header
+ _, err := v2h.ReadFrom(r)
+ if err != nil {
+ return err
+ }
+
+ // Sanity-check the CARv2 header
+ if v2h.DataOffset < HeaderSize {
+ return fmt.Errorf("malformed CARv2; data offset too small: %d", v2h.DataOffset)
+ }
+ if v2h.DataSize < 1 {
+ return fmt.Errorf("malformed CARv2; data payload size too small: %d", v2h.DataSize)
+ }
+
+ // Seek to the beginning of the inner CARv1 payload
+ _, err = reader.Seek(int64(v2h.DataOffset), io.SeekStart)
+ if err != nil {
+ return err
+ }
+
+ // Set dataSize and dataOffset which are then used during index loading logic to decide
+ // where to stop and adjust section offset respectively.
+ // Note that we could use a LimitReader here and re-define reader with it. However, it means
+ // the internalio.ToByteReadSeeker will be less efficient since LimitReader does not
+ // implement ByteReader nor ReadSeeker.
+ dataSize = int64(v2h.DataSize)
+ dataOffset = int64(v2h.DataOffset)
+
+ // Read the inner CARv1 header to skip it and sanity check it.
+ v1h, err := carv1.ReadHeader(reader, o.MaxAllowedHeaderSize)
+ if err != nil {
+ return err
+ }
+ if v1h.Version != 1 {
+ return fmt.Errorf("expected data payload header version of 1; got %d", v1h.Version)
+ }
+ default:
+ return fmt.Errorf("expected either version 1 or 2; got %d", pragma.Version)
+ }
+
+ // Record the start of each section, with first section starring from current position in the
+ // reader, i.e. right after the header, since we have only read the header so far.
+ var sectionOffset int64
+
+ // The Seek call below is equivalent to getting the reader.offset directly.
+ // We get it through Seek to only depend on APIs of a typical io.Seeker.
+ // This would also reduce refactoring in case the utility reader is moved.
+ if sectionOffset, err = reader.Seek(0, io.SeekCurrent); err != nil {
+ return err
+ }
+
+ // Subtract the data offset; if CARv1 this would be zero otherwise the value will come from the
+ // CARv2 header.
+ sectionOffset -= dataOffset
+
+ records := make([]index.Record, 0)
+ for {
+ // Read the section's length.
+ sectionLen, err := varint.ReadUvarint(reader)
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return err
+ }
+
+ // Null padding; by default it's an error.
+ if sectionLen == 0 {
+ if o.ZeroLengthSectionAsEOF {
+ break
+ } else {
+ return fmt.Errorf("carv1 null padding not allowed by default; see ZeroLengthSectionAsEOF")
+ }
+ }
+
+ // Read the CID.
+ cidLen, c, err := cid.CidFromReader(reader)
+ if err != nil {
+ return err
+ }
+
+ if o.StoreIdentityCIDs || c.Prefix().MhType != multihash.IDENTITY {
+ if uint64(cidLen) > o.MaxIndexCidSize {
+ return &ErrCidTooLarge{MaxSize: o.MaxIndexCidSize, CurrentSize: uint64(cidLen)}
+ }
+ records = append(records, index.Record{Cid: c, Offset: uint64(sectionOffset)})
+ }
+
+ // Seek to the next section by skipping the block.
+ // The section length includes the CID, so subtract it.
+ remainingSectionLen := int64(sectionLen) - int64(cidLen)
+ if sectionOffset, err = reader.Seek(remainingSectionLen, io.SeekCurrent); err != nil {
+ return err
+ }
+ // Subtract the data offset which will be non-zero when reader represents a CARv2.
+ sectionOffset -= dataOffset
+
+ // Check if we have reached the end of data payload and if so treat it as an EOF.
+ // Note, dataSize will be non-zero only if we are reading from a CARv2.
+ if dataSize != 0 && sectionOffset >= dataSize {
+ break
+ }
+ }
+
+ if err := idx.Load(records); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// GenerateIndexFromFile walks a CAR file at the give path and generates an index of cid->byte offset.
+// The index can be stored using index.WriteTo. Both CARv1 and CARv2 formats are accepted.
+//
+// Note, the index is re-generated every time even if the given CAR file is in CARv2 format and
+// already has an index. To read existing index when available see ReadOrGenerateIndex.
+//
+// See: GenerateIndex.
+func GenerateIndexFromFile(path string, opts ...Option) (index.Index, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return GenerateIndex(f, opts...)
+}
+
+// ReadOrGenerateIndex accepts both CARv1 and CARv2 formats, and reads or generates an index for it.
+// When the given reader is in CARv1 format an index is always generated.
+// For a payload in CARv2 format, an index is only generated if Header.HasIndex returns false.
+// An error is returned for all other formats, i.e. pragma with versions other than 1 or 2.
+//
+// Note, the returned index lives entirely in memory and will not depend on the
+// given reader to fulfill index lookup.
+func ReadOrGenerateIndex(rs io.ReadSeeker, opts ...Option) (index.Index, error) {
+ // Read version.
+ version, err := ReadVersion(rs, opts...)
+ if err != nil {
+ return nil, err
+ }
+ // Seek to the beginning, since reading the version changes the reader's offset.
+ if _, err := rs.Seek(0, io.SeekStart); err != nil {
+ return nil, err
+ }
+
+ switch version {
+ case 1:
+ // Simply generate the index, since there can't be a pre-existing one.
+ return GenerateIndex(rs, opts...)
+ case 2:
+ // Read CARv2 format
+ v2r, err := NewReader(internalio.ToReaderAt(rs), opts...)
+ if err != nil {
+ return nil, err
+ }
+ // If index is present, then no need to generate; decode and return it.
+ if v2r.Header.HasIndex() {
+ ir, err := v2r.IndexReader()
+ if err != nil {
+ return nil, err
+ }
+ return index.ReadFrom(ir)
+ }
+ // Otherwise, generate index from CARv1 payload wrapped within CARv2 format.
+ dr, err := v2r.DataReader()
+ if err != nil {
+ return nil, err
+ }
+ return GenerateIndex(dr, opts...)
+ default:
+ return nil, fmt.Errorf("unknown version %v", version)
+ }
+}
diff --git a/ipld/car/v2/index_gen_test.go b/ipld/car/v2/index_gen_test.go
new file mode 100644
index 000000000..48d30b426
--- /dev/null
+++ b/ipld/car/v2/index_gen_test.go
@@ -0,0 +1,274 @@
+package car_test
+
+import (
+ "io"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-multihash"
+ "github.com/multiformats/go-varint"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGenerateIndex(t *testing.T) {
+ type testCase struct {
+ name string
+ carPath string
+ opts []carv2.Option
+ wantIndexer func(t *testing.T) index.Index
+ wantErr bool
+ }
+ tests := []testCase{
+ {
+ name: "CarV1IsIndexedAsExpected",
+ carPath: "testdata/sample-v1.car",
+ wantIndexer: func(t *testing.T) index.Index {
+ v1, err := os.Open("testdata/sample-v1.car")
+ require.NoError(t, err)
+ t.Cleanup(func() { assert.NoError(t, v1.Close()) })
+ want, err := carv2.GenerateIndex(v1)
+ require.NoError(t, err)
+ return want
+ },
+ },
+ {
+ name: "CarV2WithIndexIsReturnedAsExpected",
+ carPath: "testdata/sample-wrapped-v2.car",
+ wantIndexer: func(t *testing.T) index.Index {
+ v2, err := os.Open("testdata/sample-wrapped-v2.car")
+ require.NoError(t, err)
+ t.Cleanup(func() { assert.NoError(t, v2.Close()) })
+ reader, err := carv2.NewReader(v2)
+ require.NoError(t, err)
+ ir, err := reader.IndexReader()
+ require.NoError(t, err)
+ want, err := index.ReadFrom(ir)
+ require.NoError(t, err)
+ return want
+ },
+ },
+ {
+ name: "CarV1WithZeroLenSectionIsGeneratedAsExpected",
+ carPath: "testdata/sample-v1-with-zero-len-section.car",
+ opts: []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)},
+ wantIndexer: func(t *testing.T) index.Index {
+ v1, err := os.Open("testdata/sample-v1-with-zero-len-section.car")
+ require.NoError(t, err)
+ t.Cleanup(func() { assert.NoError(t, v1.Close()) })
+ want, err := carv2.GenerateIndex(v1, carv2.ZeroLengthSectionAsEOF(true))
+ require.NoError(t, err)
+ return want
+ },
+ },
+ {
+ name: "AnotherCarV1WithZeroLenSectionIsGeneratedAsExpected",
+ carPath: "testdata/sample-v1-with-zero-len-section2.car",
+ opts: []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)},
+ wantIndexer: func(t *testing.T) index.Index {
+ v1, err := os.Open("testdata/sample-v1-with-zero-len-section2.car")
+ require.NoError(t, err)
+ t.Cleanup(func() { assert.NoError(t, v1.Close()) })
+ want, err := carv2.GenerateIndex(v1, carv2.ZeroLengthSectionAsEOF(true))
+ require.NoError(t, err)
+ return want
+ },
+ },
+ {
+ name: "CarV1WithZeroLenSectionWithoutOptionIsError",
+ carPath: "testdata/sample-v1-with-zero-len-section.car",
+ wantErr: true,
+ },
+ {
+ name: "CarOtherThanV1OrV2IsError",
+ carPath: "testdata/sample-rootless-v42.car",
+ wantIndexer: func(t *testing.T) index.Index { return nil },
+ wantErr: true,
+ },
+ }
+
+ requireWant := func(tt testCase, got index.Index, gotErr error) {
+ if tt.wantErr {
+ require.Error(t, gotErr)
+ } else {
+ require.NoError(t, gotErr)
+ var want index.Index
+ if tt.wantIndexer != nil {
+ want = tt.wantIndexer(t)
+ }
+ if want == nil {
+ require.Nil(t, got)
+ } else {
+ require.Equal(t, want, got)
+ }
+ }
+ }
+
+ for _, tt := range tests {
+ t.Run("ReadOrGenerateIndex_"+tt.name, func(t *testing.T) {
+ carFile, err := os.Open(tt.carPath)
+ require.NoError(t, err)
+ t.Cleanup(func() { assert.NoError(t, carFile.Close()) })
+ got, gotErr := carv2.ReadOrGenerateIndex(carFile, tt.opts...)
+ requireWant(tt, got, gotErr)
+ })
+ t.Run("GenerateIndexFromFile_"+tt.name, func(t *testing.T) {
+ got, gotErr := carv2.GenerateIndexFromFile(tt.carPath, tt.opts...)
+ requireWant(tt, got, gotErr)
+ })
+ t.Run("LoadIndex_"+tt.name, func(t *testing.T) {
+ carFile, err := os.Open(tt.carPath)
+ require.NoError(t, err)
+ got, err := index.New(multicodec.CarMultihashIndexSorted)
+ require.NoError(t, err)
+ gotErr := carv2.LoadIndex(got, carFile, tt.opts...)
+ requireWant(tt, got, gotErr)
+ })
+ t.Run("GenerateIndex_"+tt.name, func(t *testing.T) {
+ carFile, err := os.Open(tt.carPath)
+ require.NoError(t, err)
+ got, gotErr := carv2.GenerateIndex(carFile, tt.opts...)
+ requireWant(tt, got, gotErr)
+ })
+ }
+}
+
+func TestMultihashIndexSortedConsistencyWithIndexSorted(t *testing.T) {
+ path := "testdata/sample-v1.car"
+
+ sortedIndex, err := carv2.GenerateIndexFromFile(path)
+ require.NoError(t, err)
+ require.Equal(t, multicodec.CarMultihashIndexSorted, sortedIndex.Codec())
+
+ f, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, f.Close()) })
+ br, err := carv2.NewBlockReader(f)
+ require.NoError(t, err)
+
+ subject := generateMultihashSortedIndex(t, path)
+ for {
+ wantNext, err := br.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+
+ dmh, err := multihash.Decode(wantNext.Cid().Hash())
+ require.NoError(t, err)
+ if dmh.Code == multihash.IDENTITY {
+ continue
+ }
+
+ wantCid := wantNext.Cid()
+ var wantOffsets []uint64
+ err = sortedIndex.GetAll(wantCid, func(o uint64) bool {
+ wantOffsets = append(wantOffsets, o)
+ return false
+ })
+ require.NoError(t, err)
+
+ var gotOffsets []uint64
+ err = subject.GetAll(wantCid, func(o uint64) bool {
+ gotOffsets = append(gotOffsets, o)
+ return false
+ })
+
+ require.NoError(t, err)
+ require.Equal(t, wantOffsets, gotOffsets)
+ }
+}
+
+func TestMultihashSorted_ForEachIsConsistentWithGetAll(t *testing.T) {
+ path := "testdata/sample-v1.car"
+ f, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, f.Close()) })
+
+ br, err := carv2.NewBlockReader(f)
+ require.NoError(t, err)
+ subject := generateMultihashSortedIndex(t, path)
+
+ gotForEach := make(map[string]uint64)
+ err = subject.ForEach(func(mh multihash.Multihash, offset uint64) error {
+ gotForEach[mh.String()] = offset
+ return nil
+ })
+ require.NoError(t, err)
+
+ for {
+ b, err := br.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+ c := b.Cid()
+ dmh, err := multihash.Decode(c.Hash())
+ require.NoError(t, err)
+ if dmh.Code == multihash.IDENTITY {
+ continue
+ }
+
+ wantMh := c.Hash()
+
+ var wantOffset uint64
+ err = subject.GetAll(c, func(u uint64) bool {
+ wantOffset = u
+ return false
+ })
+ require.NoError(t, err)
+
+ s := wantMh.String()
+ gotOffset, ok := gotForEach[s]
+ require.True(t, ok)
+ require.Equal(t, wantOffset, gotOffset)
+ }
+}
+
+func generateMultihashSortedIndex(t *testing.T, path string) *index.MultihashIndexSorted {
+ f, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, f.Close()) })
+ reader := internalio.ToByteReadSeeker(f)
+ header, err := carv1.ReadHeader(reader, carv1.DefaultMaxAllowedHeaderSize)
+ require.NoError(t, err)
+ require.Equal(t, uint64(1), header.Version)
+
+ idx := index.NewMultihashSorted()
+ records := make([]index.Record, 0)
+
+ var sectionOffset int64
+ sectionOffset, err = reader.Seek(0, io.SeekCurrent)
+ require.NoError(t, err)
+
+ for {
+ sectionLen, err := varint.ReadUvarint(reader)
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+
+ if sectionLen == 0 {
+ break
+ }
+
+ cidLen, c, err := cid.CidFromReader(reader)
+ require.NoError(t, err)
+ records = append(records, index.Record{Cid: c, Offset: uint64(sectionOffset)})
+ remainingSectionLen := int64(sectionLen) - int64(cidLen)
+ sectionOffset, err = reader.Seek(remainingSectionLen, io.SeekCurrent)
+ require.NoError(t, err)
+ }
+
+ err = idx.Load(records)
+ require.NoError(t, err)
+
+ return idx
+}
diff --git a/ipld/car/v2/internal/carv1/car.go b/ipld/car/v2/internal/carv1/car.go
new file mode 100644
index 000000000..469cb1274
--- /dev/null
+++ b/ipld/car/v2/internal/carv1/car.go
@@ -0,0 +1,284 @@
+package carv1
+
+import (
+ "context"
+ "fmt"
+ "io"
+
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1/util"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/boxo/ipld/merkledag"
+ cid "github.com/ipfs/go-cid"
+ cbor "github.com/ipfs/go-ipld-cbor"
+ format "github.com/ipfs/go-ipld-format"
+)
+
+const DefaultMaxAllowedHeaderSize uint64 = 32 << 20 // 32MiB
+const DefaultMaxAllowedSectionSize uint64 = 8 << 20 // 8MiB
+
+func init() {
+ cbor.RegisterCborType(CarHeader{})
+}
+
+type Store interface {
+ Put(context.Context, blocks.Block) error
+}
+
+type ReadStore interface {
+ Get(context.Context, cid.Cid) (blocks.Block, error)
+}
+
+type CarHeader struct {
+ Roots []cid.Cid
+ Version uint64
+}
+
+type carWriter struct {
+ ds format.NodeGetter
+ w io.Writer
+}
+
+func WriteCar(ctx context.Context, ds format.NodeGetter, roots []cid.Cid, w io.Writer) error {
+ h := &CarHeader{
+ Roots: roots,
+ Version: 1,
+ }
+
+ if err := WriteHeader(h, w); err != nil {
+ return fmt.Errorf("failed to write car header: %s", err)
+ }
+
+ cw := &carWriter{ds: ds, w: w}
+ seen := cid.NewSet()
+ for _, r := range roots {
+ if err := merkledag.Walk(ctx, cw.enumGetLinks, r, seen.Visit); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func ReadHeaderAt(at io.ReaderAt, maxReadBytes uint64) (*CarHeader, error) {
+ var rr io.Reader
+ switch r := at.(type) {
+ case io.Reader:
+ rr = r
+ default:
+ var err error
+ rr, err = internalio.NewOffsetReadSeeker(r, 0)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return ReadHeader(rr, maxReadBytes)
+}
+
+func ReadHeader(r io.Reader, maxReadBytes uint64) (*CarHeader, error) {
+ hb, err := util.LdRead(r, false, maxReadBytes)
+ if err != nil {
+ if err == util.ErrSectionTooLarge {
+ err = util.ErrHeaderTooLarge
+ }
+ return nil, err
+ }
+
+ var ch CarHeader
+ if err := cbor.DecodeInto(hb, &ch); err != nil {
+ return nil, fmt.Errorf("invalid header: %v", err)
+ }
+
+ return &ch, nil
+}
+
+func WriteHeader(h *CarHeader, w io.Writer) error {
+ hb, err := cbor.DumpObject(h)
+ if err != nil {
+ return err
+ }
+
+ return util.LdWrite(w, hb)
+}
+
+func HeaderSize(h *CarHeader) (uint64, error) {
+ hb, err := cbor.DumpObject(h)
+ if err != nil {
+ return 0, err
+ }
+
+ return util.LdSize(hb), nil
+}
+
+func (cw *carWriter) enumGetLinks(ctx context.Context, c cid.Cid) ([]*format.Link, error) {
+ nd, err := cw.ds.Get(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := cw.writeNode(ctx, nd); err != nil {
+ return nil, err
+ }
+
+ return nd.Links(), nil
+}
+
+func (cw *carWriter) writeNode(ctx context.Context, nd format.Node) error {
+ return util.LdWrite(cw.w, nd.Cid().Bytes(), nd.RawData())
+}
+
+type CarReader struct {
+ r io.Reader
+ Header *CarHeader
+ zeroLenAsEOF bool
+ maxAllowedSectionSize uint64
+}
+
+func NewCarReaderWithZeroLengthSectionAsEOF(r io.Reader) (*CarReader, error) {
+ return NewCarReaderWithoutDefaults(r, true, DefaultMaxAllowedHeaderSize, DefaultMaxAllowedSectionSize)
+}
+
+func NewCarReader(r io.Reader) (*CarReader, error) {
+ return NewCarReaderWithoutDefaults(r, false, DefaultMaxAllowedHeaderSize, DefaultMaxAllowedSectionSize)
+}
+
+func NewCarReaderWithoutDefaults(r io.Reader, zeroLenAsEOF bool, maxAllowedHeaderSize uint64, maxAllowedSectionSize uint64) (*CarReader, error) {
+ ch, err := ReadHeader(r, maxAllowedHeaderSize)
+ if err != nil {
+ return nil, err
+ }
+
+ if ch.Version != 1 {
+ return nil, fmt.Errorf("invalid car version: %d", ch.Version)
+ }
+
+ if len(ch.Roots) == 0 {
+ return nil, fmt.Errorf("empty car, no roots")
+ }
+
+ return &CarReader{
+ r: r,
+ Header: ch,
+ zeroLenAsEOF: zeroLenAsEOF,
+ maxAllowedSectionSize: maxAllowedSectionSize,
+ }, nil
+}
+
+func (cr *CarReader) Next() (blocks.Block, error) {
+ c, data, err := util.ReadNode(cr.r, cr.zeroLenAsEOF, cr.maxAllowedSectionSize)
+ if err != nil {
+ return nil, err
+ }
+
+ hashed, err := c.Prefix().Sum(data)
+ if err != nil {
+ return nil, err
+ }
+
+ if !hashed.Equals(c) {
+ return nil, fmt.Errorf("mismatch in content integrity, name: %s, data: %s", c, hashed)
+ }
+
+ return blocks.NewBlockWithCid(data, c)
+}
+
+type batchStore interface {
+ PutMany(context.Context, []blocks.Block) error
+}
+
+func LoadCar(s Store, r io.Reader) (*CarHeader, error) {
+ ctx := context.TODO()
+ cr, err := NewCarReader(r)
+ if err != nil {
+ return nil, err
+ }
+
+ if bs, ok := s.(batchStore); ok {
+ return loadCarFast(ctx, bs, cr)
+ }
+
+ return loadCarSlow(ctx, s, cr)
+}
+
+func loadCarFast(ctx context.Context, s batchStore, cr *CarReader) (*CarHeader, error) {
+ var buf []blocks.Block
+ for {
+ blk, err := cr.Next()
+ if err != nil {
+ if err == io.EOF {
+ if len(buf) > 0 {
+ if err := s.PutMany(ctx, buf); err != nil {
+ return nil, err
+ }
+ }
+ return cr.Header, nil
+ }
+ return nil, err
+ }
+
+ buf = append(buf, blk)
+
+ if len(buf) > 1000 {
+ if err := s.PutMany(ctx, buf); err != nil {
+ return nil, err
+ }
+ buf = buf[:0]
+ }
+ }
+}
+
+func loadCarSlow(ctx context.Context, s Store, cr *CarReader) (*CarHeader, error) {
+ for {
+ blk, err := cr.Next()
+ if err != nil {
+ if err == io.EOF {
+ return cr.Header, nil
+ }
+ return nil, err
+ }
+
+ if err := s.Put(ctx, blk); err != nil {
+ return nil, err
+ }
+ }
+}
+
+// Matches checks whether two headers match.
+// Two headers are considered matching if:
+// 1. They have the same version number, and
+// 2. They contain the same root CIDs in any order.
+//
+// Note, this function explicitly ignores the order of roots.
+// If order of roots matter use reflect.DeepEqual instead.
+func (h CarHeader) Matches(other CarHeader) bool {
+ if h.Version != other.Version {
+ return false
+ }
+ thisLen := len(h.Roots)
+ if thisLen != len(other.Roots) {
+ return false
+ }
+ // Headers with a single root are popular.
+ // Implement a fast execution path for popular cases.
+ if thisLen == 1 {
+ return h.Roots[0].Equals(other.Roots[0])
+ }
+
+ // Check other contains all roots.
+ // TODO: should this be optimised for cases where the number of roots are large since it has O(N^2) complexity?
+ for _, r := range h.Roots {
+ if !other.containsRoot(r) {
+ return false
+ }
+ }
+ return true
+}
+
+func (h *CarHeader) containsRoot(root cid.Cid) bool {
+ for _, r := range h.Roots {
+ if r.Equals(root) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/ipld/car/v2/internal/carv1/car_test.go b/ipld/car/v2/internal/carv1/car_test.go
new file mode 100644
index 000000000..21602114e
--- /dev/null
+++ b/ipld/car/v2/internal/carv1/car_test.go
@@ -0,0 +1,333 @@
+package carv1
+
+import (
+ "bytes"
+ "context"
+ "encoding/hex"
+ "io"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ipfs/boxo/ipld/merkledag"
+ dstest "github.com/ipfs/boxo/ipld/merkledag/test"
+ cid "github.com/ipfs/go-cid"
+ format "github.com/ipfs/go-ipld-format"
+)
+
+func assertAddNodes(t *testing.T, ds format.DAGService, nds ...format.Node) {
+ for _, nd := range nds {
+ if err := ds.Add(context.Background(), nd); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func TestRoundtrip(t *testing.T) {
+ dserv := dstest.Mock()
+ a := merkledag.NewRawNode([]byte("aaaa"))
+ b := merkledag.NewRawNode([]byte("bbbb"))
+ c := merkledag.NewRawNode([]byte("cccc"))
+
+ nd1 := &merkledag.ProtoNode{}
+ nd1.AddNodeLink("cat", a)
+
+ nd2 := &merkledag.ProtoNode{}
+ nd2.AddNodeLink("first", nd1)
+ nd2.AddNodeLink("dog", b)
+
+ nd3 := &merkledag.ProtoNode{}
+ nd3.AddNodeLink("second", nd2)
+ nd3.AddNodeLink("bear", c)
+
+ assertAddNodes(t, dserv, a, b, c, nd1, nd2, nd3)
+
+ buf := new(bytes.Buffer)
+ if err := WriteCar(context.Background(), dserv, []cid.Cid{nd3.Cid()}, buf); err != nil {
+ t.Fatal(err)
+ }
+
+ bserv := dstest.Bserv()
+ ch, err := LoadCar(bserv.Blockstore(), buf)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(ch.Roots) != 1 {
+ t.Fatal("should have one root")
+ }
+
+ if !ch.Roots[0].Equals(nd3.Cid()) {
+ t.Fatal("got wrong cid")
+ }
+
+ bs := bserv.Blockstore()
+ for _, nd := range []format.Node{a, b, c, nd1, nd2, nd3} {
+ has, err := bs.Has(context.TODO(), nd.Cid())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !has {
+ t.Fatal("should have cid in blockstore")
+ }
+ }
+}
+
+func TestEOFHandling(t *testing.T) {
+ // fixture is a clean single-block, single-root CAR
+ fixture, err := hex.DecodeString("3aa265726f6f747381d82a58250001711220151fe9e73c6267a7060c6f6c4cca943c236f4b196723489608edb42a8b8fa80b6776657273696f6e012c01711220151fe9e73c6267a7060c6f6c4cca943c236f4b196723489608edb42a8b8fa80ba165646f646779f5")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ load := func(t *testing.T, byts []byte) *CarReader {
+ cr, err := NewCarReader(bytes.NewReader(byts))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ blk, err := cr.Next()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if blk.Cid().String() != "bafyreiavd7u6opdcm6tqmddpnrgmvfb4enxuwglhenejmchnwqvixd5ibm" {
+ t.Fatal("unexpected CID")
+ }
+
+ return cr
+ }
+
+ t.Run("CleanEOF", func(t *testing.T) {
+ cr := load(t, fixture)
+
+ blk, err := cr.Next()
+ if err != io.EOF {
+ t.Fatal("Didn't get expected EOF")
+ }
+ if blk != nil {
+ t.Fatal("EOF returned expected block")
+ }
+ })
+
+ t.Run("BadVarint", func(t *testing.T) {
+ fixtureBadVarint := append(fixture, 160)
+ cr := load(t, fixtureBadVarint)
+
+ blk, err := cr.Next()
+ if err != io.ErrUnexpectedEOF {
+ t.Fatal("Didn't get unexpected EOF")
+ }
+ if blk != nil {
+ t.Fatal("EOF returned unexpected block")
+ }
+ })
+
+ t.Run("TruncatedBlock", func(t *testing.T) {
+ fixtureTruncatedBlock := append(fixture, 100, 0, 0)
+ cr := load(t, fixtureTruncatedBlock)
+
+ blk, err := cr.Next()
+ if err != io.ErrUnexpectedEOF {
+ t.Fatal("Didn't get unexpected EOF")
+ }
+ if blk != nil {
+ t.Fatal("EOF returned unexpected block")
+ }
+ })
+}
+
+func TestBadHeaders(t *testing.T) {
+ testCases := []struct {
+ name string
+ hex string
+ errStr string // either the whole error string
+ errPfx string // or just the prefix
+ }{
+ {
+ "{version:2}",
+ "0aa16776657273696f6e02",
+ "invalid car version: 2",
+ "",
+ },
+ {
+ // an unfortunate error because we don't use a pointer
+ "{roots:[baeaaaa3bmjrq]}",
+ "13a165726f6f747381d82a480001000003616263",
+ "invalid car version: 0",
+ "",
+ },
+ {
+ "{version:\"1\",roots:[baeaaaa3bmjrq]}",
+ "1da265726f6f747381d82a4800010000036162636776657273696f6e6131",
+ "", "invalid header: ",
+ },
+ {
+ "{version:1}",
+ "0aa16776657273696f6e01",
+ "empty car, no roots",
+ "",
+ },
+ {
+ "{version:1,roots:{cid:baeaaaa3bmjrq}}",
+ "20a265726f6f7473a163636964d82a4800010000036162636776657273696f6e01",
+ "",
+ "invalid header: ",
+ },
+ {
+ "{version:1,roots:[baeaaaa3bmjrq],blip:true}",
+ "22a364626c6970f565726f6f747381d82a4800010000036162636776657273696f6e01",
+ "",
+ "invalid header: ",
+ },
+ {
+ "[1,[]]",
+ "03820180",
+ "",
+ "invalid header: ",
+ },
+ {
+ // this is an unfortunate error, it'd be nice to catch it better but it's
+ // very unlikely we'd ever see this in practice
+ "null",
+ "01f6",
+ "",
+ "invalid car version: 0",
+ },
+ }
+
+ makeCar := func(t *testing.T, byts string) error {
+ fixture, err := hex.DecodeString(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = NewCarReader(bytes.NewReader(fixture))
+ return err
+ }
+
+ t.Run("Sanity check {version:1,roots:[baeaaaa3bmjrq]}", func(t *testing.T) {
+ err := makeCar(t, "1ca265726f6f747381d82a4800010000036162636776657273696f6e01")
+ if err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ err := makeCar(t, tc.hex)
+ if err == nil {
+ t.Fatal("expected error from bad header, didn't get one")
+ }
+ if tc.errStr != "" {
+ if err.Error() != tc.errStr {
+ t.Fatalf("bad error: %v", err)
+ }
+ } else {
+ if !strings.HasPrefix(err.Error(), tc.errPfx) {
+ t.Fatalf("bad error: %v", err)
+ }
+ }
+ })
+ }
+}
+
+func TestCarHeaderMatchess(t *testing.T) {
+ oneCid := merkledag.NewRawNode([]byte("fish")).Cid()
+ anotherCid := merkledag.NewRawNode([]byte("lobster")).Cid()
+ tests := []struct {
+ name string
+ one CarHeader
+ other CarHeader
+ want bool
+ }{
+ {
+ "SameVersionNilRootsIsMatching",
+ CarHeader{nil, 1},
+ CarHeader{nil, 1},
+ true,
+ },
+ {
+ "SameVersionEmptyRootsIsMatching",
+ CarHeader{[]cid.Cid{}, 1},
+ CarHeader{[]cid.Cid{}, 1},
+ true,
+ },
+ {
+ "SameVersionNonEmptySameRootsIsMatching",
+ CarHeader{[]cid.Cid{oneCid}, 1},
+ CarHeader{[]cid.Cid{oneCid}, 1},
+ true,
+ },
+ {
+ "SameVersionNonEmptySameRootsInDifferentOrderIsMatching",
+ CarHeader{[]cid.Cid{oneCid, anotherCid}, 1},
+ CarHeader{[]cid.Cid{anotherCid, oneCid}, 1},
+ true,
+ },
+ {
+ "SameVersionDifferentRootsIsNotMatching",
+ CarHeader{[]cid.Cid{oneCid}, 1},
+ CarHeader{[]cid.Cid{anotherCid}, 1},
+ false,
+ },
+ {
+ "DifferentVersionDifferentRootsIsNotMatching",
+ CarHeader{[]cid.Cid{oneCid}, 0},
+ CarHeader{[]cid.Cid{anotherCid}, 1},
+ false,
+ },
+ {
+ "MismatchingVersionIsNotMatching",
+ CarHeader{nil, 0},
+ CarHeader{nil, 1},
+ false,
+ },
+ {
+ "ZeroValueHeadersAreMatching",
+ CarHeader{},
+ CarHeader{},
+ true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := tt.one.Matches(tt.other)
+ require.Equal(t, tt.want, got, "Matches() = %v, want %v", got, tt.want)
+ })
+ }
+}
+
+func TestReadingZeroLengthSectionWithoutOptionSetIsError(t *testing.T) {
+ f, err := os.Open("../../testdata/sample-v1-with-zero-len-section.car")
+ require.NoError(t, err)
+ subject, err := NewCarReader(f)
+ require.NoError(t, err)
+
+ for {
+ _, err := subject.Next()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ require.EqualError(t, err, "varints malformed, could not reach the end")
+ return
+ }
+ }
+ require.Fail(t, "expected error when reading file with zero section without option set")
+}
+
+func TestReadingZeroLengthSectionWithOptionSetIsSuccess(t *testing.T) {
+ f, err := os.Open("../../testdata/sample-v1-with-zero-len-section.car")
+ require.NoError(t, err)
+ subject, err := NewCarReaderWithZeroLengthSectionAsEOF(f)
+ require.NoError(t, err)
+
+ for {
+ _, err := subject.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+ }
+}
diff --git a/ipld/car/v2/internal/carv1/doc.go b/ipld/car/v2/internal/carv1/doc.go
new file mode 100644
index 000000000..821ca2f0a
--- /dev/null
+++ b/ipld/car/v2/internal/carv1/doc.go
@@ -0,0 +1,2 @@
+// Forked from CARv1 to avoid dependency to ipld-prime 0.9.0 due to outstanding upgrades in filecoin.
+package carv1
diff --git a/ipld/car/v2/internal/carv1/util/util.go b/ipld/car/v2/internal/carv1/util/util.go
new file mode 100644
index 000000000..eb20da652
--- /dev/null
+++ b/ipld/car/v2/internal/carv1/util/util.go
@@ -0,0 +1,98 @@
+package util
+
+import (
+ "errors"
+ "io"
+
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+
+ "github.com/multiformats/go-varint"
+
+ cid "github.com/ipfs/go-cid"
+)
+
+var ErrSectionTooLarge = errors.New("invalid section data, length of read beyond allowable maximum")
+var ErrHeaderTooLarge = errors.New("invalid header data, length of read beyond allowable maximum")
+
+type BytesReader interface {
+ io.Reader
+ io.ByteReader
+}
+
+func ReadNode(r io.Reader, zeroLenAsEOF bool, maxReadBytes uint64) (cid.Cid, []byte, error) {
+ data, err := LdRead(r, zeroLenAsEOF, maxReadBytes)
+ if err != nil {
+ return cid.Cid{}, nil, err
+ }
+
+ n, c, err := cid.CidFromBytes(data)
+ if err != nil {
+ return cid.Cid{}, nil, err
+ }
+
+ return c, data[n:], nil
+}
+
+func LdWrite(w io.Writer, d ...[]byte) error {
+ var sum uint64
+ for _, s := range d {
+ sum += uint64(len(s))
+ }
+
+ buf := make([]byte, 8)
+ n := varint.PutUvarint(buf, sum)
+ _, err := w.Write(buf[:n])
+ if err != nil {
+ return err
+ }
+
+ for _, s := range d {
+ _, err = w.Write(s)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func LdSize(d ...[]byte) uint64 {
+ var sum uint64
+ for _, s := range d {
+ sum += uint64(len(s))
+ }
+ s := varint.UvarintSize(sum)
+ return sum + uint64(s)
+}
+
+func LdReadSize(r io.Reader, zeroLenAsEOF bool, maxReadBytes uint64) (uint64, error) {
+ l, err := varint.ReadUvarint(internalio.ToByteReader(r))
+ if err != nil {
+ // If the length of bytes read is non-zero when the error is EOF then signal an unclean EOF.
+ if l > 0 && err == io.EOF {
+ return 0, io.ErrUnexpectedEOF
+ }
+ return 0, err
+ } else if l == 0 && zeroLenAsEOF {
+ return 0, io.EOF
+ }
+
+ if l > maxReadBytes { // Don't OOM
+ return 0, ErrSectionTooLarge
+ }
+ return l, nil
+}
+
+func LdRead(r io.Reader, zeroLenAsEOF bool, maxReadBytes uint64) ([]byte, error) {
+ l, err := LdReadSize(r, zeroLenAsEOF, maxReadBytes)
+ if err != nil {
+ return nil, err
+ }
+
+ buf := make([]byte, l)
+ if _, err := io.ReadFull(r, buf); err != nil {
+ return nil, err
+ }
+
+ return buf, nil
+}
diff --git a/ipld/car/v2/internal/carv1/util/util_test.go b/ipld/car/v2/internal/carv1/util/util_test.go
new file mode 100644
index 000000000..594312f99
--- /dev/null
+++ b/ipld/car/v2/internal/carv1/util/util_test.go
@@ -0,0 +1,28 @@
+package util_test
+
+import (
+ "bytes"
+ crand "crypto/rand"
+ "math/rand"
+ "testing"
+
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1/util"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestLdSize(t *testing.T) {
+ for i := 0; i < 5; i++ {
+ var buf bytes.Buffer
+ data := make([][]byte, 5)
+ for j := 0; j < 5; j++ {
+ data[j] = make([]byte, rand.Intn(30))
+ _, err := crand.Read(data[j])
+ require.NoError(t, err)
+ }
+ size := util.LdSize(data...)
+ err := util.LdWrite(&buf, data...)
+ require.NoError(t, err)
+ require.Equal(t, uint64(len(buf.Bytes())), size)
+ }
+}
diff --git a/ipld/car/v2/internal/errsort/search.go b/ipld/car/v2/internal/errsort/search.go
new file mode 100644
index 000000000..fb8861794
--- /dev/null
+++ b/ipld/car/v2/internal/errsort/search.go
@@ -0,0 +1,54 @@
+/*
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+package errsort
+
+// Search is like sort.Search but accepts an erroring closure.
+// If it errors the search is terminated immediately
+func Search(n int, f func(int) (bool, error)) (int, error) {
+ // Define f(-1) == false and f(n) == true.
+ // Invariant: f(i-1) == false, f(j) == true.
+ i, j := 0, n
+ for i < j {
+ h := int(uint(i+j) >> 1) // avoid overflow when computing h
+ // i ≤ h < j
+ less, err := f(h)
+ if err != nil {
+ return 0, err
+ }
+ if !less {
+ i = h + 1 // preserves f(i-1) == false
+ } else {
+ j = h // preserves f(j) == true
+ }
+ }
+ // i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i.
+ return i, nil
+}
diff --git a/ipld/car/v2/internal/io/converter.go b/ipld/car/v2/internal/io/converter.go
new file mode 100644
index 000000000..2e2844bd0
--- /dev/null
+++ b/ipld/car/v2/internal/io/converter.go
@@ -0,0 +1,155 @@
+package io
+
+import (
+ "errors"
+ "io"
+ "sync"
+)
+
+var (
+ _ io.ByteReader = (*readerPlusByte)(nil)
+ _ io.ByteReader = (*readSeekerPlusByte)(nil)
+ _ io.ByteReader = (*discardingReadSeekerPlusByte)(nil)
+ _ io.ReadSeeker = (*discardingReadSeekerPlusByte)(nil)
+ _ io.ReadSeeker = (*readerAtSeeker)(nil)
+ _ io.ReaderAt = (*readSeekerAt)(nil)
+)
+
+type (
+ readerPlusByte struct {
+ io.Reader
+
+ byteBuf [1]byte // escapes via io.Reader.Read; preallocate
+ }
+
+ readSeekerPlusByte struct {
+ io.ReadSeeker
+
+ byteBuf [1]byte // escapes via io.Reader.Read; preallocate
+ }
+
+ discardingReadSeekerPlusByte struct {
+ io.Reader
+ offset int64
+
+ byteBuf [1]byte // escapes via io.Reader.Read; preallocate
+ }
+
+ ByteReadSeeker interface {
+ io.ReadSeeker
+ io.ByteReader
+ }
+
+ readSeekerAt struct {
+ rs io.ReadSeeker
+ mu sync.Mutex
+ }
+
+ readerAtSeeker struct {
+ ra io.ReaderAt
+ position int64
+ mu sync.Mutex
+ }
+)
+
+func ToByteReader(r io.Reader) io.ByteReader {
+ if br, ok := r.(io.ByteReader); ok {
+ return br
+ }
+ return &readerPlusByte{Reader: r}
+}
+
+func ToByteReadSeeker(r io.Reader) ByteReadSeeker {
+ if brs, ok := r.(ByteReadSeeker); ok {
+ return brs
+ }
+ if rs, ok := r.(io.ReadSeeker); ok {
+ return &readSeekerPlusByte{ReadSeeker: rs}
+ }
+ return &discardingReadSeekerPlusByte{Reader: r}
+}
+
+func ToReadSeeker(ra io.ReaderAt) io.ReadSeeker {
+ if rs, ok := ra.(io.ReadSeeker); ok {
+ return rs
+ }
+ return &readerAtSeeker{ra: ra}
+}
+
+func ToReaderAt(rs io.ReadSeeker) io.ReaderAt {
+ if ra, ok := rs.(io.ReaderAt); ok {
+ return ra
+ }
+ return &readSeekerAt{rs: rs}
+}
+
+func (rb *readerPlusByte) ReadByte() (byte, error) {
+ _, err := io.ReadFull(rb, rb.byteBuf[:])
+ return rb.byteBuf[0], err
+}
+
+func (rsb *readSeekerPlusByte) ReadByte() (byte, error) {
+ _, err := io.ReadFull(rsb, rsb.byteBuf[:])
+ return rsb.byteBuf[0], err
+}
+
+func (drsb *discardingReadSeekerPlusByte) ReadByte() (byte, error) {
+ _, err := io.ReadFull(drsb, drsb.byteBuf[:])
+ return drsb.byteBuf[0], err
+}
+
+func (drsb *discardingReadSeekerPlusByte) Read(p []byte) (read int, err error) {
+ read, err = drsb.Reader.Read(p)
+ drsb.offset += int64(read)
+ return
+}
+
+func (drsb *discardingReadSeekerPlusByte) Seek(offset int64, whence int) (int64, error) {
+ switch whence {
+ case io.SeekStart:
+ n := offset - drsb.offset
+ if n < 0 {
+ return 0, errors.New("unsupported rewind via whence: io.SeekStart")
+ }
+ _, err := io.CopyN(io.Discard, drsb, n)
+ return drsb.offset, err
+ case io.SeekCurrent:
+ _, err := io.CopyN(io.Discard, drsb, offset)
+ return drsb.offset, err
+ default:
+ return 0, errors.New("unsupported whence: io.SeekEnd")
+ }
+}
+
+func (ras *readerAtSeeker) Read(p []byte) (n int, err error) {
+ ras.mu.Lock()
+ defer ras.mu.Unlock()
+ n, err = ras.ra.ReadAt(p, ras.position)
+ ras.position += int64(n)
+ return n, err
+}
+
+func (ras *readerAtSeeker) Seek(offset int64, whence int) (int64, error) {
+ ras.mu.Lock()
+ defer ras.mu.Unlock()
+ switch whence {
+ case io.SeekStart:
+ ras.position = offset
+ case io.SeekCurrent:
+ ras.position += offset
+ case io.SeekEnd:
+ return 0, errors.New("unsupported whence: io.SeekEnd")
+ default:
+ return 0, errors.New("unsupported whence")
+ }
+ return ras.position, nil
+}
+
+func (rsa *readSeekerAt) ReadAt(p []byte, off int64) (n int, err error) {
+ rsa.mu.Lock()
+ defer rsa.mu.Unlock()
+ if _, err := rsa.rs.Seek(off, io.SeekStart); err != nil {
+ return 0, err
+ }
+ return rsa.rs.Read(p)
+}
diff --git a/ipld/car/v2/internal/io/offset_read_seeker.go b/ipld/car/v2/internal/io/offset_read_seeker.go
new file mode 100644
index 000000000..b3899ab78
--- /dev/null
+++ b/ipld/car/v2/internal/io/offset_read_seeker.go
@@ -0,0 +1,122 @@
+package io
+
+import (
+ "errors"
+ "io"
+)
+
+var (
+ _ io.ReaderAt = (*offsetReadSeeker)(nil)
+ _ io.ReadSeeker = (*offsetReadSeeker)(nil)
+)
+
+// offsetReadSeeker implements Read, and ReadAt on a section
+// of an underlying io.ReaderAt.
+// The main difference between io.SectionReader and offsetReadSeeker is that
+// NewOffsetReadSeeker does not require the user to know the number of readable bytes.
+//
+// It also partially implements Seek, where the implementation panics if io.SeekEnd is passed.
+// This is because, offsetReadSeeker does not know the end of the file therefore cannot seek relative
+// to it.
+type offsetReadSeeker struct {
+ r io.ReaderAt
+ base int64
+ off int64
+ b [1]byte // avoid alloc in ReadByte
+}
+
+type ReadSeekerAt interface {
+ io.Reader
+ io.ReaderAt
+ io.Seeker
+ io.ByteReader
+}
+
+// NewOffsetReadSeeker returns an ReadSeekerAt that reads from r
+// starting offset offset off and stops with io.EOF when r reaches its end.
+// The Seek function will panic if whence io.SeekEnd is passed.
+func NewOffsetReadSeeker(r io.ReaderAt, off int64) (ReadSeekerAt, error) {
+ if or, ok := r.(*offsetReadSeeker); ok {
+ oldBase := or.base
+ newBase := or.base + off
+ if newBase < oldBase {
+ return nil, errors.New("NewOffsetReadSeeker overflow int64")
+ }
+ return &offsetReadSeeker{
+ r: or.r,
+ base: newBase,
+ off: newBase,
+ }, nil
+ }
+ return &offsetReadSeeker{
+ r: r,
+ base: off,
+ off: off,
+ }, nil
+}
+
+func (o *offsetReadSeeker) Read(p []byte) (n int, err error) {
+ n, err = o.r.ReadAt(p, o.off)
+ oldOffset := o.off
+ off := oldOffset + int64(n)
+ if off < oldOffset {
+ return 0, errors.New("ReadAt offset overflow")
+ }
+ o.off = off
+ return
+}
+
+func (o *offsetReadSeeker) ReadAt(p []byte, off int64) (n int, err error) {
+ if off < 0 {
+ return 0, io.EOF
+ }
+ oldOffset := off
+ off += o.base
+ if off < oldOffset {
+ return 0, errors.New("ReadAt offset overflow")
+ }
+ return o.r.ReadAt(p, off)
+}
+
+func (o *offsetReadSeeker) ReadByte() (byte, error) {
+ _, err := o.Read(o.b[:])
+ return o.b[0], err
+}
+
+func (o *offsetReadSeeker) Offset() int64 {
+ return o.off
+}
+
+func (o *offsetReadSeeker) Seek(offset int64, whence int) (int64, error) {
+ switch whence {
+ case io.SeekStart:
+ oldOffset := offset
+ off := offset + o.base
+ if off < oldOffset {
+ return 0, errors.New("Seek offset overflow")
+ }
+ o.off = off
+ case io.SeekCurrent:
+ oldOffset := o.off
+ if offset < 0 {
+ if -offset > oldOffset {
+ return 0, errors.New("Seek offset underflow")
+ }
+ o.off = oldOffset + offset
+ } else {
+ off := oldOffset + offset
+ if off < oldOffset {
+ return 0, errors.New("Seek offset overflow")
+ }
+ o.off = off
+ }
+ case io.SeekEnd:
+ panic("unsupported whence: SeekEnd")
+ }
+ return o.Position(), nil
+}
+
+// Position returns the current position of this reader relative to the initial offset.
+func (o *offsetReadSeeker) Position() int64 {
+ return o.off - o.base
+}
diff --git a/ipld/car/v2/internal/io/offset_write_seeker.go b/ipld/car/v2/internal/io/offset_write_seeker.go
new file mode 100644
index 000000000..7e0f6ba58
--- /dev/null
+++ b/ipld/car/v2/internal/io/offset_write_seeker.go
@@ -0,0 +1,41 @@
+package io
+
+import "io"
+
+var (
+ _ io.Writer = (*OffsetWriteSeeker)(nil)
+ _ io.WriteSeeker = (*OffsetWriteSeeker)(nil)
+)
+
+type OffsetWriteSeeker struct {
+ w io.WriterAt
+ base int64
+ offset int64
+}
+
+func NewOffsetWriter(w io.WriterAt, off int64) *OffsetWriteSeeker {
+ return &OffsetWriteSeeker{w, off, off}
+}
+
+func (ow *OffsetWriteSeeker) Write(b []byte) (n int, err error) {
+ n, err = ow.w.WriteAt(b, ow.offset)
+ ow.offset += int64(n)
+ return
+}
+
+func (ow *OffsetWriteSeeker) Seek(offset int64, whence int) (int64, error) {
+ switch whence {
+ case io.SeekStart:
+ ow.offset = offset + ow.base
+ case io.SeekCurrent:
+ ow.offset += offset
+ case io.SeekEnd:
+ panic("unsupported whence: SeekEnd")
+ }
+ return ow.Position(), nil
+}
+
+// Position returns the current position of this writer relative to the initial offset, i.e. the number of bytes written.
+func (ow *OffsetWriteSeeker) Position() int64 {
+ return ow.offset - ow.base
+}
diff --git a/ipld/car/v2/internal/loader/counting_loader.go b/ipld/car/v2/internal/loader/counting_loader.go
new file mode 100644
index 000000000..e428993ea
--- /dev/null
+++ b/ipld/car/v2/internal/loader/counting_loader.go
@@ -0,0 +1,61 @@
+package loader
+
+import (
+ "bytes"
+ "io"
+
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/linking"
+ "github.com/multiformats/go-varint"
+)
+
+// counter tracks how much data has been read.
+type counter struct {
+ totalRead uint64
+}
+
+func (c *counter) Size() uint64 {
+ return c.totalRead
+}
+
+// ReadCounter provides an externally consumable interface to the
+// additional data tracked about the linksystem.
+type ReadCounter interface {
+ Size() uint64
+}
+
+type countingReader struct {
+ r io.Reader
+ c *counter
+}
+
+func (c *countingReader) Read(p []byte) (int, error) {
+ n, err := c.r.Read(p)
+ c.c.totalRead += uint64(n)
+ return n, err
+}
+
+// CountingLinkSystem wraps an ipld linksystem with to track the size of
+// data loaded in a `counter` object. Each time nodes are loaded from the
+// link system which trigger block reads, the size of the block as it would
+// appear in a CAR file is added to the counter (included the size of the
+// CID and the varint length for the block data).
+func CountingLinkSystem(ls ipld.LinkSystem) (ipld.LinkSystem, ReadCounter) {
+ c := counter{}
+ clc := ls
+ clc.StorageReadOpener = func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) {
+ r, err := ls.StorageReadOpener(lc, l)
+ if err != nil {
+ return nil, err
+ }
+ buf := bytes.NewBuffer(nil)
+ n, err := buf.ReadFrom(r)
+ if err != nil {
+ return nil, err
+ }
+ size := varint.ToUvarint(uint64(n) + uint64(len(l.Binary())))
+ c.totalRead += uint64(len(size)) + uint64(len(l.Binary()))
+ return &countingReader{buf, &c}, nil
+ }
+ return clc, &c
+}
diff --git a/ipld/car/v2/internal/loader/writing_loader.go b/ipld/car/v2/internal/loader/writing_loader.go
new file mode 100644
index 000000000..8be1af3f0
--- /dev/null
+++ b/ipld/car/v2/internal/loader/writing_loader.go
@@ -0,0 +1,126 @@
+package loader
+
+import (
+ "bytes"
+ "io"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/go-cid"
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/linking"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-varint"
+)
+
+type writerOutput struct {
+ w io.Writer
+ size uint64
+ code multicodec.Code
+ rcrds map[cid.Cid]index.Record
+}
+
+func (w *writerOutput) Size() uint64 {
+ return w.size
+}
+
+func (w *writerOutput) Index() (index.Index, error) {
+ idx, err := index.New(w.code)
+ if err != nil {
+ return nil, err
+ }
+ rcrds := make([]index.Record, 0, len(w.rcrds))
+ for _, r := range w.rcrds {
+ rcrds = append(rcrds, r)
+ }
+ if err := idx.Load(rcrds); err != nil {
+ return nil, err
+ }
+
+ return idx, nil
+}
+
+// An IndexTracker tracks the records loaded/written, calculate an
+// index based on them.
+type IndexTracker interface {
+ ReadCounter
+ Index() (index.Index, error)
+}
+
+type writingReader struct {
+ r io.Reader
+ len int64
+ cid string
+ wo *writerOutput
+}
+
+func (w *writingReader) Read(p []byte) (int, error) {
+ if w.wo != nil {
+ // write the cid
+ size := varint.ToUvarint(uint64(w.len) + uint64(len(w.cid)))
+ if _, err := w.wo.w.Write(size); err != nil {
+ return 0, err
+ }
+ if _, err := w.wo.w.Write([]byte(w.cid)); err != nil {
+ return 0, err
+ }
+ cpy := bytes.NewBuffer(w.r.(*bytes.Buffer).Bytes())
+ if _, err := cpy.WriteTo(w.wo.w); err != nil {
+ return 0, err
+ }
+ _, c, err := cid.CidFromBytes([]byte(w.cid))
+ if err != nil {
+ return 0, err
+ }
+ w.wo.rcrds[c] = index.Record{
+ Cid: c,
+ Offset: w.wo.size,
+ }
+ w.wo.size += uint64(w.len) + uint64(len(size)+len(w.cid))
+
+ w.wo = nil
+ }
+
+ return w.r.Read(p)
+}
+
+// TeeingLinkSystem wraps an IPLD.LinkSystem so that each time a block is loaded from it,
+// that block is also written as a CAR block to the provided io.Writer. Metadata
+// (the size of data written) is provided in the second return value.
+// The `initialOffset` is used to calculate the offsets recorded for the index, and will be
+//
+// included in the `.Size()` of the IndexTracker.
+//
+// An indexCodec of `index.CarIndexNoIndex` can be used to not track these offsets.
+func TeeingLinkSystem(ls ipld.LinkSystem, w io.Writer, initialOffset uint64, indexCodec multicodec.Code) (ipld.LinkSystem, IndexTracker) {
+ wo := writerOutput{
+ w: w,
+ size: initialOffset,
+ code: indexCodec,
+ rcrds: make(map[cid.Cid]index.Record),
+ }
+
+ tls := ls
+ tls.StorageReadOpener = func(lc linking.LinkContext, l ipld.Link) (io.Reader, error) {
+ _, c, err := cid.CidFromBytes([]byte(l.Binary()))
+ if err != nil {
+ return nil, err
+ }
+
+ // if we've already read this cid in this session, don't re-write it.
+ if _, ok := wo.rcrds[c]; ok {
+ return ls.StorageReadOpener(lc, l)
+ }
+
+ r, err := ls.StorageReadOpener(lc, l)
+ if err != nil {
+ return nil, err
+ }
+ buf := bytes.NewBuffer(nil)
+ n, err := buf.ReadFrom(r)
+ if err != nil {
+ return nil, err
+ }
+ return &writingReader{buf, n, l.Binary(), &wo}, nil
+ }
+ return tls, &wo
+}
diff --git a/ipld/car/v2/internal/store/identity.go b/ipld/car/v2/internal/store/identity.go
new file mode 100644
index 000000000..d61a57c48
--- /dev/null
+++ b/ipld/car/v2/internal/store/identity.go
@@ -0,0 +1,17 @@
+package store
+
+import (
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multihash"
+)
+
+// IsIdentity inspects the CID and determines whether it is an IDENTITY CID.
+func IsIdentity(key cid.Cid) (digest []byte, ok bool, err error) {
+ dmh, err := multihash.Decode(key.Hash())
+ if err != nil {
+ return nil, false, err
+ }
+ ok = dmh.Code == multihash.IDENTITY
+ digest = dmh.Digest
+ return digest, ok, nil
+}
diff --git a/ipld/car/v2/internal/store/index.go b/ipld/car/v2/internal/store/index.go
new file mode 100644
index 000000000..b29ec8e86
--- /dev/null
+++ b/ipld/car/v2/internal/store/index.go
@@ -0,0 +1,108 @@
+package store
+
+import (
+ "bytes"
+ "io"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1/util"
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-varint"
+)
+
+// FindCid can be used to either up the existence, size and offset of a block
+// if it exists in CAR as specified by the index; and optionally the data bytes
+// of the block.
+func FindCid(
+ reader io.ReaderAt,
+ idx index.Index,
+ key cid.Cid,
+ useWholeCids bool,
+ zeroLenAsEOF bool,
+ maxReadBytes uint64,
+ readBytes bool,
+) ([]byte, int64, int, error) {
+
+ var fnData []byte
+ var fnOffset int64
+ var fnLen int = -1
+ var fnErr error
+ err := idx.GetAll(key, func(offset uint64) bool {
+ reader, err := internalio.NewOffsetReadSeeker(reader, int64(offset))
+ if err != nil {
+ fnErr = err
+ return false
+ }
+ var readCid cid.Cid
+ if readBytes {
+ readCid, fnData, err = util.ReadNode(reader, zeroLenAsEOF, maxReadBytes)
+ if err != nil {
+ fnErr = err
+ return false
+ }
+ fnLen = len(fnData)
+ } else {
+ sectionLen, err := varint.ReadUvarint(reader)
+ if err != nil {
+ fnErr = err
+ return false
+ }
+ var cidLen int
+ cidLen, readCid, err = cid.CidFromReader(reader)
+ if err != nil {
+ fnErr = err
+ return false
+ }
+ fnLen = int(sectionLen) - cidLen
+ fnOffset = int64(offset) + reader.(interface{ Position() int64 }).Position()
+ }
+ if useWholeCids {
+ if !readCid.Equals(key) {
+ fnLen = -1
+ return true // continue looking
+ }
+ return false
+ } else {
+ if !bytes.Equal(readCid.Hash(), key.Hash()) {
+ // weird, bad index, continue looking
+ fnLen = -1
+ return true
+ }
+ return false
+ }
+ })
+ if err != nil {
+ return nil, -1, -1, err
+ }
+ if fnErr != nil {
+ return nil, -1, -1, fnErr
+ }
+ if fnLen == -1 {
+ return nil, -1, -1, index.ErrNotFound
+ }
+ return fnData, fnOffset, fnLen, nil
+}
+
+// Finalize will write the index to the writer at the offset specified in the header. It should only
+// be used for a CARv2 and when the CAR interface is being closed.
+func Finalize(writer io.WriterAt, header carv2.Header, idx *InsertionIndex, dataSize uint64, storeIdentityCIDs bool, indexCodec multicodec.Code) error {
+ // TODO check if add index option is set and don't write the index then set index offset to zero.
+ header = header.WithDataSize(dataSize)
+ header.Characteristics.SetFullyIndexed(storeIdentityCIDs)
+
+ // TODO if index not needed don't bother flattening it.
+ fi, err := idx.Flatten(indexCodec)
+ if err != nil {
+ return err
+ }
+ if _, err := index.WriteTo(fi, internalio.NewOffsetWriter(writer, int64(header.IndexOffset))); err != nil {
+ return err
+ }
+ if _, err := header.WriteTo(internalio.NewOffsetWriter(writer, carv2.PragmaSize)); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/ipld/car/v2/internal/store/indexcheck.go b/ipld/car/v2/internal/store/indexcheck.go
new file mode 100644
index 000000000..89c563bb5
--- /dev/null
+++ b/ipld/car/v2/internal/store/indexcheck.go
@@ -0,0 +1,84 @@
+package store
+
+import (
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/go-cid"
+)
+
+// ShouldPut returns true if the block should be put into the CAR according to the options provided
+// and the index. It returns false if the block should not be put into the CAR, either because it
+// is an identity block and StoreIdentityCIDs is false, or because it already exists and
+// BlockstoreAllowDuplicatePuts is false.
+func ShouldPut(
+ idx *InsertionIndex,
+ c cid.Cid,
+ maxIndexCidSize uint64,
+ storeIdentityCIDs bool,
+ blockstoreAllowDuplicatePuts bool,
+ blockstoreUseWholeCIDs bool,
+) (bool, error) {
+
+ // If StoreIdentityCIDs option is disabled then treat IDENTITY CIDs like IdStore.
+ if !storeIdentityCIDs {
+ // Check for IDENTITY CID. If IDENTITY, ignore and move to the next block.
+ if _, ok, err := IsIdentity(c); err != nil {
+ return false, err
+ } else if ok {
+ return false, nil
+ }
+ }
+
+ // Check if its size is too big.
+ // If larger than maximum allowed size, return error.
+ // Note, we need to check this regardless of whether we have IDENTITY CID or not.
+ // Since multhihash codes other than IDENTITY can result in large digests.
+ cSize := uint64(len(c.Bytes()))
+ if cSize > maxIndexCidSize {
+ return false, &carv2.ErrCidTooLarge{MaxSize: maxIndexCidSize, CurrentSize: cSize}
+ }
+
+ if !blockstoreAllowDuplicatePuts {
+ if blockstoreUseWholeCIDs {
+ has, err := idx.HasExactCID(c)
+ if err != nil {
+ return false, err
+ }
+ return !has, nil // deduplicated by CID
+ }
+ if !blockstoreUseWholeCIDs {
+ _, err := idx.Get(c)
+ if err == nil {
+ return false, nil // deduplicated by hash
+ }
+ }
+ }
+
+ return true, nil
+}
+
+// Has returns true if the block exists in in the store according to the various
+// rules associated with the options. Similar to ShouldPut, but for the simpler
+// Has() case.
+func Has(
+ idx *InsertionIndex,
+ c cid.Cid,
+ maxIndexCidSize uint64,
+ storeIdentityCIDs bool,
+ blockstoreAllowDuplicatePuts bool,
+ blockstoreUseWholeCIDs bool,
+) (bool, error) {
+
+ // If StoreIdentityCIDs option is disabled then treat IDENTITY CIDs like IdStore.
+ if !storeIdentityCIDs {
+ if _, ok, err := IsIdentity(c); err != nil {
+ return false, err
+ } else if ok {
+ return true, nil
+ }
+ }
+
+ if blockstoreUseWholeCIDs {
+ return idx.HasExactCID(c)
+ }
+ return idx.HasMultihash(c.Hash())
+}
diff --git a/ipld/car/v2/internal/store/insertionindex.go b/ipld/car/v2/internal/store/insertionindex.go
new file mode 100644
index 000000000..a27844552
--- /dev/null
+++ b/ipld/car/v2/internal/store/insertionindex.go
@@ -0,0 +1,264 @@
+package store
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-multihash"
+ "github.com/petar/GoLLRB/llrb"
+ cbor "github.com/whyrusleeping/cbor/go"
+)
+
+// This index is intended to be efficient for random-access, in-memory lookups
+// and is not intended to be an index type that is attached to a CARv2.
+// See flatten() for conversion of this data to a known, existing index type.
+
+var (
+ errUnsupported = errors.New("not supported")
+ insertionIndexCodec = multicodec.Code(0x300003)
+)
+
+type InsertionIndex struct {
+ items llrb.LLRB
+}
+
+func NewInsertionIndex() *InsertionIndex {
+ return &InsertionIndex{}
+}
+
+type recordDigest struct {
+ digest []byte
+ index.Record
+}
+
+func (r recordDigest) Less(than llrb.Item) bool {
+ other, ok := than.(recordDigest)
+ if !ok {
+ return false
+ }
+ return bytes.Compare(r.digest, other.digest) < 0
+}
+
+func newRecordDigest(r index.Record) recordDigest {
+ d, err := multihash.Decode(r.Hash())
+ if err != nil {
+ panic(err)
+ }
+
+ return recordDigest{d.Digest, r}
+}
+
+func newRecordFromCid(c cid.Cid, at uint64) recordDigest {
+ d, err := multihash.Decode(c.Hash())
+ if err != nil {
+ panic(err)
+ }
+
+ return recordDigest{d.Digest, index.Record{Cid: c, Offset: at}}
+}
+
+func (ii *InsertionIndex) InsertNoReplace(key cid.Cid, n uint64) {
+ ii.items.InsertNoReplace(newRecordFromCid(key, n))
+}
+
+func (ii *InsertionIndex) Get(c cid.Cid) (uint64, error) {
+ record, err := ii.getRecord(c)
+ if err != nil {
+ return 0, err
+ }
+ return record.Offset, nil
+}
+
+func (ii *InsertionIndex) getRecord(c cid.Cid) (index.Record, error) {
+ d, err := multihash.Decode(c.Hash())
+ if err != nil {
+ return index.Record{}, err
+ }
+ entry := recordDigest{digest: d.Digest}
+ e := ii.items.Get(entry)
+ if e == nil {
+ return index.Record{}, index.ErrNotFound
+ }
+ r, ok := e.(recordDigest)
+ if !ok {
+ return index.Record{}, errUnsupported
+ }
+
+ return r.Record, nil
+}
+
+func (ii *InsertionIndex) GetAll(c cid.Cid, fn func(uint64) bool) error {
+ d, err := multihash.Decode(c.Hash())
+ if err != nil {
+ return err
+ }
+ entry := recordDigest{digest: d.Digest}
+
+ any := false
+ iter := func(i llrb.Item) bool {
+ existing := i.(recordDigest)
+ if !bytes.Equal(existing.digest, entry.digest) {
+ // We've already looked at all entries with matching digests.
+ return false
+ }
+ any = true
+ return fn(existing.Record.Offset)
+ }
+ ii.items.AscendGreaterOrEqual(entry, iter)
+ if !any {
+ return index.ErrNotFound
+ }
+ return nil
+}
+
+func (ii *InsertionIndex) Marshal(w io.Writer) (uint64, error) {
+ l := uint64(0)
+ if err := binary.Write(w, binary.LittleEndian, int64(ii.items.Len())); err != nil {
+ return l, err
+ }
+ l += 8
+
+ var err error
+ iter := func(i llrb.Item) bool {
+ if err = cbor.Encode(w, i.(recordDigest).Record); err != nil {
+ return false
+ }
+ return true
+ }
+ ii.items.AscendGreaterOrEqual(ii.items.Min(), iter)
+ return l, err
+}
+
+func (ii *InsertionIndex) Unmarshal(r io.Reader) error {
+ var length int64
+ if err := binary.Read(r, binary.LittleEndian, &length); err != nil {
+ return err
+ }
+ d := cbor.NewDecoder(r)
+ for i := int64(0); i < length; i++ {
+ var rec index.Record
+ if err := d.Decode(&rec); err != nil {
+ return err
+ }
+ ii.items.InsertNoReplace(newRecordDigest(rec))
+ }
+ return nil
+}
+
+func (ii *InsertionIndex) ForEach(f func(multihash.Multihash, uint64) error) error {
+ var err error
+ ii.items.AscendGreaterOrEqual(ii.items.Min(), func(i llrb.Item) bool {
+ r := i.(recordDigest).Record
+ err = f(r.Cid.Hash(), r.Offset)
+ return err == nil
+ })
+ return err
+}
+
+func (ii *InsertionIndex) ForEachCid(f func(cid.Cid, uint64) error) error {
+ var err error
+ ii.items.AscendGreaterOrEqual(ii.items.Min(), func(i llrb.Item) bool {
+ r := i.(recordDigest).Record
+ err = f(r.Cid, r.Offset)
+ return err == nil
+ })
+ return err
+}
+
+func (ii *InsertionIndex) Codec() multicodec.Code {
+ return insertionIndexCodec
+}
+
+func (ii *InsertionIndex) Load(rs []index.Record) error {
+ for _, r := range rs {
+ rec := newRecordDigest(r)
+ if rec.digest == nil {
+ return fmt.Errorf("invalid entry: %v", r)
+ }
+ ii.items.InsertNoReplace(rec)
+ }
+ return nil
+}
+
+// flatten returns a formatted index in the given codec for more efficient subsequent loading.
+func (ii *InsertionIndex) Flatten(codec multicodec.Code) (index.Index, error) {
+ si, err := index.New(codec)
+ if err != nil {
+ return nil, err
+ }
+ rcrds := make([]index.Record, ii.items.Len())
+
+ idx := 0
+ iter := func(i llrb.Item) bool {
+ rcrds[idx] = i.(recordDigest).Record
+ idx++
+ return true
+ }
+ ii.items.AscendGreaterOrEqual(ii.items.Min(), iter)
+
+ if err := si.Load(rcrds); err != nil {
+ return nil, err
+ }
+ return si, nil
+}
+
+// note that hasExactCID is very similar to GetAll,
+// but it's separate as it allows us to compare Record.Cid directly,
+// whereas GetAll just provides Record.Offset.
+
+func (ii *InsertionIndex) HasExactCID(c cid.Cid) (bool, error) {
+ d, err := multihash.Decode(c.Hash())
+ if err != nil {
+ return false, err
+ }
+ entry := recordDigest{digest: d.Digest}
+
+ found := false
+ iter := func(i llrb.Item) bool {
+ existing := i.(recordDigest)
+ if !bytes.Equal(existing.digest, entry.digest) {
+ // We've already looked at all entries with matching digests.
+ return false
+ }
+ if existing.Record.Cid == c {
+ // We found an exact match.
+ found = true
+ return false
+ }
+ // Continue looking in ascending order.
+ return true
+ }
+ ii.items.AscendGreaterOrEqual(entry, iter)
+ return found, nil
+}
+
+func (ii *InsertionIndex) HasMultihash(mh multihash.Multihash) (bool, error) {
+ d, err := multihash.Decode(mh)
+ if err != nil {
+ return false, err
+ }
+ entry := recordDigest{digest: d.Digest}
+
+ found := false
+ iter := func(i llrb.Item) bool {
+ existing := i.(recordDigest)
+ if !bytes.Equal(existing.digest, entry.digest) {
+ // We've already looked at all entries with matching digests.
+ return false
+ }
+ if bytes.Equal(existing.Record.Cid.Hash(), mh) {
+ found = true
+ return false
+ }
+ // Continue looking in ascending order.
+ return true
+ }
+ ii.items.AscendGreaterOrEqual(entry, iter)
+ return found, nil
+}
diff --git a/ipld/car/v2/internal/store/resume.go b/ipld/car/v2/internal/store/resume.go
new file mode 100644
index 000000000..0ba94570a
--- /dev/null
+++ b/ipld/car/v2/internal/store/resume.go
@@ -0,0 +1,198 @@
+package store
+
+import (
+ "errors"
+ "fmt"
+ "io"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-varint"
+)
+
+type ReaderWriterAt interface {
+ io.ReaderAt
+ io.WriterAt
+}
+
+// ResumableVersion performs two tasks - check if there is a valid header at the start of the,
+// reader, then check whether the version of that header matches what we expect.
+func ResumableVersion(reader io.Reader, writeAsV1 bool) error {
+ version, err := carv2.ReadVersion(reader)
+ if err != nil {
+ // The file is not a valid CAR file and cannot resume from it.
+ // Or the write must have failed before pragma was written.
+ return err
+ }
+
+ switch {
+ case version == 1 && writeAsV1:
+ case version == 2 && !writeAsV1:
+ default:
+ // The file is not the expected version and we cannot resume from it.
+ return fmt.Errorf("cannot resume on CAR file with version %v", version)
+ }
+ return nil
+}
+
+// Resume will attempt to resume a CARv2 or CARv1 file by checking that there exists an existing
+// CAR and that the CAR header details match what is being requested for resumption.
+// Resumption of a CARv2 involves "unfinalizing" the header by resetting it back to a bare state
+// and then truncating the file to remove the index. Truncation is important because it allows a
+// non-finalized CARv2 to be resumed from as the header won't contain the DataSize of the payload
+// body and if the file also contains an index, we cannot determine the end of the payload.
+// Therefore, when using a resumed, existing and finalized, CARv2, whose body may not extend
+// beyond the index and then closing without finalization (e.g. due to a crash), the file will no
+// longer be parseable because we won't have DataSize, and we won't be able to determine it by
+// parsing the payload to EOF.
+func Resume(
+ rw ReaderWriterAt,
+ dataReader io.ReaderAt,
+ dataWriter *internalio.OffsetWriteSeeker,
+ idx *InsertionIndex,
+ roots []cid.Cid,
+ dataOffset uint64,
+ v1 bool,
+ maxAllowedHeaderSize uint64,
+ zeroLengthSectionAsEOF bool,
+) error {
+
+ var headerInFile carv2.Header
+ var v1r internalio.ReadSeekerAt
+
+ if !v1 {
+ if _, ok := rw.(interface{ Truncate(size int64) error }); !ok {
+ return fmt.Errorf("cannot resume a CARv2 without the ability to truncate (e.g. an io.File)")
+ }
+
+ // Check if file was finalized by trying to read the CARv2 header.
+ // We check because if finalized the CARv1 reader behaviour needs to be adjusted since
+ // EOF will not signify end of CARv1 payload. i.e. index is most likely present.
+ r, err := internalio.NewOffsetReadSeeker(rw, carv2.PragmaSize)
+ if err != nil {
+ return err
+ }
+ _, err = headerInFile.ReadFrom(r)
+
+ // If reading CARv2 header succeeded, and CARv1 offset in header is not zero then the file is
+ // most-likely finalized. Check padding and truncate the file to remove index.
+ // Otherwise, carry on reading the v1 payload at offset determined from b.header.
+ if err == nil && headerInFile.DataOffset != 0 {
+ if headerInFile.DataOffset != dataOffset {
+ // Assert that the padding on file matches the given WithDataPadding option.
+ wantPadding := headerInFile.DataOffset - carv2.PragmaSize - carv2.HeaderSize
+ gotPadding := dataOffset - carv2.PragmaSize - carv2.HeaderSize
+ return fmt.Errorf(
+ "cannot resume from file with mismatched CARv1 offset; "+
+ "`WithDataPadding` option must match the padding on file. "+
+ "Expected padding value of %v but got %v", wantPadding, gotPadding,
+ )
+ } else if headerInFile.DataSize == 0 {
+ // If CARv1 size is zero, since CARv1 offset wasn't, then the CARv2 header was
+ // most-likely partially written. Since we write the header last in Finalize then the
+ // file most-likely contains the index and we cannot know where it starts, therefore
+ // can't resume.
+ return errors.New("corrupt CARv2 header; cannot resume from file")
+ }
+ }
+
+ v1r, err = internalio.NewOffsetReadSeeker(dataReader, 0)
+ if err != nil {
+ return err
+ }
+ } else {
+ var err error
+ v1r, err = internalio.NewOffsetReadSeeker(rw, 0)
+ if err != nil {
+ return err
+ }
+ }
+
+ header, err := carv1.ReadHeader(v1r, maxAllowedHeaderSize)
+ if err != nil {
+ // Cannot read the CARv1 header; the file is most likely corrupt.
+ return fmt.Errorf("error reading car header: %w", err)
+ }
+ if !header.Matches(carv1.CarHeader{Roots: roots, Version: 1}) {
+ // Cannot resume if version and root does not match.
+ return errors.New("cannot resume on file with mismatching data header")
+ }
+
+ if headerInFile.DataOffset != 0 {
+ // If header in file contains the size of car v1, then the index is most likely present.
+ // Since we will need to re-generate the index, as the one in file is flattened, truncate
+ // the file so that the Readonly.backing has the right set of bytes to deal with.
+ // This effectively means resuming from a finalized file will wipe its index even if there
+ // are no blocks put unless the user calls finalize.
+ if err := rw.(interface{ Truncate(size int64) error }).Truncate(int64(headerInFile.DataOffset + headerInFile.DataSize)); err != nil {
+ return err
+ }
+ }
+
+ if !v1 {
+ // Now that CARv2 header is present on file, clear it to avoid incorrect size and offset in
+ // header in case blocksotre is closed without finalization and is resumed from.
+ wat, ok := rw.(io.WriterAt)
+ if !ok { // how would we get this far??
+ return errors.New("cannot resume from file without io.WriterAt")
+ }
+ if _, err := new(carv2.Header).WriteTo(internalio.NewOffsetWriter(wat, carv2.PragmaSize)); err != nil {
+ return fmt.Errorf("could not un-finalize: %w", err)
+ }
+ }
+
+ // TODO See how we can reduce duplicate code here.
+ // The code here comes from car.GenerateIndex.
+ // Copied because we need to populate an insertindex, not a sorted index.
+ // Producing a sorted index via generate, then converting it to insertindex is not possible.
+ // Because Index interface does not expose internal records.
+ // This may be done as part of https://github.com/ipld/go-car/issues/95
+
+ offset, err := carv1.HeaderSize(header)
+ if err != nil {
+ return err
+ }
+ sectionOffset := int64(0)
+ if sectionOffset, err = v1r.Seek(int64(offset), io.SeekStart); err != nil {
+ return err
+ }
+
+ for {
+ // Grab the length of the section.
+ // Note that ReadUvarint wants a ByteReader.
+ length, err := varint.ReadUvarint(v1r)
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return err
+ }
+
+ // Null padding; by default it's an error.
+ if length == 0 {
+ if zeroLengthSectionAsEOF {
+ break
+ } else {
+ return fmt.Errorf("carv1 null padding not allowed by default; see WithZeroLegthSectionAsEOF")
+ }
+ }
+
+ // Grab the CID.
+ n, c, err := cid.CidFromReader(v1r)
+ if err != nil {
+ return err
+ }
+ idx.InsertNoReplace(c, uint64(sectionOffset))
+
+ // Seek to the next section by skipping the block.
+ // The section length includes the CID, so subtract it.
+ if sectionOffset, err = v1r.Seek(int64(length)-int64(n), io.SeekCurrent); err != nil {
+ return err
+ }
+ }
+ // Seek to the end of last skipped block where the writer should resume writing.
+ _, err = dataWriter.Seek(sectionOffset, io.SeekStart)
+ return err
+}
diff --git a/ipld/car/v2/options.go b/ipld/car/v2/options.go
new file mode 100644
index 000000000..5a8c5dc86
--- /dev/null
+++ b/ipld/car/v2/options.go
@@ -0,0 +1,240 @@
+package car
+
+import (
+ "math"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipld/go-ipld-prime/traversal"
+ "github.com/multiformats/go-multicodec"
+
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+)
+
+// DefaultMaxIndexCidSize specifies the maximum size in byptes accepted as a section CID by CARv2 index.
+const DefaultMaxIndexCidSize = 2 << 10 // 2 KiB
+
+// DefaultMaxAllowedHeaderSize specifies the default maximum size that a CARv1
+// decode (including within a CARv2 container) will allow a header to be without
+// erroring. This is to prevent OOM errors where a header prefix includes a
+// too-large size specifier.
+// Currently set to 32 MiB.
+const DefaultMaxAllowedHeaderSize = carv1.DefaultMaxAllowedHeaderSize
+
+// DefaultMaxAllowedHeaderSize specifies the default maximum size that a CARv1
+// decode (including within a CARv2 container) will allow a section to be
+// without erroring. This is to prevent OOM errors where a section prefix
+// includes a too-large size specifier.
+// Typically IPLD blocks should be under 2 MiB (ideally under 1 MiB), so unless
+// atypical data is expected, this should not be a large value.
+// Currently set to 8 MiB.
+const DefaultMaxAllowedSectionSize = carv1.DefaultMaxAllowedSectionSize
+
+// Option describes an option which affects behavior when interacting with CAR files.
+type Option func(*Options)
+
+// ReadOption hints that an API wants options related only to reading CAR files.
+type ReadOption = Option
+
+// WriteOption hints that an API wants options related only to reading CAR files.
+type WriteOption = Option
+
+// ReadWriteOption is either a ReadOption or a WriteOption.
+// Deprecated: use Option instead.
+type ReadWriteOption = Option
+
+// Options holds the configured options after applying a number of
+// Option funcs.
+//
+// This type should not be used directly by end users; it's only exposed as a
+// side effect of Option.
+type Options struct {
+ DataPadding uint64
+ IndexPadding uint64
+ IndexCodec multicodec.Code
+ ZeroLengthSectionAsEOF bool
+ MaxIndexCidSize uint64
+ StoreIdentityCIDs bool
+
+ BlockstoreAllowDuplicatePuts bool
+ BlockstoreUseWholeCIDs bool
+ MaxTraversalLinks uint64
+ WriteAsCarV1 bool
+ TraversalPrototypeChooser traversal.LinkTargetNodePrototypeChooser
+ TrustedCAR bool
+
+ MaxAllowedHeaderSize uint64
+ MaxAllowedSectionSize uint64
+}
+
+// ApplyOptions applies given opts and returns the resulting Options.
+// This function should not be used directly by end users; it's only exposed as a
+// side effect of Option.
+func ApplyOptions(opt ...Option) Options {
+ opts := Options{
+ MaxTraversalLinks: math.MaxInt64, //default: traverse all
+ MaxAllowedHeaderSize: carv1.DefaultMaxAllowedHeaderSize,
+ MaxAllowedSectionSize: carv1.DefaultMaxAllowedSectionSize,
+ }
+ for _, o := range opt {
+ o(&opts)
+ }
+ // Set defaults for zero valued fields.
+ if opts.IndexCodec == 0 {
+ opts.IndexCodec = multicodec.CarMultihashIndexSorted
+ }
+ if opts.MaxIndexCidSize == 0 {
+ opts.MaxIndexCidSize = DefaultMaxIndexCidSize
+ }
+ return opts
+}
+
+// ZeroLengthSectionAsEOF sets whether to allow the CARv1 decoder to treat
+// a zero-length section as the end of the input CAR file. For example, this can
+// be useful to allow "null padding" after a CARv1 without knowing where the
+// padding begins.
+func ZeroLengthSectionAsEOF(enable bool) Option {
+ return func(o *Options) {
+ o.ZeroLengthSectionAsEOF = enable
+ }
+}
+
+// UseDataPadding sets the padding to be added between CARv2 header and its data payload on Finalize.
+func UseDataPadding(p uint64) Option {
+ return func(o *Options) {
+ o.DataPadding = p
+ }
+}
+
+// UseIndexPadding sets the padding between data payload and its index on Finalize.
+func UseIndexPadding(p uint64) Option {
+ return func(o *Options) {
+ o.IndexPadding = p
+ }
+}
+
+// UseIndexCodec sets the codec used for index generation.
+func UseIndexCodec(c multicodec.Code) Option {
+ return func(o *Options) {
+ o.IndexCodec = c
+ }
+}
+
+// WithoutIndex flags that no index should be included in generation.
+func WithoutIndex() Option {
+ return func(o *Options) {
+ o.IndexCodec = index.CarIndexNone
+ }
+}
+
+// StoreIdentityCIDs sets whether to persist sections that are referenced by
+// CIDs with multihash.IDENTITY digest.
+// When writing CAR files with this option, Characteristics.IsFullyIndexed will
+// be set.
+//
+// By default, the blockstore interface will always return true for Has() called
+// with identity CIDs, but when this option is turned on, it will defer to the
+// index.
+//
+// When creating an index (or loading a CARv1 as a blockstore), when this option
+// is on, identity CIDs will be included in the index.
+//
+// This option is disabled by default.
+func StoreIdentityCIDs(b bool) Option {
+ return func(o *Options) {
+ o.StoreIdentityCIDs = b
+ }
+}
+
+// MaxIndexCidSize specifies the maximum allowed size for indexed CIDs in bytes.
+// Indexing a CID with larger than the allowed size results in ErrCidTooLarge error.
+func MaxIndexCidSize(s uint64) Option {
+ return func(o *Options) {
+ o.MaxIndexCidSize = s
+ }
+}
+
+// WithTraversalPrototypeChooser specifies the prototype chooser that should be used
+// when performing traversals in writes from a linksystem.
+func WithTraversalPrototypeChooser(t traversal.LinkTargetNodePrototypeChooser) Option {
+ return func(o *Options) {
+ o.TraversalPrototypeChooser = t
+ }
+}
+
+// WithTrustedCAR specifies whether CIDs match the block data as they are read
+// from the CAR files.
+func WithTrustedCAR(t bool) Option {
+ return func(o *Options) {
+ o.TrustedCAR = t
+ }
+}
+
+// MaxAllowedHeaderSize overrides the default maximum size (of 32 MiB) that a
+// CARv1 decode (including within a CARv2 container) will allow a header to be
+// without erroring.
+func MaxAllowedHeaderSize(max uint64) Option {
+ return func(o *Options) {
+ o.MaxAllowedHeaderSize = max
+ }
+}
+
+// MaxAllowedSectionSize overrides the default maximum size (of 8 MiB) that a
+// CARv1 decode (including within a CARv2 container) will allow a header to be
+// without erroring.
+// Typically IPLD blocks should be under 2 MiB (ideally under 1 MiB), so unless
+// atypical data is expected, this should not be a large value.
+func MaxAllowedSectionSize(max uint64) Option {
+ return func(o *Options) {
+ o.MaxAllowedSectionSize = max
+ }
+}
+
+// --------------------------------------------------- storage interface options
+
+// UseWholeCIDs is a read option which makes a CAR storage interface (blockstore
+// or storage) identify blocks by whole CIDs, and not just their multihashes.
+// The default is to use multihashes, which matches the current semantics of
+// go-ipfs-blockstore v1.
+//
+// Enabling this option affects a number of methods, including read-only ones:
+//
+// • Get, Has, and HasSize will only return a block only if the entire CID is
+// present in the CAR file.
+//
+// • AllKeysChan will return the original whole CIDs, instead of with their
+// multicodec set to "raw" to just provide multihashes.
+//
+// • If AllowDuplicatePuts isn't set, Put and PutMany will deduplicate by the
+// whole CID, allowing different CIDs with equal multihashes.
+//
+// Note that this option only affects the storage interfaces (blockstore
+// or storage), and is ignored by the root go-car/v2 package.
+func UseWholeCIDs(enable bool) Option {
+ return func(o *Options) {
+ o.BlockstoreUseWholeCIDs = enable
+ }
+}
+
+// WriteAsCarV1 is a write option which makes a CAR interface (blockstore or
+// storage) write the output as a CARv1 only, with no CARv2 header or index.
+// Indexing is used internally during write but is discarded upon finalization.
+//
+// Note that this option only affects the storage interfaces (blockstore
+// or storage), and is ignored by the root go-car/v2 package.
+func WriteAsCarV1(asCarV1 bool) Option {
+ return func(o *Options) {
+ o.WriteAsCarV1 = asCarV1
+ }
+}
+
+// AllowDuplicatePuts is a write option which makes a CAR interface (blockstore
+// or storage) not deduplicate blocks in Put and PutMany. The default is to
+// deduplicate, which matches the current semantics of go-ipfs-blockstore v1.
+//
+// Note that this option only affects the storage interfaces (blockstore
+// or storage), and is ignored by the root go-car/v2 package.
+func AllowDuplicatePuts(allow bool) Option {
+ return func(o *Options) {
+ o.BlockstoreAllowDuplicatePuts = allow
+ }
+}
diff --git a/ipld/car/v2/options_test.go b/ipld/car/v2/options_test.go
new file mode 100644
index 000000000..8964b981f
--- /dev/null
+++ b/ipld/car/v2/options_test.go
@@ -0,0 +1,50 @@
+package car_test
+
+import (
+ "math"
+ "testing"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/blockstore"
+ "github.com/multiformats/go-multicodec"
+ "github.com/stretchr/testify/require"
+)
+
+func TestApplyOptions_SetsExpectedDefaults(t *testing.T) {
+ require.Equal(t, carv2.Options{
+ IndexCodec: multicodec.CarMultihashIndexSorted,
+ MaxIndexCidSize: carv2.DefaultMaxIndexCidSize,
+ MaxTraversalLinks: math.MaxInt64,
+ MaxAllowedHeaderSize: 32 << 20,
+ MaxAllowedSectionSize: 8 << 20,
+ }, carv2.ApplyOptions())
+}
+
+func TestApplyOptions_AppliesOptions(t *testing.T) {
+ require.Equal(t,
+ carv2.Options{
+ DataPadding: 123,
+ IndexPadding: 456,
+ IndexCodec: multicodec.CarIndexSorted,
+ ZeroLengthSectionAsEOF: true,
+ MaxIndexCidSize: 789,
+ StoreIdentityCIDs: true,
+ BlockstoreAllowDuplicatePuts: true,
+ BlockstoreUseWholeCIDs: true,
+ MaxTraversalLinks: math.MaxInt64,
+ MaxAllowedHeaderSize: 101,
+ MaxAllowedSectionSize: 202,
+ },
+ carv2.ApplyOptions(
+ carv2.UseDataPadding(123),
+ carv2.UseIndexPadding(456),
+ carv2.UseIndexCodec(multicodec.CarIndexSorted),
+ carv2.ZeroLengthSectionAsEOF(true),
+ carv2.MaxIndexCidSize(789),
+ carv2.StoreIdentityCIDs(true),
+ carv2.MaxAllowedHeaderSize(101),
+ carv2.MaxAllowedSectionSize(202),
+ blockstore.AllowDuplicatePuts(true),
+ blockstore.UseWholeCIDs(true),
+ ))
+}
diff --git a/ipld/car/v2/reader.go b/ipld/car/v2/reader.go
new file mode 100644
index 000000000..30092bc43
--- /dev/null
+++ b/ipld/car/v2/reader.go
@@ -0,0 +1,368 @@
+package car
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "math"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1/util"
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-multihash"
+ "github.com/multiformats/go-varint"
+ "golang.org/x/exp/mmap"
+)
+
+// Reader represents a reader of CARv2.
+type Reader struct {
+ Header Header
+ Version uint64
+ r io.ReaderAt
+ roots []cid.Cid
+ opts Options
+ closer io.Closer
+}
+
+// OpenReader is a wrapper for NewReader which opens the file at path.
+func OpenReader(path string, opts ...Option) (*Reader, error) {
+ f, err := mmap.Open(path)
+ if err != nil {
+ return nil, err
+ }
+
+ r, err := NewReader(f, opts...)
+ if err != nil {
+ return nil, err
+ }
+
+ r.closer = f
+ return r, nil
+}
+
+// NewReader constructs a new reader that reads either CARv1 or CARv2 from the given r.
+// Upon instantiation, the reader inspects the payload and provides appropriate read operations
+// for both CARv1 and CARv2.
+//
+// Note that any other version other than 1 or 2 will result in an error. The caller may use
+// Reader.Version to get the actual version r represents. In the case where r represents a CARv1
+// Reader.Header will not be populated and is left as zero-valued.
+func NewReader(r io.ReaderAt, opts ...Option) (*Reader, error) {
+ cr := &Reader{
+ r: r,
+ }
+ cr.opts = ApplyOptions(opts...)
+
+ or, err := internalio.NewOffsetReadSeeker(r, 0)
+ if err != nil {
+ return nil, err
+ }
+ cr.Version, err = ReadVersion(or, opts...)
+ if err != nil {
+ return nil, err
+ }
+
+ if cr.Version != 1 && cr.Version != 2 {
+ return nil, fmt.Errorf("invalid car version: %d", cr.Version)
+ }
+
+ if cr.Version == 2 {
+ if err := cr.readV2Header(); err != nil {
+ return nil, err
+ }
+ }
+
+ return cr, nil
+}
+
+// Roots returns the root CIDs.
+// The root CIDs are extracted lazily from the data payload header.
+func (r *Reader) Roots() ([]cid.Cid, error) {
+ if r.roots != nil {
+ return r.roots, nil
+ }
+ dr, err := r.DataReader()
+ if err != nil {
+ return nil, err
+ }
+ header, err := carv1.ReadHeader(dr, r.opts.MaxAllowedHeaderSize)
+ if err != nil {
+ return nil, err
+ }
+ r.roots = header.Roots
+ return r.roots, nil
+}
+
+func (r *Reader) readV2Header() (err error) {
+ headerSection := io.NewSectionReader(r.r, PragmaSize, HeaderSize)
+ _, err = r.Header.ReadFrom(headerSection)
+ return
+}
+
+// SectionReader implements both io.ReadSeeker and io.ReaderAt.
+// It is the interface version of io.SectionReader, but note that the
+// implementation is not guaranteed to be an io.SectionReader.
+type SectionReader interface {
+ io.Reader
+ io.Seeker
+ io.ReaderAt
+}
+
+// DataReader provides a reader containing the data payload in CARv1 format.
+func (r *Reader) DataReader() (SectionReader, error) {
+ if r.Version == 2 {
+ return io.NewSectionReader(r.r, int64(r.Header.DataOffset), int64(r.Header.DataSize)), nil
+ }
+ return internalio.NewOffsetReadSeeker(r.r, 0)
+}
+
+// IndexReader provides an io.Reader containing the index for the data payload if the index is
+// present. Otherwise, returns nil.
+// Note, this function will always return nil if the backing payload represents a CARv1.
+func (r *Reader) IndexReader() (io.Reader, error) {
+ if r.Version == 1 || !r.Header.HasIndex() {
+ return nil, nil
+ }
+ return internalio.NewOffsetReadSeeker(r.r, int64(r.Header.IndexOffset))
+}
+
+// Stats is returned by an Inspect() call
+type Stats struct {
+ Version uint64
+ Header Header
+ Roots []cid.Cid
+ RootsPresent bool
+ BlockCount uint64
+ CodecCounts map[multicodec.Code]uint64
+ MhTypeCounts map[multicodec.Code]uint64
+ AvgCidLength uint64
+ MaxCidLength uint64
+ MinCidLength uint64
+ AvgBlockLength uint64
+ MaxBlockLength uint64
+ MinBlockLength uint64
+ IndexCodec multicodec.Code
+}
+
+// Inspect does a quick scan of a CAR, performing basic validation of the format
+// and returning a Stats object that provides a high-level description of the
+// contents of the CAR.
+// Inspect works for CARv1 and CARv2 contents. A CARv1 will return an
+// uninitialized Header value.
+//
+// If validateBlockHash is true, all block data in the payload will be hashed
+// and compared to the CID for that block and an error will return if there
+// is a mismatch. If false, block data will be skipped over and not checked.
+// Performing a full block hash validation is similar to using a BlockReader and
+// calling Next over all blocks.
+//
+// Inspect will perform a basic check of a CARv2 index, where present, but this
+// does not guarantee that the index is correct. Attempting to read index data
+// from untrusted sources is not recommended. If required, further validation of
+// an index can be performed by loading the index and performing a ForEach() and
+// sanity checking that the offsets are within the data payload section of the
+// CAR. However, re-generation of index data in this case is the recommended
+// course of action.
+//
+// Beyond the checks performed by Inspect, a valid / good CAR is somewhat
+// use-case dependent. Factors to consider include:
+//
+// - Bad indexes, including incorrect offsets, duplicate entries, or other
+// faulty data. Indexes should be re-generated, regardless, if you need to use
+// them and have any reason to not trust the source.
+//
+// - Blocks use codecs that your system doesn't have access to—which may mean
+// you can't traverse a DAG or use the contained data. Stats.CodecCounts
+// contains a list of codecs found in the CAR so this can be checked.
+//
+// - CIDs use multihashes that your system doesn't have access to—which will
+// mean you can't validate block hashes are correct (using validateBlockHash
+// in this case will result in a failure). Stats.MhTypeCounts contains a
+// list of multihashes found in the CAR so this can be checked.
+//
+// - The presence of IDENTITY CIDs, which may not be supported (or desired) by
+// the consumer of the CAR. Stats.CodecCounts can determine the presence
+// of IDENTITY CIDs.
+//
+// - Roots: the number of roots, duplicates, and whether they are related to the
+// blocks contained within the CAR. Stats contains a list of Roots and a
+// RootsPresent bool so further checks can be performed.
+//
+// - DAG completeness is not checked. Any properties relating to the DAG, or
+// DAGs contained within a CAR are the responsibility of the user to check.
+func (r *Reader) Inspect(validateBlockHash bool) (Stats, error) {
+ stats := Stats{
+ Version: r.Version,
+ Header: r.Header,
+ CodecCounts: make(map[multicodec.Code]uint64),
+ MhTypeCounts: make(map[multicodec.Code]uint64),
+ }
+
+ var totalCidLength uint64
+ var totalBlockLength uint64
+ var minCidLength uint64 = math.MaxUint64
+ var minBlockLength uint64 = math.MaxUint64
+
+ dr, err := r.DataReader()
+ if err != nil {
+ return Stats{}, err
+ }
+ bdr := internalio.ToByteReader(dr)
+
+ // read roots, not using Roots(), because we need the offset setup in the data trader
+ header, err := carv1.ReadHeader(dr, r.opts.MaxAllowedHeaderSize)
+ if err != nil {
+ return Stats{}, err
+ }
+ stats.Roots = header.Roots
+ var rootsPresentCount int
+ rootsPresent := make([]bool, len(stats.Roots))
+
+ // read block sections
+ for {
+ sectionLength, err := varint.ReadUvarint(bdr)
+ if err != nil {
+ if err == io.EOF {
+ // if the length of bytes read is non-zero when the error is EOF then signal an unclean EOF.
+ if sectionLength > 0 {
+ return Stats{}, io.ErrUnexpectedEOF
+ }
+ // otherwise, this is a normal ending
+ break
+ }
+ return Stats{}, err
+ }
+ if sectionLength == 0 && r.opts.ZeroLengthSectionAsEOF {
+ // normal ending for this read mode
+ break
+ }
+ if sectionLength > r.opts.MaxAllowedSectionSize {
+ return Stats{}, util.ErrSectionTooLarge
+ }
+
+ // decode just the CID bytes
+ cidLen, c, err := cid.CidFromReader(dr)
+ if err != nil {
+ return Stats{}, err
+ }
+
+ if sectionLength < uint64(cidLen) {
+ // this case is handled different in the normal ReadNode() path since it
+ // slurps in the whole section bytes and decodes CID from there - so an
+ // error should come from a failing io.ReadFull
+ return Stats{}, errors.New("section length shorter than CID length")
+ }
+
+ // is this a root block? (also account for duplicate root CIDs)
+ if rootsPresentCount < len(stats.Roots) {
+ for i, r := range stats.Roots {
+ if !rootsPresent[i] && c == r {
+ rootsPresent[i] = true
+ rootsPresentCount++
+ }
+ }
+ }
+
+ cp := c.Prefix()
+ codec := multicodec.Code(cp.Codec)
+ count := stats.CodecCounts[codec]
+ stats.CodecCounts[codec] = count + 1
+ mhtype := multicodec.Code(cp.MhType)
+ count = stats.MhTypeCounts[mhtype]
+ stats.MhTypeCounts[mhtype] = count + 1
+
+ blockLength := sectionLength - uint64(cidLen)
+
+ if validateBlockHash {
+ // Use multihash.SumStream to avoid having to copy the entire block content into memory.
+ // The SumStream uses a buffered copy to write bytes into the hasher which will take
+ // advantage of streaming hash calculation depending on the hash function.
+ // TODO: introduce SumStream in go-cid to simplify the code here.
+ blockReader := io.LimitReader(dr, int64(blockLength))
+ mhl := cp.MhLength
+ if mhtype == multicodec.Identity {
+ mhl = -1
+ }
+ mh, err := multihash.SumStream(blockReader, cp.MhType, mhl)
+ if err != nil {
+ return Stats{}, err
+ }
+ var gotCid cid.Cid
+ switch cp.Version {
+ case 0:
+ gotCid = cid.NewCidV0(mh)
+ case 1:
+ gotCid = cid.NewCidV1(cp.Codec, mh)
+ default:
+ return Stats{}, fmt.Errorf("invalid cid version: %d", cp.Version)
+ }
+ if !gotCid.Equals(c) {
+ return Stats{}, fmt.Errorf("mismatch in content integrity, expected: %s, got: %s", c, gotCid)
+ }
+ } else {
+ // otherwise, skip over it
+ if _, err := dr.Seek(int64(blockLength), io.SeekCurrent); err != nil {
+ return Stats{}, err
+ }
+ }
+
+ stats.BlockCount++
+ totalCidLength += uint64(cidLen)
+ totalBlockLength += blockLength
+ if uint64(cidLen) < minCidLength {
+ minCidLength = uint64(cidLen)
+ }
+ if uint64(cidLen) > stats.MaxCidLength {
+ stats.MaxCidLength = uint64(cidLen)
+ }
+ if blockLength < minBlockLength {
+ minBlockLength = blockLength
+ }
+ if blockLength > stats.MaxBlockLength {
+ stats.MaxBlockLength = blockLength
+ }
+ }
+
+ stats.RootsPresent = len(stats.Roots) == rootsPresentCount
+ if stats.BlockCount > 0 {
+ stats.MinCidLength = minCidLength
+ stats.MinBlockLength = minBlockLength
+ stats.AvgCidLength = totalCidLength / stats.BlockCount
+ stats.AvgBlockLength = totalBlockLength / stats.BlockCount
+ }
+
+ if stats.Version != 1 && stats.Header.HasIndex() {
+ idxr, err := r.IndexReader()
+ if err != nil {
+ return Stats{}, err
+ }
+ stats.IndexCodec, err = index.ReadCodec(idxr)
+ if err != nil {
+ return Stats{}, err
+ }
+ }
+
+ return stats, nil
+}
+
+// Close closes the underlying reader if it was opened by OpenReader.
+func (r *Reader) Close() error {
+ if r.closer != nil {
+ return r.closer.Close()
+ }
+ return nil
+}
+
+// ReadVersion reads the version from the pragma.
+// This function accepts both CARv1 and CARv2 payloads.
+func ReadVersion(r io.Reader, opts ...Option) (uint64, error) {
+ o := ApplyOptions(opts...)
+ header, err := carv1.ReadHeader(r, o.MaxAllowedHeaderSize)
+ if err != nil {
+ return 0, err
+ }
+ return header.Version, nil
+}
diff --git a/ipld/car/v2/reader_test.go b/ipld/car/v2/reader_test.go
new file mode 100644
index 000000000..75a4c274f
--- /dev/null
+++ b/ipld/car/v2/reader_test.go
@@ -0,0 +1,592 @@
+package car_test
+
+import (
+ "bytes"
+ "encoding/hex"
+ "io"
+ "os"
+ "strings"
+ "testing"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multicodec"
+ "github.com/stretchr/testify/require"
+)
+
+func TestReadVersion(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ want uint64
+ wantErr bool
+ }{
+ {
+ name: "CarV1VersionIsOne",
+ path: "testdata/sample-v1.car",
+ want: 1,
+ },
+ {
+ name: "CarV2VersionIsTwo",
+ path: "testdata/sample-rw-bs-v2.car",
+ want: 2,
+ },
+ {
+ name: "CarV1VersionWithZeroLenSectionIsOne",
+ path: "testdata/sample-v1-with-zero-len-section.car",
+ want: 1,
+ },
+ {
+ name: "AnotherCarV1VersionWithZeroLenSectionIsOne",
+ path: "testdata/sample-v1-with-zero-len-section2.car",
+ want: 1,
+ },
+ {
+ name: "WrappedCarV1InCarV2VersionIsTwo",
+ path: "testdata/sample-wrapped-v2.car",
+ want: 2,
+ },
+ {
+ name: "FutureVersionWithCorrectPragmaIsAsExpected",
+ path: "testdata/sample-rootless-v42.car",
+ want: 42,
+ },
+ {
+ name: "CarV1WithValidHeaderButCorruptSectionIsOne",
+ path: "testdata/sample-v1-tailing-corrupt-section.car",
+ want: 1,
+ },
+ {
+ name: "CarV2WithValidHeaderButCorruptSectionAndIndexIsTwo",
+ path: "testdata/sample-v2-corrupt-data-and-index.car",
+ want: 2,
+ },
+ {
+ name: "CarFileWithCorruptPragmaIsError",
+ path: "testdata/sample-corrupt-pragma.car",
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ f, err := os.Open(tt.path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, f.Close()) })
+
+ got, err := carv2.ReadVersion(f)
+ if tt.wantErr {
+ require.Error(t, err, "ReadVersion() error = %v, wantErr %v", err, tt.wantErr)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, tt.want, got, "ReadVersion() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestReaderFailsOnUnknownVersion(t *testing.T) {
+ _, err := carv2.OpenReader("testdata/sample-rootless-v42.car")
+ require.EqualError(t, err, "invalid car version: 42")
+}
+
+func TestReaderFailsOnCorruptPragma(t *testing.T) {
+ _, err := carv2.OpenReader("testdata/sample-corrupt-pragma.car")
+ require.EqualError(t, err, "unexpected EOF")
+}
+
+func TestReader_WithCarV1Consistency(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ zerLenAsEOF bool
+ }{
+ {
+ name: "CarV1WithoutZeroLengthSection",
+ path: "testdata/sample-v1.car",
+ },
+ {
+ name: "CarV1WithZeroLenSection",
+ path: "testdata/sample-v1-with-zero-len-section.car",
+ zerLenAsEOF: true,
+ },
+ {
+ name: "AnotherCarV1WithZeroLenSection",
+ path: "testdata/sample-v1-with-zero-len-section2.car",
+ zerLenAsEOF: true,
+ },
+ {
+ name: "CarV1WithZeroLenSectionWithoutOption",
+ path: "testdata/sample-v1-with-zero-len-section.car",
+ },
+ {
+ name: "AnotherCarV1WithZeroLenSectionWithoutOption",
+ path: "testdata/sample-v1-with-zero-len-section2.car",
+ },
+ {
+ name: "CorruptCarV1",
+ path: "testdata/sample-v1-tailing-corrupt-section.car",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ subject, err := carv2.OpenReader(tt.path, carv2.ZeroLengthSectionAsEOF(tt.zerLenAsEOF))
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, subject.Close()) })
+ wantReader := requireNewCarV1ReaderFromV1File(t, tt.path, tt.zerLenAsEOF)
+
+ require.Equal(t, uint64(1), subject.Version)
+ gotRoots, err := subject.Roots()
+ require.NoError(t, err)
+ require.Equal(t, wantReader.Header.Roots, gotRoots)
+ ir, err := subject.IndexReader()
+ require.Nil(t, ir)
+ require.NoError(t, err)
+ })
+ }
+}
+
+func TestReader_WithCarV2Consistency(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ zerLenAsEOF bool
+ }{
+ {
+ name: "CarV2WrappingV1",
+ path: "testdata/sample-wrapped-v2.car",
+ },
+ {
+ name: "CarV2ProducedByBlockstore",
+ path: "testdata/sample-rw-bs-v2.car",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ subject, err := carv2.OpenReader(tt.path, carv2.ZeroLengthSectionAsEOF(tt.zerLenAsEOF))
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, subject.Close()) })
+ wantReader := requireNewCarV1ReaderFromV2File(t, tt.path, tt.zerLenAsEOF)
+
+ require.Equal(t, uint64(2), subject.Version)
+ gotRoots, err := subject.Roots()
+ require.NoError(t, err)
+ require.Equal(t, wantReader.Header.Roots, gotRoots)
+
+ gotIndexReader, err := subject.IndexReader()
+ require.NoError(t, err)
+ require.NotNil(t, gotIndexReader)
+ gotIndex, err := index.ReadFrom(gotIndexReader)
+ require.NoError(t, err)
+ dr, err := subject.DataReader()
+ require.NoError(t, err)
+ wantIndex, err := carv2.GenerateIndex(dr)
+ require.NoError(t, err)
+ require.Equal(t, wantIndex, gotIndex)
+ })
+ }
+}
+
+func TestOpenReader_DoesNotPanicForReadersCreatedBeforeClosure(t *testing.T) {
+ subject, err := carv2.OpenReader("testdata/sample-wrapped-v2.car")
+ require.NoError(t, err)
+ dReaderBeforeClosure, err := subject.DataReader()
+ require.NoError(t, err)
+ iReaderBeforeClosure, err := subject.IndexReader()
+ require.NoError(t, err)
+ require.NoError(t, subject.Close())
+
+ buf := make([]byte, 1)
+ panicTest := func(r io.Reader) {
+ _, err := r.Read(buf)
+ require.EqualError(t, err, "mmap: closed")
+ }
+
+ require.NotPanics(t, func() { panicTest(dReaderBeforeClosure) })
+ require.NotPanics(t, func() { panicTest(iReaderBeforeClosure) })
+}
+
+func TestOpenReader_DoesNotPanicForReadersCreatedAfterClosure(t *testing.T) {
+ subject, err := carv2.OpenReader("testdata/sample-wrapped-v2.car")
+ require.NoError(t, err)
+ require.NoError(t, subject.Close())
+ dReaderAfterClosure, err := subject.DataReader()
+ require.NoError(t, err)
+ iReaderAfterClosure, err := subject.IndexReader()
+ require.NoError(t, err)
+
+ buf := make([]byte, 1)
+ panicTest := func(r io.Reader) {
+ _, err := r.Read(buf)
+ require.EqualError(t, err, "mmap: closed")
+ }
+
+ require.NotPanics(t, func() { panicTest(dReaderAfterClosure) })
+ require.NotPanics(t, func() { panicTest(iReaderAfterClosure) })
+}
+
+func TestReader_ReturnsNilWhenThereIsNoIndex(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ }{
+ {
+ name: "IndexlessCarV2",
+ path: "testdata/sample-v2-indexless.car",
+ },
+ {
+ name: "CarV1",
+ path: "testdata/sample-v1.car",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ subject, err := carv2.OpenReader(tt.path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, subject.Close()) })
+ ir, err := subject.IndexReader()
+ require.NoError(t, err)
+ require.Nil(t, ir)
+ })
+ }
+}
+
+func requireNewCarV1ReaderFromV2File(t *testing.T, carV12Path string, zerLenAsEOF bool) *carv1.CarReader {
+ f, err := os.Open(carV12Path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, f.Close()) })
+
+ _, err = f.Seek(carv2.PragmaSize, io.SeekStart)
+ require.NoError(t, err)
+ header := carv2.Header{}
+ _, err = header.ReadFrom(f)
+ require.NoError(t, err)
+ return requireNewCarV1Reader(t, io.NewSectionReader(f, int64(header.DataOffset), int64(header.DataSize)), zerLenAsEOF)
+}
+
+func requireNewCarV1ReaderFromV1File(t *testing.T, carV1Path string, zerLenAsEOF bool) *carv1.CarReader {
+ f, err := os.Open(carV1Path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, f.Close()) })
+ return requireNewCarV1Reader(t, f, zerLenAsEOF)
+}
+
+func requireNewCarV1Reader(t *testing.T, r io.Reader, zerLenAsEOF bool) *carv1.CarReader {
+ var cr *carv1.CarReader
+ var err error
+ if zerLenAsEOF {
+ cr, err = carv1.NewCarReaderWithZeroLengthSectionAsEOF(r)
+ } else {
+ cr, err = carv1.NewCarReader(r)
+ }
+ require.NoError(t, err)
+ return cr
+}
+
+func TestInspect(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ carHex string
+ zerLenAsEOF bool
+ expectedStats carv2.Stats
+ }{
+ {
+ name: "IndexlessCarV2",
+ path: "testdata/sample-v2-indexless.car",
+ expectedStats: carv2.Stats{
+ Version: 2,
+ Header: carv2.Header{
+ Characteristics: carv2.Characteristics{0, 0},
+ DataOffset: 51,
+ DataSize: 479907,
+ IndexOffset: 0,
+ },
+ Roots: []cid.Cid{mustCidDecode("bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy")},
+ RootsPresent: true,
+ AvgBlockLength: 417, // 417.6644423260248
+ MinBlockLength: 1,
+ MaxBlockLength: 1342,
+ AvgCidLength: 37, // 37.86939942802669
+ MinCidLength: 14,
+ MaxCidLength: 38,
+ BlockCount: 1049,
+ CodecCounts: map[multicodec.Code]uint64{
+ multicodec.Raw: 6,
+ multicodec.DagCbor: 1043,
+ },
+ MhTypeCounts: map[multicodec.Code]uint64{
+ multicodec.Identity: 6,
+ multicodec.Blake2b256: 1043,
+ },
+ },
+ },
+ {
+ // same payload as IndexlessCarV2, so only difference is the Version & Header
+ name: "CarV1",
+ path: "testdata/sample-v1.car",
+ expectedStats: carv2.Stats{
+ Version: 1,
+ Header: carv2.Header{},
+ Roots: []cid.Cid{mustCidDecode("bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy")},
+ RootsPresent: true,
+ AvgBlockLength: 417, // 417.6644423260248
+ MinBlockLength: 1,
+ MaxBlockLength: 1342,
+ AvgCidLength: 37, // 37.86939942802669
+ MinCidLength: 14,
+ MaxCidLength: 38,
+ BlockCount: 1049,
+ CodecCounts: map[multicodec.Code]uint64{
+ multicodec.Raw: 6,
+ multicodec.DagCbor: 1043,
+ },
+ MhTypeCounts: map[multicodec.Code]uint64{
+ multicodec.Identity: 6,
+ multicodec.Blake2b256: 1043,
+ },
+ },
+ },
+ {
+ // same payload as IndexlessCarV2, so only difference is the Header
+ name: "CarV2ProducedByBlockstore",
+ path: "testdata/sample-rw-bs-v2.car",
+ expectedStats: carv2.Stats{
+ Version: 2,
+ Header: carv2.Header{
+ DataOffset: 1464,
+ DataSize: 273,
+ IndexOffset: 1737,
+ },
+ Roots: []cid.Cid{
+ mustCidDecode("bafkreifuosuzujyf4i6psbneqtwg2fhplc2wxptc5euspa2gn3bwhnihfu"),
+ mustCidDecode("bafkreifc4hca3inognou377hfhvu2xfchn2ltzi7yu27jkaeujqqqdbjju"),
+ mustCidDecode("bafkreig5lvr4l6b4fr3un4xvzeyt3scevgsqjgrhlnwxw2unwbn5ro276u"),
+ },
+ RootsPresent: true,
+ BlockCount: 3,
+ CodecCounts: map[multicodec.Code]uint64{multicodec.Raw: 3},
+ MhTypeCounts: map[multicodec.Code]uint64{multicodec.Sha2_256: 3},
+ AvgCidLength: 36,
+ MaxCidLength: 36,
+ MinCidLength: 36,
+ AvgBlockLength: 6,
+ MaxBlockLength: 9,
+ MinBlockLength: 4,
+ IndexCodec: multicodec.CarMultihashIndexSorted,
+ },
+ },
+ // same as CarV1 but with a zero-byte EOF to test options
+ {
+ name: "CarV1VersionWithZeroLenSectionIsOne",
+ path: "testdata/sample-v1-with-zero-len-section.car",
+ zerLenAsEOF: true,
+ expectedStats: carv2.Stats{
+ Version: 1,
+ Header: carv2.Header{},
+ Roots: []cid.Cid{mustCidDecode("bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy")},
+ RootsPresent: true,
+ AvgBlockLength: 417, // 417.6644423260248
+ MinBlockLength: 1,
+ MaxBlockLength: 1342,
+ AvgCidLength: 37, // 37.86939942802669
+ MinCidLength: 14,
+ MaxCidLength: 38,
+ BlockCount: 1049,
+ CodecCounts: map[multicodec.Code]uint64{
+ multicodec.Raw: 6,
+ multicodec.DagCbor: 1043,
+ },
+ MhTypeCounts: map[multicodec.Code]uint64{
+ multicodec.Identity: 6,
+ multicodec.Blake2b256: 1043,
+ },
+ },
+ },
+ {
+ // A case where this _could_ be a valid CAR if we allowed identity CIDs
+ // and not matching block contents to exist, there's no block bytes in
+ // this. It will only fail if you don't validate the CID matches the,
+ // bytes (see TestInspectError for that case).
+ name: "IdentityCID",
+ // 47 {version:1,roots:[identity cid]} 25 identity cid (dag-json {"identity":"block"})
+ carHex: "2f a265726f6f747381d82a581a0001a90200147b226964656e74697479223a22626c6f636b227d6776657273696f6e01 19 01a90200147b226964656e74697479223a22626c6f636b227d",
+ expectedStats: carv2.Stats{
+ Version: 1,
+ Roots: []cid.Cid{mustCidDecode("baguqeaaupmrgszdfnz2gs5dzei5ceytmn5rwwit5")},
+ RootsPresent: true,
+ BlockCount: 1,
+ CodecCounts: map[multicodec.Code]uint64{multicodec.DagJson: 1},
+ MhTypeCounts: map[multicodec.Code]uint64{multicodec.Identity: 1},
+ AvgCidLength: 25,
+ MaxCidLength: 25,
+ MinCidLength: 25,
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var reader *carv2.Reader
+ var err error
+ if tt.path != "" {
+ reader, err = carv2.OpenReader(tt.path, carv2.ZeroLengthSectionAsEOF(tt.zerLenAsEOF))
+ require.NoError(t, err)
+ } else {
+ byts, err := hex.DecodeString(strings.ReplaceAll(tt.carHex, " ", ""))
+ require.NoError(t, err)
+ reader, err = carv2.NewReader(bytes.NewReader(byts), carv2.ZeroLengthSectionAsEOF(tt.zerLenAsEOF))
+ require.NoError(t, err)
+ }
+ t.Cleanup(func() { require.NoError(t, reader.Close()) })
+ stats, err := reader.Inspect(false)
+ require.NoError(t, err)
+ require.Equal(t, tt.expectedStats, stats)
+ })
+ }
+}
+
+func TestInspectError(t *testing.T) {
+ tests := []struct {
+ name string
+ carHex string
+ expectedOpenError string
+ expectedInspectError string
+ validateBlockHash bool
+ }{
+ {
+ name: "BadCidV0",
+ carHex: "3aa265726f6f747381d8305825000130302030303030303030303030303030303030303030303030303030303030303030306776657273696f6e010130",
+ expectedInspectError: "expected 1 as the cid version number, got: 48",
+ },
+ {
+ name: "BadHeaderLength",
+ carHex: "e0e0e0e0a7060c6f6c4cca943c236f4b196723489608edb42a8b8fa80b6776657273696f6e19",
+ expectedOpenError: "invalid header data, length of read beyond allowable maximum",
+ },
+ {
+ name: "BadSectionLength",
+ carHex: "11a265726f6f7473806776657273696f6e01e0e0e0e0a7060155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca00000000000000000000",
+ expectedInspectError: "invalid section data, length of read beyond allowable maximum",
+ },
+ {
+ name: "BadSectionLength2",
+ carHex: "3aa265726f6f747381d8305825000130302030303030303030303030303030303030303030303030303030303030303030306776657273696f6e01200130302030303030303030303030303030303030303030303030303030303030303030303030303030303030",
+ expectedInspectError: "section length shorter than CID length",
+ validateBlockHash: true,
+ },
+ {
+ name: "BadSectionLength3",
+ carHex: "11a265726f6f7473f66776657273696f6e0180",
+ expectedInspectError: "unexpected EOF",
+ },
+ {
+ name: "BadBlockHash(SanityCheck)", // this should pass because we don't ask the CID be validated even though it doesn't match
+ // header cid data
+ carHex: "11a265726f6f7473806776657273696f6e 012e0155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca ffffffffffffffffffff",
+ },
+ {
+ name: "BadBlockHash", // same as above, but we ask for CID validation
+ // header cid data
+ carHex: "11a265726f6f7473806776657273696f6e 012e0155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca ffffffffffffffffffff",
+ validateBlockHash: true,
+ expectedInspectError: "mismatch in content integrity, expected: bafkreiab2rek7wjiazkfrt3hbnqpljmu24226alszdlh6ivic2abgjubzi, got: bafkreiaaqoxrddiyuy6gxnks6ioqytxhq5a7tchm2mm5htigznwiljukmm",
+ },
+ {
+ name: "IdentityCID", // a case where this _could_ be a valid CAR if we allowed identity CIDs and not matching block contents to exist, there's no block bytes in this
+ // 47 {version:1,roots:[identity cid]} 25 identity cid (dag-json {"identity":"block"})
+ carHex: "2f a265726f6f747381d82a581a0001a90200147b226964656e74697479223a22626c6f636b227d6776657273696f6e01 19 01a90200147b226964656e74697479223a22626c6f636b227d",
+ validateBlockHash: true,
+ expectedInspectError: "mismatch in content integrity, expected: baguqeaaupmrgszdfnz2gs5dzei5ceytmn5rwwit5, got: baguqeaaa",
+ },
+ // the bad index tests are manually constructed from this single-block CARv2 by adjusting the Uint32 and Uint64 values in the index:
+ // pragma carv2 header carv1 icodec count codec count (swi) width dataLen mh offset
+ // 0aa16776657273696f6e02 00000000000000000000000000000000330000000000000041000000000000007400000000000000 11a265726f6f7473806776657273696f6e012e0155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca00000000000000000000 8108 01000000 1200000000000000 01000000 28000000 2800000000000000 01d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca 1200000000000000
+ // we don't test any further into the index, to do that, a user should do a ForEach across the loaded index (and sanity check the offsets)
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ car, _ := hex.DecodeString(strings.ReplaceAll(tt.carHex, " ", ""))
+ reader, err := carv2.NewReader(bytes.NewReader(car))
+ if tt.expectedOpenError != "" {
+ require.Error(t, err)
+ require.Equal(t, tt.expectedOpenError, err.Error())
+ return
+ } else {
+ require.NoError(t, err)
+ }
+ t.Cleanup(func() { require.NoError(t, reader.Close()) })
+ _, err = reader.Inspect(tt.validateBlockHash)
+ if tt.expectedInspectError != "" {
+ require.Error(t, err)
+ require.Equal(t, tt.expectedInspectError, err.Error())
+ } else {
+ require.NoError(t, err)
+ }
+ })
+ }
+}
+
+func TestIndex_ReadFromCorruptIndex(t *testing.T) {
+ tests := []struct {
+ name string
+ givenCarHex string
+ wantErr string
+ }{
+ {
+ name: "BadIndexCountOverflow",
+ // pragma carv2 header carv1 icodec count codec count (swi) width dataLen mh offset
+ givenCarHex: "0aa16776657273696f6e02 00000000000000000000000000000000330000000000000041000000000000007400000000000000 11a265726f6f7473806776657273696f6e012e0155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca00000000000000000000 8108 ffffffff 1200000000000000 01000000 28000000 2800000000000000 01d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca 1200000000000000",
+ wantErr: "index too big; MultihashIndexSorted count is overflowing int32",
+ },
+ {
+ name: "BadIndexCountTooMany",
+ // pragma carv2 header carv1 icodec count codec count (swi) width dataLen mh offset
+ givenCarHex: "0aa16776657273696f6e02 00000000000000000000000000000000330000000000000041000000000000007400000000000000 11a265726f6f7473806776657273696f6e012e0155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca00000000000000000000 8108 ffffff7f 1200000000000000 01000000 28000000 2800000000000000 01d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca 1200000000000000",
+ wantErr: "unexpected EOF",
+ },
+ {
+ name: "BadIndexMultiWidthOverflow",
+ // pragma carv2 header carv1 icodec count codec count (swi) width dataLen mh offset
+ givenCarHex: "0aa16776657273696f6e02 00000000000000000000000000000000330000000000000041000000000000007400000000000000 11a265726f6f7473806776657273696f6e012e0155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca00000000000000000000 8108 01000000 1200000000000000 ffffffff 28000000 2800000000000000 01d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca 1200000000000000",
+ wantErr: "index too big; multiWidthIndex count is overflowing int32",
+ },
+ {
+ name: "BadIndexMultiWidthTooMany",
+ // pragma carv2 header carv1 icodec count codec count (swi) width dataLen mh offset
+ givenCarHex: "0aa16776657273696f6e02 00000000000000000000000000000000330000000000000041000000000000007400000000000000 11a265726f6f7473806776657273696f6e012e0155122001d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca00000000000000000000 8108 01000000 1200000000000000 ffffff7f 28000000 2800000000000000 01d448afd928065458cf670b60f5a594d735af0172c8d67f22a81680132681ca 1200000000000000",
+ wantErr: "unexpected EOF",
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ car, _ := hex.DecodeString(strings.ReplaceAll(test.givenCarHex, " ", ""))
+ reader, err := carv2.NewReader(bytes.NewReader(car))
+ require.NoError(t, err)
+
+ ir, err := reader.IndexReader()
+ require.NoError(t, err)
+ require.NotNil(t, ir)
+
+ gotIdx, err := index.ReadFrom(ir)
+ if test.wantErr == "" {
+ require.NoError(t, err)
+ require.NotNil(t, gotIdx)
+ } else {
+ require.Error(t, err)
+ require.Equal(t, test.wantErr, err.Error())
+ require.Nil(t, gotIdx)
+ }
+ })
+ }
+}
+
+func mustCidDecode(s string) cid.Cid {
+ c, err := cid.Decode(s)
+ if err != nil {
+ panic(err)
+ }
+ return c
+}
diff --git a/ipld/car/v2/selective.go b/ipld/car/v2/selective.go
new file mode 100644
index 000000000..33ac016bc
--- /dev/null
+++ b/ipld/car/v2/selective.go
@@ -0,0 +1,289 @@
+package car
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "math"
+ "os"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/loader"
+ "github.com/ipfs/go-cid"
+ ipld "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/datamodel"
+ "github.com/ipld/go-ipld-prime/linking"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ "github.com/ipld/go-ipld-prime/node/basicnode"
+ "github.com/ipld/go-ipld-prime/traversal"
+ "github.com/ipld/go-ipld-prime/traversal/selector"
+)
+
+// ErrSizeMismatch is returned when a written traversal realizes the written header size does not
+// match the actual number of car bytes written.
+var ErrSizeMismatch = fmt.Errorf("car-error-sizemismatch")
+
+// ErrOffsetImpossible is returned when specified paddings or offsets of either a wrapped carv1
+// or index cannot be satisfied based on the data being written.
+var ErrOffsetImpossible = fmt.Errorf("car-error-offsetimpossible")
+
+// MaxTraversalLinks changes the allowed number of links a selector traversal
+// can execute before failing.
+//
+// Note that setting this option may cause an error to be returned from selector
+// execution when building a SelectiveCar.
+func MaxTraversalLinks(MaxTraversalLinks uint64) Option {
+ return func(sco *Options) {
+ sco.MaxTraversalLinks = MaxTraversalLinks
+ }
+}
+
+// NewSelectiveWriter walks through the proposed dag traversal to learn its total size in order to be able to
+// stream out a car to a writer in the expected traversal order in one go.
+func NewSelectiveWriter(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, opts ...Option) (Writer, error) {
+ cls, cntr := loader.CountingLinkSystem(*ls)
+
+ c1h := carv1.CarHeader{Roots: []cid.Cid{root}, Version: 1}
+ headSize, err := carv1.HeaderSize(&c1h)
+ if err != nil {
+ return nil, err
+ }
+ if err := traverse(ctx, &cls, root, selector, ApplyOptions(opts...)); err != nil {
+ return nil, err
+ }
+ tc := traversalCar{
+ size: headSize + cntr.Size(),
+ ctx: ctx,
+ root: root,
+ selector: selector,
+ ls: ls,
+ opts: ApplyOptions(opts...),
+ }
+ return &tc, nil
+}
+
+// TraverseToFile writes a car file matching a given root and selector to the
+// path at `destination` using one read of each block.
+func TraverseToFile(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, destination string, opts ...Option) error {
+ tc := traversalCar{
+ size: 0,
+ ctx: ctx,
+ root: root,
+ selector: selector,
+ ls: ls,
+ opts: ApplyOptions(opts...),
+ }
+
+ fp, err := os.Create(destination)
+ if err != nil {
+ return err
+ }
+ defer fp.Close()
+
+ _, err = tc.WriteTo(fp)
+ if err != nil {
+ return err
+ }
+
+ // fix header size.
+ if _, err = fp.Seek(0, 0); err != nil {
+ return err
+ }
+
+ tc.size = uint64(tc.size)
+ if _, err = tc.WriteV2Header(fp); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// TraverseV1 walks through the proposed dag traversal and writes a carv1 to the provided io.Writer
+func TraverseV1(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, selector ipld.Node, writer io.Writer, opts ...Option) (uint64, error) {
+ opts = append(opts, WithoutIndex())
+ tc := traversalCar{
+ size: 0,
+ ctx: ctx,
+ root: root,
+ selector: selector,
+ ls: ls,
+ opts: ApplyOptions(opts...),
+ }
+
+ len, _, err := tc.WriteV1(writer)
+ return len, err
+}
+
+// Writer is an interface allowing writing a car prepared by PrepareTraversal
+type Writer interface {
+ io.WriterTo
+}
+
+var _ Writer = (*traversalCar)(nil)
+
+type traversalCar struct {
+ size uint64
+ ctx context.Context
+ root cid.Cid
+ selector ipld.Node
+ ls *ipld.LinkSystem
+ opts Options
+}
+
+func (tc *traversalCar) WriteTo(w io.Writer) (int64, error) {
+ n, err := tc.WriteV2Header(w)
+ if err != nil {
+ return n, err
+ }
+ v1s, idx, err := tc.WriteV1(w)
+ n += int64(v1s)
+
+ if err != nil {
+ return n, err
+ }
+
+ // index padding, then index
+ if tc.opts.IndexCodec != index.CarIndexNone {
+ if tc.opts.IndexPadding > 0 {
+ buf := make([]byte, tc.opts.IndexPadding)
+ pn, err := w.Write(buf)
+ n += int64(pn)
+ if err != nil {
+ return n, err
+ }
+ }
+ in, err := index.WriteTo(idx, w)
+ n += int64(in)
+ if err != nil {
+ return n, err
+ }
+ }
+
+ return n, err
+}
+
+func (tc *traversalCar) WriteV2Header(w io.Writer) (int64, error) {
+ n, err := w.Write(Pragma)
+ if err != nil {
+ return int64(n), err
+ }
+
+ h := NewHeader(tc.size)
+ if p := tc.opts.DataPadding; p > 0 {
+ h = h.WithDataPadding(p)
+ }
+ if p := tc.opts.IndexPadding; p > 0 {
+ h = h.WithIndexPadding(p)
+ }
+ if tc.opts.IndexCodec == index.CarIndexNone {
+ h.IndexOffset = 0
+ }
+ hn, err := h.WriteTo(w)
+ if err != nil {
+ return int64(n) + hn, err
+ }
+ hn += int64(n)
+
+ // We include the initial data padding after the carv2 header
+ if h.DataOffset > uint64(hn) {
+ // TODO: buffer writes if this needs to be big.
+ buf := make([]byte, h.DataOffset-uint64(hn))
+ n, err = w.Write(buf)
+ hn += int64(n)
+ if err != nil {
+ return hn, err
+ }
+ } else if h.DataOffset < uint64(hn) {
+ return hn, ErrOffsetImpossible
+ }
+
+ return hn, nil
+}
+
+func (tc *traversalCar) WriteV1(w io.Writer) (uint64, index.Index, error) {
+ // write the v1 header
+ c1h := carv1.CarHeader{Roots: []cid.Cid{tc.root}, Version: 1}
+ if err := carv1.WriteHeader(&c1h, w); err != nil {
+ return 0, nil, err
+ }
+ v1Size, err := carv1.HeaderSize(&c1h)
+ if err != nil {
+ return v1Size, nil, err
+ }
+
+ // write the block.
+ wls, writer := loader.TeeingLinkSystem(*tc.ls, w, v1Size, tc.opts.IndexCodec)
+ err = traverse(tc.ctx, &wls, tc.root, tc.selector, tc.opts)
+ v1Size = writer.Size()
+ if err != nil {
+ return v1Size, nil, err
+ }
+ if tc.size != 0 && tc.size != v1Size {
+ return v1Size, nil, ErrSizeMismatch
+ }
+ tc.size = v1Size
+
+ if tc.opts.IndexCodec == index.CarIndexNone {
+ return v1Size, nil, nil
+ }
+ idx, err := writer.Index()
+ return v1Size, idx, err
+}
+
+func traverse(ctx context.Context, ls *ipld.LinkSystem, root cid.Cid, s ipld.Node, opts Options) error {
+ sel, err := selector.CompileSelector(s)
+ if err != nil {
+ return err
+ }
+
+ chooser := func(_ ipld.Link, _ linking.LinkContext) (ipld.NodePrototype, error) {
+ return basicnode.Prototype.Any, nil
+ }
+ if opts.TraversalPrototypeChooser != nil {
+ chooser = opts.TraversalPrototypeChooser
+ }
+
+ progress := traversal.Progress{
+ Cfg: &traversal.Config{
+ Ctx: ctx,
+ LinkSystem: *ls,
+ LinkTargetNodePrototypeChooser: chooser,
+ LinkVisitOnlyOnce: !opts.BlockstoreAllowDuplicatePuts,
+ },
+ }
+ if opts.MaxTraversalLinks < math.MaxInt64 {
+ progress.Budget = &traversal.Budget{
+ NodeBudget: math.MaxInt64,
+ LinkBudget: int64(opts.MaxTraversalLinks),
+ }
+ }
+
+ lnk := cidlink.Link{Cid: root}
+ ls.TrustedStorage = true
+ rp, err := chooser(lnk, ipld.LinkContext{})
+ if err != nil {
+ return err
+ }
+ rootNode, err := ls.Load(ipld.LinkContext{}, lnk, rp)
+ if err != nil {
+ return fmt.Errorf("root blk load failed: %s", err)
+ }
+ err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, node ipld.Node) error {
+ if lbn, ok := node.(datamodel.LargeBytesNode); ok {
+ s, err := lbn.AsLargeBytes()
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(io.Discard, s)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("walk failed: %s", err)
+ }
+ return nil
+}
diff --git a/ipld/car/v2/selective_test.go b/ipld/car/v2/selective_test.go
new file mode 100644
index 000000000..02afde0eb
--- /dev/null
+++ b/ipld/car/v2/selective_test.go
@@ -0,0 +1,137 @@
+package car_test
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "os"
+ "path"
+ "testing"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/blockstore"
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-unixfsnode"
+ "github.com/ipfs/go-unixfsnode/data/builder"
+ dagpb "github.com/ipld/go-codec-dagpb"
+ "github.com/ipld/go-ipld-prime/datamodel"
+ "github.com/ipld/go-ipld-prime/linking"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+ "github.com/ipld/go-ipld-prime/storage/bsadapter"
+ sb "github.com/ipld/go-ipld-prime/traversal/selector/builder"
+ selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
+ "github.com/stretchr/testify/require"
+
+ _ "github.com/ipld/go-ipld-prime/codec/dagcbor"
+ _ "github.com/ipld/go-ipld-prime/codec/raw"
+)
+
+func TestPrepareTraversal(t *testing.T) {
+ from, err := blockstore.OpenReadOnly("testdata/sample-unixfs-v2.car")
+ require.NoError(t, err)
+ ls := cidlink.DefaultLinkSystem()
+ bsa := bsadapter.Adapter{Wrapped: from}
+ ls.SetReadStorage(&bsa)
+
+ rts, _ := from.Roots()
+ writer, err := car.NewSelectiveWriter(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively)
+ require.NoError(t, err)
+
+ buf := bytes.Buffer{}
+ n, err := writer.WriteTo(&buf)
+ require.NoError(t, err)
+ require.Equal(t, int64(len(buf.Bytes())), n)
+
+ fi, _ := os.Stat("testdata/sample-unixfs-v2.car")
+ require.Equal(t, fi.Size(), n)
+
+ // Headers should be equal
+ h1, _ := car.OpenReader("testdata/sample-unixfs-v2.car")
+ h1h := bytes.Buffer{}
+ h1h.Write(car.Pragma)
+ h1.Header.WriteTo(&h1h)
+ require.Equal(t, buf.Bytes()[:h1h.Len()], h1h.Bytes())
+}
+
+func TestFileTraversal(t *testing.T) {
+ from, err := blockstore.OpenReadOnly("testdata/sample-unixfs-v2.car")
+ require.NoError(t, err)
+ ls := cidlink.DefaultLinkSystem()
+ bsa := bsadapter.Adapter{Wrapped: from}
+ ls.SetReadStorage(&bsa)
+
+ rts, _ := from.Roots()
+ outDir := t.TempDir()
+ err = car.TraverseToFile(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively, path.Join(outDir, "out.car"))
+ require.NoError(t, err)
+
+ require.FileExists(t, path.Join(outDir, "out.car"))
+
+ fa, _ := os.Stat("testdata/sample-unixfs-v2.car")
+ fb, _ := os.Stat(path.Join(outDir, "out.car"))
+ require.Equal(t, fa.Size(), fb.Size())
+}
+
+func TestV1Traversal(t *testing.T) {
+ from, err := blockstore.OpenReadOnly("testdata/sample-v1.car")
+ require.NoError(t, err)
+ ls := cidlink.DefaultLinkSystem()
+ bsa := bsadapter.Adapter{Wrapped: from}
+ ls.SetReadStorage(&bsa)
+
+ rts, _ := from.Roots()
+ w := bytes.NewBuffer(nil)
+ n, err := car.TraverseV1(context.Background(), &ls, rts[0], selectorparse.CommonSelector_ExploreAllRecursively, w)
+ require.NoError(t, err)
+ require.Equal(t, int64(len(w.Bytes())), int64(n))
+
+ fa, _ := os.Stat("testdata/sample-v1.car")
+ require.Equal(t, fa.Size(), int64(n))
+}
+
+func TestPartialTraversal(t *testing.T) {
+ store := cidlink.Memory{Bag: make(map[string][]byte)}
+ ls := cidlink.DefaultLinkSystem()
+ ls.StorageReadOpener = store.OpenRead
+ ls.StorageWriteOpener = store.OpenWrite
+ unixfsnode.AddUnixFSReificationToLinkSystem(&ls)
+
+ // write a unixfs file.
+ initBuf := bytes.Buffer{}
+ _, _ = initBuf.Write(make([]byte, 1000000))
+ rt, _, err := builder.BuildUnixFSFile(&initBuf, "", &ls)
+ require.NoError(t, err)
+
+ // read a subset of the file.
+ _, rts, err := cid.CidFromBytes([]byte(rt.Binary()))
+ require.NoError(t, err)
+ ssb := sb.NewSelectorSpecBuilder(basicnode.Prototype.Any)
+ sel := ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(0, 256*1000))
+ buf := bytes.Buffer{}
+ chooser := dagpb.AddSupportToChooser(func(l datamodel.Link, lc linking.LinkContext) (datamodel.NodePrototype, error) {
+ return basicnode.Prototype.Any, nil
+ })
+ _, err = car.TraverseV1(context.Background(), &ls, rts, sel.Node(), &buf, car.WithTraversalPrototypeChooser(chooser))
+ require.NoError(t, err)
+
+ fb := len(buf.Bytes())
+ require.Less(t, fb, 1000000)
+
+ loaded, err := car.NewBlockReader(&buf)
+ require.NoError(t, err)
+ fnd := make(map[cid.Cid]struct{})
+ var b blocks.Block
+ for err == nil {
+ b, err = loaded.Next()
+ if err == io.EOF {
+ break
+ }
+ if _, ok := fnd[b.Cid()]; ok {
+ require.Fail(t, "duplicate block present", b.Cid())
+ }
+ fnd[b.Cid()] = struct{}{}
+ }
+ require.Equal(t, 2, len(fnd))
+}
diff --git a/ipld/car/v2/storage/doc.go b/ipld/car/v2/storage/doc.go
new file mode 100644
index 000000000..21e049b21
--- /dev/null
+++ b/ipld/car/v2/storage/doc.go
@@ -0,0 +1,75 @@
+// Package storage provides a CAR abstraction for the
+// github.com/ipld/go-ipld-prime/storage interfaces in the form of a StorageCar.
+//
+// THIS PACKAGE IS EXPERIMENTAL. Breaking changes may be introduced in
+// semver-minor releases before this package stabilizes. Use with caution and
+// prefer the blockstore API if stability is required.
+//
+// StorageCar as ReadableStorage provides basic Get and Has operations. It also
+// implements StreamingReadableStorage for the more efficient GetStreaming
+// operation which is easily supported by the CAR format.
+//
+// StorageCar as WritableStorage provides the Put operation. It does not
+// implement StreamingWritableStorage because the CAR format requires CIDs to
+// be written before the blocks themselves, which is not possible with
+// StreamingWritableStorage without buffering. Therefore, the PutStream function
+// in github.com/ipld/go-ipld-prime/storage will provide equivalent
+// functionality if it were to be implemented here.
+//
+// StorageCar can be used with an IPLD LinkSystem, defined by
+// github.com/ipld/go-ipld-prime/linking, with the
+// linking.SetReadStorage and linking.SetWriteStorage functions, to provide
+// read and/or write to and/or from a CAR format as required.
+//
+// The focus of the StorageCar interfaces is to use the minimal possible IO
+// interface for the operation(s) being performed.
+//
+// • OpenReadable requires an io.ReaderAt as seeking is required for
+// random-access reads as a ReadableStore.
+//
+// • NewWritable requires an io.Writer when used to write a CARv1 as this format
+// can be written in a continuous stream as blocks are written through a
+// WritableStore (i.e. when the WriteAsCarV1 option is turned on). When used to
+// write a CARv2, the default mode, a random-access io.WriterAt is required as
+// the CARv2 header must be written after the payload is finalized and index
+// written in order to indicate payload location in the output. The plain
+// Writable store may be used to stream CARv1 contents without buffering;
+// only storing CIDs in memory for de-duplication (where required) and to still
+// allow Has operations.
+//
+// • NewReadableWritable requires an io.ReaderAt and an io.Writer as it combines
+// the functionality of a NewWritable with OpenReadable, being able to random-
+// access read any written blocks.
+//
+// • OpenReadableWritable requires an io.ReaderAt, an io.Writer and an
+// io.WriterAt as it extends the NewReadableWritable functionality with the
+// ability to resume an existing CAR. In addition, if the CAR being resumed is
+// a CARv2, the IO object being provided must have a Truncate() method (e.g.
+// an io.File) in order to properly manage CAR lifecycle and avoid writing a
+// corrupt CAR.
+//
+// The following options are available to customize the behavior of the
+// StorageCar:
+//
+// • WriteAsCarV1
+//
+// • StoreIdentityCIDs
+//
+// • AllowDuplicatePuts
+//
+// • UseWholeCIDs
+//
+// • ZeroLengthSectionAsEOF
+//
+// • UseIndexCodec
+//
+// • UseDataPadding
+//
+// • UseIndexPadding
+//
+// • MaxIndexCidSize
+//
+// • MaxAllowedHeaderSize
+//
+// • MaxAllowedSectionSize
+package storage
diff --git a/ipld/car/v2/storage/notfound.go b/ipld/car/v2/storage/notfound.go
new file mode 100644
index 000000000..82153e2f2
--- /dev/null
+++ b/ipld/car/v2/storage/notfound.go
@@ -0,0 +1,39 @@
+package storage
+
+import "github.com/ipfs/go-cid"
+
+// compatible with the go-ipld-format ErrNotFound, match against
+// interface{NotFound() bool}
+// this could go into go-ipld-prime, but for now we'll just exercise the
+// feature-test pattern
+
+type ErrNotFound struct {
+ Cid cid.Cid
+}
+
+func (e ErrNotFound) Error() string {
+ if e.Cid == cid.Undef {
+ return "ipld: could not find node"
+ }
+ return "ipld: could not find " + e.Cid.String()
+}
+
+func (e ErrNotFound) NotFound() bool {
+ return true
+}
+
+func (e ErrNotFound) Is(err error) bool {
+ switch err.(type) {
+ case ErrNotFound:
+ return true
+ default:
+ return false
+ }
+}
+
+func IsNotFound(err error) bool {
+ if nf, ok := err.(interface{ NotFound() bool }); ok {
+ return nf.NotFound()
+ }
+ return false
+}
diff --git a/ipld/car/v2/storage/storage.go b/ipld/car/v2/storage/storage.go
new file mode 100644
index 000000000..46156e8df
--- /dev/null
+++ b/ipld/car/v2/storage/storage.go
@@ -0,0 +1,505 @@
+package storage
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "sync"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1/util"
+ internalio "github.com/ipfs/boxo/ipld/car/v2/internal/io"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/store"
+ "github.com/ipfs/go-cid"
+ ipldstorage "github.com/ipld/go-ipld-prime/storage"
+)
+
+var errClosed = errors.New("cannot use a CARv2 storage after closing")
+
+type ReaderAtWriterAt interface {
+ io.ReaderAt
+ io.Writer
+ io.WriterAt
+}
+
+type ReadableCar interface {
+ ipldstorage.ReadableStorage
+ ipldstorage.StreamingReadableStorage
+ Roots() []cid.Cid
+}
+
+// WritableCar is compatible with storage.WritableStorage but also returns
+// the roots of the CAR. It does not implement ipld.StreamingWritableStorage
+// as the CAR format does not support streaming data followed by its CID, so
+// any streaming implementation would perform buffering and copy the
+// existing storage.PutStream() implementation.
+type WritableCar interface {
+ ipldstorage.WritableStorage
+ Roots() []cid.Cid
+ Finalize() error
+}
+
+var _ ReadableCar = (*StorageCar)(nil)
+var _ WritableCar = (*StorageCar)(nil)
+
+type StorageCar struct {
+ idx index.Index
+ reader io.ReaderAt
+ writer positionedWriter
+ dataWriter *internalio.OffsetWriteSeeker
+ header carv2.Header
+ roots []cid.Cid
+ opts carv2.Options
+
+ closed bool
+ mu sync.RWMutex
+}
+
+type positionedWriter interface {
+ io.Writer
+ Position() int64
+}
+
+// OpenReadable opens a CARv1 or CARv2 file for reading as a ReadableStorage
+// and StreamingReadableStorage as defined by
+// github.com/ipld/go-ipld-prime/storage.
+//
+// The returned ReadableStorage is compatible with a linksystem SetReadStorage
+// method as defined by github.com/ipld/go-ipld-prime/linking
+// to provide a block source backed by a CAR.
+//
+// When opening a CAR, an initial scan is performed to generate an index, or
+// load an index from a CARv2 index where available. This index data is kept in
+// memory while the CAR is being used in order to provide efficient random
+// Get access to blocks and Has operations.
+//
+// The Readable supports StreamingReadableStorage, which allows for efficient
+// GetStreaming operations straight out of the underlying CAR where the
+// linksystem can make use of it.
+func OpenReadable(reader io.ReaderAt, opts ...carv2.Option) (ReadableCar, error) {
+ sc := &StorageCar{opts: carv2.ApplyOptions(opts...)}
+
+ rr := internalio.ToReadSeeker(reader)
+ header, err := carv1.ReadHeader(rr, sc.opts.MaxAllowedHeaderSize)
+ if err != nil {
+ return nil, err
+ }
+ switch header.Version {
+ case 1:
+ sc.roots = header.Roots
+ sc.reader = reader
+ rr.Seek(0, io.SeekStart)
+ sc.idx = store.NewInsertionIndex()
+ if err := carv2.LoadIndex(sc.idx, rr, opts...); err != nil {
+ return nil, err
+ }
+ case 2:
+ v2r, err := carv2.NewReader(reader, opts...)
+ if err != nil {
+ return nil, err
+ }
+ sc.roots, err = v2r.Roots()
+ if err != nil {
+ return nil, err
+ }
+ if v2r.Header.HasIndex() {
+ ir, err := v2r.IndexReader()
+ if err != nil {
+ return nil, err
+ }
+ sc.idx, err = index.ReadFrom(ir)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ dr, err := v2r.DataReader()
+ if err != nil {
+ return nil, err
+ }
+ sc.idx = store.NewInsertionIndex()
+ if err := carv2.LoadIndex(sc.idx, dr, opts...); err != nil {
+ return nil, err
+ }
+ }
+ if sc.reader, err = v2r.DataReader(); err != nil {
+ return nil, err
+ }
+ default:
+ return nil, fmt.Errorf("unsupported CAR version: %v", header.Version)
+ }
+
+ return sc, nil
+}
+
+// NewWritable creates a new WritableStorage as defined by
+// github.com/ipld/go-ipld-prime/storage that writes a CARv1 or CARv2 format to
+// the given io.Writer.
+//
+// The returned WritableStorage is compatible with a linksystem SetWriteStorage
+// method as defined by github.com/ipld/go-ipld-prime/linking
+// to provide a block sink backed by a CAR.
+//
+// The WritableStorage supports Put operations, which will write
+// blocks to the CAR in the order they are received.
+//
+// When writing a CARv2 format (the default), the provided writer must be
+// compatible with io.WriterAt in order to provide random access as the CARv2
+// header must be written after the blocks in order to indicate the size of the
+// CARv2 data payload.
+//
+// A CARv1 (generated using the WriteAsCarV1 option) only requires an io.Writer
+// and can therefore stream CAR contents as blocks are written while still
+// providing Has operations and the ability to avoid writing duplicate blocks
+// as required.
+//
+// When writing a CARv2 format, it is important to call the Finalize method on
+// the returned WritableStorage in order to write the CARv2 header and index.
+func NewWritable(writer io.Writer, roots []cid.Cid, opts ...carv2.Option) (WritableCar, error) {
+ sc, err := newWritable(writer, roots, opts...)
+ if err != nil {
+ return nil, err
+ }
+ return sc.init()
+}
+
+func newWritable(writer io.Writer, roots []cid.Cid, opts ...carv2.Option) (*StorageCar, error) {
+ sc := &StorageCar{
+ writer: &positionTrackingWriter{w: writer},
+ idx: store.NewInsertionIndex(),
+ header: carv2.NewHeader(0),
+ opts: carv2.ApplyOptions(opts...),
+ roots: roots,
+ }
+
+ if p := sc.opts.DataPadding; p > 0 {
+ sc.header = sc.header.WithDataPadding(p)
+ }
+ if p := sc.opts.IndexPadding; p > 0 {
+ sc.header = sc.header.WithIndexPadding(p)
+ }
+
+ offset := int64(sc.header.DataOffset)
+ if sc.opts.WriteAsCarV1 {
+ offset = 0
+ }
+
+ if writerAt, ok := writer.(io.WriterAt); ok {
+ sc.dataWriter = internalio.NewOffsetWriter(writerAt, offset)
+ } else {
+ if !sc.opts.WriteAsCarV1 {
+ return nil, fmt.Errorf("cannot write as CARv2 to a non-seekable writer")
+ }
+ }
+
+ return sc, nil
+}
+
+func newReadableWritable(rw ReaderAtWriterAt, roots []cid.Cid, opts ...carv2.Option) (*StorageCar, error) {
+ sc, err := newWritable(rw, roots, opts...)
+ if err != nil {
+ return nil, err
+ }
+
+ sc.reader = rw
+ if !sc.opts.WriteAsCarV1 {
+ sc.reader, err = internalio.NewOffsetReadSeeker(rw, int64(sc.header.DataOffset))
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return sc, nil
+}
+
+// NewReadableWritable creates a new StorageCar that is able to provide both
+// StorageReader and StorageWriter functionality.
+//
+// The returned StorageCar is compatible with a linksystem SetReadStorage and
+// SetWriteStorage methods as defined by github.com/ipld/go-ipld-prime/linking.
+//
+// When writing a CARv2 format, it is important to call the Finalize method on
+// the returned WritableStorage in order to write the CARv2 header and index.
+func NewReadableWritable(rw ReaderAtWriterAt, roots []cid.Cid, opts ...carv2.Option) (*StorageCar, error) {
+ sc, err := newReadableWritable(rw, roots, opts...)
+ if err != nil {
+ return nil, err
+ }
+ if _, err := sc.init(); err != nil {
+ return nil, err
+ }
+ return sc, nil
+}
+
+// OpenReadableWritable creates a new StorageCar that is able to provide both
+// StorageReader and StorageWriter functionality.
+//
+// The returned StorageCar is compatible with a linksystem SetReadStorage and
+// SetWriteStorage methods as defined by github.com/ipld/go-ipld-prime/linking.
+//
+// It attempts to resume a CARv2 file that was previously written to by
+// NewWritable, or NewReadableWritable.
+func OpenReadableWritable(rw ReaderAtWriterAt, roots []cid.Cid, opts ...carv2.Option) (*StorageCar, error) {
+ sc, err := newReadableWritable(rw, roots, opts...)
+ if err != nil {
+ return nil, err
+ }
+
+ // attempt to resume
+ rs, err := internalio.NewOffsetReadSeeker(rw, 0)
+ if err != nil {
+ return nil, err
+ }
+ if err := store.ResumableVersion(rs, sc.opts.WriteAsCarV1); err != nil {
+ return nil, err
+ }
+ if err := store.Resume(
+ rw,
+ sc.reader,
+ sc.dataWriter,
+ sc.idx.(*store.InsertionIndex),
+ roots,
+ sc.header.DataOffset,
+ sc.opts.WriteAsCarV1,
+ sc.opts.MaxAllowedHeaderSize,
+ sc.opts.ZeroLengthSectionAsEOF,
+ ); err != nil {
+ return nil, err
+ }
+ return sc, nil
+}
+
+func (sc *StorageCar) init() (WritableCar, error) {
+ if !sc.opts.WriteAsCarV1 {
+ if _, err := sc.writer.Write(carv2.Pragma); err != nil {
+ return nil, err
+ }
+ }
+ var w io.Writer = sc.dataWriter
+ if sc.dataWriter == nil {
+ w = sc.writer
+ }
+ if err := carv1.WriteHeader(&carv1.CarHeader{Roots: sc.roots, Version: 1}, w); err != nil {
+ return nil, err
+ }
+ return sc, nil
+}
+
+// Roots returns the roots of the CAR.
+func (sc *StorageCar) Roots() []cid.Cid {
+ return sc.roots
+}
+
+// Put adds a block to the CAR, where the block is identified by the given CID
+// provided in string form. The keyStr value must be a valid CID binary string
+// (not a multibase string representation), i.e. generated with CID#KeyString().
+func (sc *StorageCar) Put(ctx context.Context, keyStr string, data []byte) error {
+ keyCid, err := cid.Cast([]byte(keyStr))
+ if err != nil {
+ return fmt.Errorf("bad CID key: %w", err)
+ }
+
+ sc.mu.Lock()
+ defer sc.mu.Unlock()
+
+ if sc.closed {
+ return errClosed
+ }
+
+ idx, ok := sc.idx.(*store.InsertionIndex)
+ if !ok || sc.writer == nil {
+ return fmt.Errorf("cannot put into a read-only CAR")
+ }
+
+ if should, err := store.ShouldPut(
+ idx,
+ keyCid,
+ sc.opts.MaxIndexCidSize,
+ sc.opts.StoreIdentityCIDs,
+ sc.opts.BlockstoreAllowDuplicatePuts,
+ sc.opts.BlockstoreUseWholeCIDs,
+ ); err != nil {
+ return err
+ } else if !should {
+ return nil
+ }
+
+ w := sc.writer
+ if sc.dataWriter != nil {
+ w = sc.dataWriter
+ }
+ n := uint64(w.Position())
+ if err := util.LdWrite(w, keyCid.Bytes(), data); err != nil {
+ return err
+ }
+ idx.InsertNoReplace(keyCid, n)
+
+ return nil
+}
+
+// Has returns true if the CAR contains a block identified by the given CID
+// provided in string form. The keyStr value must be a valid CID binary string
+// (not a multibase string representation), i.e. generated with CID#KeyString().
+func (sc *StorageCar) Has(ctx context.Context, keyStr string) (bool, error) {
+ keyCid, err := cid.Cast([]byte(keyStr))
+ if err != nil {
+ return false, fmt.Errorf("bad CID key: %w", err)
+ }
+
+ sc.mu.RLock()
+ defer sc.mu.RUnlock()
+
+ if sc.closed {
+ return false, errClosed
+ }
+
+ if idx, ok := sc.idx.(*store.InsertionIndex); ok && sc.writer != nil {
+ // writable CAR, fast path using InsertionIndex
+ return store.Has(
+ idx,
+ keyCid,
+ sc.opts.MaxIndexCidSize,
+ sc.opts.StoreIdentityCIDs,
+ sc.opts.BlockstoreAllowDuplicatePuts,
+ sc.opts.BlockstoreUseWholeCIDs,
+ )
+ }
+
+ if !sc.opts.StoreIdentityCIDs {
+ // If we don't store identity CIDs then we can return them straight away as if they are here,
+ // otherwise we need to check for their existence.
+ // Note, we do this without locking, since there is no shared information to lock for in order to perform the check.
+ if _, ok, err := store.IsIdentity(keyCid); err != nil {
+ return false, err
+ } else if ok {
+ return true, nil
+ }
+ }
+
+ _, _, size, err := store.FindCid(
+ sc.reader,
+ sc.idx,
+ keyCid,
+ sc.opts.BlockstoreUseWholeCIDs,
+ sc.opts.ZeroLengthSectionAsEOF,
+ sc.opts.MaxAllowedSectionSize,
+ false,
+ )
+ if errors.Is(err, index.ErrNotFound) {
+ return false, nil
+ } else if err != nil {
+ return false, err
+ }
+ return size > -1, nil
+}
+
+// Get returns the block bytes identified by the given CID provided in string
+// form. The keyStr value must be a valid CID binary string (not a multibase
+// string representation), i.e. generated with CID#KeyString().
+func (sc *StorageCar) Get(ctx context.Context, keyStr string) ([]byte, error) {
+ rdr, err := sc.GetStream(ctx, keyStr)
+ if err != nil {
+ return nil, err
+ }
+ return io.ReadAll(rdr)
+}
+
+// GetStream returns a stream of the block bytes identified by the given CID
+// provided in string form. The keyStr value must be a valid CID binary string
+// (not a multibase string representation), i.e. generated with CID#KeyString().
+func (sc *StorageCar) GetStream(ctx context.Context, keyStr string) (io.ReadCloser, error) {
+ if sc.reader == nil {
+ return nil, fmt.Errorf("cannot read from a write-only CAR")
+ }
+
+ keyCid, err := cid.Cast([]byte(keyStr))
+ if err != nil {
+ return nil, fmt.Errorf("bad CID key: %w", err)
+ }
+
+ if !sc.opts.StoreIdentityCIDs {
+ // If we don't store identity CIDs then we can return them straight away as if they are here,
+ // otherwise we need to check for their existence.
+ // Note, we do this without locking, since there is no shared information to lock for in order to perform the check.
+ if digest, ok, err := store.IsIdentity(keyCid); err != nil {
+ return nil, err
+ } else if ok {
+ return io.NopCloser(bytes.NewReader(digest)), nil
+ }
+ }
+
+ sc.mu.RLock()
+ defer sc.mu.RUnlock()
+
+ if sc.closed {
+ return nil, errClosed
+ }
+
+ _, offset, size, err := store.FindCid(
+ sc.reader,
+ sc.idx,
+ keyCid,
+ sc.opts.BlockstoreUseWholeCIDs,
+ sc.opts.ZeroLengthSectionAsEOF,
+ sc.opts.MaxAllowedSectionSize,
+ false,
+ )
+ if errors.Is(err, index.ErrNotFound) {
+ return nil, ErrNotFound{Cid: keyCid}
+ } else if err != nil {
+ return nil, err
+ }
+ return io.NopCloser(io.NewSectionReader(sc.reader, offset, int64(size))), nil
+}
+
+// Finalize writes the CAR index to the underlying writer if the CAR being
+// written is a CARv2. It also writes a finalized CARv2 header which details
+// payload location. This should be called on a writable StorageCar in order to
+// avoid data loss.
+func (sc *StorageCar) Finalize() error {
+ idx, ok := sc.idx.(*store.InsertionIndex)
+ if !ok || sc.writer == nil {
+ // ignore this, it's not writable
+ return nil
+ }
+
+ if sc.opts.WriteAsCarV1 {
+ return nil
+ }
+
+ wat, ok := sc.writer.(*positionTrackingWriter).w.(io.WriterAt)
+ if !ok { // should should already be checked at construction if this is a writable
+ return fmt.Errorf("cannot finalize a CARv2 without an io.WriterAt")
+ }
+
+ sc.mu.Lock()
+ defer sc.mu.Unlock()
+
+ if sc.closed {
+ // Allow duplicate Finalize calls, just like Close.
+ // Still error, just like ReadOnly.Close; it should be discarded.
+ return fmt.Errorf("called Finalize on a closed storage CAR")
+ }
+
+ sc.closed = true
+
+ return store.Finalize(wat, sc.header, idx, uint64(sc.dataWriter.Position()), sc.opts.StoreIdentityCIDs, sc.opts.IndexCodec)
+}
+
+type positionTrackingWriter struct {
+ w io.Writer
+ offset int64
+}
+
+func (ptw *positionTrackingWriter) Write(p []byte) (int, error) {
+ written, err := ptw.w.Write(p)
+ ptw.offset += int64(written)
+ return written, err
+}
+
+func (ptw *positionTrackingWriter) Position() int64 {
+ return ptw.offset
+}
diff --git a/ipld/car/v2/storage/storage_test.go b/ipld/car/v2/storage/storage_test.go
new file mode 100644
index 000000000..827846262
--- /dev/null
+++ b/ipld/car/v2/storage/storage_test.go
@@ -0,0 +1,1276 @@
+package storage_test
+
+// TODO: test readable can't write and writable can't read
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha512"
+ "errors"
+ "fmt"
+ "io"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ carv2 "github.com/ipfs/boxo/ipld/car/v2"
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/ipfs/boxo/ipld/car/v2/storage"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multicodec"
+ "github.com/multiformats/go-multihash"
+ "github.com/stretchr/testify/require"
+)
+
+var rng = rand.New(rand.NewSource(1413))
+var rngLk sync.Mutex
+
+func TestReadable(t *testing.T) {
+ tests := []struct {
+ name string
+ inputPath string
+ opts []carv2.Option
+ noIdCids bool
+ }{
+ {
+ "OpenedWithCarV1",
+ "../testdata/sample-v1.car",
+ []carv2.Option{carv2.UseWholeCIDs(true), carv2.StoreIdentityCIDs(true)},
+ false,
+ },
+ {
+ "OpenedWithCarV1_NoIdentityCID",
+ "../testdata/sample-v1.car",
+ []carv2.Option{carv2.UseWholeCIDs(true)},
+ false,
+ },
+ {
+ "OpenedWithCarV2",
+ "../testdata/sample-wrapped-v2.car",
+ []carv2.Option{carv2.UseWholeCIDs(true), carv2.StoreIdentityCIDs(true)},
+ // index already exists, but was made without identity CIDs, but opening with StoreIdentityCIDs(true) means we check the index
+ true,
+ },
+ {
+ "OpenedWithCarV2_NoIdentityCID",
+ "../testdata/sample-wrapped-v2.car",
+ []carv2.Option{carv2.UseWholeCIDs(true)},
+ false,
+ },
+ {
+ "OpenedWithCarV1ZeroLenSection",
+ "../testdata/sample-v1-with-zero-len-section.car",
+ []carv2.Option{carv2.UseWholeCIDs(true), carv2.ZeroLengthSectionAsEOF(true)},
+ false,
+ },
+ {
+ "OpenedWithAnotherCarV1ZeroLenSection",
+ "../testdata/sample-v1-with-zero-len-section2.car",
+ []carv2.Option{carv2.UseWholeCIDs(true), carv2.ZeroLengthSectionAsEOF(true)},
+ false,
+ },
+ {
+ "IndexlessV2",
+ "../testdata/sample-v2-indexless.car",
+ []carv2.Option{carv2.UseWholeCIDs(true)},
+ false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ // Setup new StorageCar
+ inputReader, err := os.Open(tt.inputPath)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, inputReader.Close()) })
+ readable, err := storage.OpenReadable(inputReader, tt.opts...)
+ require.NoError(t, err)
+
+ // Setup BlockReader to compare against
+ actualReader, err := os.Open(tt.inputPath)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, actualReader.Close()) })
+ actual, err := carv2.NewBlockReader(actualReader, tt.opts...)
+ require.NoError(t, err)
+
+ // Assert roots match v1 payload.
+ require.Equal(t, actual.Roots, readable.Roots())
+
+ for {
+ wantBlock, err := actual.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+
+ key := wantBlock.Cid()
+
+ // Assert StorageCar contains key.
+ has, err := readable.Has(ctx, key.KeyString())
+ require.NoError(t, err)
+ if key.Prefix().MhType == uint64(multicodec.Identity) && tt.noIdCids {
+ // fixture wasn't made with StoreIdentityCIDs, but we opened it with StoreIdentityCIDs,
+ // so they aren't there to find
+ require.False(t, has)
+ } else {
+ require.True(t, has)
+ }
+
+ // Assert block itself matches v1 payload block.
+ if has {
+ gotBlock, err := readable.Get(ctx, key.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, wantBlock.RawData(), gotBlock)
+
+ reader, err := readable.GetStream(ctx, key.KeyString())
+ require.NoError(t, err)
+ data, err := io.ReadAll(reader)
+ require.NoError(t, err)
+ require.Equal(t, wantBlock.RawData(), data)
+ }
+ }
+
+ // test not exists
+ c := randCid()
+ has, err := readable.Has(ctx, c.KeyString())
+ require.NoError(t, err)
+ require.False(t, has)
+
+ _, err = readable.Get(ctx, c.KeyString())
+ require.True(t, errors.Is(err, storage.ErrNotFound{}))
+ require.True(t, storage.IsNotFound(err))
+ require.Contains(t, err.Error(), c.String())
+
+ // random identity, should only find this if we _don't_ store identity CIDs
+ storeIdentity := carv2.ApplyOptions(tt.opts...).StoreIdentityCIDs
+ c = randIdentityCid()
+
+ has, err = readable.Has(ctx, c.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, !storeIdentity, has)
+
+ got, err := readable.Get(ctx, c.KeyString())
+ if !storeIdentity {
+ require.NoError(t, err)
+ mh, err := multihash.Decode(c.Hash())
+ require.NoError(t, err)
+ require.Equal(t, mh.Digest, got)
+ } else {
+ require.True(t, errors.Is(err, storage.ErrNotFound{}))
+ require.True(t, storage.IsNotFound(err))
+ require.Contains(t, err.Error(), c.String())
+ }
+ })
+ }
+}
+
+func TestReadableBadVersion(t *testing.T) {
+ f, err := os.Open("../testdata/sample-rootless-v42.car")
+ require.NoError(t, err)
+ t.Cleanup(func() { f.Close() })
+ subject, err := storage.OpenReadable(f)
+ require.Errorf(t, err, "unsupported car version: 42")
+ require.Nil(t, subject)
+}
+
+func TestWritable(t *testing.T) {
+ originalCarV1Path := "../testdata/sample-v1.car"
+
+ variants := []struct {
+ name string
+ compareCarV1 string
+ options []carv2.Option
+ expectedV1StartOffset int64
+ }{
+ {"carv2_noopt", "sample-v1-noidentity.car", []carv2.Option{}, int64(carv2.PragmaSize + carv2.HeaderSize)},
+ {"carv2_identity", "sample-v1.car", []carv2.Option{carv2.StoreIdentityCIDs(true)}, int64(carv2.PragmaSize + carv2.HeaderSize)},
+ {"carv1", "sample-v1-noidentity.car", []carv2.Option{carv2.WriteAsCarV1(true)}, int64(0)},
+ {"carv1_identity", "sample-v1.car", []carv2.Option{carv2.WriteAsCarV1(true), carv2.StoreIdentityCIDs(true)}, int64(0)},
+ }
+
+ for _, mode := range []string{"WithRead", "WithoutRead"} {
+ t.Run(mode, func(t *testing.T) {
+ for _, variant := range variants {
+ t.Run(variant.name, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ opts := carv2.ApplyOptions(variant.options...)
+
+ // Setup input file using standard CarV1 reader
+ srcFile, err := os.Open(originalCarV1Path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, srcFile.Close()) })
+ r, err := carv1.NewCarReader(srcFile)
+ require.NoError(t, err)
+
+ path := filepath.Join(t.TempDir(), fmt.Sprintf("writable_%s_%s.car", mode, variant.name))
+ var dstFile *os.File
+
+ var writable *storage.StorageCar
+ if mode == "WithoutRead" {
+ dstFile, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+ require.NoError(t, err)
+ t.Cleanup(func() { dstFile.Close() })
+ var writer io.Writer = &writerOnly{dstFile}
+ if !opts.WriteAsCarV1 {
+ writer = &writerAtOnly{dstFile}
+ }
+ w, err := storage.NewWritable(writer, r.Header.Roots, variant.options...)
+ require.NoError(t, err)
+ writable = w.(*storage.StorageCar)
+ } else {
+ dstFile, err = os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
+ require.NoError(t, err)
+ t.Cleanup(func() { dstFile.Close() })
+ writable, err = storage.NewReadableWritable(dstFile, r.Header.Roots, variant.options...)
+ require.NoError(t, err)
+ }
+
+ require.Equal(t, r.Header.Roots, writable.Roots())
+
+ cids := make([]cid.Cid, 0)
+ var idCidCount int
+ for {
+ // read from source
+ b, err := r.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+
+ // write to dest
+ err = writable.Put(ctx, b.Cid().KeyString(), b.RawData())
+ require.NoError(t, err)
+ cids = append(cids, b.Cid())
+
+ dmh, err := multihash.Decode(b.Cid().Hash())
+ require.NoError(t, err)
+ if dmh.Code == multihash.IDENTITY {
+ idCidCount++
+ }
+
+ if mode == "WithRead" {
+ // writable is a ReadableWritable / StorageCar
+
+ // read back out the one we just wrote
+ gotBlock, err := writable.Get(ctx, b.Cid().KeyString())
+ require.NoError(t, err)
+ require.Equal(t, b.RawData(), gotBlock)
+
+ reader, err := writable.GetStream(ctx, b.Cid().KeyString())
+ require.NoError(t, err)
+ data, err := io.ReadAll(reader)
+ require.NoError(t, err)
+ require.Equal(t, b.RawData(), data)
+
+ // try reading a random one:
+ candIndex := rng.Intn(len(cids))
+ var candidate cid.Cid
+ for _, c := range cids {
+ if candIndex == 0 {
+ candidate = c
+ break
+ }
+ candIndex--
+ }
+ has, err := writable.Has(ctx, candidate.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+
+ // not exists
+ c := randCid()
+ has, err = writable.Has(ctx, c.KeyString())
+ require.NoError(t, err)
+ require.False(t, has)
+ _, err = writable.Get(ctx, c.KeyString())
+ require.True(t, errors.Is(err, storage.ErrNotFound{}))
+ require.True(t, storage.IsNotFound(err))
+ require.Contains(t, err.Error(), c.String())
+
+ // random identity, should only find this if we _don't_ store identity CIDs
+ c = randIdentityCid()
+ has, err = writable.Has(ctx, c.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, !opts.StoreIdentityCIDs, has)
+
+ got, err := writable.Get(ctx, c.KeyString())
+ if !opts.StoreIdentityCIDs {
+ require.NoError(t, err)
+ mh, err := multihash.Decode(c.Hash())
+ require.NoError(t, err)
+ require.Equal(t, mh.Digest, got)
+ } else {
+ require.True(t, errors.Is(err, storage.ErrNotFound{}))
+ require.True(t, storage.IsNotFound(err))
+ require.Contains(t, err.Error(), c.String())
+ }
+ }
+ }
+
+ err = writable.Finalize()
+ require.NoError(t, err)
+
+ err = dstFile.Close()
+ require.NoError(t, err)
+
+ // test header version using carv2 reader
+ reopen, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, reopen.Close()) })
+ rd, err := carv2.NewReader(reopen)
+ require.NoError(t, err)
+ require.Equal(t, opts.WriteAsCarV1, rd.Version == 1)
+
+ // now compare the binary contents of the written file to the expected file
+ comparePath := filepath.Join("../testdata/", variant.compareCarV1)
+ compareStat, err := os.Stat(comparePath)
+ require.NoError(t, err)
+
+ wrote, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, wrote.Close()) })
+ _, err = wrote.Seek(variant.expectedV1StartOffset, io.SeekStart)
+ require.NoError(t, err)
+ hasher := sha512.New()
+ gotWritten, err := io.Copy(hasher, io.LimitReader(wrote, compareStat.Size()))
+ require.NoError(t, err)
+ gotSum := hasher.Sum(nil)
+
+ hasher.Reset()
+ compareV1, err := os.Open(comparePath)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, compareV1.Close()) })
+ wantWritten, err := io.Copy(hasher, compareV1)
+ require.NoError(t, err)
+ wantSum := hasher.Sum(nil)
+
+ require.Equal(t, wantWritten, gotWritten)
+ require.Equal(t, wantSum, gotSum)
+ })
+ }
+ })
+ }
+}
+
+func TestCannotWriteableV2WithoutWriterAt(t *testing.T) {
+ w, err := storage.NewWritable(&writerOnly{os.Stdout}, []cid.Cid{})
+ require.Error(t, err)
+ require.Nil(t, w)
+}
+
+func TestErrorsWhenWritingCidTooLarge(t *testing.T) {
+ maxAllowedCidSize := uint64(20)
+
+ path := filepath.Join(t.TempDir(), "writable-with-id-enabled-too-large.car")
+ out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, out.Close()) })
+ subject, err := storage.NewWritable(out, []cid.Cid{}, carv2.MaxIndexCidSize(maxAllowedCidSize))
+ require.NoError(t, err)
+
+ // normal block but shorten the CID to make it acceptable
+ testCid, testData := randBlock()
+ mh, err := multihash.Decode(testCid.Hash())
+ require.NoError(t, err)
+ dig := mh.Digest[:10]
+ shortMh, err := multihash.Encode(dig, mh.Code)
+ require.NoError(t, err)
+ testCid = cid.NewCidV1(mh.Code, shortMh)
+
+ err = subject.Put(context.TODO(), testCid.KeyString(), testData)
+ require.NoError(t, err)
+
+ // standard CID but too long for options
+ testCid, testData = randBlock()
+ err = subject.Put(context.TODO(), testCid.KeyString(), testData)
+ require.Equal(t, &carv2.ErrCidTooLarge{MaxSize: maxAllowedCidSize, CurrentSize: uint64(testCid.ByteLen())}, err)
+}
+
+func TestConcurrentUse(t *testing.T) {
+ dst, err := os.OpenFile(filepath.Join(t.TempDir(), "readwrite.car"), os.O_CREATE|os.O_RDWR, 0644)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, dst.Close()) })
+ wbs, err := storage.NewReadableWritable(dst, nil)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ require.NoError(t, err)
+ t.Cleanup(func() { wbs.Finalize() })
+
+ var wg sync.WaitGroup
+ for i := 0; i < 100; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+
+ testCid, testData := randBlock()
+
+ has, err := wbs.Has(ctx, testCid.KeyString())
+ require.NoError(t, err)
+ require.False(t, has)
+
+ err = wbs.Put(ctx, testCid.KeyString(), testData)
+ require.NoError(t, err)
+
+ got, err := wbs.Get(ctx, testCid.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, testData, got)
+ }()
+ }
+ wg.Wait()
+}
+
+func TestNullPadding(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ paddedV1, err := os.ReadFile("../testdata/sample-v1-with-zero-len-section.car")
+ require.NoError(t, err)
+
+ readable, err := storage.OpenReadable(bufferReaderAt(paddedV1), carv2.ZeroLengthSectionAsEOF(true))
+ require.NoError(t, err)
+
+ roots := readable.Roots()
+ require.Len(t, roots, 1)
+ has, err := readable.Has(ctx, roots[0].KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+
+ actual, err := carv2.NewBlockReader(bytes.NewReader(paddedV1), carv2.ZeroLengthSectionAsEOF(true))
+ require.NoError(t, err)
+
+ for {
+ wantBlock, err := actual.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+ b, err := readable.Get(ctx, wantBlock.Cid().KeyString())
+ require.NoError(t, err)
+ require.Equal(t, wantBlock.RawData(), b)
+ }
+}
+
+func TestPutSameHashes(t *testing.T) {
+ tdir := t.TempDir()
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ // This writable allows duplicate puts, and identifies by multihash as per the default.
+ pathAllowDups := filepath.Join(tdir, "writable-allowdup.car")
+ dstAllowDups, err := os.OpenFile(pathAllowDups, os.O_CREATE|os.O_RDWR, 0644)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, dstAllowDups.Close()) })
+ wbsAllowDups, err := storage.NewReadableWritable(dstAllowDups, nil, carv2.AllowDuplicatePuts(true))
+ require.NoError(t, err)
+
+ // This writable deduplicates puts by CID.
+ pathByCID := filepath.Join(tdir, "writable-dedup-wholecid.car")
+ dstByCID, err := os.OpenFile(pathByCID, os.O_CREATE|os.O_RDWR, 0644)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, dstByCID.Close()) })
+ wbsByCID, err := storage.NewReadableWritable(dstByCID, nil, carv2.UseWholeCIDs(true))
+ require.NoError(t, err)
+
+ // This writable deduplicates puts by multihash
+ pathByHash := filepath.Join(tdir, "writable-dedup-byhash.car")
+ dstByHash, err := os.OpenFile(pathByHash, os.O_CREATE|os.O_RDWR, 0644)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, dstByHash.Close()) })
+ wbsByHash, err := storage.NewReadableWritable(dstByHash, nil)
+ require.NoError(t, err)
+
+ var blockList []struct {
+ cid cid.Cid
+ data []byte
+ }
+
+ appendBlock := func(data []byte, version, codec uint64) {
+ c, err := cid.Prefix{
+ Version: version,
+ Codec: codec,
+ MhType: multihash.SHA2_256,
+ MhLength: -1,
+ }.Sum(data)
+ require.NoError(t, err)
+ blockList = append(blockList, struct {
+ cid cid.Cid
+ data []byte
+ }{c, data})
+ }
+
+ // Two raw blocks, meaning we have two unique multihashes.
+ // However, we have multiple CIDs for each multihash.
+ // We also have two duplicate CIDs.
+ data1 := []byte("foo bar")
+ appendBlock(data1, 0, cid.DagProtobuf)
+ appendBlock(data1, 1, cid.DagProtobuf)
+ appendBlock(data1, 1, cid.DagCBOR)
+ appendBlock(data1, 1, cid.DagCBOR) // duplicate CID
+
+ data2 := []byte("foo bar baz")
+ appendBlock(data2, 0, cid.DagProtobuf)
+ appendBlock(data2, 1, cid.DagProtobuf)
+ appendBlock(data2, 1, cid.DagProtobuf) // duplicate CID
+ appendBlock(data2, 1, cid.DagCBOR)
+
+ countBlocks := func(path string) int {
+ f, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, f.Close()) })
+ rdr, err := carv2.NewBlockReader(f)
+ require.NoError(t, err)
+
+ n := 0
+ for {
+ _, err := rdr.Next()
+ if err == io.EOF {
+ break
+ }
+ n++
+ }
+ return n
+ }
+
+ putBlockList := func(writable *storage.StorageCar) {
+ for i, block := range blockList {
+ // Has should never error here.
+ // The first block should be missing.
+ // Others might not, given the duplicate hashes.
+ has, err := writable.Has(ctx, block.cid.KeyString())
+ require.NoError(t, err)
+ if i == 0 {
+ require.False(t, has)
+ }
+
+ err = writable.Put(ctx, block.cid.KeyString(), block.data)
+ require.NoError(t, err)
+
+ // Has and Get need to work right after a Put
+ has, err = writable.Has(ctx, block.cid.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+
+ got, err := writable.Get(ctx, block.cid.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, block.data, got)
+ }
+ }
+
+ putBlockList(wbsAllowDups)
+ err = wbsAllowDups.Finalize()
+ require.NoError(t, err)
+ require.Equal(t, len(blockList), countBlocks(pathAllowDups))
+
+ // Put the same list of blocks to the CAR that deduplicates by CID.
+ // We should end up with two fewer blocks, as two are entire CID duplicates.
+ putBlockList(wbsByCID)
+ err = wbsByCID.Finalize()
+ require.NoError(t, err)
+ require.Equal(t, len(blockList)-2, countBlocks(pathByCID))
+
+ // Put the same list of blocks to the CAR that deduplicates by CID.
+ // We should end up with just two blocks, as the original set of blocks only
+ // has two distinct multihashes.
+ putBlockList(wbsByHash)
+ err = wbsByHash.Finalize()
+ require.NoError(t, err)
+ require.Equal(t, 2, countBlocks(pathByHash))
+}
+
+func TestReadableCantWrite(t *testing.T) {
+ inp, err := os.Open("../testdata/sample-v1.car")
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, inp.Close()) })
+ readable, err := storage.OpenReadable(inp)
+ require.NoError(t, err)
+ require.ErrorContains(t, readable.(*storage.StorageCar).Put(context.Background(), randCid().KeyString(), []byte("bar")), "read-only")
+ // Finalize() is nonsense for a readable, but it should be safe
+ require.NoError(t, readable.(*storage.StorageCar).Finalize())
+}
+
+func TestWritableCantRead(t *testing.T) {
+ // an io.Writer with no io.WriterAt capabilities
+ path := filepath.Join(t.TempDir(), "writable.car")
+ out, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, out.Close()) })
+
+ // This should fail because the writer is not an io.WriterAt
+ _, err = storage.NewWritable(&writerOnly{out}, nil)
+ require.ErrorContains(t, err, "CARv2")
+ require.ErrorContains(t, err, "non-seekable")
+
+ writable, err := storage.NewWritable(&writerOnly{out}, nil, carv2.WriteAsCarV1(true))
+ require.NoError(t, err)
+
+ _, err = writable.(*storage.StorageCar).Get(context.Background(), randCid().KeyString())
+ require.ErrorContains(t, err, "write-only")
+
+ _, err = writable.(*storage.StorageCar).GetStream(context.Background(), randCid().KeyString())
+ require.ErrorContains(t, err, "write-only")
+
+ require.NoError(t, writable.Finalize())
+}
+
+func TestReadWriteWithPaddingWorksAsExpected(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ testCid1, testData1 := randBlock()
+ testCid2, testData2 := randBlock()
+
+ wantRoots := []cid.Cid{testCid1, testCid2}
+ path := filepath.Join(t.TempDir(), "readwrite-with-padding.car")
+ writer, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, writer.Close()) })
+
+ wantCarV1Padding := uint64(1413)
+ wantIndexPadding := uint64(1314)
+ subject, err := storage.NewReadableWritable(
+ writer,
+ wantRoots,
+ carv2.UseDataPadding(wantCarV1Padding),
+ carv2.UseIndexPadding(wantIndexPadding))
+ require.NoError(t, err)
+ require.NoError(t, subject.Put(ctx, testCid1.KeyString(), testData1))
+ require.NoError(t, subject.Put(ctx, testCid2.KeyString(), testData2))
+ require.NoError(t, subject.Finalize())
+
+ // Assert CARv2 header contains right offsets.
+ gotCarV2, err := carv2.OpenReader(path)
+ t.Cleanup(func() { gotCarV2.Close() })
+ require.NoError(t, err)
+ wantCarV1Offset := carv2.PragmaSize + carv2.HeaderSize + wantCarV1Padding
+ wantIndexOffset := wantCarV1Offset + gotCarV2.Header.DataSize + wantIndexPadding
+ require.Equal(t, wantCarV1Offset, gotCarV2.Header.DataOffset)
+ require.Equal(t, wantIndexOffset, gotCarV2.Header.IndexOffset)
+ require.NoError(t, gotCarV2.Close())
+
+ f, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { f.Close() })
+
+ // Assert reading CARv1 directly at offset and size is as expected.
+ gotCarV1, err := carv1.NewCarReader(io.NewSectionReader(f, int64(wantCarV1Offset), int64(gotCarV2.Header.DataSize)))
+ require.NoError(t, err)
+ require.Equal(t, wantRoots, gotCarV1.Header.Roots)
+ gotBlock, err := gotCarV1.Next()
+ require.NoError(t, err)
+ require.Equal(t, testCid1, gotBlock.Cid())
+ require.Equal(t, testData1, gotBlock.RawData())
+ gotBlock, err = gotCarV1.Next()
+ require.NoError(t, err)
+ require.Equal(t, testCid2, gotBlock.Cid())
+ require.Equal(t, testData2, gotBlock.RawData())
+
+ _, err = gotCarV1.Next()
+ require.Equal(t, io.EOF, err)
+
+ // Assert reading index directly from file is parsable and has expected CIDs.
+ stat, err := f.Stat()
+ require.NoError(t, err)
+ indexSize := stat.Size() - int64(wantIndexOffset)
+ gotIdx, err := index.ReadFrom(io.NewSectionReader(f, int64(wantIndexOffset), indexSize))
+ require.NoError(t, err)
+ _, err = index.GetFirst(gotIdx, testCid1)
+ require.NoError(t, err)
+ _, err = index.GetFirst(gotIdx, testCid2)
+ require.NoError(t, err)
+}
+
+func TestResumption(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ srcPath := "../testdata/sample-v1.car"
+
+ v1f, err := os.Open(srcPath)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v1f.Close()) })
+ rd, err := carv2.NewReader(v1f)
+ require.NoError(t, err)
+ roots, err := rd.Roots()
+ require.NoError(t, err)
+
+ blockSource := func() <-chan simpleBlock {
+ v1f, err := os.Open(srcPath)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v1f.Close()) })
+ r, err := carv1.NewCarReader(v1f)
+ require.NoError(t, err)
+ ret := make(chan simpleBlock)
+
+ go func() {
+ for {
+ b, err := r.Next()
+ if err == io.EOF {
+ close(ret)
+ break
+ }
+ require.NoError(t, err)
+ ret <- simpleBlock{cid: b.Cid(), data: b.RawData()}
+ }
+ }()
+
+ return ret
+ }
+
+ path := filepath.Join(t.TempDir(), "readwrite-resume.car")
+ writer, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, writer.Close()) })
+ // Create an incomplete CARv2 file with no blocks put.
+ subject, err := storage.NewReadableWritable(writer, roots, carv2.UseWholeCIDs(true))
+ require.NoError(t, err)
+
+ // For each block resume on the same file, putting blocks one at a time.
+ var wantBlockCountSoFar, idCidCount int
+ wantBlocks := make(map[cid.Cid]simpleBlock)
+ for b := range blockSource() {
+ wantBlockCountSoFar++
+ wantBlocks[b.cid] = b
+
+ dmh, err := multihash.Decode(b.cid.Hash())
+ require.NoError(t, err)
+ if dmh.Code == multihash.IDENTITY {
+ idCidCount++
+ }
+
+ // 30% chance of subject failing; more concretely: re-instantiating the StorageCar with the same
+ // file without calling Finalize. The higher this percentage the slower the test runs
+ // considering the number of blocks in the original CARv1 test payload.
+ resume := rng.Float32() <= 0.3
+ // If testing resume case, then flip a coin to decide whether to finalize before the StorageCar
+ // re-instantiation or not. Note, both cases should work for resumption since we do not
+ // limit resumption to unfinalized files.
+ finalizeBeforeResumption := rng.Float32() <= 0.5
+ if resume {
+ if finalizeBeforeResumption {
+ require.NoError(t, subject.Finalize())
+ }
+
+ _, err := writer.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ subject, err = storage.OpenReadableWritable(writer, roots, carv2.UseWholeCIDs(true))
+ require.NoError(t, err)
+ }
+ require.NoError(t, subject.Put(ctx, b.cid.KeyString(), b.data))
+
+ // With 10% chance test read operations on an resumed read-write StorageCar.
+ // We don't test on every put to reduce test runtime.
+ testRead := rng.Float32() <= 0.1
+ if testRead {
+ // Assert read operations on the read-write StorageCar are as expected when resumed from an
+ // existing file
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ t.Cleanup(cancel)
+ for k, wantBlock := range wantBlocks {
+ has, err := subject.Has(ctx, k.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+ gotBlock, err := subject.Get(ctx, k.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, wantBlock.data, gotBlock)
+ }
+ // Assert the number of blocks in file are as expected calculated via AllKeysChan
+ require.Equal(t, wantBlockCountSoFar, len(wantBlocks))
+ }
+ }
+
+ // Finalize the StorageCar to complete partially written CARv2 file.
+ subject, err = storage.OpenReadableWritable(writer, roots, carv2.UseWholeCIDs(true))
+ require.NoError(t, err)
+ require.NoError(t, subject.Finalize())
+
+ // Assert resumed from file is a valid CARv2 with index.
+ v2f, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v2f.Close()) })
+ v2r, err := carv2.NewReader(v2f)
+ require.NoError(t, err)
+ require.True(t, v2r.Header.HasIndex())
+
+ // Assert CARv1 payload in file matches the original CARv1 payload.
+ _, err = v1f.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ wantPayloadReader, err := carv1.NewCarReader(v1f)
+ require.NoError(t, err)
+
+ dr, err := v2r.DataReader()
+ require.NoError(t, err)
+ gotPayloadReader, err := carv1.NewCarReader(dr)
+ require.NoError(t, err)
+
+ require.Equal(t, wantPayloadReader.Header, gotPayloadReader.Header)
+ for {
+ wantNextBlock, wantErr := wantPayloadReader.Next()
+ if wantErr == io.EOF {
+ gotNextBlock, gotErr := gotPayloadReader.Next()
+ require.Equal(t, wantErr, gotErr)
+ require.Nil(t, gotNextBlock)
+ break
+ }
+ require.NoError(t, wantErr)
+
+ dmh, err := multihash.Decode(wantNextBlock.Cid().Hash())
+ require.NoError(t, err)
+ if dmh.Code == multihash.IDENTITY {
+ continue
+ }
+
+ gotNextBlock, gotErr := gotPayloadReader.Next()
+ require.NoError(t, gotErr)
+ require.Equal(t, wantNextBlock, gotNextBlock)
+ }
+
+ // Assert index in resumed from file is identical to index generated from the data payload portion of the generated CARv2 file.
+ _, err = v1f.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ ir, err := v2r.IndexReader()
+ require.NoError(t, err)
+ gotIdx, err := index.ReadFrom(ir)
+ require.NoError(t, err)
+ dr, err = v2r.DataReader()
+ require.NoError(t, err)
+ wantIdx, err := carv2.GenerateIndex(dr)
+ require.NoError(t, err)
+ require.Equal(t, wantIdx, gotIdx)
+}
+
+func TestResumptionV1(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ srcPath := "../testdata/sample-v1.car"
+
+ v1f, err := os.Open(srcPath)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v1f.Close()) })
+ rd, err := carv2.NewReader(v1f)
+ require.NoError(t, err)
+ roots, err := rd.Roots()
+ require.NoError(t, err)
+
+ blockSource := func() <-chan simpleBlock {
+ v1f, err := os.Open(srcPath)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v1f.Close()) })
+ r, err := carv1.NewCarReader(v1f)
+ require.NoError(t, err)
+ ret := make(chan simpleBlock)
+
+ go func() {
+ for {
+ b, err := r.Next()
+ if err == io.EOF {
+ close(ret)
+ break
+ }
+ require.NoError(t, err)
+ ret <- simpleBlock{cid: b.Cid(), data: b.RawData()}
+ }
+ }()
+
+ return ret
+ }
+
+ path := filepath.Join(t.TempDir(), "readwrite-resume-v1.car")
+ writer, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, writer.Close()) })
+ // Create an incomplete CARv2 file with no blocks put.
+ subject, err := storage.NewReadableWritable(writer, roots, carv2.UseWholeCIDs(true), carv2.WriteAsCarV1(true))
+ require.NoError(t, err)
+
+ // For each block resume on the same file, putting blocks one at a time.
+ var wantBlockCountSoFar, idCidCount int
+ wantBlocks := make(map[cid.Cid]simpleBlock)
+ for b := range blockSource() {
+ wantBlockCountSoFar++
+ wantBlocks[b.cid] = b
+
+ dmh, err := multihash.Decode(b.cid.Hash())
+ require.NoError(t, err)
+ if dmh.Code == multihash.IDENTITY {
+ idCidCount++
+ }
+
+ // 30% chance of subject failing; more concretely: re-instantiating the StorageCar with the same
+ // file without calling Finalize. The higher this percentage the slower the test runs
+ // considering the number of blocks in the original CARv1 test payload.
+ resume := rng.Float32() <= 0.3
+ // If testing resume case, then flip a coin to decide whether to finalize before the StorageCar
+ // re-instantiation or not. Note, both cases should work for resumption since we do not
+ // limit resumption to unfinalized files.
+ finalizeBeforeResumption := rng.Float32() <= 0.5
+ if resume {
+ if finalizeBeforeResumption {
+ require.NoError(t, subject.Finalize())
+ }
+
+ _, err := writer.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ subject, err = storage.OpenReadableWritable(writer, roots, carv2.UseWholeCIDs(true), carv2.WriteAsCarV1(true))
+ require.NoError(t, err)
+ }
+ require.NoError(t, subject.Put(ctx, b.cid.KeyString(), b.data))
+
+ // With 10% chance test read operations on an resumed read-write StorageCar.
+ // We don't test on every put to reduce test runtime.
+ testRead := rng.Float32() <= 0.1
+ if testRead {
+ // Assert read operations on the read-write StorageCar are as expected when resumed from an
+ // existing file
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ t.Cleanup(cancel)
+ for k, wantBlock := range wantBlocks {
+ has, err := subject.Has(ctx, k.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+ gotBlock, err := subject.Get(ctx, k.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, wantBlock.data, gotBlock)
+ }
+ // Assert the number of blocks in file are as expected calculated via AllKeysChan
+ require.Equal(t, wantBlockCountSoFar, len(wantBlocks))
+ }
+ }
+
+ // Finalize the StorageCar to complete partially written CARv2 file.
+ subject, err = storage.OpenReadableWritable(writer, roots, carv2.UseWholeCIDs(true), carv2.WriteAsCarV1(true))
+ require.NoError(t, err)
+ require.NoError(t, subject.Finalize())
+
+ // Assert resumed from file is a valid CARv2 with index.
+ v2f, err := os.Open(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v2f.Close()) })
+ v2r, err := carv2.NewReader(v2f)
+ require.NoError(t, err)
+ require.False(t, v2r.Header.HasIndex())
+ require.Equal(t, uint64(1), v2r.Version)
+
+ _, err = v1f.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ wantPayloadReader, err := carv1.NewCarReader(v1f)
+ require.NoError(t, err)
+
+ dr, err := v2r.DataReader() // since this is a v1 we're just reading from the top with this
+ require.NoError(t, err)
+ gotPayloadReader, err := carv1.NewCarReader(dr)
+ require.NoError(t, err)
+
+ require.Equal(t, wantPayloadReader.Header, gotPayloadReader.Header)
+ for {
+ wantNextBlock, wantErr := wantPayloadReader.Next()
+ if wantErr == io.EOF {
+ gotNextBlock, gotErr := gotPayloadReader.Next()
+ require.Equal(t, wantErr, gotErr)
+ require.Nil(t, gotNextBlock)
+ break
+ }
+ require.NoError(t, wantErr)
+
+ dmh, err := multihash.Decode(wantNextBlock.Cid().Hash())
+ require.NoError(t, err)
+ if dmh.Code == multihash.IDENTITY {
+ continue
+ }
+
+ gotNextBlock, gotErr := gotPayloadReader.Next()
+ require.NoError(t, gotErr)
+ require.Equal(t, wantNextBlock, gotNextBlock)
+ }
+}
+
+func TestResumptionIsSupportedOnFinalizedFile(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "readwrite-resume-finalized.car")
+ v2f, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v2f.Close()) })
+ // Create an incomplete CARv2 file with no blocks put.
+ subject, err := storage.NewReadableWritable(v2f, []cid.Cid{})
+ require.NoError(t, err)
+ require.NoError(t, subject.Finalize())
+
+ reopen, err := os.OpenFile(path, os.O_RDWR, 0o666)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, reopen.Close()) })
+ subject, err = storage.NewReadableWritable(reopen, []cid.Cid{})
+ require.NoError(t, err)
+ t.Cleanup(func() { subject.Finalize() })
+}
+
+func TestReadWriteErrorsOnlyWhenFinalized(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ testCid1, testData1 := randBlock()
+ testCid2, testData2 := randBlock()
+
+ wantRoots := []cid.Cid{testCid1, testCid2}
+ path := filepath.Join(t.TempDir(), "readwrite-finalized-panic.car")
+ writer, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, writer.Close()) })
+
+ subject, err := storage.NewReadableWritable(writer, wantRoots)
+ require.NoError(t, err)
+
+ require.NoError(t, subject.Put(ctx, testCid1.KeyString(), testData1))
+ require.NoError(t, subject.Put(ctx, testCid2.KeyString(), testData2))
+
+ gotBlock, err := subject.Get(ctx, testCid1.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, testData1, gotBlock)
+
+ gotRoots := subject.Roots()
+ require.Equal(t, wantRoots, gotRoots)
+
+ has, err := subject.Has(ctx, testCid1.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+
+ require.NoError(t, subject.Finalize())
+ require.Error(t, subject.Finalize())
+
+ _, ok := (interface{})(subject).(io.Closer)
+ require.False(t, ok)
+
+ _, err = subject.Get(ctx, testCid1.KeyString())
+ require.Error(t, err)
+ require.Error(t, err)
+ _, err = subject.Has(ctx, testCid2.KeyString())
+ require.Error(t, err)
+
+ require.Error(t, subject.Put(ctx, testCid1.KeyString(), testData1))
+}
+
+func TestReadWriteResumptionMismatchingRootsIsError(t *testing.T) {
+ tmpPath := requireTmpCopy(t, "../testdata/sample-wrapped-v2.car")
+
+ origContent, err := os.ReadFile(tmpPath)
+ require.NoError(t, err)
+
+ badRoot := randCid()
+ writer, err := os.OpenFile(tmpPath, os.O_RDWR, 0o666)
+ require.NoError(t, err)
+ t.Cleanup(func() { writer.Close() })
+ subject, err := storage.OpenReadableWritable(writer, []cid.Cid{badRoot})
+ require.EqualError(t, err, "cannot resume on file with mismatching data header")
+ require.Nil(t, subject)
+ require.NoError(t, writer.Close())
+
+ newContent, err := os.ReadFile(tmpPath)
+ require.NoError(t, err)
+
+ // Expect the bad file to be left untouched; check the size first.
+ // If the sizes mismatch, printing a huge diff would not help us.
+ require.Equal(t, len(origContent), len(newContent))
+ require.Equal(t, origContent, newContent)
+}
+
+func requireTmpCopy(t *testing.T, src string) string {
+ srcF, err := os.Open(src)
+ require.NoError(t, err)
+ defer func() { require.NoError(t, srcF.Close()) }()
+ stats, err := srcF.Stat()
+ require.NoError(t, err)
+
+ dst := filepath.Join(t.TempDir(), stats.Name())
+ dstF, err := os.Create(dst)
+ require.NoError(t, err)
+ defer func() { require.NoError(t, dstF.Close()) }()
+
+ _, err = io.Copy(dstF, srcF)
+ require.NoError(t, err)
+ return dst
+}
+
+func TestReadWriteResumptionFromFileWithDifferentCarV1PaddingIsError(t *testing.T) {
+ testCid1, testData1 := randBlock()
+ wantRoots := []cid.Cid{testCid1}
+ path := filepath.Join(t.TempDir(), "readwrite-resume-with-padding.car")
+ writer, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, writer.Close()) })
+ subject, err := storage.NewReadableWritable(
+ writer,
+ wantRoots,
+ carv2.UseDataPadding(1413))
+ require.NoError(t, err)
+ require.NoError(t, subject.Put(context.TODO(), testCid1.KeyString(), testData1))
+ require.NoError(t, subject.Finalize())
+
+ subject, err = storage.OpenReadableWritable(
+ writer,
+ wantRoots,
+ carv2.UseDataPadding(1314))
+ require.EqualError(t, err, "cannot resume from file with mismatched CARv1 offset; "+
+ "`WithDataPadding` option must match the padding on file. "+
+ "Expected padding value of 1413 but got 1314")
+ require.Nil(t, subject)
+}
+
+func TestOperationsErrorWithBadCidStrings(t *testing.T) {
+ testCid, testData := randBlock()
+ path := filepath.Join(t.TempDir(), "badkeys.car")
+ writer, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, writer.Close()) })
+ subject, err := storage.NewReadableWritable(writer, []cid.Cid{})
+ require.NoError(t, err)
+
+ require.NoError(t, subject.Put(context.TODO(), testCid.KeyString(), testData))
+ require.ErrorContains(t, subject.Put(context.TODO(), fmt.Sprintf("%s/nope", testCid.KeyString()), testData), "bad CID key")
+ require.ErrorContains(t, subject.Put(context.TODO(), "nope", testData), "bad CID key")
+
+ has, err := subject.Has(context.TODO(), testCid.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+ has, err = subject.Has(context.TODO(), fmt.Sprintf("%s/nope", testCid.KeyString()))
+ require.ErrorContains(t, err, "bad CID key")
+ require.False(t, has)
+ has, err = subject.Has(context.TODO(), "nope")
+ require.ErrorContains(t, err, "bad CID key")
+ require.False(t, has)
+
+ got, err := subject.Get(context.TODO(), testCid.KeyString())
+ require.NoError(t, err)
+ require.NotNil(t, got)
+ got, err = subject.Get(context.TODO(), fmt.Sprintf("%s/nope", testCid.KeyString()))
+ require.ErrorContains(t, err, "bad CID key")
+ require.Nil(t, got)
+ got, err = subject.Get(context.TODO(), "nope")
+ require.ErrorContains(t, err, "bad CID key")
+ require.Nil(t, got)
+}
+
+func TestWholeCID(t *testing.T) {
+ for _, whole := range []bool{true, false} {
+ whole := whole
+ t.Run(fmt.Sprintf("whole=%t", whole), func(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+ path := filepath.Join(t.TempDir(), fmt.Sprintf("writable_%t.car", whole))
+ out, err := os.Create(path)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, out.Close()) })
+ store, err := storage.NewReadableWritable(out, []cid.Cid{}, carv2.UseWholeCIDs(whole))
+ require.NoError(t, err)
+
+ c1, b1 := randBlock()
+ c2, b2 := randBlock()
+
+ require.NoError(t, store.Put(ctx, c1.KeyString(), b1))
+ has, err := store.Has(ctx, c1.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+
+ pref := c1.Prefix()
+ pref.Codec = cid.DagProtobuf
+ pref.Version = 1
+ cpb1, err := pref.Sum(b1)
+ require.NoError(t, err)
+
+ has, err = store.Has(ctx, cpb1.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, has, !whole)
+
+ require.NoError(t, store.Put(ctx, c2.KeyString(), b1))
+ has, err = store.Has(ctx, c2.KeyString())
+ require.NoError(t, err)
+ require.True(t, has)
+ has, err = store.Has(ctx, cpb1.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, has, !whole)
+
+ pref = c2.Prefix()
+ pref.Codec = cid.DagProtobuf
+ pref.Version = 1
+ cpb2, err := pref.Sum(b2)
+ require.NoError(t, err)
+
+ has, err = store.Has(ctx, cpb2.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, has, !whole)
+ has, err = store.Has(ctx, cpb1.KeyString())
+ require.NoError(t, err)
+ require.Equal(t, has, !whole)
+ })
+ }
+}
+
+type writerOnly struct {
+ io.Writer
+}
+
+func (w *writerOnly) Write(p []byte) (n int, err error) {
+ return w.Writer.Write(p)
+}
+
+type writerAtOnly struct {
+ *os.File
+}
+
+func (w *writerAtOnly) WriteAt(p []byte, off int64) (n int, err error) {
+ return w.File.WriteAt(p, off)
+}
+
+func (w *writerAtOnly) Write(p []byte) (n int, err error) {
+ return w.File.Write(p)
+}
+
+func randBlock() (cid.Cid, []byte) {
+ data := make([]byte, 1024)
+ rngLk.Lock()
+ rng.Read(data)
+ rngLk.Unlock()
+ h, err := multihash.Sum(data, multihash.SHA2_512, -1)
+ if err != nil {
+ panic(err)
+ }
+ return cid.NewCidV1(cid.Raw, h), data
+}
+
+func randCid() cid.Cid {
+ b := make([]byte, 32)
+ rngLk.Lock()
+ rng.Read(b)
+ rngLk.Unlock()
+ mh, _ := multihash.Encode(b, multihash.SHA2_256)
+ return cid.NewCidV1(cid.DagProtobuf, mh)
+}
+
+func randIdentityCid() cid.Cid {
+ b := make([]byte, 32)
+ rngLk.Lock()
+ rng.Read(b)
+ rngLk.Unlock()
+ mh, _ := multihash.Encode(b, multihash.IDENTITY)
+ return cid.NewCidV1(cid.Raw, mh)
+}
+
+type bufferReaderAt []byte
+
+func (b bufferReaderAt) ReadAt(p []byte, off int64) (int, error) {
+ if off >= int64(len(b)) {
+ return 0, io.EOF
+ }
+ return copy(p, b[off:]), nil
+}
+
+type simpleBlock struct {
+ cid cid.Cid
+ data []byte
+}
diff --git a/ipld/car/v2/testdata/fuzz/FuzzBlockReader/c3c7eedeb4968a5b3131371ba9f5138a38c882a7fa06456595998e740a9f5a14 b/ipld/car/v2/testdata/fuzz/FuzzBlockReader/c3c7eedeb4968a5b3131371ba9f5138a38c882a7fa06456595998e740a9f5a14
new file mode 100644
index 000000000..ae9262d49
--- /dev/null
+++ b/ipld/car/v2/testdata/fuzz/FuzzBlockReader/c3c7eedeb4968a5b3131371ba9f5138a38c882a7fa06456595998e740a9f5a14
@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\xff\x80\xaa\x95\xa6sion\x01,\x01q\x12 \x15\x1f\xe9\xe7 dataSize {
+ // Truncate to the expected size to assure the resulting file is a correctly sized CARv1.
+ err = dst.Truncate(written)
+ }
+
+ return err
+}
+
+// AttachIndex attaches a given index to an existing CARv2 file at given path and offset.
+func AttachIndex(path string, idx index.Index, offset uint64) error {
+ // TODO: instead of offset, maybe take padding?
+ // TODO: check that the given path is indeed a CARv2.
+ // TODO: update CARv2 header according to the offset at which index is written out.
+ out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+ indexWriter := internalio.NewOffsetWriter(out, int64(offset))
+ _, err = index.WriteTo(idx, indexWriter)
+ return err
+}
+
+// ReplaceRootsInFile replaces the root CIDs in CAR file at given path with the given roots.
+// This function accepts both CARv1 and CARv2 files.
+//
+// Note that the roots are only replaced if their total serialized size exactly matches the total
+// serialized size of existing roots in CAR file.
+func ReplaceRootsInFile(path string, roots []cid.Cid, opts ...Option) (err error) {
+ f, err := os.OpenFile(path, os.O_RDWR, 0o666)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ // Close file and override return error type if it is nil.
+ if cerr := f.Close(); err == nil {
+ err = cerr
+ }
+ }()
+
+ options := ApplyOptions(opts...)
+
+ // Read header or pragma; note that both are a valid CARv1 header.
+ header, err := carv1.ReadHeader(f, options.MaxAllowedHeaderSize)
+ if err != nil {
+ return err
+ }
+
+ var currentSize int64
+ var newHeaderOffset int64
+ switch header.Version {
+ case 1:
+ // When the given file is a CARv1 :
+ // 1. The offset at which the new header should be written is zero (newHeaderOffset = 0)
+ // 2. The current header size is equal to the number of bytes read, and
+ //
+ // Note that we explicitly avoid using carv1.HeaderSize to determine the current header size.
+ // This is based on the fact that carv1.ReadHeader does not read any extra bytes.
+ // Therefore, we can avoid extra allocations of carv1.HeaderSize to determine size by simply
+ // counting the bytes read so far.
+ currentSize, err = f.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+ case 2:
+ // When the given file is a CARv2 :
+ // 1. The offset at which the new header should be written is carv2.Header.DataOffset
+ // 2. The inner CARv1 header size is equal to the number of bytes read minus carv2.Header.DataOffset
+ var v2h Header
+ if _, err = v2h.ReadFrom(f); err != nil {
+ return err
+ }
+ newHeaderOffset = int64(v2h.DataOffset)
+ if _, err = f.Seek(newHeaderOffset, io.SeekStart); err != nil {
+ return err
+ }
+ var innerV1Header *carv1.CarHeader
+ innerV1Header, err = carv1.ReadHeader(f, options.MaxAllowedHeaderSize)
+ if err != nil {
+ return err
+ }
+ if innerV1Header.Version != 1 {
+ err = fmt.Errorf("invalid data payload header: expected version 1, got %d", innerV1Header.Version)
+ }
+ var readSoFar int64
+ readSoFar, err = f.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return err
+ }
+ currentSize = readSoFar - newHeaderOffset
+ default:
+ err = fmt.Errorf("invalid car version: %d", header.Version)
+ return err
+ }
+
+ newHeader := &carv1.CarHeader{
+ Roots: roots,
+ Version: 1,
+ }
+ // Serialize the new header straight up instead of using carv1.HeaderSize.
+ // Because, carv1.HeaderSize serialises it to calculate size anyway.
+ // By serializing straight up we get the replacement bytes and size.
+ // Otherwise, we end up serializing the new header twice:
+ // once through carv1.HeaderSize, and
+ // once to write it out.
+ var buf bytes.Buffer
+ if err = carv1.WriteHeader(newHeader, &buf); err != nil {
+ return err
+ }
+ // Assert the header sizes are consistent.
+ newSize := int64(buf.Len())
+ if currentSize != newSize {
+ return fmt.Errorf("current header size (%d) must match replacement header size (%d)", currentSize, newSize)
+ }
+ // Seek to the offset at which the new header should be written.
+ if _, err = f.Seek(newHeaderOffset, io.SeekStart); err != nil {
+ return err
+ }
+ _, err = f.Write(buf.Bytes())
+ return err
+}
diff --git a/ipld/car/v2/writer_test.go b/ipld/car/v2/writer_test.go
new file mode 100644
index 000000000..517a820db
--- /dev/null
+++ b/ipld/car/v2/writer_test.go
@@ -0,0 +1,282 @@
+package car
+
+import (
+ "context"
+ "io"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/ipfs/boxo/ipld/car/v2/index"
+ "github.com/ipfs/boxo/ipld/car/v2/internal/carv1"
+ "github.com/stretchr/testify/require"
+
+ "github.com/ipfs/boxo/ipld/merkledag"
+ dstest "github.com/ipfs/boxo/ipld/merkledag/test"
+ "github.com/ipfs/go-cid"
+ format "github.com/ipfs/go-ipld-format"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestWrapV1(t *testing.T) {
+ // Produce a CARv1 file to test wrapping with.
+ dagSvc := dstest.Mock()
+ src := filepath.Join(t.TempDir(), "unwrapped-test-v1.car")
+ sf, err := os.Create(src)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, sf.Close()) })
+ require.NoError(t, carv1.WriteCar(context.Background(), dagSvc, generateRootCid(t, dagSvc), sf))
+
+ // Wrap the test CARv1 file
+ dest := filepath.Join(t.TempDir(), "wrapped-test-v1.car")
+ df, err := os.Create(dest)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, df.Close()) })
+ _, err = sf.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ require.NoError(t, WrapV1(sf, df))
+
+ // Assert wrapped file is valid CARv2 with CARv1 data payload matching the original CARv1 file.
+ subject, err := OpenReader(dest)
+ t.Cleanup(func() { require.NoError(t, subject.Close()) })
+ require.NoError(t, err)
+
+ // Assert CARv1 data payloads are identical.
+ _, err = sf.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ wantPayload, err := io.ReadAll(sf)
+ require.NoError(t, err)
+ dr, err := subject.DataReader()
+ require.NoError(t, err)
+ gotPayload, err := io.ReadAll(dr)
+ require.NoError(t, err)
+ require.Equal(t, wantPayload, gotPayload)
+
+ // Assert embedded index in CARv2 is same as index generated from the original CARv1.
+ wantIdx, err := GenerateIndexFromFile(src)
+ require.NoError(t, err)
+ ir, err := subject.IndexReader()
+ require.NoError(t, err)
+ gotIdx, err := index.ReadFrom(ir)
+ require.NoError(t, err)
+ require.Equal(t, wantIdx, gotIdx)
+}
+
+func TestExtractV1(t *testing.T) {
+ // Produce a CARv1 file to test.
+ dagSvc := dstest.Mock()
+ v1Src := filepath.Join(t.TempDir(), "original-test-v1.car")
+ v1f, err := os.Create(v1Src)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, v1f.Close()) })
+ require.NoError(t, carv1.WriteCar(context.Background(), dagSvc, generateRootCid(t, dagSvc), v1f))
+ _, err = v1f.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ wantV1, err := io.ReadAll(v1f)
+ require.NoError(t, err)
+
+ // Wrap the produced CARv1 into a CARv2 to use for testing.
+ v2path := filepath.Join(t.TempDir(), "wrapped-for-extract-test-v2.car")
+ require.NoError(t, WrapV1File(v1Src, v2path))
+
+ // Assert extract from CARv2 file is as expected.
+ dstPath := filepath.Join(t.TempDir(), "extract-file-test-v1.car")
+ require.NoError(t, ExtractV1File(v2path, dstPath))
+ gotFromFile, err := os.ReadFile(dstPath)
+ require.NoError(t, err)
+ require.Equal(t, wantV1, gotFromFile)
+
+ // Assert extract from CARv2 file in-place is as expected
+ require.NoError(t, ExtractV1File(v2path, v2path))
+ gotFromInPlaceFile, err := os.ReadFile(v2path)
+ require.NoError(t, err)
+ require.Equal(t, wantV1, gotFromInPlaceFile)
+}
+
+func TestExtractV1WithUnknownVersionIsError(t *testing.T) {
+ dstPath := filepath.Join(t.TempDir(), "extract-dst-file-test-v42.car")
+ err := ExtractV1File("testdata/sample-rootless-v42.car", dstPath)
+ require.EqualError(t, err, "source version must be 2; got: 42")
+}
+
+func TestExtractV1FromACarV1IsError(t *testing.T) {
+ dstPath := filepath.Join(t.TempDir(), "extract-dst-file-test-v1.car")
+ err := ExtractV1File("testdata/sample-v1.car", dstPath)
+ require.Equal(t, ErrAlreadyV1, err)
+}
+
+func generateRootCid(t *testing.T, adder format.NodeAdder) []cid.Cid {
+ // TODO convert this into a utility testing lib that takes an rng and generates a random DAG with some threshold for depth/breadth.
+ this := merkledag.NewRawNode([]byte("fish"))
+ that := merkledag.NewRawNode([]byte("lobster"))
+ other := merkledag.NewRawNode([]byte("🌊"))
+
+ one := &merkledag.ProtoNode{}
+ assertAddNodeLink(t, one, this, "fishmonger")
+
+ another := &merkledag.ProtoNode{}
+ assertAddNodeLink(t, another, one, "barreleye")
+ assertAddNodeLink(t, another, that, "🐡")
+
+ andAnother := &merkledag.ProtoNode{}
+ assertAddNodeLink(t, andAnother, another, "🍤")
+
+ assertAddNodes(t, adder, this, that, other, one, another, andAnother)
+ return []cid.Cid{andAnother.Cid()}
+}
+
+func assertAddNodeLink(t *testing.T, pn *merkledag.ProtoNode, fn format.Node, name string) {
+ assert.NoError(t, pn.AddNodeLink(name, fn))
+}
+
+func assertAddNodes(t *testing.T, adder format.NodeAdder, nds ...format.Node) {
+ for _, nd := range nds {
+ assert.NoError(t, adder.Add(context.Background(), nd))
+ }
+}
+
+func TestReplaceRootsInFile(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ roots []cid.Cid
+ wantErrMsg string
+ }{
+ {
+ name: "CorruptPragmaIsRejected",
+ path: "testdata/sample-corrupt-pragma.car",
+ wantErrMsg: "unexpected EOF",
+ },
+ {
+ name: "CARv42IsRejected",
+ path: "testdata/sample-rootless-v42.car",
+ wantErrMsg: "invalid car version: 42",
+ },
+ {
+ name: "CARv1RootsOfDifferentSizeAreNotReplaced",
+ path: "testdata/sample-v1.car",
+ wantErrMsg: "current header size (61) must match replacement header size (18)",
+ },
+ {
+ name: "CARv2RootsOfDifferentSizeAreNotReplaced",
+ path: "testdata/sample-wrapped-v2.car",
+ wantErrMsg: "current header size (61) must match replacement header size (18)",
+ },
+ {
+ name: "CARv1NonEmptyRootsOfDifferentSizeAreNotReplaced",
+ path: "testdata/sample-v1.car",
+ roots: []cid.Cid{requireDecodedCid(t, "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")},
+ wantErrMsg: "current header size (61) must match replacement header size (57)",
+ },
+ {
+ name: "CARv1ZeroLenNonEmptyRootsOfDifferentSizeAreNotReplaced",
+ path: "testdata/sample-v1-with-zero-len-section.car",
+ roots: []cid.Cid{merkledag.NewRawNode([]byte("fish")).Cid()},
+ wantErrMsg: "current header size (61) must match replacement header size (59)",
+ },
+ {
+ name: "CARv2NonEmptyRootsOfDifferentSizeAreNotReplaced",
+ path: "testdata/sample-wrapped-v2.car",
+ roots: []cid.Cid{merkledag.NewRawNode([]byte("fish")).Cid()},
+ wantErrMsg: "current header size (61) must match replacement header size (59)",
+ },
+ {
+ name: "CARv2IndexlessNonEmptyRootsOfDifferentSizeAreNotReplaced",
+ path: "testdata/sample-v2-indexless.car",
+ roots: []cid.Cid{merkledag.NewRawNode([]byte("fish")).Cid()},
+ wantErrMsg: "current header size (61) must match replacement header size (59)",
+ },
+ {
+ name: "CARv1SameSizeRootsAreReplaced",
+ path: "testdata/sample-v1.car",
+ roots: []cid.Cid{requireDecodedCid(t, "bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5od")},
+ },
+ {
+ name: "CARv2SameSizeRootsAreReplaced",
+ path: "testdata/sample-wrapped-v2.car",
+ roots: []cid.Cid{requireDecodedCid(t, "bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oi")},
+ },
+ {
+ name: "CARv2IndexlessSameSizeRootsAreReplaced",
+ path: "testdata/sample-v2-indexless.car",
+ roots: []cid.Cid{requireDecodedCid(t, "bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oi")},
+ },
+ {
+ name: "CARv1ZeroLenSameSizeRootsAreReplaced",
+ path: "testdata/sample-v1-with-zero-len-section.car",
+ roots: []cid.Cid{requireDecodedCid(t, "bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5o5")},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Make a copy of input files to preserve original for comparison.
+ // This also avoids modification files in testdata.
+ tmpCopy := requireTmpCopy(t, tt.path)
+ err := ReplaceRootsInFile(tmpCopy, tt.roots)
+ if tt.wantErrMsg != "" {
+ require.EqualError(t, err, tt.wantErrMsg)
+ return
+ }
+ require.NoError(t, err)
+
+ original, err := os.Open(tt.path)
+ require.NoError(t, err)
+ defer func() { require.NoError(t, original.Close()) }()
+
+ target, err := os.Open(tmpCopy)
+ require.NoError(t, err)
+ defer func() { require.NoError(t, target.Close()) }()
+
+ // Assert file size has not changed.
+ wantStat, err := original.Stat()
+ require.NoError(t, err)
+ gotStat, err := target.Stat()
+ require.NoError(t, err)
+ require.Equal(t, wantStat.Size(), gotStat.Size())
+
+ wantReader, err := NewBlockReader(original, ZeroLengthSectionAsEOF(true))
+ require.NoError(t, err)
+ gotReader, err := NewBlockReader(target, ZeroLengthSectionAsEOF(true))
+ require.NoError(t, err)
+
+ // Assert roots are replaced.
+ require.Equal(t, tt.roots, gotReader.Roots)
+
+ // Assert data blocks are identical.
+ for {
+ wantNext, wantErr := wantReader.Next()
+ gotNext, gotErr := gotReader.Next()
+ if wantErr == io.EOF {
+ require.Equal(t, io.EOF, gotErr)
+ break
+ }
+ require.NoError(t, wantErr)
+ require.NoError(t, gotErr)
+ require.Equal(t, wantNext, gotNext)
+ }
+ })
+ }
+}
+
+func requireDecodedCid(t *testing.T, s string) cid.Cid {
+ decoded, err := cid.Decode(s)
+ require.NoError(t, err)
+ return decoded
+}
+
+func requireTmpCopy(t *testing.T, src string) string {
+ srcF, err := os.Open(src)
+ require.NoError(t, err)
+ defer func() { require.NoError(t, srcF.Close()) }()
+ stats, err := srcF.Stat()
+ require.NoError(t, err)
+
+ dst := filepath.Join(t.TempDir(), stats.Name())
+ dstF, err := os.Create(dst)
+ require.NoError(t, err)
+ defer func() { require.NoError(t, dstF.Close()) }()
+
+ _, err = io.Copy(dstF, srcF)
+ require.NoError(t, err)
+ return dst
+}
diff --git a/ipld/merkledag/coding.go b/ipld/merkledag/coding.go
new file mode 100644
index 000000000..03d7ed14c
--- /dev/null
+++ b/ipld/merkledag/coding.go
@@ -0,0 +1,218 @@
+package merkledag
+
+import (
+ "fmt"
+ "sort"
+ "strings"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ pb "github.com/ipfs/boxo/ipld/merkledag/pb"
+ cid "github.com/ipfs/go-cid"
+ format "github.com/ipfs/go-ipld-format"
+ dagpb "github.com/ipld/go-codec-dagpb"
+ ipld "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/fluent/qp"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+)
+
+// Make sure the user doesn't upgrade this file.
+// We need to check *here* as well as inside the `pb` package *just* in case the
+// user replaces *all* go files in that package.
+const _ = pb.DoNotUpgradeFileEverItWillChangeYourHashes
+
+// for now, we use a PBNode intermediate thing.
+// because native go objects are nice.
+
+// pbLinkSlice is a slice of pb.PBLink, similar to LinkSlice but for sorting the
+// PB form
+type pbLinkSlice []*pb.PBLink
+
+func (pbls pbLinkSlice) Len() int { return len(pbls) }
+func (pbls pbLinkSlice) Swap(a, b int) { pbls[a], pbls[b] = pbls[b], pbls[a] }
+func (pbls pbLinkSlice) Less(a, b int) bool { return *pbls[a].Name < *pbls[b].Name }
+
+// unmarshal decodes raw data into a *Node instance.
+// The conversion uses an intermediate PBNode.
+func unmarshal(encodedBytes []byte) (*ProtoNode, error) {
+ nb := dagpb.Type.PBNode.NewBuilder()
+ if err := dagpb.DecodeBytes(nb, encodedBytes); err != nil {
+ return nil, err
+ }
+ nd := nb.Build()
+ return fromImmutableNode(&immutableProtoNode{encodedBytes, nd.(dagpb.PBNode)}), nil
+}
+
+func fromImmutableNode(encoded *immutableProtoNode) *ProtoNode {
+ n := new(ProtoNode)
+ n.encoded = encoded
+ if n.encoded.PBNode.Data.Exists() {
+ n.data = n.encoded.PBNode.Data.Must().Bytes()
+ }
+ numLinks := n.encoded.PBNode.Links.Length()
+ // links may not be sorted after deserialization, but we don't change
+ // them until we mutate this node since we're representing the current,
+ // as-serialized state
+ n.links = make([]*format.Link, numLinks)
+ linkAllocs := make([]format.Link, numLinks)
+ for i := int64(0); i < numLinks; i++ {
+ next := n.encoded.PBNode.Links.Lookup(i)
+ name := ""
+ if next.FieldName().Exists() {
+ name = next.FieldName().Must().String()
+ }
+ c := next.FieldHash().Link().(cidlink.Link).Cid
+ size := uint64(0)
+ if next.FieldTsize().Exists() {
+ size = uint64(next.FieldTsize().Must().Int())
+ }
+ link := &linkAllocs[i]
+ link.Name = name
+ link.Size = size
+ link.Cid = c
+ n.links[i] = link
+ }
+ // we don't set n.linksDirty because the order of the links list from
+ // serialized form needs to be stable, until we start mutating the ProtoNode
+ return n
+}
+func (n *ProtoNode) marshalImmutable() (*immutableProtoNode, error) {
+ links := n.Links()
+ nd, err := qp.BuildMap(dagpb.Type.PBNode, 2, func(ma ipld.MapAssembler) {
+ qp.MapEntry(ma, "Links", qp.List(int64(len(links)), func(la ipld.ListAssembler) {
+ for _, link := range links {
+ // it shouldn't be possible to get here with an undefined CID, but in
+ // case it is we're going to drop this link from the encoded form
+ // entirely
+ if link.Cid.Defined() {
+ qp.ListEntry(la, qp.Map(3, func(ma ipld.MapAssembler) {
+ qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: link.Cid}))
+ qp.MapEntry(ma, "Name", qp.String(link.Name))
+ sz := int64(link.Size)
+ if sz < 0 { // overflow, >MaxInt64 is almost certainly an error
+ sz = 0
+ }
+ qp.MapEntry(ma, "Tsize", qp.Int(sz))
+ }))
+ }
+ }
+ }))
+ if n.data != nil {
+ qp.MapEntry(ma, "Data", qp.Bytes(n.data))
+ }
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ // 1KiB can be allocated on the stack, and covers most small nodes
+ // without having to grow the buffer and cause allocations.
+ enc := make([]byte, 0, 1024)
+
+ enc, err = dagpb.AppendEncode(enc, nd)
+ if err != nil {
+ return nil, err
+ }
+ return &immutableProtoNode{enc, nd.(dagpb.PBNode)}, nil
+}
+
+// Marshal encodes a *Node instance into a new byte slice.
+// The conversion uses an intermediate PBNode.
+func (n *ProtoNode) Marshal() ([]byte, error) {
+ enc, err := n.marshalImmutable()
+ if err != nil {
+ return nil, err
+ }
+ return enc.encoded, nil
+}
+
+// GetPBNode converts *ProtoNode into it's protocol buffer variant.
+// If you plan on mutating the data of the original node, it is recommended
+// that you call ProtoNode.Copy() before calling ProtoNode.GetPBNode()
+func (n *ProtoNode) GetPBNode() *pb.PBNode {
+ pbn := &pb.PBNode{}
+ if len(n.links) > 0 {
+ pbn.Links = make([]*pb.PBLink, len(n.links))
+ }
+
+ for i, l := range n.links {
+ pbn.Links[i] = &pb.PBLink{}
+ pbn.Links[i].Name = &l.Name
+ pbn.Links[i].Tsize = &l.Size
+ if l.Cid.Defined() {
+ pbn.Links[i].Hash = l.Cid.Bytes()
+ }
+ }
+
+ // Ensure links are sorted prior to encode, regardless of `linksDirty`. They
+ // may not have come sorted if we deserialized a badly encoded form that
+ // didn't have links already sorted.
+ sort.Stable(pbLinkSlice(pbn.Links))
+
+ if len(n.data) > 0 {
+ pbn.Data = n.data
+ }
+ return pbn
+}
+
+// EncodeProtobuf returns the encoded raw data version of a Node instance.
+// It may use a cached encoded version, unless the force flag is given.
+func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) {
+ if n.encoded == nil || n.linksDirty || force {
+ if n.linksDirty {
+ // there was a mutation involving links, make sure we sort before we build
+ // and cache a `Node` form that captures the current state
+ sort.Stable(LinkSlice(n.links))
+ n.linksDirty = false
+ }
+ n.cached = cid.Undef
+ var err error
+ n.encoded, err = n.marshalImmutable()
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ if !n.cached.Defined() {
+ c, err := n.CidBuilder().Sum(n.encoded.encoded)
+ if err != nil {
+ return nil, err
+ }
+
+ n.cached = c
+ }
+
+ return n.encoded.encoded, nil
+}
+
+// DecodeProtobuf decodes raw data and returns a new Node instance.
+func DecodeProtobuf(encoded []byte) (*ProtoNode, error) {
+ n, err := unmarshal(encoded)
+ if err != nil {
+ return nil, fmt.Errorf("incorrectly formatted merkledag node: %s", err)
+ }
+ return n, nil
+}
+
+// DecodeProtobufBlock is a block decoder for protobuf IPLD nodes conforming to
+// node.DecodeBlockFunc
+func DecodeProtobufBlock(b blocks.Block) (format.Node, error) {
+ c := b.Cid()
+ if c.Type() != cid.DagProtobuf {
+ return nil, fmt.Errorf("this function can only decode protobuf nodes")
+ }
+
+ decnd, err := DecodeProtobuf(b.RawData())
+ if err != nil {
+ if strings.Contains(err.Error(), "Unmarshal failed") {
+ return nil, fmt.Errorf("the block referred to by '%s' was not a valid merkledag node", c)
+ }
+ return nil, fmt.Errorf("failed to decode Protocol Buffers: %v", err)
+ }
+
+ decnd.cached = c
+ decnd.builder = c.Prefix()
+ return decnd, nil
+}
+
+// Type assertion
+var _ format.DecodeBlockFunc = DecodeProtobufBlock
diff --git a/ipld/merkledag/coding_test.go b/ipld/merkledag/coding_test.go
new file mode 100644
index 000000000..a213a912c
--- /dev/null
+++ b/ipld/merkledag/coding_test.go
@@ -0,0 +1,52 @@
+package merkledag_test
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+
+ "github.com/ipfs/boxo/ipld/merkledag"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+var benchInput []byte
+
+func init() {
+ someData := bytes.Repeat([]byte("some plaintext data\n"), 10)
+ // make a test CID -- doesn't matter just to add as a link
+ someCid, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4})
+
+ node := &merkledag.ProtoNode{}
+ node.SetData(someData)
+ for i := 0; i < 10; i++ {
+ node.AddRawLink(fmt.Sprintf("%d", i), &ipld.Link{
+ Size: 10,
+ Cid: someCid,
+ })
+ }
+
+ enc, err := node.EncodeProtobuf(true)
+ if err != nil {
+ panic(err)
+ }
+ benchInput = enc
+}
+
+func BenchmarkRoundtrip(b *testing.B) {
+ b.ReportAllocs()
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ node, err := merkledag.DecodeProtobuf(benchInput)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ enc, err := node.EncodeProtobuf(true)
+ if err != nil {
+ b.Fatal(err)
+ }
+ _ = enc
+ }
+ })
+}
diff --git a/ipld/merkledag/dagutils/diff.go b/ipld/merkledag/dagutils/diff.go
new file mode 100644
index 000000000..49a3ae627
--- /dev/null
+++ b/ipld/merkledag/dagutils/diff.go
@@ -0,0 +1,204 @@
+package dagutils
+
+import (
+ "context"
+ "fmt"
+ "path"
+
+ "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+)
+
+// ChangeType denotes type of change in Change
+type ChangeType int
+
+// These constants define the changes that can be applied to a DAG.
+const (
+ Add ChangeType = iota
+ Remove
+ Mod
+)
+
+// Change represents a change to a DAG and contains a reference to the old and
+// new CIDs.
+type Change struct {
+ Type ChangeType
+ Path string
+ Before cid.Cid
+ After cid.Cid
+}
+
+// String prints a human-friendly line about a change.
+func (c *Change) String() string {
+ switch c.Type {
+ case Add:
+ return fmt.Sprintf("Added %s at %s", c.After.String(), c.Path)
+ case Remove:
+ return fmt.Sprintf("Removed %s from %s", c.Before.String(), c.Path)
+ case Mod:
+ return fmt.Sprintf("Changed %s to %s at %s", c.Before.String(), c.After.String(), c.Path)
+ default:
+ panic("nope")
+ }
+}
+
+// ApplyChange applies the requested changes to the given node in the given dag.
+func ApplyChange(ctx context.Context, ds ipld.DAGService, nd *dag.ProtoNode, cs []*Change) (*dag.ProtoNode, error) {
+ e := NewDagEditor(nd, ds)
+ for _, c := range cs {
+ switch c.Type {
+ case Add:
+ child, err := ds.Get(ctx, c.After)
+ if err != nil {
+ return nil, err
+ }
+
+ childpb, ok := child.(*dag.ProtoNode)
+ if !ok {
+ return nil, dag.ErrNotProtobuf
+ }
+
+ err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ case Remove:
+ err := e.RmLink(ctx, c.Path)
+ if err != nil {
+ return nil, err
+ }
+
+ case Mod:
+ err := e.RmLink(ctx, c.Path)
+ if err != nil {
+ return nil, err
+ }
+ child, err := ds.Get(ctx, c.After)
+ if err != nil {
+ return nil, err
+ }
+
+ childpb, ok := child.(*dag.ProtoNode)
+ if !ok {
+ return nil, dag.ErrNotProtobuf
+ }
+
+ err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil)
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ return e.Finalize(ctx, ds)
+}
+
+// Diff returns a set of changes that transform node 'a' into node 'b'.
+// It only traverses links in the following cases:
+// 1. two node's links number are greater than 0.
+// 2. both of two nodes are ProtoNode.
+// Otherwise, it compares the cid and emits a Mod change object.
+func Diff(ctx context.Context, ds ipld.DAGService, a, b ipld.Node) ([]*Change, error) {
+ if a.Cid() == b.Cid() {
+ return []*Change{}, nil
+ }
+
+ cleanA, okA := a.Copy().(*dag.ProtoNode)
+ cleanB, okB := b.Copy().(*dag.ProtoNode)
+
+ linksA := a.Links()
+ linksB := b.Links()
+
+ if !okA || !okB || (len(linksA) == 0 && len(linksB) == 0) {
+ return []*Change{{Type: Mod, Before: a.Cid(), After: b.Cid()}}, nil
+ }
+
+ var out []*Change
+ for _, linkA := range linksA {
+ linkB, _, err := b.ResolveLink([]string{linkA.Name})
+ if err != nil {
+ continue
+ }
+
+ cleanA.RemoveNodeLink(linkA.Name)
+ cleanB.RemoveNodeLink(linkA.Name)
+
+ if linkA.Cid == linkB.Cid {
+ continue
+ }
+
+ nodeA, err := linkA.GetNode(ctx, ds)
+ if err != nil {
+ return nil, err
+ }
+
+ nodeB, err := linkB.GetNode(ctx, ds)
+ if err != nil {
+ return nil, err
+ }
+
+ sub, err := Diff(ctx, ds, nodeA, nodeB)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, c := range sub {
+ c.Path = path.Join(linkA.Name, c.Path)
+ }
+
+ out = append(out, sub...)
+ }
+
+ for _, l := range cleanA.Links() {
+ out = append(out, &Change{Type: Remove, Path: l.Name, Before: l.Cid})
+ }
+
+ for _, l := range cleanB.Links() {
+ out = append(out, &Change{Type: Add, Path: l.Name, After: l.Cid})
+ }
+
+ return out, nil
+}
+
+// Conflict represents two incompatible changes and is returned by MergeDiffs().
+type Conflict struct {
+ A *Change
+ B *Change
+}
+
+// MergeDiffs takes two slice of changes and adds them to a single slice.
+// When a Change from b happens to the same path of an existing change in a,
+// a conflict is created and b is not added to the merged slice.
+// A slice of Conflicts is returned and contains pointers to the
+// Changes involved (which share the same path).
+func MergeDiffs(a, b []*Change) ([]*Change, []Conflict) {
+ paths := make(map[string]*Change)
+ for _, c := range b {
+ paths[c.Path] = c
+ }
+
+ var changes []*Change
+ var conflicts []Conflict
+
+ // NOTE: we avoid iterating over maps here to ensure iteration order is determistic. We
+ // include changes from a first, then b.
+ for _, changeA := range a {
+ if changeB, ok := paths[changeA.Path]; ok {
+ conflicts = append(conflicts, Conflict{changeA, changeB})
+ } else {
+ changes = append(changes, changeA)
+ }
+ delete(paths, changeA.Path)
+ }
+
+ for _, c := range b {
+ if _, ok := paths[c.Path]; ok {
+ changes = append(changes, c)
+ }
+ }
+
+ return changes, conflicts
+}
diff --git a/ipld/merkledag/dagutils/diff_test.go b/ipld/merkledag/dagutils/diff_test.go
new file mode 100644
index 000000000..f9e28b149
--- /dev/null
+++ b/ipld/merkledag/dagutils/diff_test.go
@@ -0,0 +1,127 @@
+package dagutils
+
+import (
+ "context"
+ "testing"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+func TestMergeDiffs(t *testing.T) {
+ node1 := dag.NodeWithData([]byte("one"))
+ node2 := dag.NodeWithData([]byte("two"))
+ node3 := dag.NodeWithData([]byte("three"))
+ node4 := dag.NodeWithData([]byte("four"))
+
+ changesA := []*Change{
+ {Add, "one", cid.Cid{}, node1.Cid()},
+ {Remove, "two", node2.Cid(), cid.Cid{}},
+ {Mod, "three", node3.Cid(), node4.Cid()},
+ }
+
+ changesB := []*Change{
+ {Mod, "two", node2.Cid(), node3.Cid()},
+ {Add, "four", cid.Cid{}, node4.Cid()},
+ }
+
+ changes, conflicts := MergeDiffs(changesA, changesB)
+ if len(changes) != 3 {
+ t.Fatal("unexpected merge changes")
+ }
+
+ expect := []*Change{
+ changesA[0],
+ changesA[2],
+ changesB[1],
+ }
+
+ for i, change := range changes {
+ if change.Type != expect[i].Type {
+ t.Error("unexpected diff change type")
+ }
+
+ if change.Path != expect[i].Path {
+ t.Error("unexpected diff change path")
+ }
+
+ if change.Before != expect[i].Before {
+ t.Error("unexpected diff change before")
+ }
+
+ if change.After != expect[i].After {
+ t.Error("unexpected diff change before")
+ }
+ }
+
+ if len(conflicts) != 1 {
+ t.Fatal("unexpected merge conflicts")
+ }
+
+ if conflicts[0].A != changesA[1] {
+ t.Error("unexpected merge conflict a")
+ }
+
+ if conflicts[0].B != changesB[0] {
+ t.Error("unexpected merge conflict b")
+ }
+}
+
+func TestDiff(t *testing.T) {
+ ctx := context.Background()
+ ds := mdtest.Mock()
+
+ rootA := &dag.ProtoNode{}
+ rootB := &dag.ProtoNode{}
+
+ child1 := dag.NodeWithData([]byte("one"))
+ child2 := dag.NodeWithData([]byte("two"))
+ child3 := dag.NodeWithData([]byte("three"))
+ child4 := dag.NodeWithData([]byte("four"))
+
+ rootA.AddNodeLink("one", child1)
+ rootA.AddNodeLink("two", child2)
+
+ rootB.AddNodeLink("one", child3)
+ rootB.AddNodeLink("four", child4)
+
+ nodes := []ipld.Node{child1, child2, child3, child4, rootA, rootB}
+ if err := ds.AddMany(ctx, nodes); err != nil {
+ t.Fatal("failed to add nodes")
+ }
+
+ changes, err := Diff(ctx, ds, rootA, rootB)
+ if err != nil {
+ t.Fatal("unexpected diff error")
+ }
+
+ if len(changes) != 3 {
+ t.Fatal("unexpected diff changes")
+ }
+
+ expect := []Change{
+ {Mod, "one", child1.Cid(), child3.Cid()},
+ {Remove, "two", child2.Cid(), cid.Cid{}},
+ {Add, "four", cid.Cid{}, child4.Cid()},
+ }
+
+ for i, change := range changes {
+ if change.Type != expect[i].Type {
+ t.Error("unexpected diff change type")
+ }
+
+ if change.Path != expect[i].Path {
+ t.Error("unexpected diff change path")
+ }
+
+ if change.Before != expect[i].Before {
+ t.Error("unexpected diff change before")
+ }
+
+ if change.After != expect[i].After {
+ t.Error("unexpected diff change before")
+ }
+ }
+}
diff --git a/ipld/merkledag/dagutils/diffenum.go b/ipld/merkledag/dagutils/diffenum.go
new file mode 100644
index 000000000..75f86b952
--- /dev/null
+++ b/ipld/merkledag/dagutils/diffenum.go
@@ -0,0 +1,99 @@
+package dagutils
+
+import (
+ "context"
+ "fmt"
+
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+)
+
+// DiffEnumerate fetches every object in the graph pointed to by 'to' that is
+// not in 'from'. This can be used to more efficiently fetch a graph if you can
+// guarantee you already have the entirety of 'from'
+func DiffEnumerate(ctx context.Context, dserv ipld.NodeGetter, from, to cid.Cid) error {
+ fnd, err := dserv.Get(ctx, from)
+ if err != nil {
+ return fmt.Errorf("get %s: %s", from, err)
+ }
+
+ tnd, err := dserv.Get(ctx, to)
+ if err != nil {
+ return fmt.Errorf("get %s: %s", to, err)
+ }
+
+ diff := getLinkDiff(fnd, tnd)
+
+ sset := cid.NewSet()
+ for _, c := range diff {
+ // Since we're already assuming we have everything in the 'from' graph,
+ // add all those cids to our 'already seen' set to avoid potentially
+ // enumerating them later
+ if c.bef.Defined() {
+ sset.Add(c.bef)
+ }
+ }
+ for _, c := range diff {
+ if !c.bef.Defined() {
+ if sset.Has(c.aft) {
+ continue
+ }
+ err := mdag.Walk(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit, mdag.Concurrent())
+ if err != nil {
+ return err
+ }
+ } else {
+ err := DiffEnumerate(ctx, dserv, c.bef, c.aft)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+// if both bef and aft are not nil, then that signifies bef was replaces with aft.
+// if bef is nil and aft is not, that means aft was newly added
+// if aft is nil and bef is not, that means bef was deleted
+type diffpair struct {
+ bef, aft cid.Cid
+}
+
+// getLinkDiff returns a changeset between nodes 'a' and 'b'. Currently does
+// not log deletions as our usecase doesnt call for this.
+func getLinkDiff(a, b ipld.Node) []diffpair {
+ ina := make(map[string]*ipld.Link)
+ inb := make(map[string]*ipld.Link)
+ var aonly []cid.Cid
+ for _, l := range b.Links() {
+ inb[l.Cid.KeyString()] = l
+ }
+ for _, l := range a.Links() {
+ var key = l.Cid.KeyString()
+ ina[key] = l
+ if inb[key] == nil {
+ aonly = append(aonly, l.Cid)
+ }
+ }
+
+ var out []diffpair
+ var aindex int
+
+ for _, l := range b.Links() {
+ if ina[l.Cid.KeyString()] != nil {
+ continue
+ }
+
+ if aindex < len(aonly) {
+ out = append(out, diffpair{bef: aonly[aindex], aft: l.Cid})
+ aindex++
+ } else {
+ out = append(out, diffpair{aft: l.Cid})
+ continue
+ }
+ }
+ return out
+}
diff --git a/ipld/merkledag/dagutils/diffenum_test.go b/ipld/merkledag/dagutils/diffenum_test.go
new file mode 100644
index 000000000..b96e6f759
--- /dev/null
+++ b/ipld/merkledag/dagutils/diffenum_test.go
@@ -0,0 +1,249 @@
+package dagutils
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+)
+
+func buildNode(name string, desc map[string]ndesc, out map[string]ipld.Node) ipld.Node {
+ this := desc[name]
+ nd := new(dag.ProtoNode)
+ nd.SetData([]byte(name))
+ for k, v := range this {
+ child, ok := out[v]
+ if !ok {
+ child = buildNode(v, desc, out)
+ out[v] = child
+ }
+
+ if err := nd.AddNodeLink(k, child); err != nil {
+ panic(err)
+ }
+ }
+
+ return nd
+}
+
+type ndesc map[string]string
+
+func mkGraph(desc map[string]ndesc) map[string]ipld.Node {
+ out := make(map[string]ipld.Node)
+ for name := range desc {
+ if _, ok := out[name]; ok {
+ continue
+ }
+
+ out[name] = buildNode(name, desc, out)
+ }
+ return out
+}
+
+var tg1 = map[string]ndesc{
+ "a1": {
+ "foo": "b",
+ },
+ "b": {},
+ "a2": {
+ "foo": "b",
+ "bar": "c",
+ },
+ "c": {},
+}
+
+var tg2 = map[string]ndesc{
+ "a1": {
+ "foo": "b",
+ },
+ "b": {},
+ "a2": {
+ "foo": "b",
+ "bar": "c",
+ },
+ "c": {"baz": "d"},
+ "d": {},
+}
+
+var tg3 = map[string]ndesc{
+ "a1": {
+ "foo": "b",
+ "bar": "c",
+ },
+ "b": {},
+ "a2": {
+ "foo": "b",
+ "bar": "d",
+ },
+ "c": {},
+ "d": {},
+}
+
+var tg4 = map[string]ndesc{
+ "a1": {
+ "key1": "b",
+ "key2": "c",
+ },
+ "a2": {
+ "key1": "b",
+ "key2": "d",
+ },
+}
+
+var tg5 = map[string]ndesc{
+ "a1": {
+ "key1": "a",
+ "key2": "b",
+ },
+ "a2": {
+ "key1": "c",
+ "key2": "d",
+ },
+}
+
+func TestNameMatching(t *testing.T) {
+ nds := mkGraph(tg4)
+
+ diff := getLinkDiff(nds["a1"], nds["a2"])
+ if len(diff) != 1 {
+ t.Fatal(fmt.Errorf("node diff didn't match by name"))
+ }
+}
+
+func TestNameMatching2(t *testing.T) {
+ nds := mkGraph(tg5)
+
+ diff := getLinkDiff(nds["a1"], nds["a2"])
+ if len(diff) != 2 {
+ t.Fatal(fmt.Errorf("incorrect number of link diff elements"))
+ }
+ if !(diff[0].bef.Equals(nds["a1"].Links()[0].Cid) && diff[0].aft.Equals(nds["a2"].Links()[0].Cid)) {
+ t.Fatal(fmt.Errorf("node diff didn't match by name"))
+ }
+}
+
+func TestDiffEnumBasic(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nds := mkGraph(tg1)
+
+ ds := mdtest.Mock()
+ lgds := &getLogger{ds: ds}
+
+ for _, nd := range nds {
+ err := ds.Add(ctx, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()})
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+type getLogger struct {
+ ds ipld.NodeGetter
+ log []cid.Cid
+}
+
+func (gl *getLogger) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) {
+ nd, err := gl.ds.Get(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+ gl.log = append(gl.log, c)
+ return nd, nil
+}
+
+func (gl *getLogger) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption {
+ outCh := make(chan *ipld.NodeOption, len(cids))
+ nds := gl.ds.GetMany(ctx, cids)
+ for no := range nds {
+ if no.Err == nil {
+ gl.log = append(gl.log, no.Node.Cid())
+ }
+ select {
+ case outCh <- no:
+ default:
+ panic("too many responses")
+ }
+ }
+ return nds
+}
+
+func assertCidList(a, b []cid.Cid) error {
+ if len(a) != len(b) {
+ return fmt.Errorf("got different number of cids than expected")
+ }
+ for i, c := range a {
+ if !c.Equals(b[i]) {
+ return fmt.Errorf("expected %s, got %s", c, b[i])
+ }
+ }
+ return nil
+}
+
+func TestDiffEnumFail(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nds := mkGraph(tg2)
+
+ ds := mdtest.Mock()
+ lgds := &getLogger{ds: ds}
+
+ for _, s := range []string{"a1", "a2", "b", "c"} {
+ err := ds.Add(ctx, nds[s])
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid())
+ if !ipld.IsNotFound(err) {
+ t.Fatal("expected err not found")
+ }
+
+ err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+}
+
+func TestDiffEnumRecurse(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nds := mkGraph(tg3)
+
+ ds := mdtest.Mock()
+ lgds := &getLogger{ds: ds}
+
+ for _, s := range []string{"a1", "a2", "b", "c", "d"} {
+ err := ds.Add(ctx, nds[s])
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid(), nds["d"].Cid()})
+ if err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/ipld/merkledag/dagutils/utils.go b/ipld/merkledag/dagutils/utils.go
new file mode 100644
index 000000000..4f6500b25
--- /dev/null
+++ b/ipld/merkledag/dagutils/utils.go
@@ -0,0 +1,234 @@
+package dagutils
+
+import (
+ "context"
+ "errors"
+ "strings"
+
+ bserv "github.com/ipfs/boxo/blockservice"
+ bstore "github.com/ipfs/boxo/blockstore"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ ds "github.com/ipfs/go-datastore"
+ syncds "github.com/ipfs/go-datastore/sync"
+ ipld "github.com/ipfs/go-ipld-format"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+)
+
+// Editor represents a ProtoNode tree editor and provides methods to
+// modify it.
+type Editor struct {
+ root *dag.ProtoNode
+
+ // tmp is a temporary in memory (for now) dagstore for all of the
+ // intermediary nodes to be stored in
+ tmp ipld.DAGService
+
+ // src is the dagstore with *all* of the data on it, it is used to pull
+ // nodes from for modification (nil is a valid value)
+ src ipld.DAGService
+}
+
+// NewMemoryDagService returns a new, thread-safe in-memory DAGService.
+func NewMemoryDagService() ipld.DAGService {
+ // build mem-datastore for editor's intermediary nodes
+ bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
+ bsrv := bserv.New(bs, offline.Exchange(bs))
+ return dag.NewDAGService(bsrv)
+}
+
+// NewDagEditor returns an ProtoNode editor.
+//
+// * root is the node to be modified
+// * source is the dagstore to pull nodes from (optional)
+func NewDagEditor(root *dag.ProtoNode, source ipld.DAGService) *Editor {
+ return &Editor{
+ root: root,
+ tmp: NewMemoryDagService(),
+ src: source,
+ }
+}
+
+// GetNode returns the a copy of the root node being edited.
+func (e *Editor) GetNode() *dag.ProtoNode {
+ return e.root.Copy().(*dag.ProtoNode)
+}
+
+// GetDagService returns the DAGService used by this editor.
+func (e *Editor) GetDagService() ipld.DAGService {
+ return e.tmp
+}
+
+func addLink(ctx context.Context, ds ipld.DAGService, root *dag.ProtoNode, childname string, childnd ipld.Node) (*dag.ProtoNode, error) {
+ if childname == "" {
+ return nil, errors.New("cannot create link with no name")
+ }
+
+ // ensure that the node we are adding is in the dagservice
+ err := ds.Add(ctx, childnd)
+ if err != nil {
+ return nil, err
+ }
+
+ _ = ds.Remove(ctx, root.Cid())
+
+ // ensure no link with that name already exists
+ _ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound
+
+ if err := root.AddNodeLink(childname, childnd); err != nil {
+ return nil, err
+ }
+
+ if err := ds.Add(ctx, root); err != nil {
+ return nil, err
+ }
+ return root, nil
+}
+
+// InsertNodeAtPath inserts a new node in the tree and replaces the current root with the new one.
+func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert ipld.Node, create func() *dag.ProtoNode) error {
+ splpath := strings.Split(pth, "/")
+ nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create)
+ if err != nil {
+ return err
+ }
+ e.root = nd
+ return nil
+}
+
+func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert ipld.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) {
+ if len(path) == 1 {
+ return addLink(ctx, e.tmp, root, path[0], toinsert)
+ }
+
+ nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0])
+ if err != nil {
+ // if 'create' is true, we create directories on the way down as needed
+ if err == dag.ErrLinkNotFound && create != nil {
+ nd = create()
+ err = nil // no longer an error case
+ } else if ipld.IsNotFound(err) {
+ // try finding it in our source dagstore
+ nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0])
+ }
+
+ // if we receive an ErrNotFound, then our second 'GetLinkedNode' call
+ // also fails, we want to error out
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create)
+ if err != nil {
+ return nil, err
+ }
+
+ _ = e.tmp.Remove(ctx, root.Cid())
+
+ _ = root.RemoveNodeLink(path[0])
+ err = root.AddNodeLink(path[0], ndprime)
+ if err != nil {
+ return nil, err
+ }
+
+ err = e.tmp.Add(ctx, root)
+ if err != nil {
+ return nil, err
+ }
+
+ return root, nil
+}
+
+// RmLink removes the link with the given name and updates the root node of
+// the editor.
+func (e *Editor) RmLink(ctx context.Context, pth string) error {
+ splpath := strings.Split(pth, "/")
+ nd, err := e.rmLink(ctx, e.root, splpath)
+ if err != nil {
+ return err
+ }
+ e.root = nd
+ return nil
+}
+
+func (e *Editor) rmLink(ctx context.Context, root *dag.ProtoNode, path []string) (*dag.ProtoNode, error) {
+ if len(path) == 1 {
+ // base case, remove node in question
+ err := root.RemoveNodeLink(path[0])
+ if err != nil {
+ return nil, err
+ }
+
+ err = e.tmp.Add(ctx, root)
+ if err != nil {
+ return nil, err
+ }
+
+ return root, nil
+ }
+
+ // search for node in both tmp dagstore and source dagstore
+ nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0])
+ if ipld.IsNotFound(err) {
+ nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0])
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ nnode, err := e.rmLink(ctx, nd, path[1:])
+ if err != nil {
+ return nil, err
+ }
+
+ _ = e.tmp.Remove(ctx, root.Cid())
+
+ _ = root.RemoveNodeLink(path[0])
+ err = root.AddNodeLink(path[0], nnode)
+ if err != nil {
+ return nil, err
+ }
+
+ err = e.tmp.Add(ctx, root)
+ if err != nil {
+ return nil, err
+ }
+
+ return root, nil
+}
+
+// Finalize writes the new DAG to the given DAGService and returns the modified
+// root node.
+func (e *Editor) Finalize(ctx context.Context, ds ipld.DAGService) (*dag.ProtoNode, error) {
+ nd := e.GetNode()
+ err := copyDag(ctx, nd, e.tmp, ds)
+ return nd, err
+}
+
+func copyDag(ctx context.Context, nd ipld.Node, from, to ipld.DAGService) error {
+ // TODO(#4609): make this batch.
+ err := to.Add(ctx, nd)
+ if err != nil {
+ return err
+ }
+
+ for _, lnk := range nd.Links() {
+ child, err := lnk.GetNode(ctx, from)
+ if err != nil {
+ if ipld.IsNotFound(err) {
+ // not found means we didnt modify it, and it should
+ // already be in the target datastore
+ continue
+ }
+ return err
+ }
+
+ err = copyDag(ctx, child, from, to)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/ipld/merkledag/dagutils/utils_test.go b/ipld/merkledag/dagutils/utils_test.go
new file mode 100644
index 000000000..3fd3e823f
--- /dev/null
+++ b/ipld/merkledag/dagutils/utils_test.go
@@ -0,0 +1,114 @@
+package dagutils
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+)
+
+func TestAddLink(t *testing.T) {
+ ctx, context := context.WithCancel(context.Background())
+ defer context()
+
+ ds := mdtest.Mock()
+ fishnode := dag.NodeWithData([]byte("fishcakes!"))
+
+ err := ds.Add(ctx, fishnode)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nd := new(dag.ProtoNode)
+ nnode, err := addLink(ctx, ds, nd, "fish", fishnode)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fnprime, err := nnode.GetLinkedNode(ctx, ds, "fish")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fnpkey := fnprime.Cid()
+ if !fnpkey.Equals(fishnode.Cid()) {
+ t.Fatal("wrong child node found!")
+ }
+}
+
+func assertNodeAtPath(t *testing.T, ds ipld.DAGService, root *dag.ProtoNode, pth string, exp cid.Cid) {
+ parts := strings.Split(pth, "/")
+ cur := root
+ for _, e := range parts {
+ nxt, err := cur.GetLinkedProtoNode(context.Background(), ds, e)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cur = nxt
+ }
+
+ curc := cur.Cid()
+ if !curc.Equals(exp) {
+ t.Fatal("node not as expected at end of path")
+ }
+}
+
+func TestInsertNode(t *testing.T) {
+ root := new(dag.ProtoNode)
+ e := NewDagEditor(root, nil)
+
+ testInsert(t, e, "a", "anodefortesting", false, "")
+ testInsert(t, e, "a/b", "data", false, "")
+ testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name")
+ testInsert(t, e, "a/b/c/d/e", "foo", true, "")
+ testInsert(t, e, "a/b/c/d/f", "baz", true, "")
+ testInsert(t, e, "a/b/c/d/f", "bar", true, "")
+
+ testInsert(t, e, "", "bar", true, "cannot create link with no name")
+ testInsert(t, e, "////", "slashes", true, "cannot create link with no name")
+
+ c := e.GetNode().Cid()
+
+ if c.String() != "QmZ8yeT9uD6ouJPNAYt62XffYuXBT6b4mP4obRSE9cJrSt" {
+ t.Fatal("output was different than expected: ", c)
+ }
+}
+
+func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
+ child := dag.NodeWithData([]byte(data))
+ err := e.tmp.Add(context.Background(), child)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var c func() *dag.ProtoNode
+ if create {
+ c = func() *dag.ProtoNode {
+ return &dag.ProtoNode{}
+ }
+ }
+
+ err = e.InsertNodeAtPath(context.Background(), path, child, c)
+ if experr != "" {
+ var got string
+ if err != nil {
+ got = err.Error()
+ }
+ if got != experr {
+ t.Fatalf("expected '%s' but got '%s'", experr, got)
+ }
+ return
+ }
+
+ if err != nil {
+ t.Fatal(err, path, data, create, experr)
+ }
+
+ assertNodeAtPath(t, e.tmp, e.root, path, child.Cid())
+}
diff --git a/ipld/merkledag/errservice.go b/ipld/merkledag/errservice.go
new file mode 100644
index 000000000..f4607615a
--- /dev/null
+++ b/ipld/merkledag/errservice.go
@@ -0,0 +1,47 @@
+package merkledag
+
+import (
+ "context"
+
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// ErrorService implements ipld.DAGService, returning 'Err' for every call.
+type ErrorService struct {
+ Err error
+}
+
+var _ ipld.DAGService = (*ErrorService)(nil)
+
+// Add returns the cs.Err.
+func (cs *ErrorService) Add(ctx context.Context, nd ipld.Node) error {
+ return cs.Err
+}
+
+// AddMany returns the cs.Err.
+func (cs *ErrorService) AddMany(ctx context.Context, nds []ipld.Node) error {
+ return cs.Err
+}
+
+// Get returns the cs.Err.
+func (cs *ErrorService) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) {
+ return nil, cs.Err
+}
+
+// GetMany many returns the cs.Err.
+func (cs *ErrorService) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption {
+ ch := make(chan *ipld.NodeOption)
+ close(ch)
+ return ch
+}
+
+// Remove returns the cs.Err.
+func (cs *ErrorService) Remove(ctx context.Context, c cid.Cid) error {
+ return cs.Err
+}
+
+// RemoveMany returns the cs.Err.
+func (cs *ErrorService) RemoveMany(ctx context.Context, cids []cid.Cid) error {
+ return cs.Err
+}
diff --git a/ipld/merkledag/merkledag.go b/ipld/merkledag/merkledag.go
new file mode 100644
index 000000000..3b1891794
--- /dev/null
+++ b/ipld/merkledag/merkledag.go
@@ -0,0 +1,580 @@
+// Package merkledag implements the IPFS Merkle DAG data structures.
+package merkledag
+
+import (
+ "context"
+ "fmt"
+ "sync"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ bserv "github.com/ipfs/boxo/blockservice"
+ cid "github.com/ipfs/go-cid"
+ ipldcbor "github.com/ipfs/go-ipld-cbor"
+ format "github.com/ipfs/go-ipld-format"
+ legacy "github.com/ipfs/go-ipld-legacy"
+ dagpb "github.com/ipld/go-codec-dagpb"
+
+ // blank import is used to register the IPLD raw codec
+ _ "github.com/ipld/go-ipld-prime/codec/raw"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+)
+
+// TODO: We should move these registrations elsewhere. Really, most of the IPLD
+// functionality should go in a `go-ipld` repo but that will take a lot of work
+// and design.
+func init() {
+ format.Register(cid.DagProtobuf, DecodeProtobufBlock)
+ format.Register(cid.Raw, DecodeRawBlock)
+ format.Register(cid.DagCBOR, ipldcbor.DecodeBlock)
+
+ legacy.RegisterCodec(cid.DagProtobuf, dagpb.Type.PBNode, ProtoNodeConverter)
+ legacy.RegisterCodec(cid.Raw, basicnode.Prototype.Bytes, RawNodeConverter)
+}
+
+// contextKey is a type to use as value for the ProgressTracker contexts.
+type contextKey string
+
+const progressContextKey contextKey = "progress"
+
+// NewDAGService constructs a new DAGService (using the default implementation).
+// Note that the default implementation is also an ipld.LinkGetter.
+func NewDAGService(bs bserv.BlockService) *dagService {
+ return &dagService{Blocks: bs}
+}
+
+// dagService is an IPFS Merkle DAG service.
+// - the root is virtual (like a forest)
+// - stores nodes' data in a BlockService
+// TODO: should cache Nodes that are in memory, and be
+//
+// able to free some of them when vm pressure is high
+type dagService struct {
+ Blocks bserv.BlockService
+}
+
+// Add adds a node to the dagService, storing the block in the BlockService
+func (n *dagService) Add(ctx context.Context, nd format.Node) error {
+ if n == nil { // FIXME remove this assertion. protect with constructor invariant
+ return fmt.Errorf("dagService is nil")
+ }
+
+ return n.Blocks.AddBlock(ctx, nd)
+}
+
+func (n *dagService) AddMany(ctx context.Context, nds []format.Node) error {
+ blks := make([]blocks.Block, len(nds))
+ for i, nd := range nds {
+ blks[i] = nd
+ }
+ return n.Blocks.AddBlocks(ctx, blks)
+}
+
+// Get retrieves a node from the dagService, fetching the block in the BlockService
+func (n *dagService) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
+ if n == nil {
+ return nil, fmt.Errorf("dagService is nil")
+ }
+
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ b, err := n.Blocks.GetBlock(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+
+ return legacy.DecodeNode(ctx, b)
+}
+
+// GetLinks return the links for the node, the node doesn't necessarily have
+// to exist locally.
+func (n *dagService) GetLinks(ctx context.Context, c cid.Cid) ([]*format.Link, error) {
+ if c.Type() == cid.Raw {
+ return nil, nil
+ }
+ node, err := n.Get(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+ return node.Links(), nil
+}
+
+func (n *dagService) Remove(ctx context.Context, c cid.Cid) error {
+ return n.Blocks.DeleteBlock(ctx, c)
+}
+
+// RemoveMany removes multiple nodes from the DAG. It will likely be faster than
+// removing them individually.
+//
+// This operation is not atomic. If it returns an error, some nodes may or may
+// not have been removed.
+func (n *dagService) RemoveMany(ctx context.Context, cids []cid.Cid) error {
+ // TODO(#4608): make this batch all the way down.
+ for _, c := range cids {
+ if err := n.Blocks.DeleteBlock(ctx, c); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// GetLinksDirect creates a function to get the links for a node, from
+// the node, bypassing the LinkService. If the node does not exist
+// locally (and can not be retrieved) an error will be returned.
+func GetLinksDirect(serv format.NodeGetter) GetLinks {
+ return func(ctx context.Context, c cid.Cid) ([]*format.Link, error) {
+ nd, err := serv.Get(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+ return nd.Links(), nil
+ }
+}
+
+type sesGetter struct {
+ bs *bserv.Session
+}
+
+// Get gets a single node from the DAG.
+func (sg *sesGetter) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
+ blk, err := sg.bs.GetBlock(ctx, c)
+
+ if err != nil {
+ return nil, err
+ }
+
+ return legacy.DecodeNode(ctx, blk)
+}
+
+// GetMany gets many nodes at once, batching the request if possible.
+func (sg *sesGetter) GetMany(ctx context.Context, keys []cid.Cid) <-chan *format.NodeOption {
+ return getNodesFromBG(ctx, sg.bs, keys)
+}
+
+// WrapSession wraps a blockservice session to satisfy the format.NodeGetter interface
+func WrapSession(s *bserv.Session) format.NodeGetter {
+ return &sesGetter{s}
+}
+
+// Session returns a NodeGetter using a new session for block fetches.
+func (n *dagService) Session(ctx context.Context) format.NodeGetter {
+ return WrapSession(bserv.NewSession(ctx, n.Blocks))
+}
+
+// FetchGraph fetches all nodes that are children of the given node
+func FetchGraph(ctx context.Context, root cid.Cid, serv format.DAGService) error {
+ return FetchGraphWithDepthLimit(ctx, root, -1, serv)
+}
+
+// FetchGraphWithDepthLimit fetches all nodes that are children to the given
+// node down to the given depth. maxDepth=0 means "only fetch root",
+// maxDepth=1 means "fetch root and its direct children" and so on...
+// maxDepth=-1 means unlimited.
+func FetchGraphWithDepthLimit(ctx context.Context, root cid.Cid, depthLim int, serv format.DAGService) error {
+ var ng format.NodeGetter = NewSession(ctx, serv)
+
+ set := make(map[cid.Cid]int)
+
+ // Visit function returns true when:
+ // * The element is not in the set and we're not over depthLim
+ // * The element is in the set but recorded depth is deeper
+ // than currently seen (if we find it higher in the tree we'll need
+ // to explore deeper than before).
+ // depthLim = -1 means we only return true if the element is not in the
+ // set.
+ visit := func(c cid.Cid, depth int) bool {
+ oldDepth, ok := set[c]
+
+ if (ok && depthLim < 0) || (depthLim >= 0 && depth > depthLim) {
+ return false
+ }
+
+ if !ok || oldDepth > depth {
+ set[c] = depth
+ return true
+ }
+ return false
+ }
+
+ // If we have a ProgressTracker, we wrap the visit function to handle it
+ v, _ := ctx.Value(progressContextKey).(*ProgressTracker)
+ if v == nil {
+ return WalkDepth(ctx, GetLinksDirect(ng), root, visit, Concurrent())
+ }
+
+ visitProgress := func(c cid.Cid, depth int) bool {
+ if visit(c, depth) {
+ v.Increment()
+ return true
+ }
+ return false
+ }
+ return WalkDepth(ctx, GetLinksDirect(ng), root, visitProgress, Concurrent())
+}
+
+// GetMany gets many nodes from the DAG at once.
+//
+// This method may not return all requested nodes (and may or may not return an
+// error indicating that it failed to do so. It is up to the caller to verify
+// that it received all nodes.
+func (n *dagService) GetMany(ctx context.Context, keys []cid.Cid) <-chan *format.NodeOption {
+ return getNodesFromBG(ctx, n.Blocks, keys)
+}
+
+func dedupKeys(keys []cid.Cid) []cid.Cid {
+ set := cid.NewSet()
+ for _, c := range keys {
+ set.Add(c)
+ }
+ if set.Len() == len(keys) {
+ return keys
+ }
+ return set.Keys()
+}
+
+func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []cid.Cid) <-chan *format.NodeOption {
+ keys = dedupKeys(keys)
+
+ out := make(chan *format.NodeOption, len(keys))
+ blocks := bs.GetBlocks(ctx, keys)
+ var count int
+
+ go func() {
+ defer close(out)
+ for {
+ select {
+ case b, ok := <-blocks:
+ if !ok {
+ if count != len(keys) {
+ out <- &format.NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
+ }
+ return
+ }
+
+ nd, err := legacy.DecodeNode(ctx, b)
+ if err != nil {
+ out <- &format.NodeOption{Err: err}
+ return
+ }
+
+ out <- &format.NodeOption{Node: nd}
+ count++
+
+ case <-ctx.Done():
+ out <- &format.NodeOption{Err: ctx.Err()}
+ return
+ }
+ }
+ }()
+ return out
+}
+
+// GetLinks is the type of function passed to the EnumerateChildren function(s)
+// for getting the children of an IPLD node.
+type GetLinks func(context.Context, cid.Cid) ([]*format.Link, error)
+
+// GetLinksWithDAG returns a GetLinks function that tries to use the given
+// NodeGetter as a LinkGetter to get the children of a given IPLD node. This may
+// allow us to traverse the DAG without actually loading and parsing the node in
+// question (if we already have the links cached).
+func GetLinksWithDAG(ng format.NodeGetter) GetLinks {
+ return func(ctx context.Context, c cid.Cid) ([]*format.Link, error) {
+ return format.GetLinks(ctx, ng, c)
+ }
+}
+
+// defaultConcurrentFetch is the default maximum number of concurrent fetches
+// that 'fetchNodes' will start at a time
+const defaultConcurrentFetch = 32
+
+// walkOptions represent the parameters of a graph walking algorithm
+type walkOptions struct {
+ SkipRoot bool
+ Concurrency int
+ ErrorHandler func(c cid.Cid, err error) error
+}
+
+// WalkOption is a setter for walkOptions
+type WalkOption func(*walkOptions)
+
+func (wo *walkOptions) addHandler(handler func(c cid.Cid, err error) error) {
+ if wo.ErrorHandler != nil {
+ wo.ErrorHandler = func(c cid.Cid, err error) error {
+ return handler(c, wo.ErrorHandler(c, err))
+ }
+ } else {
+ wo.ErrorHandler = handler
+ }
+}
+
+// SkipRoot is a WalkOption indicating that the root node should skipped
+func SkipRoot() WalkOption {
+ return func(walkOptions *walkOptions) {
+ walkOptions.SkipRoot = true
+ }
+}
+
+// Concurrent is a WalkOption indicating that node fetching should be done in
+// parallel, with the default concurrency factor.
+// NOTE: When using that option, the walk order is *not* guarantee.
+// NOTE: It *does not* make multiple concurrent calls to the passed `visit` function.
+func Concurrent() WalkOption {
+ return func(walkOptions *walkOptions) {
+ walkOptions.Concurrency = defaultConcurrentFetch
+ }
+}
+
+// Concurrency is a WalkOption indicating that node fetching should be done in
+// parallel, with a specific concurrency factor.
+// NOTE: When using that option, the walk order is *not* guarantee.
+// NOTE: It *does not* make multiple concurrent calls to the passed `visit` function.
+func Concurrency(worker int) WalkOption {
+ return func(walkOptions *walkOptions) {
+ walkOptions.Concurrency = worker
+ }
+}
+
+// IgnoreErrors is a WalkOption indicating that the walk should attempt to
+// continue even when an error occur.
+func IgnoreErrors() WalkOption {
+ return func(walkOptions *walkOptions) {
+ walkOptions.addHandler(func(c cid.Cid, err error) error {
+ return nil
+ })
+ }
+}
+
+// IgnoreMissing is a WalkOption indicating that the walk should continue when
+// a node is missing.
+func IgnoreMissing() WalkOption {
+ return func(walkOptions *walkOptions) {
+ walkOptions.addHandler(func(c cid.Cid, err error) error {
+ if format.IsNotFound(err) {
+ return nil
+ }
+ return err
+ })
+ }
+}
+
+// OnMissing is a WalkOption adding a callback that will be triggered on a missing
+// node.
+func OnMissing(callback func(c cid.Cid)) WalkOption {
+ return func(walkOptions *walkOptions) {
+ walkOptions.addHandler(func(c cid.Cid, err error) error {
+ if format.IsNotFound(err) {
+ callback(c)
+ }
+ return err
+ })
+ }
+}
+
+// OnError is a WalkOption adding a custom error handler.
+// If this handler return a nil error, the walk will continue.
+func OnError(handler func(c cid.Cid, err error) error) WalkOption {
+ return func(walkOptions *walkOptions) {
+ walkOptions.addHandler(handler)
+ }
+}
+
+// WalkGraph will walk the dag in order (depth first) starting at the given root.
+func Walk(ctx context.Context, getLinks GetLinks, c cid.Cid, visit func(cid.Cid) bool, options ...WalkOption) error {
+ visitDepth := func(c cid.Cid, depth int) bool {
+ return visit(c)
+ }
+
+ return WalkDepth(ctx, getLinks, c, visitDepth, options...)
+}
+
+// WalkDepth walks the dag starting at the given root and passes the current
+// depth to a given visit function. The visit function can be used to limit DAG
+// exploration.
+func WalkDepth(ctx context.Context, getLinks GetLinks, c cid.Cid, visit func(cid.Cid, int) bool, options ...WalkOption) error {
+ opts := &walkOptions{}
+ for _, opt := range options {
+ opt(opts)
+ }
+
+ if opts.Concurrency > 1 {
+ return parallelWalkDepth(ctx, getLinks, c, visit, opts)
+ } else {
+ return sequentialWalkDepth(ctx, getLinks, c, 0, visit, opts)
+ }
+}
+
+func sequentialWalkDepth(ctx context.Context, getLinks GetLinks, root cid.Cid, depth int, visit func(cid.Cid, int) bool, options *walkOptions) error {
+ if !(options.SkipRoot && depth == 0) {
+ if !visit(root, depth) {
+ return nil
+ }
+ }
+
+ links, err := getLinks(ctx, root)
+ if err != nil && options.ErrorHandler != nil {
+ err = options.ErrorHandler(root, err)
+ }
+ if err != nil {
+ return err
+ }
+
+ for _, lnk := range links {
+ if err := sequentialWalkDepth(ctx, getLinks, lnk.Cid, depth+1, visit, options); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// ProgressTracker is used to show progress when fetching nodes.
+type ProgressTracker struct {
+ Total int
+ lk sync.Mutex
+}
+
+// DeriveContext returns a new context with value "progress" derived from
+// the given one.
+func (p *ProgressTracker) DeriveContext(ctx context.Context) context.Context {
+ return context.WithValue(ctx, progressContextKey, p)
+}
+
+// Increment adds one to the total progress.
+func (p *ProgressTracker) Increment() {
+ p.lk.Lock()
+ defer p.lk.Unlock()
+ p.Total++
+}
+
+// Value returns the current progress.
+func (p *ProgressTracker) Value() int {
+ p.lk.Lock()
+ defer p.lk.Unlock()
+ return p.Total
+}
+
+func parallelWalkDepth(ctx context.Context, getLinks GetLinks, root cid.Cid, visit func(cid.Cid, int) bool, options *walkOptions) error {
+ type cidDepth struct {
+ cid cid.Cid
+ depth int
+ }
+
+ type linksDepth struct {
+ links []*format.Link
+ depth int
+ }
+
+ feed := make(chan cidDepth)
+ out := make(chan linksDepth)
+ done := make(chan struct{})
+
+ var visitlk sync.Mutex
+ var wg sync.WaitGroup
+
+ errChan := make(chan error)
+ fetchersCtx, cancel := context.WithCancel(ctx)
+ defer wg.Wait()
+ defer cancel()
+ for i := 0; i < options.Concurrency; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for cdepth := range feed {
+ ci := cdepth.cid
+ depth := cdepth.depth
+
+ var shouldVisit bool
+
+ // bypass the root if needed
+ if !(options.SkipRoot && depth == 0) {
+ visitlk.Lock()
+ shouldVisit = visit(ci, depth)
+ visitlk.Unlock()
+ } else {
+ shouldVisit = true
+ }
+
+ if shouldVisit {
+ links, err := getLinks(ctx, ci)
+ if err != nil && options.ErrorHandler != nil {
+ err = options.ErrorHandler(root, err)
+ }
+ if err != nil {
+ select {
+ case errChan <- err:
+ case <-fetchersCtx.Done():
+ }
+ return
+ }
+
+ outLinks := linksDepth{
+ links: links,
+ depth: depth + 1,
+ }
+
+ select {
+ case out <- outLinks:
+ case <-fetchersCtx.Done():
+ return
+ }
+ }
+ select {
+ case done <- struct{}{}:
+ case <-fetchersCtx.Done():
+ }
+ }
+ }()
+ }
+ defer close(feed)
+
+ send := feed
+ var todoQueue []cidDepth
+ var inProgress int
+
+ next := cidDepth{
+ cid: root,
+ depth: 0,
+ }
+
+ for {
+ select {
+ case send <- next:
+ inProgress++
+ if len(todoQueue) > 0 {
+ next = todoQueue[0]
+ todoQueue = todoQueue[1:]
+ } else {
+ next = cidDepth{}
+ send = nil
+ }
+ case <-done:
+ inProgress--
+ if inProgress == 0 && !next.cid.Defined() {
+ return nil
+ }
+ case linksDepth := <-out:
+ for _, lnk := range linksDepth.links {
+ cd := cidDepth{
+ cid: lnk.Cid,
+ depth: linksDepth.depth,
+ }
+
+ if !next.cid.Defined() {
+ next = cd
+ send = feed
+ } else {
+ todoQueue = append(todoQueue, cd)
+ }
+ }
+ case err := <-errChan:
+ return err
+
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+}
+
+var _ format.LinkGetter = &dagService{}
+var _ format.NodeGetter = &dagService{}
+var _ format.NodeGetter = &sesGetter{}
+var _ format.DAGService = &dagService{}
diff --git a/ipld/merkledag/merkledag_test.go b/ipld/merkledag/merkledag_test.go
new file mode 100644
index 000000000..68933a867
--- /dev/null
+++ b/ipld/merkledag/merkledag_test.go
@@ -0,0 +1,1232 @@
+package merkledag_test
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ . "github.com/ipfs/boxo/ipld/merkledag"
+ mdpb "github.com/ipfs/boxo/ipld/merkledag/pb"
+ dstest "github.com/ipfs/boxo/ipld/merkledag/test"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ bserv "github.com/ipfs/boxo/blockservice"
+ bstest "github.com/ipfs/boxo/blockservice/test"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ u "github.com/ipfs/boxo/util"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+ prime "github.com/ipld/go-ipld-prime"
+ mh "github.com/multiformats/go-multihash"
+)
+
+var someCid cid.Cid = func() cid.Cid {
+ c, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4})
+ return c
+}()
+
+// makeDepthTestingGraph makes a small DAG with two levels. The level-two
+// nodes are both children of the root and of one of the level 1 nodes.
+// This is meant to test the Walk*Depth functions.
+func makeDepthTestingGraph(t *testing.T, ds ipld.DAGService) ipld.Node {
+ root := NodeWithData(nil)
+ l11 := NodeWithData([]byte("leve1_node1"))
+ l12 := NodeWithData([]byte("leve1_node2"))
+ l21 := NodeWithData([]byte("leve2_node1"))
+ l22 := NodeWithData([]byte("leve2_node2"))
+ l23 := NodeWithData([]byte("leve2_node3"))
+
+ l11.AddNodeLink(l21.Cid().String(), l21)
+ l11.AddNodeLink(l22.Cid().String(), l22)
+ l11.AddNodeLink(l23.Cid().String(), l23)
+
+ root.AddNodeLink(l11.Cid().String(), l11)
+ root.AddNodeLink(l12.Cid().String(), l12)
+ root.AddNodeLink(l23.Cid().String(), l23)
+
+ ctx := context.Background()
+ for _, n := range []ipld.Node{l23, l22, l21, l12, l11, root} {
+ err := ds.Add(ctx, n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ return root
+}
+
+// Check that all children of root are in the given set and in the datastore
+func traverseAndCheck(t *testing.T, root ipld.Node, ds ipld.DAGService, hasF func(c cid.Cid) bool) {
+ // traverse dag and check
+ for _, lnk := range root.Links() {
+ c := lnk.Cid
+ if !hasF(c) {
+ t.Fatal("missing key in set! ", lnk.Cid.String())
+ }
+ child, err := ds.Get(context.Background(), c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ traverseAndCheck(t, child, ds, hasF)
+ }
+}
+
+type brokenBuilder struct{}
+
+func (brokenBuilder) Sum([]byte) (cid.Cid, error) { return cid.Undef, errors.New("Nope!") }
+func (brokenBuilder) GetCodec() uint64 { return 0 }
+func (b brokenBuilder) WithCodec(uint64) cid.Builder { return b }
+
+// builder that will pass the basic SetCidBuilder tests but fail otherwise
+type sneakyBrokenBuilder struct{}
+
+func (sneakyBrokenBuilder) Sum(data []byte) (cid.Cid, error) {
+ if len(data) == 256 {
+ return V1CidPrefix().Sum(data)
+ }
+ return cid.Undef, errors.New("Nope!")
+}
+func (sneakyBrokenBuilder) GetCodec() uint64 { return 0 }
+func (b sneakyBrokenBuilder) WithCodec(uint64) cid.Builder { return b }
+
+func TestBadBuilderEncode(t *testing.T) {
+ n := NodeWithData([]byte("boop"))
+
+ t.Run("good builder sanity check", func(t *testing.T) {
+ if _, err := n.EncodeProtobuf(false); err != nil {
+ t.Fatal(err)
+ }
+ if err := n.SetCidBuilder(
+ &cid.Prefix{
+ MhType: mh.SHA2_256,
+ MhLength: -1,
+ Version: 1,
+ Codec: cid.DagProtobuf,
+ },
+ ); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ t.Run("hasher we can't use, should error", func(t *testing.T) {
+ if err := n.SetCidBuilder(
+ &cid.Prefix{
+ MhType: mh.SHA2_256_TRUNC254_PADDED,
+ MhLength: 256,
+ Version: 1,
+ Codec: cid.DagProtobuf,
+ },
+ ); err == nil {
+ t.Fatal("expected SetCidBuilder to error on unusable hasher")
+ }
+ if _, err := n.EncodeProtobuf(false); err != nil {
+ t.Fatalf("expected EncodeProtobuf to use safe CidBuilder: %v", err)
+ }
+ })
+
+ t.Run("broken custom builder, should error", func(t *testing.T) {
+ if err := n.SetCidBuilder(brokenBuilder{}); err == nil {
+ t.Fatal("expected SetCidBuilder to error on unusable hasher")
+ }
+ if _, err := n.EncodeProtobuf(false); err != nil {
+ t.Fatalf("expected EncodeProtobuf to use safe CidBuilder: %v", err)
+ }
+ })
+
+ t.Run("broken custom builder as pointer, should error", func(t *testing.T) {
+ if err := n.SetCidBuilder(&brokenBuilder{}); err == nil {
+ t.Fatal("expected SetCidBuilder to error on unusable hasher")
+ }
+ if _, err := n.EncodeProtobuf(false); err != nil {
+ t.Fatalf("expected EncodeProtobuf to use safe CidBuilder: %v", err)
+ }
+ })
+
+ t.Run("broken sneaky custom builder, should error", func(t *testing.T) {
+ if err := n.SetCidBuilder(sneakyBrokenBuilder{}); err != nil {
+ t.Fatalf("expected SetCidBuilder to not error with sneaky custom builder: %v", err)
+ }
+ if _, err := n.EncodeProtobuf(false); err == nil {
+ t.Fatal("expected EncodeProtobuf to fail using the sneaky custom builder")
+ }
+ if len(n.RawData()) != 0 {
+ t.Fatal("expected RawData to return zero-byte slice")
+ }
+ if n.Cid().String() != "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" {
+ t.Fatal("expected Cid to return the zero dag-pb CID")
+ }
+ if n.String() != "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" {
+ t.Fatal("expected String to return the zero dag-pb CID string")
+ }
+ })
+}
+
+func TestLinkChecking(t *testing.T) {
+ cases := []struct {
+ name string
+ fn func(*ProtoNode) error
+ }{
+ {
+ name: "AddRawLink overflow Tsize",
+ fn: func(n *ProtoNode) error {
+ return n.AddRawLink("foo", &ipld.Link{Size: math.MaxUint64, Cid: someCid})
+ },
+ },
+
+ {
+ name: "AddRawLink undefined CID",
+ fn: func(n *ProtoNode) error {
+ return n.AddRawLink("foo", &ipld.Link{Cid: cid.Undef})
+ },
+ },
+
+ {
+ name: "SetLinks overflow Tsize",
+ fn: func(n *ProtoNode) error {
+ return n.SetLinks([]*ipld.Link{{Size: math.MaxUint64, Cid: someCid}})
+ },
+ },
+
+ {
+ name: "SetLinks undefined CID",
+ fn: func(n *ProtoNode) error {
+ return n.SetLinks([]*ipld.Link{{Cid: cid.Undef}})
+ },
+ },
+
+ {
+ name: "UnmarshalJSON overflow Tsize",
+ fn: func(n *ProtoNode) error {
+ return n.UnmarshalJSON([]byte(`{"data":null,"links":[{"Name":"","Size":18446744073709549568,"Cid":{"/":"QmNPWHBrVQiiV8FpyNuEPhB9E2rbvdy9Yx79EY1EJuyf9o"}}]}`))
+ },
+ },
+
+ {
+ name: "UnmarshalJSON undefined CID",
+ fn: func(n *ProtoNode) error {
+ return n.UnmarshalJSON([]byte(`{"data":null,"links":[{"Name":"","Size":100}]}`))
+ },
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ n := NodeWithData([]byte("boop"))
+ err := tc.fn(n)
+ if err == nil {
+ t.Fatal("expected error")
+ }
+ })
+ }
+
+ t.Run("round-trip block with bad Tsize", func(t *testing.T) {
+ badblock, _ := hex.DecodeString("122f0a22122000bb3604d2ecd386227007c548249521fbb9a394e1e26460091d0a692888e7361880f0ffffffffffffff01")
+ n, err := DecodeProtobuf(badblock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // sanity
+ if len(n.Links()) != 1 {
+ t.Fatal("expected a link")
+ }
+ // sanity
+ if n.Links()[0].Size <= math.MaxInt64 {
+ t.Fatal("expected link Tsize to be oversized")
+ }
+
+ // forced round-trip
+ byts, err := n.EncodeProtobuf(true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n, err = DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(n.Links()) != 1 {
+ t.Fatal("expected a link")
+ }
+ if n.Links()[0].Size != 0 {
+ t.Fatal("expected link Tsize to be truncated on reencode")
+ }
+ })
+}
+
+func TestNode(t *testing.T) {
+
+ n1 := NodeWithData([]byte("beep"))
+ n2 := NodeWithData([]byte("boop"))
+ n3 := NodeWithData([]byte("beep boop"))
+ if err := n3.AddNodeLink("beep-link", n1); err != nil {
+ t.Error(err)
+ }
+ if err := n3.AddNodeLink("boop-link", n2); err != nil {
+ t.Error(err)
+ }
+
+ printn := func(name string, n *ProtoNode) {
+ fmt.Println(">", name)
+ fmt.Println("data:", string(n.Data()))
+
+ fmt.Println("links:")
+ for _, l := range n.Links() {
+ fmt.Println("-", l.Name, l.Size, l.Cid)
+ }
+
+ e, err := n.EncodeProtobuf(false)
+ if err != nil {
+ t.Error(err)
+ } else {
+ fmt.Println("encoded:", e)
+ }
+
+ h := n.Multihash()
+ k := n.Cid().Hash()
+ if k.String() != h.String() {
+ t.Error("Key is not equivalent to multihash")
+ } else {
+ fmt.Println("key: ", k)
+ }
+
+ SubtestNodeStat(t, n)
+ }
+
+ printn("beep", n1)
+ printn("boop", n2)
+ printn("beep boop", n3)
+}
+
+func SubtestNodeStat(t *testing.T, n *ProtoNode) {
+ enc, err := n.EncodeProtobuf(true)
+ if err != nil {
+ t.Error("n.EncodeProtobuf(true) failed")
+ return
+ }
+
+ cumSize, err := n.Size()
+ if err != nil {
+ t.Error("n.Size() failed")
+ return
+ }
+
+ k := n.Cid()
+
+ expected := ipld.NodeStat{
+ NumLinks: len(n.Links()),
+ BlockSize: len(enc),
+ LinksSize: len(enc) - len(n.Data()), // includes framing.
+ DataSize: len(n.Data()),
+ CumulativeSize: int(cumSize),
+ Hash: k.String(),
+ }
+
+ actual, err := n.Stat()
+ if err != nil {
+ t.Error("n.Stat() failed")
+ return
+ }
+
+ if expected != *actual {
+ t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual)
+ } else {
+ fmt.Printf("n.Stat correct: %s\n", actual)
+ }
+}
+
+type devZero struct{}
+
+func (devZero) Read(b []byte) (int, error) {
+ for i := range b {
+ b[i] = 0
+ }
+ return len(b), nil
+}
+
+func TestBatchFetch(t *testing.T) {
+ read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
+ runBatchFetchTest(t, read)
+}
+
+func TestBatchFetchDupBlock(t *testing.T) {
+ read := io.LimitReader(devZero{}, 1024*32)
+ runBatchFetchTest(t, read)
+}
+
+// makeTestDAG creates a simple DAG from the data in a reader.
+// First, a node is created from each 512 bytes of data from the reader
+// (like a the Size chunker would do). Then all nodes are added as children
+// to a root node, which is returned.
+func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node {
+ p := make([]byte, 512)
+ nodes := []*ProtoNode{}
+
+ for {
+ n, err := io.ReadFull(read, p)
+ if err == io.EOF {
+ break
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if n != len(p) {
+ t.Fatal("should have read 512 bytes from the reader")
+ }
+
+ protoNode := NodeWithData(p)
+ nodes = append(nodes, protoNode)
+ }
+
+ ctx := context.Background()
+ // Add a root referencing all created nodes
+ root := NodeWithData(nil)
+ for _, n := range nodes {
+ err := root.AddNodeLink(n.Cid().String(), n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = ds.Add(ctx, n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ err := ds.Add(ctx, root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return root
+}
+
+// makeTestDAGReader takes the root node as returned by makeTestDAG and
+// provides a reader that reads all the RawData from that node and its children.
+func makeTestDAGReader(t *testing.T, root ipld.Node, ds ipld.DAGService) io.Reader {
+ ctx := context.Background()
+ buf := new(bytes.Buffer)
+ buf.Write(root.RawData())
+ for _, l := range root.Links() {
+ n, err := ds.Get(ctx, l.Cid)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = buf.Write(n.RawData())
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ return buf
+}
+
+func runBatchFetchTest(t *testing.T, read io.Reader) {
+ ctx := context.Background()
+ var dagservs []ipld.DAGService
+ for _, bsi := range bstest.Mocks(5) {
+ dagservs = append(dagservs, NewDAGService(bsi))
+ }
+
+ root := makeTestDAG(t, read, dagservs[0])
+
+ t.Log("finished setup.")
+
+ dagr := makeTestDAGReader(t, root, dagservs[0])
+
+ expected, err := io.ReadAll(dagr)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = dagservs[0].Add(ctx, root)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Log("Added file to first node.")
+
+ c := root.Cid()
+
+ wg := sync.WaitGroup{}
+ errs := make(chan error)
+
+ for i := 1; i < len(dagservs); i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ first, err := dagservs[i].Get(ctx, c)
+ if err != nil {
+ errs <- err
+ }
+ fmt.Println("Got first node back.")
+
+ firstpb, ok := first.(*ProtoNode)
+ if !ok {
+ errs <- ErrNotProtobuf
+ }
+ read := makeTestDAGReader(t, firstpb, dagservs[i])
+ datagot, err := io.ReadAll(read)
+ if err != nil {
+ errs <- err
+ }
+
+ if !bytes.Equal(datagot, expected) {
+ errs <- errors.New("got bad data back")
+ }
+ }(i)
+ }
+
+ go func() {
+ wg.Wait()
+ close(errs)
+ }()
+
+ for err := range errs {
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func TestCantGet(t *testing.T) {
+ ds := dstest.Mock()
+ a := NodeWithData([]byte("A"))
+
+ c := a.Cid()
+ _, err := ds.Get(context.Background(), c)
+ if !strings.Contains(err.Error(), "not found") {
+ t.Fatal("expected err not found, got: ", err)
+ }
+}
+
+func TestFetchGraph(t *testing.T) {
+ var dservs []ipld.DAGService
+ bsis := bstest.Mocks(2)
+ for _, bsi := range bsis {
+ dservs = append(dservs, NewDAGService(bsi))
+ }
+
+ read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
+ root := makeTestDAG(t, read, dservs[0])
+
+ err := FetchGraph(context.TODO(), root.Cid(), dservs[1])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // create an offline dagstore and ensure all blocks were fetched
+ bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore()))
+
+ offlineDS := NewDAGService(bs)
+
+ err = Walk(context.Background(), offlineDS.GetLinks, root.Cid(), func(_ cid.Cid) bool { return true })
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestFetchGraphWithDepthLimit(t *testing.T) {
+ type testcase struct {
+ depthLim int
+ setLen int
+ }
+
+ tests := []testcase{
+ {1, 4},
+ {0, 1},
+ {-1, 6},
+ {2, 6},
+ {3, 6},
+ }
+
+ testF := func(t *testing.T, tc testcase) {
+ var dservs []ipld.DAGService
+ bsis := bstest.Mocks(2)
+ for _, bsi := range bsis {
+ dservs = append(dservs, NewDAGService(bsi))
+ }
+
+ root := makeDepthTestingGraph(t, dservs[0])
+
+ err := FetchGraphWithDepthLimit(context.TODO(), root.Cid(), tc.depthLim, dservs[1])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // create an offline dagstore and ensure all blocks were fetched
+ bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore()))
+
+ offlineDS := NewDAGService(bs)
+
+ set := make(map[string]int)
+ visitF := func(c cid.Cid, depth int) bool {
+ if tc.depthLim < 0 || depth <= tc.depthLim {
+ set[string(c.Bytes())] = depth
+ return true
+ }
+ return false
+
+ }
+
+ err = WalkDepth(context.Background(), offlineDS.GetLinks, root.Cid(), visitF)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(set) != tc.setLen {
+ t.Fatalf("expected %d nodes but visited %d", tc.setLen, len(set))
+ }
+ }
+
+ for _, tc := range tests {
+ t.Run(fmt.Sprintf("depth limit %d", tc.depthLim), func(t *testing.T) {
+ testF(t, tc)
+ })
+ }
+}
+
+func TestWalk(t *testing.T) {
+ bsi := bstest.Mocks(1)
+ ds := NewDAGService(bsi[0])
+
+ read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
+ root := makeTestDAG(t, read, ds)
+
+ set := cid.NewSet()
+
+ err := Walk(context.Background(), ds.GetLinks, root.Cid(), set.Visit)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ traverseAndCheck(t, root, ds, set.Has)
+}
+
+func TestFetchFailure(t *testing.T) {
+ ctx := context.Background()
+
+ ds := dstest.Mock()
+ ds_bad := dstest.Mock()
+
+ top := new(ProtoNode)
+ for i := 0; i < 10; i++ {
+ nd := NodeWithData([]byte{byte('a' + i)})
+ err := ds.Add(ctx, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = top.AddNodeLink(fmt.Sprintf("AA%d", i), nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ for i := 0; i < 10; i++ {
+ nd := NodeWithData([]byte{'f', 'a' + byte(i)})
+ err := ds_bad.Add(ctx, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = top.AddNodeLink(fmt.Sprintf("BB%d", i), nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ getters := ipld.GetDAG(ctx, ds, top)
+ for i, getter := range getters {
+ _, err := getter.Get(ctx)
+ if err != nil && i < 10 {
+ t.Fatal(err)
+ }
+ if err == nil && i >= 10 {
+ t.Fatal("should have failed request")
+ }
+ }
+}
+
+func TestUnmarshalFailure(t *testing.T) {
+ badData := []byte("hello world")
+
+ _, err := DecodeProtobuf(badData)
+ if err == nil {
+ t.Fatal("shouldnt succeed to parse this")
+ }
+
+ // now with a bad link
+ pbn := &mdpb.PBNode{Links: []*mdpb.PBLink{{Hash: []byte("not a multihash")}}}
+ badlink, err := pbn.Marshal()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = DecodeProtobuf(badlink)
+ if err == nil {
+ t.Fatal("should have failed to parse node with bad link")
+ }
+
+ n := &ProtoNode{}
+ n.Marshal()
+}
+
+func TestBasicAddGet(t *testing.T) {
+ ctx := context.Background()
+
+ ds := dstest.Mock()
+ nd := new(ProtoNode)
+
+ err := ds.Add(ctx, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := ds.Get(ctx, nd.Cid())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !nd.Cid().Equals(out.Cid()) {
+ t.Fatal("output didnt match input")
+ }
+}
+
+func TestGetRawNodes(t *testing.T) {
+ ctx := context.Background()
+
+ rn := NewRawNode([]byte("test"))
+
+ ds := dstest.Mock()
+
+ err := ds.Add(ctx, rn)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !rn.Cid().Equals(rn.Cid()) {
+ t.Fatal("output cids didnt match")
+ }
+
+ out, err := ds.Get(ctx, rn.Cid())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(out.RawData(), []byte("test")) {
+ t.Fatal("raw block should match input data")
+ }
+
+ if out.Links() != nil {
+ t.Fatal("raw blocks shouldnt have links")
+ }
+
+ if out.Tree("", -1) != nil {
+ t.Fatal("tree should return no paths in a raw block")
+ }
+
+ size, err := out.Size()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if size != 4 {
+ t.Fatal("expected size to be 4")
+ }
+
+ ns, err := out.Stat()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if ns.DataSize != 4 {
+ t.Fatal("expected size to be 4, got: ", ns.DataSize)
+ }
+
+ _, _, err = out.Resolve([]string{"foo"})
+ if err != ErrLinkNotFound {
+ t.Fatal("shouldnt find links under raw blocks")
+ }
+}
+
+func TestProtoNodeResolve(t *testing.T) {
+
+ nd := new(ProtoNode)
+ nd.SetLinks([]*ipld.Link{{Name: "foo", Cid: someCid}})
+
+ lnk, left, err := nd.ResolveLink([]string{"foo", "bar"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(left) != 1 || left[0] != "bar" {
+ t.Fatal("expected the single path element 'bar' to remain")
+ }
+
+ if lnk.Name != "foo" {
+ t.Fatal("how did we get anything else?")
+ }
+
+ tvals := nd.Tree("", -1)
+ if len(tvals) != 1 || tvals[0] != "foo" {
+ t.Fatal("expected tree to return []{\"foo\"}")
+ }
+}
+
+func TestCidRetention(t *testing.T) {
+ ctx := context.Background()
+
+ nd := new(ProtoNode)
+ nd.SetData([]byte("fooooo"))
+
+ pref := nd.Cid().Prefix()
+ pref.Version = 1
+
+ c2, err := pref.Sum(nd.RawData())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ blk, err := blocks.NewBlockWithCid(nd.RawData(), c2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ bs := dstest.Bserv()
+ err = bs.AddBlock(ctx, blk)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ds := NewDAGService(bs)
+ out, err := ds.Get(ctx, c2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !out.Cid().Equals(c2) {
+ t.Fatal("output cid didnt match")
+ }
+}
+
+func TestCidRawDoesnNeedData(t *testing.T) {
+ srv := NewDAGService(dstest.Bserv())
+ nd := NewRawNode([]byte("somedata"))
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ // there is no data for this node in the blockservice
+ // so dag service can't load it
+ links, err := srv.GetLinks(ctx, nd.Cid())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(links) != 0 {
+ t.Fatal("raw node shouldn't have any links")
+ }
+}
+
+func TestRawToJson(t *testing.T) {
+ rawData := []byte{1, 2, 3, 4}
+ nd := NewRawNode(rawData)
+ encoded, err := nd.MarshalJSON()
+ if err != nil {
+ t.Fatal(err)
+ }
+ var res interface{}
+ err = json.Unmarshal(encoded, &res)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resBytes, ok := res.(string)
+ if !ok {
+ t.Fatal("expected to marshal to a string")
+ }
+ if string(rawData) != resBytes {
+ t.Fatal("failed to round-trip bytes")
+ }
+}
+
+func TestGetManyDuplicate(t *testing.T) {
+ ctx := context.Background()
+
+ srv := NewDAGService(dstest.Bserv())
+
+ nd := NodeWithData([]byte("foo"))
+ if err := srv.Add(ctx, nd); err != nil {
+ t.Fatal(err)
+ }
+ nds := srv.GetMany(ctx, []cid.Cid{nd.Cid(), nd.Cid(), nd.Cid()})
+ out, ok := <-nds
+ if !ok {
+ t.Fatal("expecting node foo")
+ }
+ if out.Err != nil {
+ t.Fatal(out.Err)
+ }
+ if !out.Node.Cid().Equals(nd.Cid()) {
+ t.Fatal("got wrong node")
+ }
+ out, ok = <-nds
+ if ok {
+ if out.Err != nil {
+ t.Fatal(out.Err)
+ } else {
+ t.Fatal("expecting no more nodes")
+ }
+ }
+}
+
+func TestEnumerateAsyncFailsNotFound(t *testing.T) {
+ ctx := context.Background()
+
+ a := NodeWithData([]byte("foo1"))
+ b := NodeWithData([]byte("foo2"))
+ c := NodeWithData([]byte("foo3"))
+ d := NodeWithData([]byte("foo4"))
+ e := NodeWithData([]byte("foo5"))
+
+ ds := dstest.Mock()
+ for _, n := range []ipld.Node{a, b, c} {
+ err := ds.Add(ctx, n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ parent := new(ProtoNode)
+ if err := parent.AddNodeLink("a", a); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := parent.AddNodeLink("b", b); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := parent.AddNodeLink("c", c); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := parent.AddNodeLink("d", d); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := parent.AddNodeLink("e", e); err != nil {
+ t.Fatal(err)
+ }
+
+ err := ds.Add(ctx, parent)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cset := cid.NewSet()
+ err = Walk(ctx, GetLinksDirect(ds), parent.Cid(), cset.Visit)
+ if err == nil {
+ t.Fatal("this should have failed")
+ }
+}
+
+func TestLinkSorting(t *testing.T) {
+ az := "az"
+ aaaa := "aaaa"
+ bbbb := "bbbb"
+ cccc := "cccc"
+
+ azBlk := NewRawNode([]byte(az))
+ aaaaBlk := NewRawNode([]byte(aaaa))
+ bbbbBlk := NewRawNode([]byte(bbbb))
+ ccccBlk := NewRawNode([]byte(cccc))
+ pbn := &mdpb.PBNode{
+ Links: []*mdpb.PBLink{
+ {Hash: bbbbBlk.Cid().Bytes(), Name: &bbbb},
+ {Hash: azBlk.Cid().Bytes(), Name: &az},
+ {Hash: aaaaBlk.Cid().Bytes(), Name: &aaaa},
+ {Hash: ccccBlk.Cid().Bytes(), Name: &cccc},
+ },
+ }
+ byts, err := pbn.Marshal()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ mustLookupNodeString := func(t *testing.T, node prime.Node, name string) prime.Node {
+ subNode, err := node.LookupByString(name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return subNode
+ }
+
+ mustLookupNodeIndex := func(t *testing.T, node prime.Node, idx int64) prime.Node {
+ subNode, err := node.LookupByIndex(idx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return subNode
+ }
+
+ mustNodeAsString := func(t *testing.T, node prime.Node) string {
+ str, err := node.AsString()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return str
+ }
+
+ verifyUnsortedNode := func(t *testing.T, node *ProtoNode) {
+ links := node.Links()
+ if len(links) != 4 {
+ t.Errorf("wrong number of links, expected 4 but got %d", len(links))
+ }
+ if links[0].Name != bbbb {
+ t.Errorf("expected link 0 to be 'bbbb', got %s", links[0].Name)
+ }
+ if links[1].Name != az {
+ t.Errorf("expected link 0 to be 'az', got %s", links[1].Name)
+ }
+ if links[2].Name != aaaa {
+ t.Errorf("expected link 0 to be 'aaaa', got %s", links[2].Name)
+ }
+ if links[3].Name != cccc {
+ t.Errorf("expected link 0 to be 'cccc', got %s", links[3].Name)
+ }
+
+ // check the go-ipld-prime form
+ linksNode := mustLookupNodeString(t, node, "Links")
+ if linksNode.Length() != 4 {
+ t.Errorf("(Node) wrong number of links, expected 4 but got %d", len(links))
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 0), "Name")); name != bbbb {
+ t.Errorf("(Node) expected link 0 to be 'bbbb', got %s", name)
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 1), "Name")); name != az {
+ t.Errorf("(Node) expected link 0 to be 'az', got %s", name)
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 2), "Name")); name != aaaa {
+ t.Errorf("(Node) expected link 0 to be 'aaaa', got %s", name)
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 3), "Name")); name != cccc {
+ t.Errorf("(Node) expected link 0 to be 'cccc', got %s", name)
+ }
+ }
+
+ verifySortedNode := func(t *testing.T, node *ProtoNode) {
+ links := node.Links()
+ if len(links) != 4 {
+ t.Errorf("wrong number of links, expected 4 but got %d", len(links))
+ }
+ if links[0].Name != aaaa {
+ t.Errorf("expected link 0 to be 'aaaa', got %s", links[0].Name)
+ }
+ if links[1].Name != az {
+ t.Errorf("expected link 0 to be 'az', got %s", links[1].Name)
+ }
+ if links[2].Name != bbbb {
+ t.Errorf("expected link 0 to be 'bbbb', got %s", links[2].Name)
+ }
+ if links[3].Name != cccc {
+ t.Errorf("expected link 0 to be 'cccc', got %s", links[3].Name)
+ }
+
+ // check the go-ipld-prime form
+ linksNode := mustLookupNodeString(t, node, "Links")
+ if linksNode.Length() != 4 {
+ t.Errorf("(Node) wrong number of links, expected 4 but got %d", len(links))
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 0), "Name")); name != aaaa {
+ t.Errorf("(Node) expected link 0 to be 'aaaa', got %s", name)
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 1), "Name")); name != az {
+ t.Errorf("(Node) expected link 0 to be 'az', got %s", name)
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 2), "Name")); name != bbbb {
+ t.Errorf("(Node) expected link 0 to be 'bbbb', got %s", name)
+ }
+ if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 3), "Name")); name != cccc {
+ t.Errorf("(Node) expected link 0 to be 'cccc', got %s", name)
+ }
+ }
+
+ t.Run("decode", func(t *testing.T) {
+ node, err := DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ verifyUnsortedNode(t, node)
+ })
+
+ t.Run("RawData() should not mutate, should return original form", func(t *testing.T) {
+ node, err := DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rawData := node.RawData()
+ verifyUnsortedNode(t, node)
+ if !bytes.Equal(rawData, byts) {
+ t.Error("RawData() did not return original bytes")
+ }
+ })
+
+ t.Run("Size() should not mutate", func(t *testing.T) {
+ node, err := DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sz, err := node.Size()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if sz != 182 {
+ t.Errorf("expected size to be 182, got %d", sz)
+ }
+ verifyUnsortedNode(t, node)
+ })
+
+ t.Run("GetPBNode() should not mutate, returned PBNode should be sorted", func(t *testing.T) {
+ node, err := DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rtPBNode := node.GetPBNode()
+ rtByts, err := rtPBNode.Marshal()
+ if err != nil {
+ t.Fatal(err)
+ }
+ verifyUnsortedNode(t, node)
+ rtNode, err := DecodeProtobuf(rtByts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ verifySortedNode(t, rtNode)
+ })
+
+ t.Run("add and remove link should mutate", func(t *testing.T) {
+ node, err := DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = node.AddRawLink("foo", &ipld.Link{
+ Size: 10,
+ Cid: someCid,
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if err = node.RemoveNodeLink("foo"); err != nil {
+ t.Fatal(err)
+ }
+ verifySortedNode(t, node)
+ })
+
+ t.Run("update link should not mutate, returned ProtoNode should be sorted", func(t *testing.T) {
+ node, err := DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ newNode, err := node.UpdateNodeLink("self", node)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = newNode.RemoveNodeLink("self"); err != nil {
+ t.Fatal(err)
+ }
+ verifySortedNode(t, newNode)
+ verifyUnsortedNode(t, node)
+ })
+
+ t.Run("SetLinks() should mutate", func(t *testing.T) {
+ node, err := DecodeProtobuf(byts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ links := node.Links() // clone
+ node.SetLinks(links)
+ verifySortedNode(t, node)
+ })
+}
+
+func TestProgressIndicator(t *testing.T) {
+ testProgressIndicator(t, 5)
+}
+
+func TestProgressIndicatorNoChildren(t *testing.T) {
+ testProgressIndicator(t, 0)
+}
+
+func testProgressIndicator(t *testing.T, depth int) {
+ ds := dstest.Mock()
+
+ top, numChildren := mkDag(ds, depth)
+
+ v := new(ProgressTracker)
+ ctx := v.DeriveContext(context.Background())
+
+ err := FetchGraph(ctx, top, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if v.Value() != numChildren+1 {
+ t.Errorf("wrong number of children reported in progress indicator, expected %d, got %d",
+ numChildren+1, v.Value())
+ }
+}
+
+func mkDag(ds ipld.DAGService, depth int) (cid.Cid, int) {
+ ctx := context.Background()
+
+ totalChildren := 0
+ f := func() *ProtoNode {
+ p := new(ProtoNode)
+ buf := make([]byte, 16)
+ rand.Read(buf)
+
+ p.SetData(buf)
+ err := ds.Add(ctx, p)
+ if err != nil {
+ panic(err)
+ }
+ return p
+ }
+
+ for i := 0; i < depth; i++ {
+ thisf := f
+ f = func() *ProtoNode {
+ pn := mkNodeWithChildren(thisf, 10)
+ err := ds.Add(ctx, pn)
+ if err != nil {
+ panic(err)
+ }
+ totalChildren += 10
+ return pn
+ }
+ }
+
+ nd := f()
+ err := ds.Add(ctx, nd)
+ if err != nil {
+ panic(err)
+ }
+
+ return nd.Cid(), totalChildren
+}
+
+func mkNodeWithChildren(getChild func() *ProtoNode, width int) *ProtoNode {
+ cur := new(ProtoNode)
+
+ for i := 0; i < width; i++ {
+ c := getChild()
+ if err := cur.AddNodeLink(fmt.Sprint(i), c); err != nil {
+ panic(err)
+ }
+ }
+
+ return cur
+}
diff --git a/ipld/merkledag/node.go b/ipld/merkledag/node.go
new file mode 100644
index 000000000..24ad8e165
--- /dev/null
+++ b/ipld/merkledag/node.go
@@ -0,0 +1,575 @@
+package merkledag
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "math"
+ "sort"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ cid "github.com/ipfs/go-cid"
+ format "github.com/ipfs/go-ipld-format"
+ legacy "github.com/ipfs/go-ipld-legacy"
+ logging "github.com/ipfs/go-log/v2"
+ dagpb "github.com/ipld/go-codec-dagpb"
+ ipld "github.com/ipld/go-ipld-prime"
+ mh "github.com/multiformats/go-multihash"
+ mhcore "github.com/multiformats/go-multihash/core"
+)
+
+// Common errors
+var (
+ ErrNotProtobuf = fmt.Errorf("expected protobuf dag node")
+ ErrNotRawNode = fmt.Errorf("expected raw bytes node")
+ ErrLinkNotFound = fmt.Errorf("no link by that name")
+)
+
+var log = logging.Logger("merkledag")
+
+// for testing custom CidBuilders
+var zeros [256]byte
+var zeroCid = mustZeroCid()
+
+type immutableProtoNode struct {
+ encoded []byte
+ dagpb.PBNode
+}
+
+// ProtoNode represents a node in the IPFS Merkle DAG.
+// nodes have opaque data and a set of navigable links.
+// ProtoNode is a go-ipld-legacy.UniversalNode, meaning it is both
+// a go-ipld-prime node and a go-ipld-format node.
+// ProtoNode maintains compatibility with it's original implementation
+// as a go-ipld-format only node, which included some mutability, namely the
+// the ability to add/remove links in place
+//
+// TODO: We should be able to eventually replace this implementation with
+// * go-codec-dagpb for basic DagPB encode/decode to go-ipld-prime
+// * go-unixfsnode ADLs for higher level DAGPB functionality
+// For the time being however, go-unixfsnode is read only and
+// this mutable protonode implementation is needed to support go-unixfs,
+// the only library that implements both read and write for UnixFS v1.
+type ProtoNode struct {
+ links []*format.Link
+ linksDirty bool
+ data []byte
+
+ // cache encoded/marshaled value, kept to make the go-ipld-prime Node interface
+ // work (see prime.go), and to provide a cached []byte encoded form available
+ encoded *immutableProtoNode
+ cached cid.Cid
+
+ // builder specifies cid version and hashing function
+ builder cid.Builder
+}
+
+var v0CidPrefix = cid.Prefix{
+ Codec: cid.DagProtobuf,
+ MhLength: -1,
+ MhType: mh.SHA2_256,
+ Version: 0,
+}
+
+var v1CidPrefix = cid.Prefix{
+ Codec: cid.DagProtobuf,
+ MhLength: -1,
+ MhType: mh.SHA2_256,
+ Version: 1,
+}
+
+// V0CidPrefix returns a prefix for CIDv0
+func V0CidPrefix() cid.Prefix { return v0CidPrefix }
+
+// V1CidPrefix returns a prefix for CIDv1 with the default settings
+func V1CidPrefix() cid.Prefix { return v1CidPrefix }
+
+// PrefixForCidVersion returns the Protobuf prefix for a given CID version
+func PrefixForCidVersion(version int) (cid.Prefix, error) {
+ switch version {
+ case 0:
+ return v0CidPrefix, nil
+ case 1:
+ return v1CidPrefix, nil
+ default:
+ return cid.Prefix{}, fmt.Errorf("unknown CID version: %d", version)
+ }
+}
+
+// CidBuilder returns the CID Builder for this ProtoNode, it is never nil
+func (n *ProtoNode) CidBuilder() cid.Builder {
+ if n.builder == nil {
+ n.builder = v0CidPrefix
+ }
+ return n.builder
+}
+
+// SetCidBuilder sets the CID builder if it is non nil, if nil then it
+// is reset to the default value. An error will be returned if the builder
+// is not usable.
+func (n *ProtoNode) SetCidBuilder(builder cid.Builder) error {
+ if builder == nil {
+ n.builder = v0CidPrefix
+ return nil
+ }
+ switch b := builder.(type) {
+ case cid.Prefix:
+ if err := checkHasher(b.MhType, b.MhLength); err != nil {
+ return err
+ }
+ case *cid.Prefix:
+ if err := checkHasher(b.MhType, b.MhLength); err != nil {
+ return err
+ }
+ default:
+ // We have to test it's a usable hasher by invoking it and checking it
+ // doesn't error. This is only a basic check, there are still ways it may
+ // break
+ if _, err := builder.Sum(zeros[:]); err != nil {
+ return err
+ }
+ }
+ n.builder = builder.WithCodec(cid.DagProtobuf)
+ n.cached = cid.Undef
+ return nil
+}
+
+// check whether the hasher is likely to be a usable one
+func checkHasher(indicator uint64, sizeHint int) error {
+ mhLen := sizeHint
+ if mhLen <= 0 {
+ mhLen = -1
+ }
+ _, err := mhcore.GetVariableHasher(indicator, mhLen)
+ return err
+}
+
+// LinkSlice is a slice of format.Links
+type LinkSlice []*format.Link
+
+func (ls LinkSlice) Len() int { return len(ls) }
+func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] }
+func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name }
+
+// NodeWithData builds a new Protonode with the given data.
+func NodeWithData(d []byte) *ProtoNode {
+ return &ProtoNode{data: d}
+}
+
+// AddNodeLink adds a link to another node. The link will be added in
+// sorted order.
+//
+// If sorting has not already been applied to this node (because
+// it was deserialized from a form that did not have sorted links), the links
+// list will be sorted. If a ProtoNode was deserialized from a badly encoded
+// form that did not already have its links sorted, calling AddNodeLink and then
+// RemoveNodeLink for the same link, will not result in an identically encoded
+// form as the links will have been sorted.
+func (n *ProtoNode) AddNodeLink(name string, that format.Node) error {
+ lnk, err := format.MakeLink(that)
+ if err != nil {
+ return err
+ }
+
+ lnk.Name = name
+
+ n.AddRawLink(name, lnk)
+
+ return nil
+}
+
+// AddRawLink adds a copy of a link to this node. The link will be added in
+// sorted order.
+//
+// If sorting has not already been applied to this node (because
+// it was deserialized from a form that did not have sorted links), the links
+// list will be sorted. If a ProtoNode was deserialized from a badly encoded
+// form that did not already have its links sorted, calling AddRawLink and then
+// RemoveNodeLink for the same link, will not result in an identically encoded
+// form as the links will have been sorted.
+func (n *ProtoNode) AddRawLink(name string, l *format.Link) error {
+ lnk := &format.Link{
+ Name: name,
+ Size: l.Size,
+ Cid: l.Cid,
+ }
+ if err := checkLink(lnk); err != nil {
+ return err
+ }
+ n.links = append(n.links, lnk)
+ n.linksDirty = true // needs a sort
+ n.encoded = nil
+ return nil
+}
+
+// RemoveNodeLink removes a link on this node by the given name. If there are
+// no links with this name, ErrLinkNotFound will be returned. If there are more
+// than one link with this name, they will all be removed.
+func (n *ProtoNode) RemoveNodeLink(name string) error {
+ ref := n.links[:0]
+ found := false
+
+ for _, v := range n.links {
+ if v.Name != name {
+ ref = append(ref, v)
+ } else {
+ found = true
+ }
+ }
+
+ if !found {
+ return ErrLinkNotFound
+ }
+
+ n.links = ref
+ // Even though a removal won't change sorting, this node may have come from
+ // a deserialized state with badly sorted links. Now that we are mutating,
+ // we need to ensure the resulting link list is sorted when it gets consumed.
+ n.linksDirty = true
+ n.encoded = nil
+
+ return nil
+}
+
+// GetNodeLink returns a copy of the link with the given name.
+func (n *ProtoNode) GetNodeLink(name string) (*format.Link, error) {
+ for _, l := range n.links {
+ if l.Name == name {
+ return &format.Link{
+ Name: l.Name,
+ Size: l.Size,
+ Cid: l.Cid,
+ }, nil
+ }
+ }
+ return nil, ErrLinkNotFound
+}
+
+// GetLinkedProtoNode returns a copy of the ProtoNode with the given name.
+func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds format.DAGService, name string) (*ProtoNode, error) {
+ nd, err := n.GetLinkedNode(ctx, ds, name)
+ if err != nil {
+ return nil, err
+ }
+
+ pbnd, ok := nd.(*ProtoNode)
+ if !ok {
+ return nil, ErrNotProtobuf
+ }
+
+ return pbnd, nil
+}
+
+// GetLinkedNode returns a copy of the IPLD Node with the given name.
+func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds format.DAGService, name string) (format.Node, error) {
+ lnk, err := n.GetNodeLink(name)
+ if err != nil {
+ return nil, err
+ }
+
+ return lnk.GetNode(ctx, ds)
+}
+
+// Copy returns a copy of the node. The resulting node will have a properly
+// sorted Links list regardless of whether the original came from a badly
+// serialized form that didn't have a sorted list.
+// NOTE: This does not make copies of Node objects in the links.
+func (n *ProtoNode) Copy() format.Node {
+ nnode := new(ProtoNode)
+ if len(n.data) > 0 {
+ nnode.data = make([]byte, len(n.data))
+ copy(nnode.data, n.data)
+ }
+
+ if len(n.links) > 0 {
+ nnode.links = append([]*format.Link(nil), n.links...)
+ // Sort links regardless of linksDirty state, this may have come from a
+ // serialized form that had badly sorted links, in which case linksDirty
+ // will not be true.
+ sort.Stable(LinkSlice(nnode.links))
+ }
+
+ nnode.builder = n.builder
+
+ return nnode
+}
+
+// RawData returns the encoded byte form of this node.
+//
+// Note that this method may return an empty byte slice if there is an error
+// performing the encode. To check whether such an error may have occurred, use
+// node.EncodeProtobuf(false), instead (or prior to calling RawData) and check
+// for its returned error value; the result of EncodeProtobuf is cached so there
+// is minimal overhead when invoking both methods.
+func (n *ProtoNode) RawData() []byte {
+ out, err := n.EncodeProtobuf(false)
+ if err != nil {
+ log.Errorf("failed to encode dag-pb block: %s", err.Error())
+ return nil
+ }
+ return out
+}
+
+// Data returns the data stored by this node.
+func (n *ProtoNode) Data() []byte {
+ return n.data
+}
+
+// SetData stores data in this nodes.
+func (n *ProtoNode) SetData(d []byte) {
+ n.encoded = nil
+ n.cached = cid.Undef
+ n.data = d
+}
+
+// UpdateNodeLink return a copy of the node with the link name set to point to
+// that. The link will be added in sorted order. If a link of the same name
+// existed, it is removed.
+//
+// If sorting has not already been applied to this node (because
+// it was deserialized from a form that did not have sorted links), the links
+// list will be sorted in the returned copy.
+func (n *ProtoNode) UpdateNodeLink(name string, that *ProtoNode) (*ProtoNode, error) {
+ newnode := n.Copy().(*ProtoNode)
+ _ = newnode.RemoveNodeLink(name) // ignore error
+ err := newnode.AddNodeLink(name, that)
+ return newnode, err
+}
+
+// Size returns the total size of the data addressed by node,
+// including the total sizes of references.
+func (n *ProtoNode) Size() (uint64, error) {
+ b, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return 0, err
+ }
+
+ s := uint64(len(b))
+ for _, l := range n.links {
+ s += l.Size
+ }
+ return s, nil
+}
+
+// Stat returns statistics on the node.
+func (n *ProtoNode) Stat() (*format.NodeStat, error) {
+ enc, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return nil, err
+ }
+
+ cumSize, err := n.Size()
+ if err != nil {
+ return nil, err
+ }
+
+ return &format.NodeStat{
+ Hash: n.Cid().String(),
+ NumLinks: len(n.links),
+ BlockSize: len(enc),
+ LinksSize: len(enc) - len(n.data), // includes framing.
+ DataSize: len(n.data),
+ CumulativeSize: int(cumSize),
+ }, nil
+}
+
+// Loggable implements the ipfs/go-log.Loggable interface.
+func (n *ProtoNode) Loggable() map[string]interface{} {
+ return map[string]interface{}{
+ "node": n.String(),
+ }
+}
+
+// UnmarshalJSON reads the node fields from a JSON-encoded byte slice.
+func (n *ProtoNode) UnmarshalJSON(b []byte) error {
+ s := struct {
+ Data []byte `json:"data"`
+ Links []*format.Link `json:"links"`
+ }{}
+
+ err := json.Unmarshal(b, &s)
+ if err != nil {
+ return err
+ }
+
+ n.data = s.Data
+ // Links may not be sorted after deserialization, but we don't change
+ // them until we mutate this node since we're representing the current,
+ // as-serialized state. So n.linksDirty is not set here.
+ n.links = s.Links
+ for _, lnk := range s.Links {
+ if err := checkLink(lnk); err != nil {
+ return err
+ }
+ }
+
+ n.encoded = nil
+ return nil
+}
+
+func checkLink(lnk *format.Link) error {
+ if lnk.Size > math.MaxInt64 {
+ return fmt.Errorf("value of Tsize is too large: %d", lnk.Size)
+ }
+ if !lnk.Cid.Defined() {
+ return errors.New("link must have a value Cid value")
+ }
+ return nil
+}
+
+// MarshalJSON returns a JSON representation of the node.
+func (n *ProtoNode) MarshalJSON() ([]byte, error) {
+ if n.linksDirty {
+ // there was a mutation involving links, make sure we sort
+ sort.Stable(LinkSlice(n.links))
+ n.linksDirty = false
+ n.encoded = nil
+ }
+
+ out := map[string]interface{}{
+ "data": n.data,
+ "links": n.links,
+ }
+
+ return json.Marshal(out)
+}
+
+// Cid returns the node's Cid, calculated according to its prefix
+// and raw data contents.
+//
+// Note that this method may return a CID representing a zero-length byte slice
+// if there is an error performing the encode. To check whether such an error
+// may have occurred, use node.EncodeProtobuf(false), instead (or prior to
+// calling RawData) and check for its returned error value; the result of
+// EncodeProtobuf is cached so there is minimal overhead when invoking both
+// methods.
+func (n *ProtoNode) Cid() cid.Cid {
+ // re-encode if necessary and we'll get a new cached CID
+ if _, err := n.EncodeProtobuf(false); err != nil {
+ log.Errorf("failed to encode dag-pb block: %s", err.Error())
+ // error, return a zero-CID
+ c, err := n.CidBuilder().Sum([]byte{})
+ if err != nil {
+ // CidBuilder was a source of error, return _the_ dag-pb zero CIDv1
+ return zeroCid
+ }
+ return c
+ }
+ return n.cached
+}
+
+// String prints the node's Cid.
+//
+// Note that this method may return a CID representing a zero-length byte slice
+// if there is an error performing the encode. To check whether such an error
+// may have occurred, use node.EncodeProtobuf(false), instead (or prior to
+// calling RawData) and check for its returned error value; the result of
+// EncodeProtobuf is cached so there is minimal overhead when invoking both
+// methods.
+func (n *ProtoNode) String() string {
+ return n.Cid().String()
+}
+
+// Multihash hashes the encoded data of this node.
+//
+// Note that this method may return a multihash representing a zero-length byte
+// slice if there is an error performing the encode. To check whether such an
+// error may have occurred, use node.EncodeProtobuf(false), instead (or prior to
+// calling RawData) and check for its returned error value; the result of
+// EncodeProtobuf is cached so there is minimal overhead when invoking both
+// methods.
+func (n *ProtoNode) Multihash() mh.Multihash {
+ return n.Cid().Hash()
+}
+
+// Links returns a copy of the node's links.
+func (n *ProtoNode) Links() []*format.Link {
+ if n.linksDirty {
+ // there was a mutation involving links, make sure we sort
+ sort.Stable(LinkSlice(n.links))
+ n.linksDirty = false
+ n.encoded = nil
+ }
+ return append([]*format.Link(nil), n.links...)
+}
+
+// SetLinks replaces the node links with a copy of the provided links. Sorting
+// will be applied to the list.
+func (n *ProtoNode) SetLinks(links []*format.Link) error {
+ for _, lnk := range links {
+ if err := checkLink(lnk); err != nil {
+ return err
+ }
+ }
+ n.links = append([]*format.Link(nil), links...)
+ n.linksDirty = true // needs a sort
+ n.encoded = nil
+ return nil
+}
+
+// Resolve is an alias for ResolveLink.
+func (n *ProtoNode) Resolve(path []string) (interface{}, []string, error) {
+ return n.ResolveLink(path)
+}
+
+// ResolveLink consumes the first element of the path and obtains the link
+// corresponding to it from the node. It returns the link
+// and the path without the consumed element.
+func (n *ProtoNode) ResolveLink(path []string) (*format.Link, []string, error) {
+ if len(path) == 0 {
+ return nil, nil, fmt.Errorf("end of path, no more links to resolve")
+ }
+
+ lnk, err := n.GetNodeLink(path[0])
+ if err != nil {
+ return nil, nil, err
+ }
+
+ return lnk, path[1:], nil
+}
+
+// Tree returns the link names of the ProtoNode.
+// ProtoNodes are only ever one path deep, so anything different than an empty
+// string for p results in nothing. The depth parameter is ignored.
+func (n *ProtoNode) Tree(p string, depth int) []string {
+ if p != "" {
+ return nil
+ }
+
+ if n.linksDirty {
+ // there was a mutation involving links, make sure we sort
+ sort.Stable(LinkSlice(n.links))
+ n.linksDirty = false
+ n.encoded = nil
+ }
+
+ out := make([]string, 0, len(n.links))
+ for _, lnk := range n.links {
+ out = append(out, lnk.Name)
+ }
+ return out
+}
+
+func ProtoNodeConverter(b blocks.Block, nd ipld.Node) (legacy.UniversalNode, error) {
+ pbNode, ok := nd.(dagpb.PBNode)
+ if !ok {
+ return nil, ErrNotProtobuf
+ }
+ encoded := &immutableProtoNode{b.RawData(), pbNode}
+ pn := fromImmutableNode(encoded)
+ pn.cached = b.Cid()
+ pn.builder = b.Cid().Prefix()
+ return pn, nil
+}
+
+// TODO: replace with cid.MustParse() when we bump go-cid
+func mustZeroCid() cid.Cid {
+ c, err := cid.Parse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")
+ if err != nil {
+ panic(err)
+ }
+ return c
+}
+
+var _ legacy.UniversalNode = &ProtoNode{}
diff --git a/ipld/merkledag/node_test.go b/ipld/merkledag/node_test.go
new file mode 100644
index 000000000..87fb484e8
--- /dev/null
+++ b/ipld/merkledag/node_test.go
@@ -0,0 +1,194 @@
+package merkledag_test
+
+import (
+ "bytes"
+ "context"
+ "testing"
+
+ . "github.com/ipfs/boxo/ipld/merkledag"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+var sampleCid cid.Cid
+
+func init() {
+ var err error
+ // make a test CID -- doesn't matter just to add as a link
+ sampleCid, err = cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4})
+ if err != nil {
+ panic(err)
+ }
+}
+
+func TestStableCID(t *testing.T) {
+ nd := &ProtoNode{}
+ nd.SetData([]byte("foobar"))
+ nd.SetLinks([]*ipld.Link{
+ {Name: "a", Cid: sampleCid},
+ {Name: "b", Cid: sampleCid},
+ {Name: "c", Cid: sampleCid},
+ })
+ expected, err := cid.Decode("QmciCHWD9Q47VPX6naY3XsPZGnqVqbedAniGCcaHjBaCri")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !nd.Cid().Equals(expected) {
+ t.Fatalf("Got CID %s, expected CID %s", nd.Cid(), expected)
+ }
+}
+
+func TestRemoveLink(t *testing.T) {
+ nd := &ProtoNode{}
+ nd.SetLinks([]*ipld.Link{
+ {Name: "a", Cid: sampleCid},
+ {Name: "b", Cid: sampleCid},
+ {Name: "a", Cid: sampleCid},
+ {Name: "a", Cid: sampleCid},
+ {Name: "c", Cid: sampleCid},
+ {Name: "a", Cid: sampleCid},
+ })
+
+ err := nd.RemoveNodeLink("a")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(nd.Links()) != 2 {
+ t.Fatal("number of links incorrect")
+ }
+
+ if nd.Links()[0].Name != "b" {
+ t.Fatal("link order wrong")
+ }
+
+ if nd.Links()[1].Name != "c" {
+ t.Fatal("link order wrong")
+ }
+
+ // should fail
+ err = nd.RemoveNodeLink("a")
+ if err != ErrLinkNotFound {
+ t.Fatal("should have failed to remove link")
+ }
+
+ // ensure nothing else got touched
+ if len(nd.Links()) != 2 {
+ t.Fatal("number of links incorrect")
+ }
+
+ if nd.Links()[0].Name != "b" {
+ t.Fatal("link order wrong")
+ }
+
+ if nd.Links()[1].Name != "c" {
+ t.Fatal("link order wrong")
+ }
+}
+
+func TestFindLink(t *testing.T) {
+ ctx := context.Background()
+
+ ds := mdtest.Mock()
+ ndEmpty := new(ProtoNode)
+ err := ds.Add(ctx, ndEmpty)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ kEmpty := ndEmpty.Cid()
+
+ nd := &ProtoNode{}
+ nd.SetLinks([]*ipld.Link{
+ {Name: "a", Cid: kEmpty},
+ {Name: "c", Cid: kEmpty},
+ {Name: "b", Cid: kEmpty},
+ })
+
+ err = ds.Add(ctx, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ lnk, err := nd.GetNodeLink("b")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if lnk.Name != "b" {
+ t.Fatal("got wrong link back")
+ }
+
+ _, err = nd.GetNodeLink("f")
+ if err != ErrLinkNotFound {
+ t.Fatal("shouldnt have found link")
+ }
+
+ _, err = nd.GetLinkedNode(context.Background(), ds, "b")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ outnd, err := nd.UpdateNodeLink("b", nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ olnk, err := outnd.GetNodeLink("b")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if olnk.Cid.String() == kEmpty.String() {
+ t.Fatal("new link should have different hash")
+ }
+}
+
+func TestNodeCopy(t *testing.T) {
+ nd := &ProtoNode{}
+ nd.SetLinks([]*ipld.Link{
+ {Name: "a", Cid: sampleCid},
+ {Name: "c", Cid: sampleCid},
+ {Name: "b", Cid: sampleCid},
+ })
+
+ nd.SetData([]byte("testing"))
+
+ ond := nd.Copy().(*ProtoNode)
+ ond.SetData(nil)
+
+ if nd.Data() == nil {
+ t.Fatal("should be different objects")
+ }
+}
+
+func TestJsonRoundtrip(t *testing.T) {
+ nd := new(ProtoNode)
+ nd.SetLinks([]*ipld.Link{
+ {Name: "a", Cid: sampleCid},
+ {Name: "c", Cid: sampleCid},
+ {Name: "b", Cid: sampleCid},
+ })
+ nd.SetData([]byte("testing"))
+
+ jb, err := nd.MarshalJSON()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nn := new(ProtoNode)
+ err = nn.UnmarshalJSON(jb)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(nn.Data(), nd.Data()) {
+ t.Fatal("data wasnt the same")
+ }
+
+ if !nn.Cid().Equals(nd.Cid()) {
+ t.Fatal("objects differed after marshaling")
+ }
+}
diff --git a/ipld/merkledag/pb/compat_test.go b/ipld/merkledag/pb/compat_test.go
new file mode 100644
index 000000000..529cee0ca
--- /dev/null
+++ b/ipld/merkledag/pb/compat_test.go
@@ -0,0 +1,299 @@
+package merkledag_pb
+
+// mirrored in JavaScript @ https://github.com/ipld/js-dag-pb/blob/master/test/test-compat.js
+
+import (
+ "encoding/hex"
+ "encoding/json"
+ "testing"
+)
+
+var dataZero []byte = make([]byte, 0)
+var dataSome []byte = []byte{0, 1, 2, 3, 4}
+var cidBytes []byte = []byte{1, 85, 0, 5, 0, 1, 2, 3, 4}
+var zeroName string = ""
+var someName string = "some name"
+var zeroTsize uint64 = 0
+var someTsize uint64 = 1010
+var largeTsize uint64 = 9007199254740991 // JavaScript Number.MAX_SAFE_INTEGER
+
+type testCase struct {
+ name string
+ node *PBNode
+ expectedBytes string
+ expectedForm string
+}
+
+var testCases = []testCase{
+ {
+ name: "empty",
+ node: &PBNode{},
+ expectedBytes: "",
+ expectedForm: "{}",
+ },
+ {
+ name: "Data zero",
+ node: &PBNode{Data: dataZero},
+ expectedBytes: "0a00",
+ expectedForm: `{
+ "Data": ""
+}`,
+ },
+ {
+ name: "Data some",
+ node: &PBNode{Data: dataSome},
+ expectedBytes: "0a050001020304",
+ expectedForm: `{
+ "Data": "0001020304"
+}`,
+ },
+ {
+ name: "Links zero",
+ node: &PBNode{Links: make([]*PBLink, 0)},
+ expectedBytes: "",
+ expectedForm: "{}",
+ },
+ {
+ name: "Data some Links zero",
+ node: &PBNode{Data: dataSome, Links: make([]*PBLink, 0)},
+ expectedBytes: "0a050001020304",
+ expectedForm: `{
+ "Data": "0001020304"
+}`,
+ },
+ {
+ name: "Links empty",
+ node: &PBNode{Links: []*PBLink{{}}},
+ expectedBytes: "1200",
+ expectedForm: `{
+ "Links": [
+ {}
+ ]
+}`,
+ },
+ {
+ name: "Data some Links empty",
+ node: &PBNode{Data: dataSome, Links: []*PBLink{{}}},
+ expectedBytes: "12000a050001020304",
+ expectedForm: `{
+ "Data": "0001020304",
+ "Links": [
+ {}
+ ]
+}`,
+ },
+ {
+ name: "Links Hash zero",
+ node: &PBNode{Links: []*PBLink{{Hash: dataZero}}},
+ expectedBytes: "12020a00",
+ expectedForm: `{
+ "Links": [
+ {
+ "Hash": ""
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Hash some",
+ node: &PBNode{Links: []*PBLink{{Hash: cidBytes}}},
+ expectedBytes: "120b0a09015500050001020304",
+ expectedForm: `{
+ "Links": [
+ {
+ "Hash": "015500050001020304"
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Name zero",
+ node: &PBNode{Links: []*PBLink{{Name: &zeroName}}},
+ expectedBytes: "12021200",
+ expectedForm: `{
+ "Links": [
+ {
+ "Name": ""
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Hash some Name zero",
+ node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Name: &zeroName}}},
+ expectedBytes: "120d0a090155000500010203041200",
+ expectedForm: `{
+ "Links": [
+ {
+ "Hash": "015500050001020304",
+ "Name": ""
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Name some",
+ node: &PBNode{Links: []*PBLink{{Name: &someName}}},
+ expectedBytes: "120b1209736f6d65206e616d65",
+ expectedForm: `{
+ "Links": [
+ {
+ "Name": "some name"
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Hash some Name some",
+ node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Name: &someName}}},
+ expectedBytes: "12160a090155000500010203041209736f6d65206e616d65",
+ expectedForm: `{
+ "Links": [
+ {
+ "Hash": "015500050001020304",
+ "Name": "some name"
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Tsize zero",
+ node: &PBNode{Links: []*PBLink{{Tsize: &zeroTsize}}},
+ expectedBytes: "12021800",
+ expectedForm: `{
+ "Links": [
+ {
+ "Tsize": 0
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Hash some Tsize zero",
+ node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Tsize: &zeroTsize}}},
+ expectedBytes: "120d0a090155000500010203041800",
+ expectedForm: `{
+ "Links": [
+ {
+ "Hash": "015500050001020304",
+ "Tsize": 0
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Tsize some",
+ node: &PBNode{Links: []*PBLink{{Tsize: &someTsize}}},
+ expectedBytes: "120318f207",
+ expectedForm: `{
+ "Links": [
+ {
+ "Tsize": 1010
+ }
+ ]
+}`,
+ },
+ {
+ name: "Links Hash some Tsize some",
+ node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Tsize: &largeTsize}}},
+ expectedBytes: "12140a0901550005000102030418ffffffffffffff0f",
+ expectedForm: `{
+ "Links": [
+ {
+ "Hash": "015500050001020304",
+ "Tsize": 9007199254740991
+ }
+ ]
+}`,
+ },
+}
+
+func TestCompat(t *testing.T) {
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ verifyRoundTrip(t, tc)
+ })
+ }
+}
+
+func verifyRoundTrip(t *testing.T, tc testCase) {
+ actualBytes, actualForm, err := nodeRoundTripToString(t, tc.node)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if actualBytes != tc.expectedBytes {
+ t.Logf(
+ "Expected bytes: [%v]\nGot: [%v]\n",
+ tc.expectedBytes,
+ actualBytes)
+ t.Error("Did not match")
+ }
+
+ if actualForm != tc.expectedForm {
+ t.Logf(
+ "Expected form: [%v]\nGot: [%v]\n",
+ tc.expectedForm,
+ actualForm)
+ t.Error("Did not match")
+ }
+}
+
+func nodeRoundTripToString(t *testing.T, n *PBNode) (string, string, error) {
+ bytes, err := n.Marshal()
+ if err != nil {
+ return "", "", err
+ }
+ t.Logf("[%v]\n", hex.EncodeToString(bytes))
+ rt := new(PBNode)
+ if err := rt.Unmarshal(bytes); err != nil {
+ return "", "", err
+ }
+ str, err := json.MarshalIndent(cleanPBNode(t, rt), "", "\t")
+ if err != nil {
+ return "", "", err
+ }
+ return hex.EncodeToString(bytes), string(str), nil
+}
+
+// convert a PBLink into a map for clean JSON marshalling
+func cleanPBLink(t *testing.T, link *PBLink) map[string]interface{} {
+ if link == nil {
+ return nil
+ }
+ // this would be a bad pb decode
+ if link.XXX_unrecognized != nil {
+ t.Fatal("Got unexpected XXX_unrecognized")
+ }
+ nl := make(map[string]interface{})
+ if link.Hash != nil {
+ nl["Hash"] = hex.EncodeToString(link.Hash)
+ }
+ if link.Name != nil {
+ nl["Name"] = link.Name
+ }
+ if link.Tsize != nil {
+ nl["Tsize"] = link.Tsize
+ }
+ return nl
+}
+
+// convert a PBNode into a map for clean JSON marshalling
+func cleanPBNode(t *testing.T, node *PBNode) map[string]interface{} {
+ // this would be a bad pb decode
+ if node.XXX_unrecognized != nil {
+ t.Fatal("Got unexpected XXX_unrecognized")
+ }
+ nn := make(map[string]interface{})
+ if node.Data != nil {
+ nn["Data"] = hex.EncodeToString(node.Data)
+ }
+ if node.Links != nil {
+ links := make([]map[string]interface{}, len(node.Links))
+ for i, l := range node.Links {
+ links[i] = cleanPBLink(t, l)
+ }
+ nn["Links"] = links
+ }
+ return nn
+}
diff --git a/ipld/merkledag/pb/merkledag.pb.go b/ipld/merkledag/pb/merkledag.pb.go
new file mode 100644
index 000000000..45806ce92
--- /dev/null
+++ b/ipld/merkledag/pb/merkledag.pb.go
@@ -0,0 +1,1062 @@
+// Code originally generated by protoc-gen-gogo from merkledag.proto,
+// now manually managed
+
+package merkledag_pb
+
+import (
+ bytes "bytes"
+ fmt "fmt"
+ io "io"
+ math "math"
+ math_bits "math/bits"
+ reflect "reflect"
+ strings "strings"
+
+ _ "github.com/gogo/protobuf/gogoproto"
+ proto "github.com/gogo/protobuf/proto"
+)
+
+// DoNotUpgradeFileEverItWillChangeYourHashes warns users about not breaking
+// their file hashes.
+const DoNotUpgradeFileEverItWillChangeYourHashes = `
+This file does not produce canonical protobufs. Unfortunately, if we change it,
+we'll change the hashes of the files we produce.
+
+Do *not regenerate this file.
+`
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+// An IPFS MerkleDAG Link
+type PBLink struct {
+ // multihash of the target object
+ Hash []byte `protobuf:"bytes,1,opt,name=Hash" json:"Hash,omitempty"`
+ // utf string name. should be unique per object
+ Name *string `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"`
+ // cumulative size of target object
+ Tsize *uint64 `protobuf:"varint,3,opt,name=Tsize" json:"Tsize,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PBLink) Reset() { *m = PBLink{} }
+func (*PBLink) ProtoMessage() {}
+func (*PBLink) Descriptor() ([]byte, []int) {
+ return fileDescriptor_10837cc3557cec00, []int{0}
+}
+func (m *PBLink) XXX_Unmarshal(b []byte) error {
+ return m.Unmarshal(b)
+}
+func (m *PBLink) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ if deterministic {
+ return xxx_messageInfo_PBLink.Marshal(b, m, deterministic)
+ } else {
+ b = b[:cap(b)]
+ n, err := m.MarshalToSizedBuffer(b)
+ if err != nil {
+ return nil, err
+ }
+ return b[:n], nil
+ }
+}
+func (m *PBLink) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PBLink.Merge(m, src)
+}
+func (m *PBLink) XXX_Size() int {
+ return m.Size()
+}
+func (m *PBLink) XXX_DiscardUnknown() {
+ xxx_messageInfo_PBLink.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PBLink proto.InternalMessageInfo
+
+func (m *PBLink) GetHash() []byte {
+ if m != nil {
+ return m.Hash
+ }
+ return nil
+}
+
+func (m *PBLink) GetName() string {
+ if m != nil && m.Name != nil {
+ return *m.Name
+ }
+ return ""
+}
+
+func (m *PBLink) GetTsize() uint64 {
+ if m != nil && m.Tsize != nil {
+ return *m.Tsize
+ }
+ return 0
+}
+
+// An IPFS MerkleDAG Node
+type PBNode struct {
+ // refs to other objects
+ Links []*PBLink `protobuf:"bytes,2,rep,name=Links" json:"Links,omitempty"`
+ // opaque user data
+ Data []byte `protobuf:"bytes,1,opt,name=Data" json:"Data,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *PBNode) Reset() { *m = PBNode{} }
+func (*PBNode) ProtoMessage() {}
+func (*PBNode) Descriptor() ([]byte, []int) {
+ return fileDescriptor_10837cc3557cec00, []int{1}
+}
+func (m *PBNode) XXX_Unmarshal(b []byte) error {
+ return m.Unmarshal(b)
+}
+func (m *PBNode) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ if deterministic {
+ return xxx_messageInfo_PBNode.Marshal(b, m, deterministic)
+ } else {
+ b = b[:cap(b)]
+ n, err := m.MarshalToSizedBuffer(b)
+ if err != nil {
+ return nil, err
+ }
+ return b[:n], nil
+ }
+}
+func (m *PBNode) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_PBNode.Merge(m, src)
+}
+func (m *PBNode) XXX_Size() int {
+ return m.Size()
+}
+func (m *PBNode) XXX_DiscardUnknown() {
+ xxx_messageInfo_PBNode.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PBNode proto.InternalMessageInfo
+
+func (m *PBNode) GetLinks() []*PBLink {
+ if m != nil {
+ return m.Links
+ }
+ return nil
+}
+
+func (m *PBNode) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+func init() {
+ proto.RegisterType((*PBLink)(nil), "merkledag.pb.PBLink")
+ proto.RegisterType((*PBNode)(nil), "merkledag.pb.PBNode")
+}
+
+func init() { proto.RegisterFile("merkledag.proto", fileDescriptor_10837cc3557cec00) }
+
+var fileDescriptor_10837cc3557cec00 = []byte{
+ // 227 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcf, 0x4d, 0x2d, 0xca,
+ 0xce, 0x49, 0x4d, 0x49, 0x4c, 0xd7, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x41, 0x12, 0x48,
+ 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f,
+ 0xcf, 0xd7, 0x07, 0x2b, 0x4a, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x59, 0xc9,
+ 0x8d, 0x8b, 0x2d, 0xc0, 0xc9, 0x27, 0x33, 0x2f, 0x5b, 0x48, 0x88, 0x8b, 0xc5, 0x23, 0xb1, 0x38,
+ 0x43, 0x82, 0x51, 0x81, 0x51, 0x83, 0x27, 0x08, 0xcc, 0x06, 0x89, 0xf9, 0x25, 0xe6, 0xa6, 0x4a,
+ 0x30, 0x29, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x42, 0x22, 0x5c, 0xac, 0x21, 0xc5, 0x99, 0x55,
+ 0xa9, 0x12, 0xcc, 0x0a, 0x8c, 0x1a, 0x2c, 0x41, 0x10, 0x8e, 0x92, 0x07, 0xc8, 0x1c, 0xbf, 0xfc,
+ 0x94, 0x54, 0x21, 0x2d, 0x2e, 0x56, 0x90, 0x79, 0xc5, 0x12, 0x4c, 0x0a, 0xcc, 0x1a, 0xdc, 0x46,
+ 0x22, 0x7a, 0xc8, 0xce, 0xd3, 0x83, 0x58, 0x16, 0x04, 0x51, 0x02, 0x32, 0xdf, 0x25, 0xb1, 0x24,
+ 0x11, 0x66, 0x27, 0x88, 0xed, 0xa4, 0x73, 0xe3, 0xa1, 0x1c, 0xc3, 0x83, 0x87, 0x72, 0x8c, 0x1f,
+ 0x1e, 0xca, 0x31, 0xfe, 0x78, 0x28, 0xc7, 0xd8, 0xf0, 0x48, 0x8e, 0x71, 0xc5, 0x23, 0x39, 0xc6,
+ 0x1d, 0x8f, 0xe4, 0x18, 0x0f, 0x3c, 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39,
+ 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1d, 0x56, 0xb5, 0x6e, 0x0e,
+ 0x01, 0x00, 0x00,
+}
+
+func (pbLink *PBLink) VerboseEqual(that interface{}) error {
+ if that == nil {
+ if pbLink == nil {
+ return nil
+ }
+ return fmt.Errorf("that == nil && this != nil")
+ }
+
+ that1, ok := that.(*PBLink)
+ if !ok {
+ that2, ok := that.(PBLink)
+ if ok {
+ that1 = &that2
+ } else {
+ return fmt.Errorf("that is not of type *PBLink")
+ }
+ }
+ if that1 == nil {
+ if pbLink == nil {
+ return nil
+ }
+ return fmt.Errorf("that is type *PBLink but is nil && this != nil")
+ } else if pbLink == nil {
+ return fmt.Errorf("that is type *PBLink but is not nil && this == nil")
+ }
+ if !bytes.Equal(pbLink.Hash, that1.Hash) {
+ return fmt.Errorf("this.Hash(%v) is not equal to that.Hash(%v)", pbLink.Hash, that1.Hash)
+ }
+ if pbLink.Name != nil && that1.Name != nil {
+ if *pbLink.Name != *that1.Name {
+ return fmt.Errorf("this.Name(%v) is not equal to that.Name(%v)", *pbLink.Name, *that1.Name)
+ }
+ } else if pbLink.Name != nil {
+ return fmt.Errorf("this.Name == nil && that.Name != nil")
+ } else if that1.Name != nil {
+ return fmt.Errorf("this.Name(%v) is not equal to that.Name(%v)", pbLink.Name, that1.Name)
+ }
+ if pbLink.Tsize != nil && that1.Tsize != nil {
+ if *pbLink.Tsize != *that1.Tsize {
+ return fmt.Errorf("this.Tsize(%v) is not equal to that.Tsize(%v)", *pbLink.Tsize, *that1.Tsize)
+ }
+ } else if pbLink.Tsize != nil {
+ return fmt.Errorf("this.Tsize == nil && that.Tsize != nil")
+ } else if that1.Tsize != nil {
+ return fmt.Errorf("this.Tsize(%v) is not equal to that.Tsize(%v)", pbLink.Tsize, that1.Tsize)
+ }
+ if !bytes.Equal(pbLink.XXX_unrecognized, that1.XXX_unrecognized) {
+ return fmt.Errorf("XXX_unrecognized this(%v) is not equal to that(%v)", pbLink.XXX_unrecognized, that1.XXX_unrecognized)
+ }
+ return nil
+}
+func (pbLink *PBLink) Equal(that interface{}) bool {
+ if that == nil {
+ return pbLink == nil
+ }
+
+ that1, ok := that.(*PBLink)
+ if !ok {
+ that2, ok := that.(PBLink)
+ if ok {
+ that1 = &that2
+ } else {
+ return false
+ }
+ }
+ if that1 == nil {
+ return pbLink == nil
+ } else if pbLink == nil {
+ return false
+ }
+ if !bytes.Equal(pbLink.Hash, that1.Hash) {
+ return false
+ }
+ if pbLink.Name != nil && that1.Name != nil {
+ if *pbLink.Name != *that1.Name {
+ return false
+ }
+ } else if pbLink.Name != nil {
+ return false
+ } else if that1.Name != nil {
+ return false
+ }
+ if pbLink.Tsize != nil && that1.Tsize != nil {
+ if *pbLink.Tsize != *that1.Tsize {
+ return false
+ }
+ } else if pbLink.Tsize != nil {
+ return false
+ } else if that1.Tsize != nil {
+ return false
+ }
+ if !bytes.Equal(pbLink.XXX_unrecognized, that1.XXX_unrecognized) {
+ return false
+ }
+ return true
+}
+func (pbLink *PBNode) VerboseEqual(that interface{}) error {
+ if that == nil {
+ if pbLink == nil {
+ return nil
+ }
+ return fmt.Errorf("that == nil && this != nil")
+ }
+
+ that1, ok := that.(*PBNode)
+ if !ok {
+ that2, ok := that.(PBNode)
+ if ok {
+ that1 = &that2
+ } else {
+ return fmt.Errorf("that is not of type *PBNode")
+ }
+ }
+ if that1 == nil {
+ if pbLink == nil {
+ return nil
+ }
+ return fmt.Errorf("that is type *PBNode but is nil && this != nil")
+ } else if pbLink == nil {
+ return fmt.Errorf("that is type *PBNode but is not nil && this == nil")
+ }
+ if len(pbLink.Links) != len(that1.Links) {
+ return fmt.Errorf("len(this.Links)(%v) is not equal to len(that.Links)(%v)", len(pbLink.Links), len(that1.Links))
+ }
+ for i := range pbLink.Links {
+ if !pbLink.Links[i].Equal(that1.Links[i]) {
+ return fmt.Errorf("this.Links[%v](%v) is not equal to that.Links[%v](%v)", i, pbLink.Links[i], i, that1.Links[i])
+ }
+ }
+ if !bytes.Equal(pbLink.Data, that1.Data) {
+ return fmt.Errorf("this.Data(%v) is not equal to that.Data(%v)", pbLink.Data, that1.Data)
+ }
+ if !bytes.Equal(pbLink.XXX_unrecognized, that1.XXX_unrecognized) {
+ return fmt.Errorf("this.XXX_unrecognized(%v) is not equal to that.XXX_unrecognized(%v)", pbLink.XXX_unrecognized, that1.XXX_unrecognized)
+ }
+ return nil
+}
+
+func (pbNode *PBNode) Equal(that interface{}) bool {
+ if that == nil {
+ return pbNode == nil
+ }
+
+ that1, ok := that.(*PBNode)
+ if !ok {
+ that2, ok := that.(PBNode)
+ if ok {
+ that1 = &that2
+ } else {
+ return false
+ }
+ }
+ if that1 == nil {
+ return pbNode == nil
+ } else if pbNode == nil {
+ return false
+ }
+ if len(pbNode.Links) != len(that1.Links) {
+ return false
+ }
+ for i := range pbNode.Links {
+ if !pbNode.Links[i].Equal(that1.Links[i]) {
+ return false
+ }
+ }
+ if !bytes.Equal(pbNode.Data, that1.Data) {
+ return false
+ }
+ if !bytes.Equal(pbNode.XXX_unrecognized, that1.XXX_unrecognized) {
+ return false
+ }
+ return true
+}
+func (pbLink *PBLink) GoString() string {
+ if pbLink == nil {
+ return "nil"
+ }
+ s := make([]string, 0, 7)
+ s = append(s, "&merkledag_pb.PBLink{")
+ if pbLink.Hash != nil {
+ s = append(s, "Hash: "+valueToGoStringMerkledag(pbLink.Hash, "byte")+",\n")
+ }
+ if pbLink.Name != nil {
+ s = append(s, "Name: "+valueToGoStringMerkledag(pbLink.Name, "string")+",\n")
+ }
+ if pbLink.Tsize != nil {
+ s = append(s, "Tsize: "+valueToGoStringMerkledag(pbLink.Tsize, "uint64")+",\n")
+ }
+ if pbLink.XXX_unrecognized != nil {
+ s = append(s, "XXX_unrecognized:"+fmt.Sprintf("%#v", pbLink.XXX_unrecognized)+",\n")
+ }
+ s = append(s, "}")
+ return strings.Join(s, "")
+}
+func (pbNode *PBNode) GoString() string {
+ if pbNode == nil {
+ return "nil"
+ }
+ s := make([]string, 0, 6)
+ s = append(s, "&merkledag_pb.PBNode{")
+ if pbNode.Links != nil {
+ s = append(s, "Links: "+fmt.Sprintf("%#v", pbNode.Links)+",\n")
+ }
+ if pbNode.Data != nil {
+ s = append(s, "Data: "+valueToGoStringMerkledag(pbNode.Data, "byte")+",\n")
+ }
+ if pbNode.XXX_unrecognized != nil {
+ s = append(s, "XXX_unrecognized:"+fmt.Sprintf("%#v", pbNode.XXX_unrecognized)+",\n")
+ }
+ s = append(s, "}")
+ return strings.Join(s, "")
+}
+func valueToGoStringMerkledag(v interface{}, typ string) string {
+ rv := reflect.ValueOf(v)
+ if rv.IsNil() {
+ return "nil"
+ }
+ pv := reflect.Indirect(rv).Interface()
+ return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv)
+}
+func (m *PBLink) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBuffer(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *PBLink) MarshalTo(dAtA []byte) (int, error) {
+ size := m.Size()
+ return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PBLink) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if m.XXX_unrecognized != nil {
+ i -= len(m.XXX_unrecognized)
+ copy(dAtA[i:], m.XXX_unrecognized)
+ }
+ if m.Tsize != nil {
+ i = encodeVarintMerkledag(dAtA, i, uint64(*m.Tsize))
+ i--
+ dAtA[i] = 0x18
+ }
+ if m.Name != nil {
+ i -= len(*m.Name)
+ copy(dAtA[i:], *m.Name)
+ i = encodeVarintMerkledag(dAtA, i, uint64(len(*m.Name)))
+ i--
+ dAtA[i] = 0x12
+ }
+ if m.Hash != nil {
+ i -= len(m.Hash)
+ copy(dAtA[i:], m.Hash)
+ i = encodeVarintMerkledag(dAtA, i, uint64(len(m.Hash)))
+ i--
+ dAtA[i] = 0xa
+ }
+ return len(dAtA) - i, nil
+}
+
+func (m *PBNode) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBuffer(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *PBNode) MarshalTo(dAtA []byte) (int, error) {
+ size := m.Size()
+ return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PBNode) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if m.XXX_unrecognized != nil {
+ i -= len(m.XXX_unrecognized)
+ copy(dAtA[i:], m.XXX_unrecognized)
+ }
+ if m.Data != nil {
+ i -= len(m.Data)
+ copy(dAtA[i:], m.Data)
+ i = encodeVarintMerkledag(dAtA, i, uint64(len(m.Data)))
+ i--
+ dAtA[i] = 0xa
+ }
+ if len(m.Links) > 0 {
+ for iNdEx := len(m.Links) - 1; iNdEx >= 0; iNdEx-- {
+ {
+ size, err := m.Links[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+ if err != nil {
+ return 0, err
+ }
+ i -= size
+ i = encodeVarintMerkledag(dAtA, i, uint64(size))
+ }
+ i--
+ dAtA[i] = 0x12
+ }
+ }
+ return len(dAtA) - i, nil
+}
+
+func encodeVarintMerkledag(dAtA []byte, offset int, v uint64) int {
+ offset -= sovMerkledag(v)
+ base := offset
+ for v >= 1<<7 {
+ dAtA[offset] = uint8(v&0x7f | 0x80)
+ v >>= 7
+ offset++
+ }
+ dAtA[offset] = uint8(v)
+ return base
+}
+func NewPopulatedPBLink(r randyMerkledag, easy bool) *PBLink {
+ this := &PBLink{}
+ if r.Intn(5) != 0 {
+ v1 := r.Intn(100)
+ this.Hash = make([]byte, v1)
+ for i := 0; i < v1; i++ {
+ this.Hash[i] = byte(r.Intn(256))
+ }
+ }
+ if r.Intn(5) != 0 {
+ v2 := string(randStringMerkledag(r))
+ this.Name = &v2
+ }
+ if r.Intn(5) != 0 {
+ v3 := uint64(uint64(r.Uint32()))
+ this.Tsize = &v3
+ }
+ if !easy && r.Intn(10) != 0 {
+ this.XXX_unrecognized = randUnrecognizedMerkledag(r, 4)
+ }
+ return this
+}
+
+func NewPopulatedPBNode(r randyMerkledag, easy bool) *PBNode {
+ this := &PBNode{}
+ if r.Intn(5) != 0 {
+ v4 := r.Intn(100)
+ this.Data = make([]byte, v4)
+ for i := 0; i < v4; i++ {
+ this.Data[i] = byte(r.Intn(256))
+ }
+ }
+ if r.Intn(5) != 0 {
+ v5 := r.Intn(5)
+ this.Links = make([]*PBLink, v5)
+ for i := 0; i < v5; i++ {
+ this.Links[i] = NewPopulatedPBLink(r, easy)
+ }
+ }
+ if !easy && r.Intn(10) != 0 {
+ this.XXX_unrecognized = randUnrecognizedMerkledag(r, 3)
+ }
+ return this
+}
+
+type randyMerkledag interface {
+ Float32() float32
+ Float64() float64
+ Int63() int64
+ Int31() int32
+ Uint32() uint32
+ Intn(n int) int
+}
+
+func randUTF8RuneMerkledag(r randyMerkledag) rune {
+ ru := r.Intn(62)
+ if ru < 10 {
+ return rune(ru + 48)
+ } else if ru < 36 {
+ return rune(ru + 55)
+ }
+ return rune(ru + 61)
+}
+func randStringMerkledag(r randyMerkledag) string {
+ v6 := r.Intn(100)
+ tmps := make([]rune, v6)
+ for i := 0; i < v6; i++ {
+ tmps[i] = randUTF8RuneMerkledag(r)
+ }
+ return string(tmps)
+}
+func randUnrecognizedMerkledag(r randyMerkledag, maxFieldNumber int) (dAtA []byte) {
+ l := r.Intn(5)
+ for i := 0; i < l; i++ {
+ wire := r.Intn(4)
+ if wire == 3 {
+ wire = 5
+ }
+ fieldNumber := maxFieldNumber + r.Intn(100)
+ dAtA = randFieldMerkledag(dAtA, r, fieldNumber, wire)
+ }
+ return dAtA
+}
+func randFieldMerkledag(dAtA []byte, r randyMerkledag, fieldNumber int, wire int) []byte {
+ key := uint32(fieldNumber)<<3 | uint32(wire)
+ switch wire {
+ case 0:
+ dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key))
+ v7 := r.Int63()
+ if r.Intn(2) == 0 {
+ v7 *= -1
+ }
+ dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(v7))
+ case 1:
+ dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key))
+ dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)))
+ case 2:
+ dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key))
+ ll := r.Intn(100)
+ dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(ll))
+ for j := 0; j < ll; j++ {
+ dAtA = append(dAtA, byte(r.Intn(256)))
+ }
+ default:
+ dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key))
+ dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)))
+ }
+ return dAtA
+}
+func encodeVarintPopulateMerkledag(dAtA []byte, v uint64) []byte {
+ for v >= 1<<7 {
+ dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80))
+ v >>= 7
+ }
+ dAtA = append(dAtA, uint8(v))
+ return dAtA
+}
+func (m *PBLink) Size() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if m.Hash != nil {
+ l = len(m.Hash)
+ n += 1 + l + sovMerkledag(uint64(l))
+ }
+ if m.Name != nil {
+ l = len(*m.Name)
+ n += 1 + l + sovMerkledag(uint64(l))
+ }
+ if m.Tsize != nil {
+ n += 1 + sovMerkledag(uint64(*m.Tsize))
+ }
+ if m.XXX_unrecognized != nil {
+ n += len(m.XXX_unrecognized)
+ }
+ return n
+}
+
+func (m *PBNode) Size() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if m.Data != nil {
+ l = len(m.Data)
+ n += 1 + l + sovMerkledag(uint64(l))
+ }
+ if len(m.Links) > 0 {
+ for _, e := range m.Links {
+ l = e.Size()
+ n += 1 + l + sovMerkledag(uint64(l))
+ }
+ }
+ if m.XXX_unrecognized != nil {
+ n += len(m.XXX_unrecognized)
+ }
+ return n
+}
+
+func sovMerkledag(x uint64) (n int) {
+ return (math_bits.Len64(x|1) + 6) / 7
+}
+func (pbLink *PBLink) String() string {
+ if pbLink == nil {
+ return "nil"
+ }
+ s := strings.Join([]string{`&PBLink{`,
+ `Hash:` + valueToStringMerkledag(pbLink.Hash) + `,`,
+ `Name:` + valueToStringMerkledag(pbLink.Name) + `,`,
+ `Tsize:` + valueToStringMerkledag(pbLink.Tsize) + `,`,
+ `XXX_unrecognized:` + fmt.Sprintf("%v", pbLink.XXX_unrecognized) + `,`,
+ `}`,
+ }, "")
+ return s
+}
+func (pbNode *PBNode) String() string {
+ if pbNode == nil {
+ return "nil"
+ }
+ repeatedStringForLinks := "[]*PBLink{"
+ for _, f := range pbNode.Links {
+ repeatedStringForLinks += strings.Replace(f.String(), "PBLink", "PBLink", 1) + ","
+ }
+ repeatedStringForLinks += "}"
+ s := strings.Join([]string{`&PBNode{`,
+ `Data:` + valueToStringMerkledag(pbNode.Data) + `,`,
+ `Links:` + repeatedStringForLinks + `,`,
+ `XXX_unrecognized:` + fmt.Sprintf("%v", pbNode.XXX_unrecognized) + `,`,
+ `}`,
+ }, "")
+ return s
+}
+func valueToStringMerkledag(v interface{}) string {
+ rv := reflect.ValueOf(v)
+ if rv.IsNil() {
+ return "nil"
+ }
+ pv := reflect.Indirect(rv).Interface()
+ return fmt.Sprintf("*%v", pv)
+}
+func (m *PBLink) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: PBLink: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: PBLink: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Hash = append(m.Hash[:0], dAtA[iNdEx:postIndex]...)
+ if m.Hash == nil {
+ m.Hash = []byte{}
+ }
+ iNdEx = postIndex
+ case 2:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
+ }
+ var stringLen uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ stringLen |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ intStringLen := int(stringLen)
+ if intStringLen < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ postIndex := iNdEx + intStringLen
+ if postIndex < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ s := string(dAtA[iNdEx:postIndex])
+ m.Name = &s
+ iNdEx = postIndex
+ case 3:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Tsize", wireType)
+ }
+ var v uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.Tsize = &v
+ default:
+ iNdEx = preIndex
+ skippy, err := skipMerkledag(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if (iNdEx + skippy) < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *PBNode) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: PBNode: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: PBNode: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...)
+ if m.Data == nil {
+ m.Data = []byte{}
+ }
+ iNdEx = postIndex
+ case 2:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Links", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ postIndex := iNdEx + msglen
+ if postIndex < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Links = append(m.Links, &PBLink{})
+ if err := m.Links[len(m.Links)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipMerkledag(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if (iNdEx + skippy) < 0 {
+ return ErrInvalidLengthMerkledag
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func skipMerkledag(dAtA []byte) (n int, err error) {
+ l := len(dAtA)
+ iNdEx := 0
+ depth := 0
+ for iNdEx < l {
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return 0, ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return 0, io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ wireType := int(wire & 0x7)
+ switch wireType {
+ case 0:
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return 0, ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return 0, io.ErrUnexpectedEOF
+ }
+ iNdEx++
+ if dAtA[iNdEx-1] < 0x80 {
+ break
+ }
+ }
+ case 1:
+ iNdEx += 8
+ case 2:
+ var length int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return 0, ErrIntOverflowMerkledag
+ }
+ if iNdEx >= l {
+ return 0, io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ length |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if length < 0 {
+ return 0, ErrInvalidLengthMerkledag
+ }
+ iNdEx += length
+ if iNdEx < 0 {
+ return 0, ErrInvalidLengthMerkledag
+ }
+ case 3:
+ depth++
+ case 4:
+ if depth == 0 {
+ return 0, ErrUnexpectedEndOfGroupMerkledag
+ }
+ depth--
+ case 5:
+ iNdEx += 4
+ default:
+ return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+ }
+ if depth == 0 {
+ return iNdEx, nil
+ }
+ }
+ return 0, io.ErrUnexpectedEOF
+}
+
+var (
+ ErrInvalidLengthMerkledag = fmt.Errorf("proto: negative length found during unmarshaling")
+ ErrIntOverflowMerkledag = fmt.Errorf("proto: integer overflow")
+ ErrUnexpectedEndOfGroupMerkledag = fmt.Errorf("proto: unexpected end of group")
+)
diff --git a/ipld/merkledag/pb/merkledag.proto b/ipld/merkledag/pb/merkledag.proto
new file mode 100644
index 000000000..ec540e681
--- /dev/null
+++ b/ipld/merkledag/pb/merkledag.proto
@@ -0,0 +1,41 @@
+syntax = "proto2";
+
+package merkledag.pb;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.gostring_all) = true;
+option (gogoproto.equal_all) = true;
+option (gogoproto.verbose_equal_all) = true;
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) = true;
+option (gogoproto.populate_all) = true;
+option (gogoproto.testgen_all) = true;
+option (gogoproto.benchgen_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+
+
+// An IPFS MerkleDAG Link
+message PBLink {
+
+ // multihash of the target object
+ optional bytes Hash = 1;
+
+ // utf string name. should be unique per object
+ optional string Name = 2;
+
+ // cumulative size of target object
+ optional uint64 Tsize = 3;
+}
+
+// An IPFS MerkleDAG Node
+message PBNode {
+
+ // refs to other objects
+ repeated PBLink Links = 2;
+
+ // opaque user data
+ optional bytes Data = 1;
+}
diff --git a/ipld/merkledag/pb/merkledagpb_test.go b/ipld/merkledag/pb/merkledagpb_test.go
new file mode 100644
index 000000000..f72b306da
--- /dev/null
+++ b/ipld/merkledag/pb/merkledagpb_test.go
@@ -0,0 +1,486 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: merkledag.proto
+
+package merkledag_pb
+
+import (
+ fmt "fmt"
+ go_parser "go/parser"
+ math "math"
+ math_rand "math/rand"
+ testing "testing"
+ time "time"
+
+ _ "github.com/gogo/protobuf/gogoproto"
+ github_com_gogo_protobuf_jsonpb "github.com/gogo/protobuf/jsonpb"
+ github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
+ proto "github.com/gogo/protobuf/proto"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+func TestPBLinkProto(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBLink(popr, false)
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ msg := &PBLink{}
+ if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ littlefuzz := make([]byte, len(dAtA))
+ copy(littlefuzz, dAtA)
+ for i := range dAtA {
+ dAtA[i] = byte(popr.Intn(256))
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+ if len(littlefuzz) > 0 {
+ fuzzamount := 100
+ for i := 0; i < fuzzamount; i++ {
+ littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
+ littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
+ }
+ // shouldn't panic
+ _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
+ }
+}
+
+func TestPBLinkMarshalTo(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBLink(popr, false)
+ size := p.Size()
+ dAtA := make([]byte, size)
+ for i := range dAtA {
+ dAtA[i] = byte(popr.Intn(256))
+ }
+ _, err := p.MarshalTo(dAtA)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ msg := &PBLink{}
+ if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ for i := range dAtA {
+ dAtA[i] = byte(popr.Intn(256))
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+}
+
+func BenchmarkPBLinkProtoMarshal(b *testing.B) {
+ popr := math_rand.New(math_rand.NewSource(616))
+ total := 0
+ pops := make([]*PBLink, 10000)
+ for i := 0; i < 10000; i++ {
+ pops[i] = NewPopulatedPBLink(popr, false)
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
+ if err != nil {
+ panic(err)
+ }
+ total += len(dAtA)
+ }
+ b.SetBytes(int64(total / b.N))
+}
+
+func BenchmarkPBLinkProtoUnmarshal(b *testing.B) {
+ popr := math_rand.New(math_rand.NewSource(616))
+ total := 0
+ datas := make([][]byte, 10000)
+ for i := 0; i < 10000; i++ {
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBLink(popr, false))
+ if err != nil {
+ panic(err)
+ }
+ datas[i] = dAtA
+ }
+ msg := &PBLink{}
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ total += len(datas[i%10000])
+ if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
+ panic(err)
+ }
+ }
+ b.SetBytes(int64(total / b.N))
+}
+
+func TestPBNodeProto(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBNode(popr, false)
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ msg := &PBNode{}
+ if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ littlefuzz := make([]byte, len(dAtA))
+ copy(littlefuzz, dAtA)
+ for i := range dAtA {
+ dAtA[i] = byte(popr.Intn(256))
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+ if len(littlefuzz) > 0 {
+ fuzzamount := 100
+ for i := 0; i < fuzzamount; i++ {
+ littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
+ littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
+ }
+ // shouldn't panic
+ _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
+ }
+}
+
+func TestPBNodeMarshalTo(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBNode(popr, false)
+ size := p.Size()
+ dAtA := make([]byte, size)
+ for i := range dAtA {
+ dAtA[i] = byte(popr.Intn(256))
+ }
+ _, err := p.MarshalTo(dAtA)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ msg := &PBNode{}
+ if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ for i := range dAtA {
+ dAtA[i] = byte(popr.Intn(256))
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+}
+
+func BenchmarkPBNodeProtoMarshal(b *testing.B) {
+ popr := math_rand.New(math_rand.NewSource(616))
+ total := 0
+ pops := make([]*PBNode, 10000)
+ for i := 0; i < 10000; i++ {
+ pops[i] = NewPopulatedPBNode(popr, false)
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
+ if err != nil {
+ panic(err)
+ }
+ total += len(dAtA)
+ }
+ b.SetBytes(int64(total / b.N))
+}
+
+func BenchmarkPBNodeProtoUnmarshal(b *testing.B) {
+ popr := math_rand.New(math_rand.NewSource(616))
+ total := 0
+ datas := make([][]byte, 10000)
+ for i := 0; i < 10000; i++ {
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBNode(popr, false))
+ if err != nil {
+ panic(err)
+ }
+ datas[i] = dAtA
+ }
+ msg := &PBNode{}
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ total += len(datas[i%10000])
+ if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
+ panic(err)
+ }
+ }
+ b.SetBytes(int64(total / b.N))
+}
+
+func TestPBLinkJSON(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBLink(popr, true)
+ marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
+ jsondata, err := marshaler.MarshalToString(p)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ msg := &PBLink{}
+ err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
+ }
+}
+func TestPBNodeJSON(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBNode(popr, true)
+ marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
+ jsondata, err := marshaler.MarshalToString(p)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ msg := &PBNode{}
+ err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
+ }
+}
+func TestPBLinkProtoText(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBLink(popr, true)
+ dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
+ msg := &PBLink{}
+ if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+}
+
+func TestPBLinkProtoCompactText(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBLink(popr, true)
+ dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
+ msg := &PBLink{}
+ if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+}
+
+func TestPBNodeProtoText(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBNode(popr, true)
+ dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
+ msg := &PBNode{}
+ if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+}
+
+func TestPBNodeProtoCompactText(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBNode(popr, true)
+ dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
+ msg := &PBNode{}
+ if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err)
+ }
+ if !p.Equal(msg) {
+ t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
+ }
+}
+
+func TestPBLinkVerboseEqual(t *testing.T) {
+ popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+ p := NewPopulatedPBLink(popr, false)
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+ if err != nil {
+ panic(err)
+ }
+ msg := &PBLink{}
+ if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+ panic(err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err)
+ }
+}
+func TestPBNodeVerboseEqual(t *testing.T) {
+ popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+ p := NewPopulatedPBNode(popr, false)
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+ if err != nil {
+ panic(err)
+ }
+ msg := &PBNode{}
+ if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
+ panic(err)
+ }
+ if err := p.VerboseEqual(msg); err != nil {
+ t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err)
+ }
+}
+func TestPBLinkGoString(t *testing.T) {
+ popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+ p := NewPopulatedPBLink(popr, false)
+ s1 := p.GoString()
+ s2 := fmt.Sprintf("%#v", p)
+ if s1 != s2 {
+ t.Fatalf("GoString want %v got %v", s1, s2)
+ }
+ _, err := go_parser.ParseExpr(s1)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+func TestPBNodeGoString(t *testing.T) {
+ popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+ p := NewPopulatedPBNode(popr, false)
+ s1 := p.GoString()
+ s2 := fmt.Sprintf("%#v", p)
+ if s1 != s2 {
+ t.Fatalf("GoString want %v got %v", s1, s2)
+ }
+ _, err := go_parser.ParseExpr(s1)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+func TestPBLinkSize(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBLink(popr, true)
+ size2 := github_com_gogo_protobuf_proto.Size(p)
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ size := p.Size()
+ if len(dAtA) != size {
+ t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
+ }
+ if size2 != size {
+ t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
+ }
+ size3 := github_com_gogo_protobuf_proto.Size(p)
+ if size3 != size {
+ t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
+ }
+}
+
+func BenchmarkPBLinkSize(b *testing.B) {
+ popr := math_rand.New(math_rand.NewSource(616))
+ total := 0
+ pops := make([]*PBLink, 1000)
+ for i := 0; i < 1000; i++ {
+ pops[i] = NewPopulatedPBLink(popr, false)
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ total += pops[i%1000].Size()
+ }
+ b.SetBytes(int64(total / b.N))
+}
+
+func TestPBNodeSize(t *testing.T) {
+ seed := time.Now().UnixNano()
+ popr := math_rand.New(math_rand.NewSource(seed))
+ p := NewPopulatedPBNode(popr, true)
+ size2 := github_com_gogo_protobuf_proto.Size(p)
+ dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
+ if err != nil {
+ t.Fatalf("seed = %d, err = %v", seed, err)
+ }
+ size := p.Size()
+ if len(dAtA) != size {
+ t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
+ }
+ if size2 != size {
+ t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
+ }
+ size3 := github_com_gogo_protobuf_proto.Size(p)
+ if size3 != size {
+ t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
+ }
+}
+
+func BenchmarkPBNodeSize(b *testing.B) {
+ popr := math_rand.New(math_rand.NewSource(616))
+ total := 0
+ pops := make([]*PBNode, 1000)
+ for i := 0; i < 1000; i++ {
+ pops[i] = NewPopulatedPBNode(popr, false)
+ }
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ total += pops[i%1000].Size()
+ }
+ b.SetBytes(int64(total / b.N))
+}
+
+func TestPBLinkStringer(t *testing.T) {
+ popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+ p := NewPopulatedPBLink(popr, false)
+ s1 := p.String()
+ s2 := fmt.Sprintf("%v", p)
+ if s1 != s2 {
+ t.Fatalf("String want %v got %v", s1, s2)
+ }
+}
+func TestPBNodeStringer(t *testing.T) {
+ popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
+ p := NewPopulatedPBNode(popr, false)
+ s1 := p.String()
+ s2 := fmt.Sprintf("%v", p)
+ if s1 != s2 {
+ t.Fatalf("String want %v got %v", s1, s2)
+ }
+}
+
+//These tests are generated by github.com/gogo/protobuf/plugin/testgen
diff --git a/ipld/merkledag/pb/stability_test.go b/ipld/merkledag/pb/stability_test.go
new file mode 100644
index 000000000..5da833713
--- /dev/null
+++ b/ipld/merkledag/pb/stability_test.go
@@ -0,0 +1,24 @@
+package merkledag_pb
+
+import (
+ bytes "bytes"
+ "testing"
+)
+
+func TestStability(t *testing.T) {
+ correct := []byte("\x12\x87\x01\n;\x81\x869\xacH\xa4Ư\xa2\xf1X\x1a\x8b\x95%\xe2\x0f\xdah\x92\u007f+/\xf86\xf75x\xdb\x0f\xa5L)\xf7\xfd\x92\x8d\x92\xcaC\xf1\x93\xde\xe4\u007fY\x15I\xf5\x97\xa8\x11\xc8\xfag\xab\x03\x1e\xbd\x12B|CMw`mHq{>?|vd{0F7>8m[C`HSg3UcXmGs-qp-z6{Kc.tGX->H07\x18\xeaئ\xd0\b\x129\x121LZ3,V9jnmk^veYEV71EMLt9;6]}bnkU2e7GXmqisoCPV0C+ni\x18Ա\xfe\x8b\f\x12u\n'\xf2#\xc1\xc0nQ\xf9\xb5\x19\x80\xcd\xf8\x06k1\xf6#\x84\x1c\xb6\xbf\xeaY\x9b\xd8O\x84\x04\xdbKq\xe4\xae\xf2\xd6\xe9*\x16B\x12D[gVeg4=t}EGSu82+dmgvQ+Tr>_sLUJ|iZ[P2y2T67ilvEikK}\\iru?IF?mVS[Mv9KG8+\x18\x92\xa0\xf9\xa1\n\n\x11?̎\v\x06ѣ\x80nH\x12\x00\xa7\xd2w͝")
+ n := new(PBNode)
+ err := n.Unmarshal(correct)
+ if err != nil {
+ t.Fatal(err)
+ }
+ d, err := n.Marshal()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(d, correct) {
+ t.Logf("%q", d)
+ t.Fatal("protobuf not stable")
+ }
+
+}
diff --git a/ipld/merkledag/pb/upgrade_check.go b/ipld/merkledag/pb/upgrade_check.go
new file mode 100644
index 000000000..e5a6473c1
--- /dev/null
+++ b/ipld/merkledag/pb/upgrade_check.go
@@ -0,0 +1,5 @@
+package merkledag_pb
+
+// Make sure the user doesn't upgrade this package!
+// This will fail to build if the user does.
+const _ = DoNotUpgradeFileEverItWillChangeYourHashes
diff --git a/ipld/merkledag/prime.go b/ipld/merkledag/prime.go
new file mode 100644
index 000000000..5f9bbc691
--- /dev/null
+++ b/ipld/merkledag/prime.go
@@ -0,0 +1,206 @@
+package merkledag
+
+import (
+ dagpb "github.com/ipld/go-codec-dagpb"
+ "github.com/ipld/go-ipld-prime"
+)
+
+// Protonode was originally implemented as a go-ipld-format node, and included
+// functionality that does not fit well into the model for go-ipld-prime, namely
+// the ability ot modify the node in place.
+
+// In order to support the go-ipld-prime interface, all of these prime methods
+// serialize and rebuild the go-ipld-prime node as needed, so that it remains up
+// to date with mutations made via the add/remove link methods
+
+// Kind returns a value from the Kind enum describing what the
+// essential serializable kind of this node is (map, list, integer, etc).
+// Most other handling of a node requires first switching upon the kind.
+func (n *ProtoNode) Kind() ipld.Kind {
+ _, _ = n.EncodeProtobuf(false)
+ return n.encoded.Kind()
+}
+
+// LookupByString looks up a child object in this node and returns it.
+// The returned Node may be any of the Kind:
+// a primitive (string, int64, etc), a map, a list, or a link.
+//
+// If the Kind of this Node is not Kind_Map, a nil node and an error
+// will be returned.
+//
+// If the key does not exist, a nil node and an error will be returned.
+func (n *ProtoNode) LookupByString(key string) (ipld.Node, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return nil, err
+ }
+ return n.encoded.LookupByString(key)
+}
+
+// LookupByNode is the equivalent of LookupByString, but takes a reified Node
+// as a parameter instead of a plain string.
+// This mechanism is useful if working with typed maps (if the key types
+// have constraints, and you already have a reified `schema.TypedNode` value,
+// using that value can save parsing and validation costs);
+// and may simply be convenient if you already have a Node value in hand.
+//
+// (When writing generic functions over Node, a good rule of thumb is:
+// when handling a map, check for `schema.TypedNode`, and in this case prefer
+// the LookupByNode(Node) method; otherwise, favor LookupByString; typically
+// implementations will have their fastest paths thusly.)
+func (n *ProtoNode) LookupByNode(key ipld.Node) (ipld.Node, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return nil, err
+ }
+ return n.encoded.LookupByNode(key)
+}
+
+// LookupByIndex is the equivalent of LookupByString but for indexing into a list.
+// As with LookupByString, the returned Node may be any of the Kind:
+// a primitive (string, int64, etc), a map, a list, or a link.
+//
+// If the Kind of this Node is not Kind_List, a nil node and an error
+// will be returned.
+//
+// If idx is out of range, a nil node and an error will be returned.
+func (n *ProtoNode) LookupByIndex(idx int64) (ipld.Node, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return nil, err
+ }
+ return n.encoded.LookupByIndex(idx)
+}
+
+// LookupBySegment is will act as either LookupByString or LookupByIndex,
+// whichever is contextually appropriate.
+//
+// Using LookupBySegment may imply an "atoi" conversion if used on a list node,
+// or an "itoa" conversion if used on a map node. If an "itoa" conversion
+// takes place, it may error, and this method may return that error.
+func (n *ProtoNode) LookupBySegment(seg ipld.PathSegment) (ipld.Node, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return nil, err
+ }
+ return n.encoded.LookupBySegment(seg)
+}
+
+// Note that when using codegenerated types, there may be a fifth variant
+// of lookup method on maps: `Get($GeneratedTypeKey) $GeneratedTypeValue`!
+// MapIterator returns an iterator which yields key-value pairs
+// traversing the node.
+// If the node kind is anything other than a map, nil will be returned.
+//
+// The iterator will yield every entry in the map; that is, it
+// can be expected that itr.Next will be called node.Length times
+// before itr.Done becomes true.
+func (n *ProtoNode) MapIterator() ipld.MapIterator {
+ _, _ = n.EncodeProtobuf(false)
+ return n.encoded.MapIterator()
+}
+
+// ListIterator returns an iterator which yields key-value pairs
+// traversing the node.
+// If the node kind is anything other than a list, nil will be returned.
+//
+// The iterator will yield every entry in the list; that is, it
+// can be expected that itr.Next will be called node.Length times
+// before itr.Done becomes true.
+func (n *ProtoNode) ListIterator() ipld.ListIterator {
+ _, _ = n.EncodeProtobuf(false)
+ return n.encoded.ListIterator()
+}
+
+// Length returns the length of a list, or the number of entries in a map,
+// or -1 if the node is not of list nor map kind.
+func (n *ProtoNode) Length() int64 {
+ _, _ = n.EncodeProtobuf(false)
+ return n.encoded.Length()
+}
+
+// Absent nodes are returned when traversing a struct field that is
+// defined by a schema but unset in the data. (Absent nodes are not
+// possible otherwise; you'll only see them from `schema.TypedNode`.)
+// The absent flag is necessary so iterating over structs can
+// unambiguously make the distinction between values that are
+// present-and-null versus values that are absent.
+//
+// Absent nodes respond to `Kind()` as `ipld.Kind_Null`,
+// for lack of any better descriptive value; you should therefore
+// always check IsAbsent rather than just a switch on kind
+// when it may be important to handle absent values distinctly.
+func (n *ProtoNode) IsAbsent() bool {
+ _, _ = n.EncodeProtobuf(false)
+ return n.encoded.IsAbsent()
+}
+
+func (n *ProtoNode) IsNull() bool {
+ _, _ = n.EncodeProtobuf(false)
+ return n.encoded.IsNull()
+}
+
+func (n *ProtoNode) AsBool() (bool, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return false, err
+ }
+ return n.encoded.AsBool()
+}
+
+func (n *ProtoNode) AsInt() (int64, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return 0, err
+ }
+ return n.encoded.AsInt()
+}
+
+func (n *ProtoNode) AsFloat() (float64, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return 0, err
+ }
+ return n.encoded.AsFloat()
+}
+
+func (n *ProtoNode) AsString() (string, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return "", err
+ }
+ return n.encoded.AsString()
+}
+
+func (n *ProtoNode) AsBytes() ([]byte, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return nil, err
+ }
+ return n.encoded.AsBytes()
+}
+
+func (n *ProtoNode) AsLink() (ipld.Link, error) {
+ _, err := n.EncodeProtobuf(false)
+ if err != nil {
+ return nil, err
+ }
+ return n.encoded.AsLink()
+}
+
+// Prototype returns a NodePrototype which can describe some properties of this node's implementation,
+// and also be used to get a NodeBuilder,
+// which can be use to create new nodes with the same implementation as this one.
+//
+// For typed nodes, the NodePrototype will also implement schema.Type.
+//
+// For Advanced Data Layouts, the NodePrototype will encapsulate any additional
+// parameters and configuration of the ADL, and will also (usually)
+// implement NodePrototypeSupportingAmend.
+//
+// Calling this method should not cause an allocation.
+func (n *ProtoNode) Prototype() ipld.NodePrototype {
+ return dagpb.Type.PBNode
+}
+
+var _ ipld.Node = &ProtoNode{}
diff --git a/ipld/merkledag/raw.go b/ipld/merkledag/raw.go
new file mode 100644
index 000000000..68e2473e1
--- /dev/null
+++ b/ipld/merkledag/raw.go
@@ -0,0 +1,126 @@
+package merkledag
+
+import (
+ "encoding/json"
+ "fmt"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ u "github.com/ipfs/boxo/util"
+ legacy "github.com/ipfs/go-ipld-legacy"
+ ipld "github.com/ipld/go-ipld-prime"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+
+ cid "github.com/ipfs/go-cid"
+ format "github.com/ipfs/go-ipld-format"
+)
+
+// RawNode represents a node which only contains data.
+type RawNode struct {
+ blocks.Block
+
+ // Always a node/basic Bytes.
+ // We can't reference a specific type, as it's not exposed there.
+ // If we find that the interface indirection really matters,
+ // then we could possibly use dagpb.Bytes.
+ ipld.Node
+}
+
+var _ legacy.UniversalNode = &RawNode{}
+
+// NewRawNode creates a RawNode using the default sha2-256 hash function.
+func NewRawNode(data []byte) *RawNode {
+ h := u.Hash(data)
+ c := cid.NewCidV1(cid.Raw, h)
+ blk, _ := blocks.NewBlockWithCid(data, c)
+ return &RawNode{blk, basicnode.NewBytes(data)}
+}
+
+// DecodeRawBlock is a block decoder for raw IPLD nodes conforming to `node.DecodeBlockFunc`.
+func DecodeRawBlock(block blocks.Block) (format.Node, error) {
+ if block.Cid().Type() != cid.Raw {
+ return nil, fmt.Errorf("raw nodes cannot be decoded from non-raw blocks: %d", block.Cid().Type())
+ }
+ // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is.
+ return &RawNode{block, basicnode.NewBytes(block.RawData())}, nil
+}
+
+var _ format.DecodeBlockFunc = DecodeRawBlock
+
+// NewRawNodeWPrefix creates a RawNode using the provided cid builder
+func NewRawNodeWPrefix(data []byte, builder cid.Builder) (*RawNode, error) {
+ builder = builder.WithCodec(cid.Raw)
+ c, err := builder.Sum(data)
+ if err != nil {
+ return nil, err
+ }
+ blk, err := blocks.NewBlockWithCid(data, c)
+ if err != nil {
+ return nil, err
+ }
+ // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is.
+ return &RawNode{blk, basicnode.NewBytes(data)}, nil
+}
+
+// Links returns nil.
+func (rn *RawNode) Links() []*format.Link {
+ return nil
+}
+
+// ResolveLink returns an error.
+func (rn *RawNode) ResolveLink(path []string) (*format.Link, []string, error) {
+ return nil, nil, ErrLinkNotFound
+}
+
+// Resolve returns an error.
+func (rn *RawNode) Resolve(path []string) (interface{}, []string, error) {
+ return nil, nil, ErrLinkNotFound
+}
+
+// Tree returns nil.
+func (rn *RawNode) Tree(p string, depth int) []string {
+ return nil
+}
+
+// Copy performs a deep copy of this node and returns it as an format.Node
+func (rn *RawNode) Copy() format.Node {
+ copybuf := make([]byte, len(rn.RawData()))
+ copy(copybuf, rn.RawData())
+ nblk, err := blocks.NewBlockWithCid(rn.RawData(), rn.Cid())
+ if err != nil {
+ // programmer error
+ panic("failure attempting to clone raw block: " + err.Error())
+ }
+ // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is.
+ return &RawNode{nblk, basicnode.NewBytes(nblk.RawData())}
+}
+
+// Size returns the size of this node
+func (rn *RawNode) Size() (uint64, error) {
+ return uint64(len(rn.RawData())), nil
+}
+
+// Stat returns some Stats about this node.
+func (rn *RawNode) Stat() (*format.NodeStat, error) {
+ return &format.NodeStat{
+ Hash: rn.Cid().String(),
+ NumLinks: 0,
+ BlockSize: len(rn.RawData()),
+ LinksSize: 0,
+ CumulativeSize: len(rn.RawData()),
+ DataSize: len(rn.RawData()),
+ }, nil
+}
+
+// MarshalJSON is required for our "ipfs dag" commands.
+func (rn *RawNode) MarshalJSON() ([]byte, error) {
+ return json.Marshal(string(rn.RawData()))
+}
+
+func RawNodeConverter(b blocks.Block, nd ipld.Node) (legacy.UniversalNode, error) {
+ if nd.Kind() != ipld.Kind_Bytes {
+ return nil, ErrNotRawNode
+ }
+ return &RawNode{b, nd}, nil
+}
+
+var _ legacy.UniversalNode = (*RawNode)(nil)
diff --git a/ipld/merkledag/readonly.go b/ipld/merkledag/readonly.go
new file mode 100644
index 000000000..36242fbeb
--- /dev/null
+++ b/ipld/merkledag/readonly.go
@@ -0,0 +1,20 @@
+package merkledag
+
+import (
+ "fmt"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// ErrReadOnly is used when a read-only datastructure is written to.
+var ErrReadOnly = fmt.Errorf("cannot write to readonly DAGService")
+
+// NewReadOnlyDagService takes a NodeGetter, and returns a full DAGService
+// implementation that returns ErrReadOnly when its 'write' methods are
+// invoked.
+func NewReadOnlyDagService(ng ipld.NodeGetter) ipld.DAGService {
+ return &ComboService{
+ Read: ng,
+ Write: &ErrorService{ErrReadOnly},
+ }
+}
diff --git a/ipld/merkledag/readonly_test.go b/ipld/merkledag/readonly_test.go
new file mode 100644
index 000000000..0d00eebfb
--- /dev/null
+++ b/ipld/merkledag/readonly_test.go
@@ -0,0 +1,64 @@
+package merkledag_test
+
+import (
+ "context"
+ "testing"
+
+ . "github.com/ipfs/boxo/ipld/merkledag"
+ dstest "github.com/ipfs/boxo/ipld/merkledag/test"
+
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+func TestReadonlyProperties(t *testing.T) {
+ ds := dstest.Mock()
+ ro := NewReadOnlyDagService(ds)
+
+ ctx := context.Background()
+ nds := []ipld.Node{
+ NewRawNode([]byte("foo1")),
+ NewRawNode([]byte("foo2")),
+ NewRawNode([]byte("foo3")),
+ NewRawNode([]byte("foo4")),
+ }
+ cids := []cid.Cid{
+ nds[0].Cid(),
+ nds[1].Cid(),
+ nds[2].Cid(),
+ nds[3].Cid(),
+ }
+
+ // add to the actual underlying datastore
+ if err := ds.Add(ctx, nds[2]); err != nil {
+ t.Fatal(err)
+ }
+ if err := ds.Add(ctx, nds[3]); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ro.Add(ctx, nds[0]); err != ErrReadOnly {
+ t.Fatal("expected ErrReadOnly")
+ }
+ if err := ro.Add(ctx, nds[2]); err != ErrReadOnly {
+ t.Fatal("expected ErrReadOnly")
+ }
+
+ if err := ro.AddMany(ctx, nds[0:1]); err != ErrReadOnly {
+ t.Fatal("expected ErrReadOnly")
+ }
+
+ if err := ro.Remove(ctx, cids[3]); err != ErrReadOnly {
+ t.Fatal("expected ErrReadOnly")
+ }
+ if err := ro.RemoveMany(ctx, cids[1:2]); err != ErrReadOnly {
+ t.Fatal("expected ErrReadOnly")
+ }
+
+ if _, err := ro.Get(ctx, cids[0]); !ipld.IsNotFound(err) {
+ t.Fatal("expected ErrNotFound")
+ }
+ if _, err := ro.Get(ctx, cids[3]); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/ipld/merkledag/rwservice.go b/ipld/merkledag/rwservice.go
new file mode 100644
index 000000000..a916350a6
--- /dev/null
+++ b/ipld/merkledag/rwservice.go
@@ -0,0 +1,47 @@
+package merkledag
+
+import (
+ "context"
+
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// ComboService implements ipld.DAGService, using 'Read' for all fetch methods,
+// and 'Write' for all methods that add new objects.
+type ComboService struct {
+ Read ipld.NodeGetter
+ Write ipld.DAGService
+}
+
+var _ ipld.DAGService = (*ComboService)(nil)
+
+// Add writes a new node using the Write DAGService.
+func (cs *ComboService) Add(ctx context.Context, nd ipld.Node) error {
+ return cs.Write.Add(ctx, nd)
+}
+
+// AddMany adds nodes using the Write DAGService.
+func (cs *ComboService) AddMany(ctx context.Context, nds []ipld.Node) error {
+ return cs.Write.AddMany(ctx, nds)
+}
+
+// Get fetches a node using the Read DAGService.
+func (cs *ComboService) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) {
+ return cs.Read.Get(ctx, c)
+}
+
+// GetMany fetches nodes using the Read DAGService.
+func (cs *ComboService) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption {
+ return cs.Read.GetMany(ctx, cids)
+}
+
+// Remove deletes a node using the Write DAGService.
+func (cs *ComboService) Remove(ctx context.Context, c cid.Cid) error {
+ return cs.Write.Remove(ctx, c)
+}
+
+// RemoveMany deletes nodes using the Write DAGService.
+func (cs *ComboService) RemoveMany(ctx context.Context, cids []cid.Cid) error {
+ return cs.Write.RemoveMany(ctx, cids)
+}
diff --git a/ipld/merkledag/session.go b/ipld/merkledag/session.go
new file mode 100644
index 000000000..c7bbff169
--- /dev/null
+++ b/ipld/merkledag/session.go
@@ -0,0 +1,21 @@
+package merkledag
+
+import (
+ "context"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// SessionMaker is an object that can generate a new fetching session.
+type SessionMaker interface {
+ Session(context.Context) ipld.NodeGetter
+}
+
+// NewSession returns a session backed NodeGetter if the given NodeGetter
+// implements SessionMaker.
+func NewSession(ctx context.Context, g ipld.NodeGetter) ipld.NodeGetter {
+ if sm, ok := g.(SessionMaker); ok {
+ return sm.Session(ctx)
+ }
+ return g
+}
diff --git a/ipld/merkledag/test/utils.go b/ipld/merkledag/test/utils.go
new file mode 100644
index 000000000..302d194d6
--- /dev/null
+++ b/ipld/merkledag/test/utils.go
@@ -0,0 +1,23 @@
+package mdutils
+
+import (
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+
+ bsrv "github.com/ipfs/boxo/blockservice"
+ blockstore "github.com/ipfs/boxo/blockstore"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// Mock returns a new thread-safe, mock DAGService.
+func Mock() ipld.DAGService {
+ return dag.NewDAGService(Bserv())
+}
+
+// Bserv returns a new, thread-safe, mock BlockService.
+func Bserv() bsrv.BlockService {
+ bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
+ return bsrv.New(bstore, offline.Exchange(bstore))
+}
diff --git a/ipld/merkledag/traverse/traverse.go b/ipld/merkledag/traverse/traverse.go
new file mode 100644
index 000000000..dbc426fa9
--- /dev/null
+++ b/ipld/merkledag/traverse/traverse.go
@@ -0,0 +1,222 @@
+// Package traverse provides merkledag traversal functions
+package traverse
+
+import (
+ "context"
+ "errors"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// Order is an identifier for traversal algorithm orders
+type Order int
+
+// These constants define different traversing methods
+const (
+ // DFSPre defines depth-first pre-order
+ DFSPre Order = iota
+ // DFSPost defines depth-first post-order
+ DFSPost
+ // BFS defines breadth-first order
+ BFS
+)
+
+// Options specifies a series of traversal options
+type Options struct {
+ DAG ipld.NodeGetter // the dagservice to fetch nodes
+ Order Order // what order to traverse in
+ Func Func // the function to perform at each step
+ ErrFunc ErrFunc // see ErrFunc. Optional
+
+ SkipDuplicates bool // whether to skip duplicate nodes
+}
+
+// State is a current traversal state
+type State struct {
+ Node ipld.Node
+ Depth int
+}
+
+type traversal struct {
+ opts Options
+ seen map[string]struct{}
+}
+
+func (t *traversal) shouldSkip(n ipld.Node) (bool, error) {
+ if t.opts.SkipDuplicates {
+ k := n.Cid()
+ if _, found := t.seen[k.KeyString()]; found {
+ return true, nil
+ }
+ t.seen[k.KeyString()] = struct{}{}
+ }
+
+ return false, nil
+}
+
+func (t *traversal) callFunc(next State) error {
+ return t.opts.Func(next)
+}
+
+// getNode returns the node for link. If it return an error,
+// stop processing. if it returns a nil node, just skip it.
+//
+// the error handling is a little complicated.
+func (t *traversal) getNode(link *ipld.Link) (ipld.Node, error) {
+
+ getNode := func(l *ipld.Link) (ipld.Node, error) {
+ next, err := l.GetNode(context.TODO(), t.opts.DAG)
+ if err != nil {
+ return nil, err
+ }
+
+ skip, err := t.shouldSkip(next)
+ if skip {
+ next = nil
+ }
+ return next, err
+ }
+
+ next, err := getNode(link)
+ if err != nil && t.opts.ErrFunc != nil { // attempt recovery.
+ err = t.opts.ErrFunc(err)
+ next = nil // skip regardless
+ }
+ return next, err
+}
+
+// Func is the type of the function called for each dag.Node visited by Traverse.
+// The traversal argument contains the current traversal state.
+// If an error is returned, processing stops.
+type Func func(current State) error
+
+// ErrFunc is provided to handle problems when walking to the Node. Traverse
+// will call ErrFunc with the error encountered. ErrFunc can decide how to
+// handle that error, and return an error back to Traversal with how to proceed:
+// - nil - skip the Node and its children, but continue processing
+// - all other errors halt processing immediately.
+//
+// If ErrFunc is nil, Traversal will stop, as if:
+//
+// opts.ErrFunc = func(err error) { return err }
+type ErrFunc func(err error) error
+
+// Traverse initiates a DAG traversal with the given options starting at
+// the given root.
+func Traverse(root ipld.Node, o Options) error {
+ t := traversal{
+ opts: o,
+ seen: map[string]struct{}{},
+ }
+
+ state := State{
+ Node: root,
+ Depth: 0,
+ }
+
+ switch o.Order {
+ default:
+ return dfsPreTraverse(state, &t)
+ case DFSPre:
+ return dfsPreTraverse(state, &t)
+ case DFSPost:
+ return dfsPostTraverse(state, &t)
+ case BFS:
+ return bfsTraverse(state, &t)
+ }
+}
+
+type dfsFunc func(state State, t *traversal) error
+
+func dfsPreTraverse(state State, t *traversal) error {
+ if err := t.callFunc(state); err != nil {
+ return err
+ }
+ return dfsDescend(dfsPreTraverse, state, t)
+}
+
+func dfsPostTraverse(state State, t *traversal) error {
+ if err := dfsDescend(dfsPostTraverse, state, t); err != nil {
+ return err
+ }
+ return t.callFunc(state)
+}
+
+func dfsDescend(df dfsFunc, curr State, t *traversal) error {
+ for _, l := range curr.Node.Links() {
+ node, err := t.getNode(l)
+ if err != nil {
+ return err
+ }
+ if node == nil { // skip
+ continue
+ }
+
+ next := State{
+ Node: node,
+ Depth: curr.Depth + 1,
+ }
+ if err := df(next, t); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func bfsTraverse(root State, t *traversal) error {
+
+ if skip, err := t.shouldSkip(root.Node); skip || err != nil {
+ return err
+ }
+
+ var q queue
+ q.enq(root)
+ for q.len() > 0 {
+ curr := q.deq()
+ if curr.Node == nil {
+ return errors.New("failed to dequeue though queue not empty")
+ }
+
+ // call user's func
+ if err := t.callFunc(curr); err != nil {
+ return err
+ }
+
+ for _, l := range curr.Node.Links() {
+ node, err := t.getNode(l)
+ if err != nil {
+ return err
+ }
+ if node == nil { // skip
+ continue
+ }
+
+ q.enq(State{
+ Node: node,
+ Depth: curr.Depth + 1,
+ })
+ }
+ }
+ return nil
+}
+
+type queue struct {
+ s []State
+}
+
+func (q *queue) enq(n State) {
+ q.s = append(q.s, n)
+}
+
+func (q *queue) deq() State {
+ if len(q.s) < 1 {
+ return State{}
+ }
+ n := q.s[0]
+ q.s = q.s[1:]
+ return n
+}
+
+func (q *queue) len() int {
+ return len(q.s)
+}
diff --git a/ipld/merkledag/traverse/traverse_test.go b/ipld/merkledag/traverse/traverse_test.go
new file mode 100644
index 000000000..c19f1b054
--- /dev/null
+++ b/ipld/merkledag/traverse/traverse_test.go
@@ -0,0 +1,418 @@
+package traverse
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "testing"
+
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ mdagtest "github.com/ipfs/boxo/ipld/merkledag/test"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+func TestDFSPreNoSkip(t *testing.T) {
+ ds := mdagtest.Mock()
+ opts := Options{Order: DFSPre, DAG: ds}
+
+ testWalkOutputs(t, newFan(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+1 /a/ab
+1 /a/ac
+1 /a/ad
+`))
+
+ testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+
+ testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+2 /a/aa/aab
+1 /a/ab
+2 /a/ab/aba
+2 /a/ab/abb
+`))
+
+ testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+}
+
+func TestDFSPreSkip(t *testing.T) {
+ ds := mdagtest.Mock()
+ opts := Options{Order: DFSPre, SkipDuplicates: true, DAG: ds}
+
+ testWalkOutputs(t, newFan(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+1 /a/ab
+1 /a/ac
+1 /a/ad
+`))
+
+ testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+
+ testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+2 /a/aa/aab
+1 /a/ab
+2 /a/ab/aba
+2 /a/ab/abb
+`))
+
+ testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+}
+
+func TestDFSPostNoSkip(t *testing.T) {
+ ds := mdagtest.Mock()
+ opts := Options{Order: DFSPost, DAG: ds}
+
+ testWalkOutputs(t, newFan(t, ds), opts, []byte(`
+1 /a/aa
+1 /a/ab
+1 /a/ac
+1 /a/ad
+0 /a
+`))
+
+ testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+2 /a/aa/aaa
+1 /a/aa
+0 /a
+`))
+
+ testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
+2 /a/aa/aaa
+2 /a/aa/aab
+1 /a/aa
+2 /a/ab/aba
+2 /a/ab/abb
+1 /a/ab
+0 /a
+`))
+
+ testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+2 /a/aa/aaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+2 /a/aa/aaa
+1 /a/aa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+2 /a/aa/aaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+2 /a/aa/aaa
+1 /a/aa
+0 /a
+`))
+}
+
+func TestDFSPostSkip(t *testing.T) {
+ ds := mdagtest.Mock()
+ opts := Options{Order: DFSPost, SkipDuplicates: true, DAG: ds}
+
+ testWalkOutputs(t, newFan(t, ds), opts, []byte(`
+1 /a/aa
+1 /a/ab
+1 /a/ac
+1 /a/ad
+0 /a
+`))
+
+ testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+2 /a/aa/aaa
+1 /a/aa
+0 /a
+`))
+
+ testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
+2 /a/aa/aaa
+2 /a/aa/aab
+1 /a/aa
+2 /a/ab/aba
+2 /a/ab/abb
+1 /a/ab
+0 /a
+`))
+
+ testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
+4 /a/aa/aaa/aaaa/aaaaa
+3 /a/aa/aaa/aaaa
+2 /a/aa/aaa
+1 /a/aa
+0 /a
+`))
+}
+
+func TestBFSNoSkip(t *testing.T) {
+ ds := mdagtest.Mock()
+ opts := Options{Order: BFS, DAG: ds}
+
+ testWalkOutputs(t, newFan(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+1 /a/ab
+1 /a/ac
+1 /a/ad
+`))
+
+ testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+
+ testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+1 /a/ab
+2 /a/aa/aaa
+2 /a/aa/aab
+2 /a/ab/aba
+2 /a/ab/abb
+`))
+
+ testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+1 /a/aa
+2 /a/aa/aaa
+2 /a/aa/aaa
+2 /a/aa/aaa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+3 /a/aa/aaa/aaaa
+3 /a/aa/aaa/aaaa
+3 /a/aa/aaa/aaaa
+3 /a/aa/aaa/aaaa
+3 /a/aa/aaa/aaaa
+3 /a/aa/aaa/aaaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+}
+
+func TestBFSSkip(t *testing.T) {
+ ds := mdagtest.Mock()
+ opts := Options{Order: BFS, SkipDuplicates: true, DAG: ds}
+
+ testWalkOutputs(t, newFan(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+1 /a/ab
+1 /a/ac
+1 /a/ad
+`))
+
+ testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+
+ testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+1 /a/ab
+2 /a/aa/aaa
+2 /a/aa/aab
+2 /a/ab/aba
+2 /a/ab/abb
+`))
+
+ testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
+0 /a
+1 /a/aa
+2 /a/aa/aaa
+3 /a/aa/aaa/aaaa
+4 /a/aa/aaa/aaaa/aaaaa
+`))
+}
+
+func testWalkOutputs(t *testing.T, root ipld.Node, opts Options, expect []byte) {
+ expect = bytes.TrimLeft(expect, "\n")
+
+ buf := new(bytes.Buffer)
+ walk := func(current State) error {
+ s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.(*mdag.ProtoNode).Data())
+ t.Logf("walk: %s", s)
+ buf.Write([]byte(s))
+ return nil
+ }
+
+ opts.Func = walk
+ if err := Traverse(root, opts); err != nil {
+ t.Error(err)
+ return
+ }
+
+ actual := buf.Bytes()
+ if !bytes.Equal(actual, expect) {
+ t.Error("error: outputs differ")
+ t.Logf("expect:\n%s", expect)
+ t.Logf("actual:\n%s", actual)
+ } else {
+ t.Logf("expect matches actual:\n%s", expect)
+ }
+}
+
+func newFan(t *testing.T, ds ipld.DAGService) ipld.Node {
+ a := mdag.NodeWithData([]byte("/a"))
+ addLink(t, ds, a, child(t, ds, a, "aa"))
+ addLink(t, ds, a, child(t, ds, a, "ab"))
+ addLink(t, ds, a, child(t, ds, a, "ac"))
+ addLink(t, ds, a, child(t, ds, a, "ad"))
+ return a
+}
+
+func newLinkedList(t *testing.T, ds ipld.DAGService) ipld.Node {
+ a := mdag.NodeWithData([]byte("/a"))
+ aa := child(t, ds, a, "aa")
+ aaa := child(t, ds, aa, "aaa")
+ aaaa := child(t, ds, aaa, "aaaa")
+ aaaaa := child(t, ds, aaaa, "aaaaa")
+ addLink(t, ds, aaaa, aaaaa)
+ addLink(t, ds, aaa, aaaa)
+ addLink(t, ds, aa, aaa)
+ addLink(t, ds, a, aa)
+ return a
+}
+
+func newBinaryTree(t *testing.T, ds ipld.DAGService) ipld.Node {
+ a := mdag.NodeWithData([]byte("/a"))
+ aa := child(t, ds, a, "aa")
+ ab := child(t, ds, a, "ab")
+ addLink(t, ds, aa, child(t, ds, aa, "aaa"))
+ addLink(t, ds, aa, child(t, ds, aa, "aab"))
+ addLink(t, ds, ab, child(t, ds, ab, "aba"))
+ addLink(t, ds, ab, child(t, ds, ab, "abb"))
+ addLink(t, ds, a, aa)
+ addLink(t, ds, a, ab)
+ return a
+}
+
+func newBinaryDAG(t *testing.T, ds ipld.DAGService) ipld.Node {
+ a := mdag.NodeWithData([]byte("/a"))
+ aa := child(t, ds, a, "aa")
+ aaa := child(t, ds, aa, "aaa")
+ aaaa := child(t, ds, aaa, "aaaa")
+ aaaaa := child(t, ds, aaaa, "aaaaa")
+ addLink(t, ds, aaaa, aaaaa)
+ addLink(t, ds, aaaa, aaaaa)
+ addLink(t, ds, aaa, aaaa)
+ addLink(t, ds, aaa, aaaa)
+ addLink(t, ds, aa, aaa)
+ addLink(t, ds, aa, aaa)
+ addLink(t, ds, a, aa)
+ addLink(t, ds, a, aa)
+ return a
+}
+
+func addLink(t *testing.T, ds ipld.DAGService, a, b ipld.Node) {
+ to := string(a.(*mdag.ProtoNode).Data()) + "2" + string(b.(*mdag.ProtoNode).Data())
+ if err := ds.Add(context.Background(), b); err != nil {
+ t.Error(err)
+ }
+ if err := a.(*mdag.ProtoNode).AddNodeLink(to, b.(*mdag.ProtoNode)); err != nil {
+ t.Error(err)
+ }
+}
+
+func child(t *testing.T, ds ipld.DAGService, a ipld.Node, name string) ipld.Node {
+ return mdag.NodeWithData([]byte(string(a.(*mdag.ProtoNode).Data()) + "/" + name))
+}
diff --git a/ipns/README.md b/ipns/README.md
new file mode 100644
index 000000000..0afa0be00
--- /dev/null
+++ b/ipns/README.md
@@ -0,0 +1,31 @@
+## Usage
+
+To create a new IPNS record:
+
+```go
+import (
+ "time"
+
+ ipns "github.com/ipfs/boxo/ipns"
+ crypto "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+// Generate a private key to sign the IPNS record with. Most of the time,
+// however, you'll want to retrieve an already-existing key from IPFS using the
+// go-ipfs/core/coreapi CoreAPI.KeyAPI() interface.
+privateKey, publicKey, err := crypto.GenerateKeyPair(crypto.RSA, 2048)
+if err != nil {
+ panic(err)
+}
+
+// Create an IPNS record that expires in one hour and points to the IPFS address
+// /ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5
+ipnsRecord, err := ipns.Create(privateKey, []byte("/ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5"), 0, time.Now().Add(1*time.Hour))
+if err != nil {
+ panic(err)
+}
+```
+
+Once you have the record, you’ll need to use IPFS to *publish* it.
+
+There are several other major operations you can do with `go-ipns`. Check out the [API docs](https://pkg.go.dev/github.com/ipfs/boxo/ipns) or look at the tests in this repo for examples.
diff --git a/ipns/errors.go b/ipns/errors.go
new file mode 100644
index 000000000..d78aafffa
--- /dev/null
+++ b/ipns/errors.go
@@ -0,0 +1,44 @@
+package ipns
+
+import (
+ "errors"
+)
+
+// ErrExpiredRecord should be returned when an ipns record is
+// invalid due to being too old
+var ErrExpiredRecord = errors.New("expired record")
+
+// ErrUnrecognizedValidity is returned when an IpnsRecord has an
+// unknown validity type.
+var ErrUnrecognizedValidity = errors.New("unrecognized validity type")
+
+// ErrInvalidPath should be returned when an ipns record path
+// is not in a valid format
+var ErrInvalidPath = errors.New("record path invalid")
+
+// ErrSignature should be returned when an ipns record fails
+// signature verification
+var ErrSignature = errors.New("record signature verification failed")
+
+// ErrKeyFormat should be returned when an ipns record key is
+// incorrectly formatted (not a peer ID)
+var ErrKeyFormat = errors.New("record key could not be parsed into peer ID")
+
+// ErrPublicKeyNotFound should be returned when the public key
+// corresponding to the ipns record path cannot be retrieved
+// from the peer store
+var ErrPublicKeyNotFound = errors.New("public key not found in peer store")
+
+// ErrPublicKeyMismatch should be returned when the public key embedded in the
+// record doesn't match the expected public key.
+var ErrPublicKeyMismatch = errors.New("public key in record did not match expected pubkey")
+
+// ErrBadRecord should be returned when an ipns record cannot be unmarshalled
+var ErrBadRecord = errors.New("record could not be unmarshalled")
+
+// 10 KiB limit defined in https://github.com/ipfs/specs/pull/319
+const MaxRecordSize int = 10 << (10 * 1)
+
+// ErrRecordSize should be returned when an ipns record is
+// invalid due to being too big
+var ErrRecordSize = errors.New("record exceeds allowed size limit")
diff --git a/ipns/examples/embed.go b/ipns/examples/embed.go
new file mode 100644
index 000000000..1f33d514d
--- /dev/null
+++ b/ipns/examples/embed.go
@@ -0,0 +1,27 @@
+package examples
+
+import (
+ "time"
+
+ pb "github.com/ipfs/boxo/ipns/pb"
+
+ "github.com/ipfs/boxo/ipns"
+ "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+// CreateEntryWithEmbed shows how you can create an IPNS entry
+// and embed it with a public key. For ed25519 keys this is not needed
+// so attempting to embed with an ed25519 key, will not actually embed the key
+func CreateEntryWithEmbed(ipfsPath string, publicKey crypto.PubKey, privateKey crypto.PrivKey) (*pb.IpnsEntry, error) {
+ ipfsPathByte := []byte(ipfsPath)
+ eol := time.Now().Add(time.Hour * 48)
+ entry, err := ipns.Create(privateKey, ipfsPathByte, 1, eol, 0)
+ if err != nil {
+ return nil, err
+ }
+ err = ipns.EmbedPublicKey(publicKey, entry)
+ if err != nil {
+ return nil, err
+ }
+ return entry, nil
+}
diff --git a/ipns/examples/examples_test.go b/ipns/examples/examples_test.go
new file mode 100644
index 000000000..0fe182d9f
--- /dev/null
+++ b/ipns/examples/examples_test.go
@@ -0,0 +1,60 @@
+package examples_test
+
+import (
+ "testing"
+
+ "github.com/ipfs/boxo/ipns/examples"
+ "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+var testPath = "/ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5"
+
+func TestKeyGeneration(t *testing.T) {
+ _, err := generateRSAKey()
+ if err != nil {
+ t.Error(err)
+ }
+
+ _, err = generateEDKey()
+ if err != nil {
+ t.Error(err)
+ }
+}
+
+func TestEmbeddedEntryCreation(t *testing.T) {
+ rk, err := generateRSAKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ek, err := generateEDKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = examples.CreateEntryWithEmbed(testPath, rk.GetPublic(), rk)
+ if err != nil {
+ t.Error(err)
+ }
+
+ _, err = examples.CreateEntryWithEmbed(testPath, ek.GetPublic(), ek)
+ if err != nil {
+ t.Error(err)
+ }
+
+}
+func generateRSAKey() (crypto.PrivKey, error) {
+ k, err := examples.GenerateRSAKeyPair(2048)
+ if err != nil {
+ return nil, err
+ }
+ return k, nil
+}
+
+func generateEDKey() (crypto.PrivKey, error) {
+ // ED25519 uses 256bit keys, and ignore the bit param
+ k, err := examples.GenerateEDKeyPair()
+ if err != nil {
+ return nil, err
+ }
+ return k, nil
+}
diff --git a/ipns/examples/key.go b/ipns/examples/key.go
new file mode 100644
index 000000000..94f219b8d
--- /dev/null
+++ b/ipns/examples/key.go
@@ -0,0 +1,24 @@
+package examples
+
+import (
+ "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+// GenerateRSAKeyPair is used to generate an RSA key pair
+func GenerateRSAKeyPair(bits int) (crypto.PrivKey, error) {
+ priv, _, err := crypto.GenerateKeyPair(crypto.RSA, bits)
+ if err != nil {
+ return nil, err
+ }
+ return priv, nil
+}
+
+// GenerateEDKeyPair is used to generate an ED25519 keypair
+func GenerateEDKeyPair() (crypto.PrivKey, error) {
+ // ED25519 ignores the bit param and uses 256bit keys
+ priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 256)
+ if err != nil {
+ return nil, err
+ }
+ return priv, nil
+}
diff --git a/ipns/ipns.go b/ipns/ipns.go
new file mode 100644
index 000000000..87ddadb03
--- /dev/null
+++ b/ipns/ipns.go
@@ -0,0 +1,398 @@
+package ipns
+
+import (
+ "bytes"
+ "fmt"
+ "sort"
+ "time"
+
+ "github.com/multiformats/go-multicodec"
+ "github.com/pkg/errors"
+
+ "github.com/ipld/go-ipld-prime"
+ _ "github.com/ipld/go-ipld-prime/codec/dagcbor" // used to import the DagCbor encoder/decoder
+ ipldcodec "github.com/ipld/go-ipld-prime/multicodec"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+
+ "github.com/gogo/protobuf/proto"
+
+ pb "github.com/ipfs/boxo/ipns/pb"
+
+ u "github.com/ipfs/boxo/util"
+ ic "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+const (
+ validity = "Validity"
+ validityType = "ValidityType"
+ value = "Value"
+ sequence = "Sequence"
+ ttl = "TTL"
+)
+
+// Create creates a new IPNS entry and signs it with the given private key.
+//
+// This function does not embed the public key. If you want to do that, use
+// `EmbedPublicKey`.
+func Create(sk ic.PrivKey, val []byte, seq uint64, eol time.Time, ttl time.Duration) (*pb.IpnsEntry, error) {
+ entry := new(pb.IpnsEntry)
+
+ entry.Value = val
+ typ := pb.IpnsEntry_EOL
+ entry.ValidityType = &typ
+ entry.Sequence = &seq
+ entry.Validity = []byte(u.FormatRFC3339(eol))
+
+ ttlNs := uint64(ttl.Nanoseconds())
+ entry.Ttl = proto.Uint64(ttlNs)
+
+ cborData, err := createCborDataForIpnsEntry(entry)
+ if err != nil {
+ return nil, err
+ }
+ entry.Data = cborData
+
+ // For now we still create V1 signatures. These are deprecated, and not
+ // used during verification anymore (Validate func requires SignatureV2),
+ // but setting it here allows legacy nodes (e.g., go-ipfs < v0.9.0) to
+ // still resolve IPNS published by modern nodes.
+ sig1, err := sk.Sign(ipnsEntryDataForSigV1(entry))
+ if err != nil {
+ return nil, errors.Wrap(err, "could not compute signature data")
+ }
+ entry.SignatureV1 = sig1
+
+ sig2Data, err := ipnsEntryDataForSigV2(entry)
+ if err != nil {
+ return nil, err
+ }
+ sig2, err := sk.Sign(sig2Data)
+ if err != nil {
+ return nil, err
+ }
+ entry.SignatureV2 = sig2
+
+ return entry, nil
+}
+
+func createCborDataForIpnsEntry(e *pb.IpnsEntry) ([]byte, error) {
+ m := make(map[string]ipld.Node)
+ var keys []string
+ m[value] = basicnode.NewBytes(e.GetValue())
+ keys = append(keys, value)
+
+ m[validity] = basicnode.NewBytes(e.GetValidity())
+ keys = append(keys, validity)
+
+ m[validityType] = basicnode.NewInt(int64(e.GetValidityType()))
+ keys = append(keys, validityType)
+
+ m[sequence] = basicnode.NewInt(int64(e.GetSequence()))
+ keys = append(keys, sequence)
+
+ m[ttl] = basicnode.NewInt(int64(e.GetTtl()))
+ keys = append(keys, ttl)
+
+ sort.Sort(cborMapKeyString_RFC7049(keys))
+
+ newNd := basicnode.Prototype__Map{}.NewBuilder()
+ ma, err := newNd.BeginMap(int64(len(keys)))
+ if err != nil {
+ return nil, err
+ }
+
+ for _, k := range keys {
+ if err := ma.AssembleKey().AssignString(k); err != nil {
+ return nil, err
+ }
+ if err := ma.AssembleValue().AssignNode(m[k]); err != nil {
+ return nil, err
+ }
+ }
+
+ if err := ma.Finish(); err != nil {
+ return nil, err
+ }
+
+ nd := newNd.Build()
+
+ enc, err := ipldcodec.LookupEncoder(uint64(multicodec.DagCbor))
+ if err != nil {
+ return nil, err
+ }
+
+ buf := new(bytes.Buffer)
+ if err := enc(nd, buf); err != nil {
+ return nil, err
+ }
+
+ return buf.Bytes(), nil
+}
+
+// Validates validates the given IPNS entry against the given public key.
+func Validate(pk ic.PubKey, entry *pb.IpnsEntry) error {
+ // Make sure max size is respected
+ if entry.Size() > MaxRecordSize {
+ return ErrRecordSize
+ }
+
+ // Check the ipns record signature with the public key
+ if entry.GetSignatureV2() == nil {
+ // always error if no valid signature could be found
+ return ErrSignature
+ }
+
+ sig2Data, err := ipnsEntryDataForSigV2(entry)
+ if err != nil {
+ return fmt.Errorf("could not compute signature data: %w", err)
+ }
+ if ok, err := pk.Verify(sig2Data, entry.GetSignatureV2()); err != nil || !ok {
+ return ErrSignature
+ }
+
+ // TODO: If we switch from pb.IpnsEntry to a more generic IpnsRecord type then perhaps we should only check
+ // this if there is no v1 signature. In the meanwhile this helps avoid some potential rough edges around people
+ // checking the entry fields instead of doing CBOR decoding everywhere.
+ // See https://github.com/ipfs/boxo/ipns/pull/42 for next steps here
+ if err := validateCborDataMatchesPbData(entry); err != nil {
+ return err
+ }
+
+ eol, err := GetEOL(entry)
+ if err != nil {
+ return err
+ }
+ if time.Now().After(eol) {
+ return ErrExpiredRecord
+ }
+ return nil
+}
+
+// TODO: Most of this function could probably be replaced with codegen
+func validateCborDataMatchesPbData(entry *pb.IpnsEntry) error {
+ if len(entry.GetData()) == 0 {
+ return fmt.Errorf("record data is missing")
+ }
+
+ dec, err := ipldcodec.LookupDecoder(uint64(multicodec.DagCbor))
+ if err != nil {
+ return err
+ }
+
+ ndbuilder := basicnode.Prototype__Map{}.NewBuilder()
+ if err := dec(ndbuilder, bytes.NewReader(entry.GetData())); err != nil {
+ return err
+ }
+
+ fullNd := ndbuilder.Build()
+ nd, err := fullNd.LookupByString(value)
+ if err != nil {
+ return err
+ }
+ ndBytes, err := nd.AsBytes()
+ if err != nil {
+ return err
+ }
+ if !bytes.Equal(entry.GetValue(), ndBytes) {
+ return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", value)
+ }
+
+ nd, err = fullNd.LookupByString(validity)
+ if err != nil {
+ return err
+ }
+ ndBytes, err = nd.AsBytes()
+ if err != nil {
+ return err
+ }
+ if !bytes.Equal(entry.GetValidity(), ndBytes) {
+ return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", validity)
+ }
+
+ nd, err = fullNd.LookupByString(validityType)
+ if err != nil {
+ return err
+ }
+ ndInt, err := nd.AsInt()
+ if err != nil {
+ return err
+ }
+ if int64(entry.GetValidityType()) != ndInt {
+ return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", validityType)
+ }
+
+ nd, err = fullNd.LookupByString(sequence)
+ if err != nil {
+ return err
+ }
+ ndInt, err = nd.AsInt()
+ if err != nil {
+ return err
+ }
+
+ if entry.GetSequence() != uint64(ndInt) {
+ return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", sequence)
+ }
+
+ nd, err = fullNd.LookupByString("TTL")
+ if err != nil {
+ return err
+ }
+ ndInt, err = nd.AsInt()
+ if err != nil {
+ return err
+ }
+ if entry.GetTtl() != uint64(ndInt) {
+ return fmt.Errorf("field \"%v\" did not match between protobuf and CBOR", ttl)
+ }
+
+ return nil
+}
+
+// GetEOL returns the EOL of this IPNS entry
+//
+// This function returns ErrUnrecognizedValidity if the validity type of the
+// record isn't EOL. Otherwise, it returns an error if it can't parse the EOL.
+func GetEOL(entry *pb.IpnsEntry) (time.Time, error) {
+ if entry.GetValidityType() != pb.IpnsEntry_EOL {
+ return time.Time{}, ErrUnrecognizedValidity
+ }
+ return u.ParseRFC3339(string(entry.GetValidity()))
+}
+
+// EmbedPublicKey embeds the given public key in the given ipns entry. While not
+// strictly required, some nodes (e.g., DHT servers) may reject IPNS entries
+// that don't embed their public keys as they may not be able to validate them
+// efficiently.
+func EmbedPublicKey(pk ic.PubKey, entry *pb.IpnsEntry) error {
+ // Try extracting the public key from the ID. If we can, *don't* embed
+ // it.
+ id, err := peer.IDFromPublicKey(pk)
+ if err != nil {
+ return err
+ }
+ if _, err := id.ExtractPublicKey(); err != peer.ErrNoPublicKey {
+ // Either a *real* error or nil.
+ return err
+ }
+
+ // We failed to extract the public key from the peer ID, embed it in the
+ // record.
+ pkBytes, err := ic.MarshalPublicKey(pk)
+ if err != nil {
+ return err
+ }
+ entry.PubKey = pkBytes
+ return nil
+}
+
+// ExtractPublicKey extracts a public key matching `pid` from the IPNS record,
+// if possible.
+//
+// This function returns (nil, nil) when no public key can be extracted and
+// nothing is malformed.
+func ExtractPublicKey(pid peer.ID, entry *pb.IpnsEntry) (ic.PubKey, error) {
+ if entry.PubKey != nil {
+ pk, err := ic.UnmarshalPublicKey(entry.PubKey)
+ if err != nil {
+ return nil, fmt.Errorf("unmarshaling pubkey in record: %s", err)
+ }
+
+ expPid, err := peer.IDFromPublicKey(pk)
+ if err != nil {
+ return nil, fmt.Errorf("could not regenerate peerID from pubkey: %s", err)
+ }
+
+ if pid != expPid {
+ return nil, ErrPublicKeyMismatch
+ }
+ return pk, nil
+ }
+
+ return pid.ExtractPublicKey()
+}
+
+// Compare compares two IPNS entries. It returns:
+//
+// * -1 if a is older than b
+// * 0 if a and b cannot be ordered (this doesn't mean that they are equal)
+// * +1 if a is newer than b
+//
+// It returns an error when either a or b are malformed.
+//
+// NOTE: It *does not* validate the records, the caller is responsible for calling
+// `Validate` first.
+//
+// NOTE: If a and b cannot be ordered by this function, you can determine their
+// order by comparing their serialized byte representations (using
+// `bytes.Compare`). You must do this if you are implementing a libp2p record
+// validator (or you can just use the one provided for you by this package).
+func Compare(a, b *pb.IpnsEntry) (int, error) {
+ aHasV2Sig := a.GetSignatureV2() != nil
+ bHasV2Sig := b.GetSignatureV2() != nil
+
+ // Having a newer signature version is better than an older signature version
+ if aHasV2Sig && !bHasV2Sig {
+ return 1, nil
+ } else if !aHasV2Sig && bHasV2Sig {
+ return -1, nil
+ }
+
+ as := a.GetSequence()
+ bs := b.GetSequence()
+
+ if as > bs {
+ return 1, nil
+ } else if as < bs {
+ return -1, nil
+ }
+
+ at, err := u.ParseRFC3339(string(a.GetValidity()))
+ if err != nil {
+ return 0, err
+ }
+
+ bt, err := u.ParseRFC3339(string(b.GetValidity()))
+ if err != nil {
+ return 0, err
+ }
+
+ if at.After(bt) {
+ return 1, nil
+ } else if bt.After(at) {
+ return -1, nil
+ }
+
+ return 0, nil
+}
+
+func ipnsEntryDataForSigV1(e *pb.IpnsEntry) []byte {
+ return bytes.Join([][]byte{
+ e.Value,
+ e.Validity,
+ []byte(fmt.Sprint(e.GetValidityType())),
+ },
+ []byte{})
+}
+
+func ipnsEntryDataForSigV2(e *pb.IpnsEntry) ([]byte, error) {
+ dataForSig := []byte("ipns-signature:")
+ dataForSig = append(dataForSig, e.Data...)
+
+ return dataForSig, nil
+}
+
+type cborMapKeyString_RFC7049 []string
+
+func (x cborMapKeyString_RFC7049) Len() int { return len(x) }
+func (x cborMapKeyString_RFC7049) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+func (x cborMapKeyString_RFC7049) Less(i, j int) bool {
+ li, lj := len(x[i]), len(x[j])
+ if li == lj {
+ return x[i] < x[j]
+ }
+ return li < lj
+}
+
+var _ sort.Interface = (cborMapKeyString_RFC7049)(nil)
diff --git a/ipns/ipns_test.go b/ipns/ipns_test.go
new file mode 100644
index 000000000..e6d521ce4
--- /dev/null
+++ b/ipns/ipns_test.go
@@ -0,0 +1,63 @@
+package ipns
+
+import (
+ "fmt"
+ "testing"
+ "time"
+
+ u "github.com/ipfs/boxo/util"
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+func TestEmbedPublicKey(t *testing.T) {
+
+ sr := u.NewTimeSeededRand()
+ priv, pub, err := ci.GenerateKeyPairWithReader(ci.RSA, 2048, sr)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ pid, err := peer.IDFromPublicKey(pub)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ e, err := Create(priv, []byte("/a/b"), 0, time.Now().Add(1*time.Hour), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := EmbedPublicKey(pub, e); err != nil {
+ t.Fatal(err)
+ }
+ embeddedPk, err := ci.UnmarshalPublicKey(e.PubKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ embeddedPid, err := peer.IDFromPublicKey(embeddedPk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if embeddedPid != pid {
+ t.Fatalf("pid mismatch: %s != %s", pid, embeddedPid)
+ }
+}
+
+func ExampleCreate() {
+ // Generate a private key to sign the IPNS record with. Most of the time,
+ // however, you'll want to retrieve an already-existing key from IPFS using
+ // go-ipfs/core/coreapi CoreAPI.KeyAPI() interface.
+ privateKey, _, err := ci.GenerateKeyPair(ci.RSA, 2048)
+ if err != nil {
+ panic(err)
+ }
+
+ // Create an IPNS record that expires in one hour and points to the IPFS address
+ // /ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5
+ ipnsRecord, err := Create(privateKey, []byte("/ipfs/Qme1knMqwt1hKZbc1BmQFmnm9f36nyQGwXxPGVpVJ9rMK5"), 0, time.Now().Add(1*time.Hour), 0)
+ if err != nil {
+ panic(err)
+ }
+
+ fmt.Println(ipnsRecord)
+}
diff --git a/ipns/pb/Makefile b/ipns/pb/Makefile
new file mode 100644
index 000000000..eb14b5768
--- /dev/null
+++ b/ipns/pb/Makefile
@@ -0,0 +1,11 @@
+PB = $(wildcard *.proto)
+GO = $(PB:.proto=.pb.go)
+
+all: $(GO)
+
+%.pb.go: %.proto
+ protoc --proto_path=$(GOPATH)/src:. --gogofast_out=. $<
+
+clean:
+ rm -f *.pb.go
+ rm -f *.go
diff --git a/ipns/pb/ipns.pb.go b/ipns/pb/ipns.pb.go
new file mode 100644
index 000000000..1e2488852
--- /dev/null
+++ b/ipns/pb/ipns.pb.go
@@ -0,0 +1,992 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: ipns.proto
+
+package ipns_pb
+
+import (
+ fmt "fmt"
+ proto "github.com/gogo/protobuf/proto"
+ io "io"
+ math "math"
+ math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type IpnsEntry_ValidityType int32
+
+const (
+ // setting an EOL says "this record is valid until..."
+ IpnsEntry_EOL IpnsEntry_ValidityType = 0
+)
+
+var IpnsEntry_ValidityType_name = map[int32]string{
+ 0: "EOL",
+}
+
+var IpnsEntry_ValidityType_value = map[string]int32{
+ "EOL": 0,
+}
+
+func (x IpnsEntry_ValidityType) Enum() *IpnsEntry_ValidityType {
+ p := new(IpnsEntry_ValidityType)
+ *p = x
+ return p
+}
+
+func (x IpnsEntry_ValidityType) String() string {
+ return proto.EnumName(IpnsEntry_ValidityType_name, int32(x))
+}
+
+func (x *IpnsEntry_ValidityType) UnmarshalJSON(data []byte) error {
+ value, err := proto.UnmarshalJSONEnum(IpnsEntry_ValidityType_value, data, "IpnsEntry_ValidityType")
+ if err != nil {
+ return err
+ }
+ *x = IpnsEntry_ValidityType(value)
+ return nil
+}
+
+func (IpnsEntry_ValidityType) EnumDescriptor() ([]byte, []int) {
+ return fileDescriptor_4d5b16fb32bfe8ea, []int{0, 0}
+}
+
+type IpnsEntry struct {
+ Value []byte `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"`
+ SignatureV1 []byte `protobuf:"bytes,2,opt,name=signatureV1" json:"signatureV1,omitempty"`
+ ValidityType *IpnsEntry_ValidityType `protobuf:"varint,3,opt,name=validityType,enum=ipns.v1.pb.IpnsEntry_ValidityType" json:"validityType,omitempty"`
+ Validity []byte `protobuf:"bytes,4,opt,name=validity" json:"validity,omitempty"`
+ Sequence *uint64 `protobuf:"varint,5,opt,name=sequence" json:"sequence,omitempty"`
+ Ttl *uint64 `protobuf:"varint,6,opt,name=ttl" json:"ttl,omitempty"`
+ // in order for nodes to properly validate a record upon receipt, they need the public
+ // key associated with it. For old RSA keys, its easiest if we just send this as part of
+ // the record itself. For newer ed25519 keys, the public key can be embedded in the
+ // peerID, making this field unnecessary.
+ PubKey []byte `protobuf:"bytes,7,opt,name=pubKey" json:"pubKey,omitempty"`
+ SignatureV2 []byte `protobuf:"bytes,8,opt,name=signatureV2" json:"signatureV2,omitempty"`
+ Data []byte `protobuf:"bytes,9,opt,name=data" json:"data,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *IpnsEntry) Reset() { *m = IpnsEntry{} }
+func (m *IpnsEntry) String() string { return proto.CompactTextString(m) }
+func (*IpnsEntry) ProtoMessage() {}
+func (*IpnsEntry) Descriptor() ([]byte, []int) {
+ return fileDescriptor_4d5b16fb32bfe8ea, []int{0}
+}
+func (m *IpnsEntry) XXX_Unmarshal(b []byte) error {
+ return m.Unmarshal(b)
+}
+func (m *IpnsEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ if deterministic {
+ return xxx_messageInfo_IpnsEntry.Marshal(b, m, deterministic)
+ } else {
+ b = b[:cap(b)]
+ n, err := m.MarshalToSizedBuffer(b)
+ if err != nil {
+ return nil, err
+ }
+ return b[:n], nil
+ }
+}
+func (m *IpnsEntry) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_IpnsEntry.Merge(m, src)
+}
+func (m *IpnsEntry) XXX_Size() int {
+ return m.Size()
+}
+func (m *IpnsEntry) XXX_DiscardUnknown() {
+ xxx_messageInfo_IpnsEntry.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_IpnsEntry proto.InternalMessageInfo
+
+func (m *IpnsEntry) GetValue() []byte {
+ if m != nil {
+ return m.Value
+ }
+ return nil
+}
+
+func (m *IpnsEntry) GetSignatureV1() []byte {
+ if m != nil {
+ return m.SignatureV1
+ }
+ return nil
+}
+
+func (m *IpnsEntry) GetValidityType() IpnsEntry_ValidityType {
+ if m != nil && m.ValidityType != nil {
+ return *m.ValidityType
+ }
+ return IpnsEntry_EOL
+}
+
+func (m *IpnsEntry) GetValidity() []byte {
+ if m != nil {
+ return m.Validity
+ }
+ return nil
+}
+
+func (m *IpnsEntry) GetSequence() uint64 {
+ if m != nil && m.Sequence != nil {
+ return *m.Sequence
+ }
+ return 0
+}
+
+func (m *IpnsEntry) GetTtl() uint64 {
+ if m != nil && m.Ttl != nil {
+ return *m.Ttl
+ }
+ return 0
+}
+
+func (m *IpnsEntry) GetPubKey() []byte {
+ if m != nil {
+ return m.PubKey
+ }
+ return nil
+}
+
+func (m *IpnsEntry) GetSignatureV2() []byte {
+ if m != nil {
+ return m.SignatureV2
+ }
+ return nil
+}
+
+func (m *IpnsEntry) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+type IpnsSignatureV2Checker struct {
+ PubKey []byte `protobuf:"bytes,7,opt,name=pubKey" json:"pubKey,omitempty"`
+ SignatureV2 []byte `protobuf:"bytes,8,opt,name=signatureV2" json:"signatureV2,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *IpnsSignatureV2Checker) Reset() { *m = IpnsSignatureV2Checker{} }
+func (m *IpnsSignatureV2Checker) String() string { return proto.CompactTextString(m) }
+func (*IpnsSignatureV2Checker) ProtoMessage() {}
+func (*IpnsSignatureV2Checker) Descriptor() ([]byte, []int) {
+ return fileDescriptor_4d5b16fb32bfe8ea, []int{1}
+}
+func (m *IpnsSignatureV2Checker) XXX_Unmarshal(b []byte) error {
+ return m.Unmarshal(b)
+}
+func (m *IpnsSignatureV2Checker) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ if deterministic {
+ return xxx_messageInfo_IpnsSignatureV2Checker.Marshal(b, m, deterministic)
+ } else {
+ b = b[:cap(b)]
+ n, err := m.MarshalToSizedBuffer(b)
+ if err != nil {
+ return nil, err
+ }
+ return b[:n], nil
+ }
+}
+func (m *IpnsSignatureV2Checker) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_IpnsSignatureV2Checker.Merge(m, src)
+}
+func (m *IpnsSignatureV2Checker) XXX_Size() int {
+ return m.Size()
+}
+func (m *IpnsSignatureV2Checker) XXX_DiscardUnknown() {
+ xxx_messageInfo_IpnsSignatureV2Checker.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_IpnsSignatureV2Checker proto.InternalMessageInfo
+
+func (m *IpnsSignatureV2Checker) GetPubKey() []byte {
+ if m != nil {
+ return m.PubKey
+ }
+ return nil
+}
+
+func (m *IpnsSignatureV2Checker) GetSignatureV2() []byte {
+ if m != nil {
+ return m.SignatureV2
+ }
+ return nil
+}
+
+func init() {
+ proto.RegisterEnum("ipns.v1.pb.IpnsEntry_ValidityType", IpnsEntry_ValidityType_name, IpnsEntry_ValidityType_value)
+ proto.RegisterType((*IpnsEntry)(nil), "ipns.v1.pb.IpnsEntry")
+ proto.RegisterType((*IpnsSignatureV2Checker)(nil), "ipns.v1.pb.IpnsSignatureV2Checker")
+}
+
+func init() { proto.RegisterFile("ipns.proto", fileDescriptor_4d5b16fb32bfe8ea) }
+
+var fileDescriptor_4d5b16fb32bfe8ea = []byte{
+ // 272 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xca, 0x2c, 0xc8, 0x2b,
+ 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x82, 0xb0, 0xcb, 0x0c, 0xf5, 0x0a, 0x92, 0x94, 0xf6,
+ 0x30, 0x71, 0x71, 0x7a, 0x16, 0xe4, 0x15, 0xbb, 0xe6, 0x95, 0x14, 0x55, 0x0a, 0x89, 0x70, 0xb1,
+ 0x96, 0x25, 0xe6, 0x94, 0xa6, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x04, 0x41, 0x38, 0x42, 0x0a,
+ 0x5c, 0xdc, 0xc5, 0x99, 0xe9, 0x79, 0x89, 0x25, 0xa5, 0x45, 0xa9, 0x61, 0x86, 0x12, 0x4c, 0x60,
+ 0x39, 0x64, 0x21, 0x21, 0x37, 0x2e, 0x9e, 0xb2, 0xc4, 0x9c, 0xcc, 0x94, 0xcc, 0x92, 0xca, 0x90,
+ 0xca, 0x82, 0x54, 0x09, 0x66, 0x05, 0x46, 0x0d, 0x3e, 0x23, 0x25, 0x3d, 0x84, 0x45, 0x7a, 0x70,
+ 0x4b, 0xf4, 0xc2, 0x90, 0x54, 0x06, 0xa1, 0xe8, 0x13, 0x92, 0xe2, 0xe2, 0x80, 0xf1, 0x25, 0x58,
+ 0xc0, 0xd6, 0xc0, 0xf9, 0x20, 0xb9, 0xe2, 0xd4, 0xc2, 0xd2, 0xd4, 0xbc, 0xe4, 0x54, 0x09, 0x56,
+ 0x05, 0x46, 0x0d, 0x96, 0x20, 0x38, 0x5f, 0x48, 0x80, 0x8b, 0xb9, 0xa4, 0x24, 0x47, 0x82, 0x0d,
+ 0x2c, 0x0c, 0x62, 0x0a, 0x89, 0x71, 0xb1, 0x15, 0x94, 0x26, 0x79, 0xa7, 0x56, 0x4a, 0xb0, 0x83,
+ 0xcd, 0x81, 0xf2, 0x50, 0xfd, 0x62, 0x24, 0xc1, 0x81, 0xee, 0x17, 0x23, 0x21, 0x21, 0x2e, 0x96,
+ 0x94, 0xc4, 0x92, 0x44, 0x09, 0x4e, 0xb0, 0x14, 0x98, 0xad, 0x24, 0xce, 0xc5, 0x83, 0xec, 0x6a,
+ 0x21, 0x76, 0x2e, 0x66, 0x57, 0x7f, 0x1f, 0x01, 0x06, 0xa5, 0x20, 0x2e, 0x31, 0x90, 0xc7, 0x82,
+ 0x11, 0xfa, 0x9d, 0x33, 0x52, 0x93, 0xb3, 0x53, 0x8b, 0xc8, 0x77, 0x80, 0x93, 0xe8, 0x89, 0x47,
+ 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x18, 0xc5, 0x0e, 0x0a, 0xc3, 0xf8,
+ 0x82, 0x24, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbd, 0x45, 0xdd, 0x1a, 0xc2, 0x01, 0x00, 0x00,
+}
+
+func (m *IpnsEntry) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBuffer(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *IpnsEntry) MarshalTo(dAtA []byte) (int, error) {
+ size := m.Size()
+ return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *IpnsEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if m.XXX_unrecognized != nil {
+ i -= len(m.XXX_unrecognized)
+ copy(dAtA[i:], m.XXX_unrecognized)
+ }
+ if m.Data != nil {
+ i -= len(m.Data)
+ copy(dAtA[i:], m.Data)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.Data)))
+ i--
+ dAtA[i] = 0x4a
+ }
+ if m.SignatureV2 != nil {
+ i -= len(m.SignatureV2)
+ copy(dAtA[i:], m.SignatureV2)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.SignatureV2)))
+ i--
+ dAtA[i] = 0x42
+ }
+ if m.PubKey != nil {
+ i -= len(m.PubKey)
+ copy(dAtA[i:], m.PubKey)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.PubKey)))
+ i--
+ dAtA[i] = 0x3a
+ }
+ if m.Ttl != nil {
+ i = encodeVarintIpns(dAtA, i, uint64(*m.Ttl))
+ i--
+ dAtA[i] = 0x30
+ }
+ if m.Sequence != nil {
+ i = encodeVarintIpns(dAtA, i, uint64(*m.Sequence))
+ i--
+ dAtA[i] = 0x28
+ }
+ if m.Validity != nil {
+ i -= len(m.Validity)
+ copy(dAtA[i:], m.Validity)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.Validity)))
+ i--
+ dAtA[i] = 0x22
+ }
+ if m.ValidityType != nil {
+ i = encodeVarintIpns(dAtA, i, uint64(*m.ValidityType))
+ i--
+ dAtA[i] = 0x18
+ }
+ if m.SignatureV1 != nil {
+ i -= len(m.SignatureV1)
+ copy(dAtA[i:], m.SignatureV1)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.SignatureV1)))
+ i--
+ dAtA[i] = 0x12
+ }
+ if m.Value != nil {
+ i -= len(m.Value)
+ copy(dAtA[i:], m.Value)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.Value)))
+ i--
+ dAtA[i] = 0xa
+ }
+ return len(dAtA) - i, nil
+}
+
+func (m *IpnsSignatureV2Checker) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalToSizedBuffer(dAtA[:size])
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *IpnsSignatureV2Checker) MarshalTo(dAtA []byte) (int, error) {
+ size := m.Size()
+ return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *IpnsSignatureV2Checker) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+ i := len(dAtA)
+ _ = i
+ var l int
+ _ = l
+ if m.XXX_unrecognized != nil {
+ i -= len(m.XXX_unrecognized)
+ copy(dAtA[i:], m.XXX_unrecognized)
+ }
+ if m.SignatureV2 != nil {
+ i -= len(m.SignatureV2)
+ copy(dAtA[i:], m.SignatureV2)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.SignatureV2)))
+ i--
+ dAtA[i] = 0x42
+ }
+ if m.PubKey != nil {
+ i -= len(m.PubKey)
+ copy(dAtA[i:], m.PubKey)
+ i = encodeVarintIpns(dAtA, i, uint64(len(m.PubKey)))
+ i--
+ dAtA[i] = 0x3a
+ }
+ return len(dAtA) - i, nil
+}
+
+func encodeVarintIpns(dAtA []byte, offset int, v uint64) int {
+ offset -= sovIpns(v)
+ base := offset
+ for v >= 1<<7 {
+ dAtA[offset] = uint8(v&0x7f | 0x80)
+ v >>= 7
+ offset++
+ }
+ dAtA[offset] = uint8(v)
+ return base
+}
+func (m *IpnsEntry) Size() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if m.Value != nil {
+ l = len(m.Value)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.SignatureV1 != nil {
+ l = len(m.SignatureV1)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.ValidityType != nil {
+ n += 1 + sovIpns(uint64(*m.ValidityType))
+ }
+ if m.Validity != nil {
+ l = len(m.Validity)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.Sequence != nil {
+ n += 1 + sovIpns(uint64(*m.Sequence))
+ }
+ if m.Ttl != nil {
+ n += 1 + sovIpns(uint64(*m.Ttl))
+ }
+ if m.PubKey != nil {
+ l = len(m.PubKey)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.SignatureV2 != nil {
+ l = len(m.SignatureV2)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.Data != nil {
+ l = len(m.Data)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.XXX_unrecognized != nil {
+ n += len(m.XXX_unrecognized)
+ }
+ return n
+}
+
+func (m *IpnsSignatureV2Checker) Size() (n int) {
+ if m == nil {
+ return 0
+ }
+ var l int
+ _ = l
+ if m.PubKey != nil {
+ l = len(m.PubKey)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.SignatureV2 != nil {
+ l = len(m.SignatureV2)
+ n += 1 + l + sovIpns(uint64(l))
+ }
+ if m.XXX_unrecognized != nil {
+ n += len(m.XXX_unrecognized)
+ }
+ return n
+}
+
+func sovIpns(x uint64) (n int) {
+ return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozIpns(x uint64) (n int) {
+ return sovIpns(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *IpnsEntry) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: IpnsEntry: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: IpnsEntry: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...)
+ if m.Value == nil {
+ m.Value = []byte{}
+ }
+ iNdEx = postIndex
+ case 2:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field SignatureV1", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.SignatureV1 = append(m.SignatureV1[:0], dAtA[iNdEx:postIndex]...)
+ if m.SignatureV1 == nil {
+ m.SignatureV1 = []byte{}
+ }
+ iNdEx = postIndex
+ case 3:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field ValidityType", wireType)
+ }
+ var v IpnsEntry_ValidityType
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= IpnsEntry_ValidityType(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.ValidityType = &v
+ case 4:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Validity", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Validity = append(m.Validity[:0], dAtA[iNdEx:postIndex]...)
+ if m.Validity == nil {
+ m.Validity = []byte{}
+ }
+ iNdEx = postIndex
+ case 5:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Sequence", wireType)
+ }
+ var v uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.Sequence = &v
+ case 6:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Ttl", wireType)
+ }
+ var v uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ v |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ m.Ttl = &v
+ case 7:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field PubKey", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.PubKey = append(m.PubKey[:0], dAtA[iNdEx:postIndex]...)
+ if m.PubKey == nil {
+ m.PubKey = []byte{}
+ }
+ iNdEx = postIndex
+ case 8:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field SignatureV2", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.SignatureV2 = append(m.SignatureV2[:0], dAtA[iNdEx:postIndex]...)
+ if m.SignatureV2 == nil {
+ m.SignatureV2 = []byte{}
+ }
+ iNdEx = postIndex
+ case 9:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...)
+ if m.Data == nil {
+ m.Data = []byte{}
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipIpns(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if (iNdEx + skippy) < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func (m *IpnsSignatureV2Checker) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= uint64(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: IpnsSignatureV2Checker: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: IpnsSignatureV2Checker: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 7:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field PubKey", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.PubKey = append(m.PubKey[:0], dAtA[iNdEx:postIndex]...)
+ if m.PubKey == nil {
+ m.PubKey = []byte{}
+ }
+ iNdEx = postIndex
+ case 8:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field SignatureV2", wireType)
+ }
+ var byteLen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ byteLen |= int(b&0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if byteLen < 0 {
+ return ErrInvalidLengthIpns
+ }
+ postIndex := iNdEx + byteLen
+ if postIndex < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.SignatureV2 = append(m.SignatureV2[:0], dAtA[iNdEx:postIndex]...)
+ if m.SignatureV2 == nil {
+ m.SignatureV2 = []byte{}
+ }
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipIpns(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if (iNdEx + skippy) < 0 {
+ return ErrInvalidLengthIpns
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
+func skipIpns(dAtA []byte) (n int, err error) {
+ l := len(dAtA)
+ iNdEx := 0
+ depth := 0
+ for iNdEx < l {
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return 0, ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return 0, io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ wireType := int(wire & 0x7)
+ switch wireType {
+ case 0:
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return 0, ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return 0, io.ErrUnexpectedEOF
+ }
+ iNdEx++
+ if dAtA[iNdEx-1] < 0x80 {
+ break
+ }
+ }
+ case 1:
+ iNdEx += 8
+ case 2:
+ var length int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return 0, ErrIntOverflowIpns
+ }
+ if iNdEx >= l {
+ return 0, io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ length |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if length < 0 {
+ return 0, ErrInvalidLengthIpns
+ }
+ iNdEx += length
+ case 3:
+ depth++
+ case 4:
+ if depth == 0 {
+ return 0, ErrUnexpectedEndOfGroupIpns
+ }
+ depth--
+ case 5:
+ iNdEx += 4
+ default:
+ return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+ }
+ if iNdEx < 0 {
+ return 0, ErrInvalidLengthIpns
+ }
+ if depth == 0 {
+ return iNdEx, nil
+ }
+ }
+ return 0, io.ErrUnexpectedEOF
+}
+
+var (
+ ErrInvalidLengthIpns = fmt.Errorf("proto: negative length found during unmarshaling")
+ ErrIntOverflowIpns = fmt.Errorf("proto: integer overflow")
+ ErrUnexpectedEndOfGroupIpns = fmt.Errorf("proto: unexpected end of group")
+)
diff --git a/ipns/pb/ipns.proto b/ipns/pb/ipns.proto
new file mode 100644
index 000000000..bd89a34ec
--- /dev/null
+++ b/ipns/pb/ipns.proto
@@ -0,0 +1,36 @@
+syntax = "proto2";
+
+package ipns.v1.pb;
+
+option go_package = "ipns_pb";
+
+message IpnsEntry {
+ enum ValidityType {
+ // setting an EOL says "this record is valid until..."
+ EOL = 0;
+ }
+ optional bytes value = 1;
+ optional bytes signatureV1 = 2;
+
+ optional ValidityType validityType = 3;
+ optional bytes validity = 4;
+
+ optional uint64 sequence = 5;
+
+ optional uint64 ttl = 6;
+
+ // in order for nodes to properly validate a record upon receipt, they need the public
+ // key associated with it. For old RSA keys, its easiest if we just send this as part of
+ // the record itself. For newer ed25519 keys, the public key can be embedded in the
+ // peerID, making this field unnecessary.
+ optional bytes pubKey = 7;
+
+ optional bytes signatureV2 = 8;
+
+ optional bytes data = 9;
+}
+
+message IpnsSignatureV2Checker {
+ optional bytes pubKey = 7;
+ optional bytes signatureV2 = 8;
+}
diff --git a/ipns/record.go b/ipns/record.go
new file mode 100644
index 000000000..b479dab90
--- /dev/null
+++ b/ipns/record.go
@@ -0,0 +1,126 @@
+package ipns
+
+import (
+ "bytes"
+ "errors"
+
+ pb "github.com/ipfs/boxo/ipns/pb"
+
+ "github.com/gogo/protobuf/proto"
+ logging "github.com/ipfs/go-log/v2"
+ record "github.com/libp2p/go-libp2p-record"
+ ic "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ pstore "github.com/libp2p/go-libp2p/core/peerstore"
+)
+
+var log = logging.Logger("ipns")
+
+var _ record.Validator = Validator{}
+
+// RecordKey returns the libp2p record key for a given peer ID.
+func RecordKey(pid peer.ID) string {
+ return "/ipns/" + string(pid)
+}
+
+// Validator is an IPNS record validator that satisfies the libp2p record
+// validator interface.
+type Validator struct {
+ // KeyBook, if non-nil, will be used to lookup keys for validating IPNS
+ // records.
+ KeyBook pstore.KeyBook
+}
+
+// Validate validates an IPNS record.
+func (v Validator) Validate(key string, value []byte) error {
+ ns, pidString, err := record.SplitKey(key)
+ if err != nil || ns != "ipns" {
+ return ErrInvalidPath
+ }
+
+ // Parse the value into an IpnsEntry
+ entry := new(pb.IpnsEntry)
+ err = proto.Unmarshal(value, entry)
+ if err != nil {
+ return ErrBadRecord
+ }
+
+ // Get the public key defined by the ipns path
+ pid, err := peer.IDFromBytes([]byte(pidString))
+ if err != nil {
+ log.Debugf("failed to parse ipns record key %s into peer ID", pidString)
+ return ErrKeyFormat
+ }
+
+ pubk, err := v.getPublicKey(pid, entry)
+ if err != nil {
+ return err
+ }
+
+ return Validate(pubk, entry)
+}
+
+func (v Validator) getPublicKey(pid peer.ID, entry *pb.IpnsEntry) (ic.PubKey, error) {
+ switch pk, err := ExtractPublicKey(pid, entry); err {
+ case peer.ErrNoPublicKey:
+ case nil:
+ return pk, nil
+ default:
+ return nil, err
+ }
+
+ if v.KeyBook == nil {
+ log.Debugf("public key with hash %s not found in IPNS record and no peer store provided", pid)
+ return nil, ErrPublicKeyNotFound
+ }
+
+ pubk := v.KeyBook.PubKey(pid)
+ if pubk == nil {
+ log.Debugf("public key with hash %s not found in peer store", pid)
+ return nil, ErrPublicKeyNotFound
+ }
+ return pubk, nil
+}
+
+// Select selects the best record by checking which has the highest sequence
+// number and latest EOL.
+//
+// This function returns an error if any of the records fail to parse. Validate
+// your records first!
+func (v Validator) Select(k string, vals [][]byte) (int, error) {
+ var recs []*pb.IpnsEntry
+ for _, v := range vals {
+ e := new(pb.IpnsEntry)
+ if err := proto.Unmarshal(v, e); err != nil {
+ return -1, err
+ }
+ recs = append(recs, e)
+ }
+
+ return selectRecord(recs, vals)
+}
+
+func selectRecord(recs []*pb.IpnsEntry, vals [][]byte) (int, error) {
+ switch len(recs) {
+ case 0:
+ return -1, errors.New("no usable records in given set")
+ case 1:
+ return 0, nil
+ }
+
+ var i int
+ for j := 1; j < len(recs); j++ {
+ cmp, err := Compare(recs[i], recs[j])
+ if err != nil {
+ return -1, err
+ }
+ if cmp == 0 {
+ cmp = bytes.Compare(vals[i], vals[j])
+ }
+ if cmp < 0 {
+ i = j
+ }
+ }
+
+ return i, nil
+}
diff --git a/ipns/select_test.go b/ipns/select_test.go
new file mode 100644
index 000000000..5b435f62d
--- /dev/null
+++ b/ipns/select_test.go
@@ -0,0 +1,126 @@
+package ipns
+
+import (
+ "fmt"
+ "math/rand"
+ "testing"
+ "time"
+
+ pb "github.com/ipfs/boxo/ipns/pb"
+
+ "github.com/gogo/protobuf/proto"
+ u "github.com/ipfs/boxo/util"
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+func shuffle(a []*pb.IpnsEntry) {
+ for n := 0; n < 5; n++ {
+ for i := range a {
+ j := rand.Intn(len(a))
+ a[i], a[j] = a[j], a[i]
+ }
+ }
+}
+
+func AssertSelected(r *pb.IpnsEntry, from ...*pb.IpnsEntry) error {
+ shuffle(from)
+ var vals [][]byte
+ for _, r := range from {
+ data, err := proto.Marshal(r)
+ if err != nil {
+ return err
+ }
+ vals = append(vals, data)
+ }
+
+ i, err := selectRecord(from, vals)
+ if err != nil {
+ return err
+ }
+
+ if from[i] != r {
+ return fmt.Errorf("selected incorrect record %d", i)
+ }
+
+ return nil
+}
+
+func TestOrdering(t *testing.T) {
+ // select timestamp so selection is deterministic
+ ts := time.Unix(1000000, 0)
+
+ // generate a key for signing the records
+ r := u.NewSeededRand(15) // generate deterministic keypair
+ priv, _, err := ci.GenerateKeyPairWithReader(ci.RSA, 2048, r)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ e1, err := Create(priv, []byte("foo"), 1, ts.Add(time.Hour), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ e2, err := Create(priv, []byte("bar"), 2, ts.Add(time.Hour), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ e3, err := Create(priv, []byte("baz"), 3, ts.Add(time.Hour), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ e4, err := Create(priv, []byte("cat"), 3, ts.Add(time.Hour*2), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ e5, err := Create(priv, []byte("dog"), 4, ts.Add(time.Hour*3), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ e6, err := Create(priv, []byte("fish"), 4, ts.Add(time.Hour*3), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // e1 is the only record, i hope it gets this right
+ err = AssertSelected(e1, e1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // e2 has the highest sequence number
+ err = AssertSelected(e2, e1, e2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // e3 has the highest sequence number
+ err = AssertSelected(e3, e1, e2, e3)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // e4 has a higher timeout
+ err = AssertSelected(e4, e1, e2, e3, e4)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // e5 has the highest sequence number
+ err = AssertSelected(e5, e1, e2, e3, e4, e5)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // e6 should be selected as its signauture will win in the comparison
+ err = AssertSelected(e6, e1, e2, e3, e4, e5, e6)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _ = []interface{}{e1, e2, e3, e4, e5, e6}
+}
diff --git a/ipns/validate_test.go b/ipns/validate_test.go
new file mode 100644
index 000000000..e75d0aee7
--- /dev/null
+++ b/ipns/validate_test.go
@@ -0,0 +1,415 @@
+package ipns
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "math/rand"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gogo/protobuf/proto"
+ pb "github.com/ipfs/boxo/ipns/pb"
+ u "github.com/ipfs/boxo/util"
+ ipldcodec "github.com/ipld/go-ipld-prime/multicodec"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ pstore "github.com/libp2p/go-libp2p/core/peerstore"
+ "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem"
+ "github.com/multiformats/go-multicodec"
+)
+
+func testValidatorCase(t *testing.T, priv crypto.PrivKey, kbook pstore.KeyBook, key string, val []byte, eol time.Time, exp error) {
+ t.Helper()
+
+ match := func(t *testing.T, err error) {
+ t.Helper()
+ if err != exp {
+ params := fmt.Sprintf("key: %s\neol: %s\n", key, eol)
+ if exp == nil {
+ t.Fatalf("Unexpected error %s for params %s", err, params)
+ } else if err == nil {
+ t.Fatalf("Expected error %s but there was no error for params %s", exp, params)
+ } else {
+ t.Fatalf("Expected error %s but got %s for params %s", exp, err, params)
+ }
+ }
+ }
+
+ testValidatorCaseMatchFunc(t, priv, kbook, key, val, eol, match)
+}
+
+func testValidatorCaseMatchFunc(t *testing.T, priv crypto.PrivKey, kbook pstore.KeyBook, key string, val []byte, eol time.Time, matchf func(*testing.T, error)) {
+ t.Helper()
+ validator := Validator{kbook}
+
+ data := val
+ if data == nil {
+ p := []byte("/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG")
+ entry, err := Create(priv, p, 1, eol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ data, err = proto.Marshal(entry)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ matchf(t, validator.Validate(key, data))
+}
+
+func TestValidator(t *testing.T) {
+ ts := time.Now()
+
+ priv, id, _ := genKeys(t)
+ priv2, id2, _ := genKeys(t)
+ kbook, err := pstoremem.NewPeerstore()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := kbook.AddPubKey(id, priv.GetPublic()); err != nil {
+ t.Fatal(err)
+ }
+ emptyKbook, err := pstoremem.NewPeerstore()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ testValidatorCase(t, priv, kbook, "/ipns/"+string(id), nil, ts.Add(time.Hour), nil)
+ testValidatorCase(t, priv, kbook, "/ipns/"+string(id), nil, ts.Add(time.Hour*-1), ErrExpiredRecord)
+ testValidatorCase(t, priv, kbook, "/ipns/"+string(id), []byte("bad data"), ts.Add(time.Hour), ErrBadRecord)
+ testValidatorCase(t, priv, kbook, "/ipns/"+"bad key", nil, ts.Add(time.Hour), ErrKeyFormat)
+ testValidatorCase(t, priv, emptyKbook, "/ipns/"+string(id), nil, ts.Add(time.Hour), ErrPublicKeyNotFound)
+ testValidatorCase(t, priv2, kbook, "/ipns/"+string(id2), nil, ts.Add(time.Hour), ErrPublicKeyNotFound)
+ testValidatorCase(t, priv2, kbook, "/ipns/"+string(id), nil, ts.Add(time.Hour), ErrSignature)
+ testValidatorCase(t, priv, kbook, "//"+string(id), nil, ts.Add(time.Hour), ErrInvalidPath)
+ testValidatorCase(t, priv, kbook, "/wrong/"+string(id), nil, ts.Add(time.Hour), ErrInvalidPath)
+}
+
+func mustMarshal(t *testing.T, entry *pb.IpnsEntry) []byte {
+ t.Helper()
+ data, err := proto.Marshal(entry)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return data
+}
+
+func TestEmbeddedPubKeyValidate(t *testing.T) {
+ goodeol := time.Now().Add(time.Hour)
+ kbook, err := pstoremem.NewPeerstore()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ pth := []byte("/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG")
+
+ priv, _, ipnsk := genKeys(t)
+
+ entry, err := Create(priv, pth, 1, goodeol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ testValidatorCase(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, ErrPublicKeyNotFound)
+
+ pubkb, err := crypto.MarshalPublicKey(priv.GetPublic())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ entry.PubKey = pubkb
+ testValidatorCase(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, nil)
+
+ entry.PubKey = []byte("probably not a public key")
+ testValidatorCaseMatchFunc(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, func(t *testing.T, err error) {
+ if !strings.Contains(err.Error(), "unmarshaling pubkey in record:") {
+ t.Fatal("expected pubkey unmarshaling to fail")
+ }
+ })
+
+ opriv, _, _ := genKeys(t)
+ wrongkeydata, err := crypto.MarshalPublicKey(opriv.GetPublic())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ entry.PubKey = wrongkeydata
+ testValidatorCase(t, priv, kbook, ipnsk, mustMarshal(t, entry), goodeol, ErrPublicKeyMismatch)
+}
+
+func TestPeerIDPubKeyValidate(t *testing.T) {
+ t.Skip("disabled until libp2p/go-libp2p-crypto#51 is fixed")
+
+ goodeol := time.Now().Add(time.Hour)
+ kbook, err := pstoremem.NewPeerstore()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ pth := []byte("/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG")
+
+ sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ pid, err := peer.IDFromPublicKey(pk)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ipnsk := "/ipns/" + string(pid)
+
+ entry, err := Create(sk, pth, 1, goodeol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ dataNoKey, err := proto.Marshal(entry)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ testValidatorCase(t, sk, kbook, ipnsk, dataNoKey, goodeol, nil)
+}
+
+func TestOnlySignatureV2Validate(t *testing.T) {
+ goodeol := time.Now().Add(time.Hour)
+
+ sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ path1 := []byte("/path/1")
+ entry, err := Create(sk, path1, 1, goodeol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := Validate(pk, entry); err != nil {
+ t.Fatal(err)
+ }
+
+ entry.SignatureV2 = nil
+ if err := Validate(pk, entry); !errors.Is(err, ErrSignature) {
+ t.Fatal(err)
+ }
+}
+
+func TestSignatureV1Ignored(t *testing.T) {
+ goodeol := time.Now().Add(time.Hour)
+
+ sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ pid, err := peer.IDFromPublicKey(pk)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ipnsk := "/ipns/" + string(pid)
+
+ path1 := []byte("/path/1")
+ entry1, err := Create(sk, path1, 1, goodeol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ path2 := []byte("/path/2")
+ entry2, err := Create(sk, path2, 2, goodeol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := Validate(pk, entry1); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := Validate(pk, entry2); err != nil {
+ t.Fatal(err)
+ }
+
+ v := Validator{}
+ best, err := v.Select(ipnsk, [][]byte{mustMarshal(t, entry1), mustMarshal(t, entry2)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if best != 1 {
+ t.Fatal("entry2 should be better than entry1")
+ }
+
+ // Having only the v1 signature should be invalid
+ entry2.SignatureV2 = nil
+ if err := Validate(pk, entry2); !errors.Is(err, ErrSignature) {
+ t.Fatal(err)
+ }
+
+ // Record with v2 signature should always be preferred
+ best, err = v.Select(ipnsk, [][]byte{mustMarshal(t, entry1), mustMarshal(t, entry2)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if best != 0 {
+ t.Fatal("entry1 should be better than entry2")
+ }
+
+ // Having a missing v1 signature is acceptable as long as there is a valid v2 signature
+ entry1.SignatureV1 = nil
+ if err := Validate(pk, entry1); err != nil {
+ t.Fatal(err)
+ }
+
+ // Having an invalid v1 signature is acceptable as long as there is a valid v2 signature
+ entry1.SignatureV1 = []byte("garbage")
+ if err := Validate(pk, entry1); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestMaxSizeValidate(t *testing.T) {
+ goodeol := time.Now().Add(time.Hour)
+
+ sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Create record over the max size (value+other fields)
+ value := make([]byte, MaxRecordSize)
+ entry, err := Create(sk, value, 1, goodeol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Must fail with ErrRecordSize
+ if err := Validate(pk, entry); !errors.Is(err, ErrRecordSize) {
+ t.Fatal(err)
+ }
+}
+
+func TestCborDataCanonicalization(t *testing.T) {
+ goodeol := time.Now().Add(time.Hour)
+
+ sk, pk, err := crypto.GenerateEd25519Key(rand.New(rand.NewSource(42)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ path := append([]byte("/path/1"), 0x00)
+ seqnum := uint64(1)
+ entry, err := Create(sk, path, seqnum, goodeol, time.Hour)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := Validate(pk, entry); err != nil {
+ t.Fatal(err)
+ }
+
+ dec, err := ipldcodec.LookupDecoder(uint64(multicodec.DagCbor))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ndbuilder := basicnode.Prototype__Map{}.NewBuilder()
+ if err := dec(ndbuilder, bytes.NewReader(entry.GetData())); err != nil {
+ t.Fatal(err)
+ }
+
+ nd := ndbuilder.Build()
+ iter := nd.MapIterator()
+ var fields []string
+ for !iter.Done() {
+ k, v, err := iter.Next()
+ if err != nil {
+ t.Fatal(err)
+ }
+ kstr, err := k.AsString()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ switch kstr {
+ case value:
+ b, err := v.AsBytes()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(path, b) {
+ t.Fatal("value did not match")
+ }
+ case sequence:
+ s, err := v.AsInt()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if uint64(s) != seqnum {
+ t.Fatal("sequence numbers did not match")
+ }
+ case validity:
+ val, err := v.AsBytes()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(val, []byte(u.FormatRFC3339(goodeol))) {
+ t.Fatal("validity did not match")
+ }
+ case validityType:
+ vt, err := v.AsInt()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if uint64(vt) != 0 {
+ t.Fatal("validity types did not match")
+ }
+ case ttl:
+ ttlVal, err := v.AsInt()
+ if err != nil {
+ t.Fatal(err)
+ }
+ // TODO: test non-zero TTL
+ if uint64(ttlVal) != uint64(time.Hour.Nanoseconds()) {
+ t.Fatal("TTLs did not match")
+ }
+ }
+
+ fields = append(fields, kstr)
+ }
+
+ // Check for map sort order (i.e. by length then by value)
+ expectedOrder := []string{"TTL", "Value", "Sequence", "Validity", "ValidityType"}
+ if len(fields) != len(expectedOrder) {
+ t.Fatal("wrong number of fields")
+ }
+
+ for i, f := range fields {
+ expected := expectedOrder[i]
+ if f != expected {
+ t.Fatalf("expected %s, got %s", expected, f)
+ }
+ }
+}
+
+func genKeys(t *testing.T) (crypto.PrivKey, peer.ID, string) {
+ sr := u.NewTimeSeededRand()
+ priv, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, sr)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Create entry with expiry in one hour
+ pid, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ipnsKey := RecordKey(pid)
+
+ return priv, pid, ipnsKey
+}
diff --git a/keystore/keystore.go b/keystore/keystore.go
new file mode 100644
index 000000000..fc6793a1e
--- /dev/null
+++ b/keystore/keystore.go
@@ -0,0 +1,188 @@
+package keystore
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "encoding/base32"
+
+ logging "github.com/ipfs/go-log"
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+var log = logging.Logger("keystore")
+
+var codec = base32.StdEncoding.WithPadding(base32.NoPadding)
+
+// Keystore provides a key management interface
+type Keystore interface {
+ // Has returns whether or not a key exists in the Keystore
+ Has(string) (bool, error)
+ // Put stores a key in the Keystore, if a key with the same name already exists, returns ErrKeyExists
+ Put(string, ci.PrivKey) error
+ // Get retrieves a key from the Keystore if it exists, and returns ErrNoSuchKey
+ // otherwise.
+ Get(string) (ci.PrivKey, error)
+ // Delete removes a key from the Keystore
+ Delete(string) error
+ // List returns a list of key identifier
+ List() ([]string, error)
+}
+
+// ErrNoSuchKey is an error message returned when no key of a given name was found.
+var ErrNoSuchKey = fmt.Errorf("no key by the given name was found")
+
+// ErrKeyExists is an error message returned when a key already exists
+var ErrKeyExists = fmt.Errorf("key by that name already exists, refusing to overwrite")
+
+const keyFilenamePrefix = "key_"
+
+// FSKeystore is a keystore backed by files in a given directory stored on disk.
+type FSKeystore struct {
+ dir string
+}
+
+// NewFSKeystore returns a new filesystem-backed keystore.
+func NewFSKeystore(dir string) (*FSKeystore, error) {
+ err := os.Mkdir(dir, 0700)
+ switch {
+ case os.IsExist(err):
+ case err == nil:
+ default:
+ return nil, err
+ }
+ return &FSKeystore{dir}, nil
+}
+
+// Has returns whether or not a key exists in the Keystore
+func (ks *FSKeystore) Has(name string) (bool, error) {
+ name, err := encode(name)
+ if err != nil {
+ return false, err
+ }
+
+ kp := filepath.Join(ks.dir, name)
+
+ _, err = os.Stat(kp)
+
+ if os.IsNotExist(err) {
+ return false, nil
+ }
+ return err == nil, err
+}
+
+// Put stores a key in the Keystore, if a key with the same name already exists, returns ErrKeyExists
+func (ks *FSKeystore) Put(name string, k ci.PrivKey) error {
+ name, err := encode(name)
+ if err != nil {
+ return err
+ }
+
+ b, err := ci.MarshalPrivateKey(k)
+ if err != nil {
+ return err
+ }
+
+ kp := filepath.Join(ks.dir, name)
+
+ fi, err := os.OpenFile(kp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0400)
+ if err != nil {
+ if os.IsExist(err) {
+ err = ErrKeyExists
+ }
+ return err
+ }
+ defer fi.Close()
+
+ _, err = fi.Write(b)
+
+ return err
+}
+
+// Get retrieves a key from the Keystore if it exists, and returns ErrNoSuchKey
+// otherwise.
+func (ks *FSKeystore) Get(name string) (ci.PrivKey, error) {
+ name, err := encode(name)
+ if err != nil {
+ return nil, err
+ }
+
+ kp := filepath.Join(ks.dir, name)
+
+ data, err := os.ReadFile(kp)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, ErrNoSuchKey
+ }
+ return nil, err
+ }
+
+ return ci.UnmarshalPrivateKey(data)
+}
+
+// Delete removes a key from the Keystore
+func (ks *FSKeystore) Delete(name string) error {
+ name, err := encode(name)
+ if err != nil {
+ return err
+ }
+
+ kp := filepath.Join(ks.dir, name)
+
+ return os.Remove(kp)
+}
+
+// List return a list of key identifier
+func (ks *FSKeystore) List() ([]string, error) {
+ dir, err := os.Open(ks.dir)
+ if err != nil {
+ return nil, err
+ }
+
+ dirs, err := dir.Readdirnames(0)
+ if err != nil {
+ return nil, err
+ }
+
+ list := make([]string, 0, len(dirs))
+
+ for _, name := range dirs {
+ decodedName, err := decode(name)
+ if err == nil {
+ list = append(list, decodedName)
+ } else {
+ log.Errorf("Ignoring keyfile with invalid encoded filename: %s", name)
+ }
+ }
+
+ return list, nil
+}
+
+func encode(name string) (string, error) {
+ if name == "" {
+ return "", fmt.Errorf("key name must be at least one character")
+ }
+
+ encodedName := codec.EncodeToString([]byte(name))
+ log.Debugf("Encoded key name: %s to: %s", name, encodedName)
+
+ return keyFilenamePrefix + strings.ToLower(encodedName), nil
+}
+
+func decode(name string) (string, error) {
+ if !strings.HasPrefix(name, keyFilenamePrefix) {
+ return "", fmt.Errorf("key's filename has unexpected format")
+ }
+
+ nameWithoutPrefix := strings.ToUpper(name[len(keyFilenamePrefix):])
+ decodedName, err := codec.DecodeString(nameWithoutPrefix)
+ if err != nil {
+ return "", err
+ }
+
+ log.Debugf("Decoded key name: %s to: %s", name, decodedName)
+
+ return string(decodedName), nil
+}
diff --git a/keystore/keystore_test.go b/keystore/keystore_test.go
new file mode 100644
index 000000000..9a4406217
--- /dev/null
+++ b/keystore/keystore_test.go
@@ -0,0 +1,277 @@
+package keystore
+
+import (
+ "fmt"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "sort"
+ "testing"
+
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+type rr struct{}
+
+func (rr rr) Read(b []byte) (int, error) {
+ return rand.Read(b)
+}
+
+func privKeyOrFatal(t *testing.T) ci.PrivKey {
+ priv, _, err := ci.GenerateEd25519Key(rr{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return priv
+}
+
+func TestKeystoreBasics(t *testing.T) {
+ tdir, err := os.MkdirTemp("", "keystore-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ks, err := NewFSKeystore(tdir)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ l, err := ks.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(l) != 0 {
+ t.Fatal("expected no keys")
+ }
+
+ k1 := privKeyOrFatal(t)
+ k2 := privKeyOrFatal(t)
+ k3 := privKeyOrFatal(t)
+ k4 := privKeyOrFatal(t)
+
+ err = ks.Put("foo", k1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = ks.Put("bar", k2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ l, err = ks.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sort.Strings(l)
+ if l[0] != "bar" || l[1] != "foo" {
+ t.Fatal("wrong entries listed")
+ }
+
+ if err := assertDirContents(tdir, []string{"foo", "bar"}); err != nil {
+ t.Fatal(err)
+ }
+
+ err = ks.Put("foo", k3)
+ if err == nil {
+ t.Fatal("should not be able to overwrite key")
+ }
+
+ if err := assertDirContents(tdir, []string{"foo", "bar"}); err != nil {
+ t.Fatal(err)
+ }
+
+ exist, err := ks.Has("foo")
+ if !exist {
+ t.Fatal("should know it has a key named foo")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ exist, err = ks.Has("nonexistingkey")
+ if exist {
+ t.Fatal("should know it doesn't have a key named nonexistingkey")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Delete("bar"); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := assertDirContents(tdir, []string{"foo"}); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Put("beep", k3); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Put("boop", k4); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := assertDirContents(tdir, []string{"foo", "beep", "boop"}); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := assertGetKey(ks, "foo", k1); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := assertGetKey(ks, "beep", k3); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := assertGetKey(ks, "boop", k4); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Put("..///foo/", k1); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Put("", k1); err == nil {
+ t.Fatal("shouldn't be able to put a key with no name")
+ }
+
+ if err := ks.Put(".foo", k1); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestInvalidKeyFiles(t *testing.T) {
+ tdir, err := os.MkdirTemp("", "keystore-test")
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ defer os.RemoveAll(tdir)
+
+ ks, err := NewFSKeystore(tdir)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ key := privKeyOrFatal(t)
+
+ bytes, err := key.Raw()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ encodedName, err := encode("valid")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = os.WriteFile(filepath.Join(ks.dir, encodedName), bytes, 0644)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = os.WriteFile(filepath.Join(ks.dir, "z.invalid"), bytes, 0644)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ l, err := ks.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sort.Strings(l)
+ if len(l) != 1 {
+ t.Fatal("wrong entry count")
+ }
+
+ if l[0] != "valid" {
+ t.Fatal("wrong entries listed")
+ }
+
+ exist, err := ks.Has("valid")
+ if !exist {
+ t.Fatal("should know it has a key named valid")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestNonExistingKey(t *testing.T) {
+ tdir, err := os.MkdirTemp("", "keystore-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ks, err := NewFSKeystore(tdir)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ k, err := ks.Get("does-it-exist")
+ if err != ErrNoSuchKey {
+ t.Fatalf("expected: %s, got %s", ErrNoSuchKey, err)
+ }
+ if k != nil {
+ t.Fatalf("Get on nonexistant key should give nil")
+ }
+}
+
+func TestMakeKeystoreNoDir(t *testing.T) {
+ _, err := NewFSKeystore("/this/is/not/a/real/dir")
+ if err == nil {
+ t.Fatal("shouldnt be able to make a keystore in a nonexistant directory")
+ }
+}
+
+func assertGetKey(ks Keystore, name string, exp ci.PrivKey) error {
+ outK, err := ks.Get(name)
+ if err != nil {
+ return err
+ }
+
+ if !outK.Equals(exp) {
+ return fmt.Errorf("key we got out didn't match expectation")
+ }
+
+ return nil
+}
+
+func assertDirContents(dir string, exp []string) error {
+ finfos, err := os.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+
+ if len(finfos) != len(exp) {
+ return fmt.Errorf("expected %d directory entries", len(exp))
+ }
+
+ var names []string
+ for _, fi := range finfos {
+ decodedName, err := decode(fi.Name())
+ if err != nil {
+ return err
+ }
+ names = append(names, decodedName)
+ }
+
+ sort.Strings(names)
+ sort.Strings(exp)
+ if len(names) != len(exp) {
+ return fmt.Errorf("directory had wrong number of entries in it")
+ }
+
+ for i, v := range names {
+ if v != exp[i] {
+ return fmt.Errorf("had wrong entry in directory")
+ }
+ }
+ return nil
+}
diff --git a/keystore/memkeystore.go b/keystore/memkeystore.go
new file mode 100644
index 000000000..0ea62f4e1
--- /dev/null
+++ b/keystore/memkeystore.go
@@ -0,0 +1,64 @@
+package keystore
+
+import (
+ "errors"
+
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+)
+
+// MemKeystore is an in memory keystore implementation that is not persisted to
+// any backing storage.
+type MemKeystore struct {
+ keys map[string]ci.PrivKey
+}
+
+// NewMemKeystore creates a MemKeystore.
+func NewMemKeystore() *MemKeystore {
+ return &MemKeystore{make(map[string]ci.PrivKey)}
+}
+
+// Has return whether or not a key exists in the Keystore
+func (mk *MemKeystore) Has(name string) (bool, error) {
+ _, ok := mk.keys[name]
+ return ok, nil
+}
+
+// Put store a key in the Keystore
+func (mk *MemKeystore) Put(name string, k ci.PrivKey) error {
+ if name == "" {
+ return errors.New("key name must be at least one character")
+ }
+
+ _, ok := mk.keys[name]
+ if ok {
+ return ErrKeyExists
+ }
+
+ mk.keys[name] = k
+ return nil
+}
+
+// Get retrieve a key from the Keystore
+func (mk *MemKeystore) Get(name string) (ci.PrivKey, error) {
+ k, ok := mk.keys[name]
+ if !ok {
+ return nil, ErrNoSuchKey
+ }
+
+ return k, nil
+}
+
+// Delete remove a key from the Keystore
+func (mk *MemKeystore) Delete(name string) error {
+ delete(mk.keys, name)
+ return nil
+}
+
+// List return a list of key identifier
+func (mk *MemKeystore) List() ([]string, error) {
+ out := make([]string, 0, len(mk.keys))
+ for k := range mk.keys {
+ out = append(out, k)
+ }
+ return out, nil
+}
diff --git a/keystore/memkeystore_test.go b/keystore/memkeystore_test.go
new file mode 100644
index 000000000..907cbbd0e
--- /dev/null
+++ b/keystore/memkeystore_test.go
@@ -0,0 +1,99 @@
+package keystore
+
+import (
+ "sort"
+ "testing"
+)
+
+func TestMemKeyStoreBasics(t *testing.T) {
+ ks := NewMemKeystore()
+
+ l, err := ks.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(l) != 0 {
+ t.Fatal("expected no keys")
+ }
+
+ k1 := privKeyOrFatal(t)
+ k2 := privKeyOrFatal(t)
+ k3 := privKeyOrFatal(t)
+ k4 := privKeyOrFatal(t)
+
+ err = ks.Put("foo", k1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = ks.Put("bar", k2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ l, err = ks.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sort.Strings(l)
+ if l[0] != "bar" || l[1] != "foo" {
+ t.Fatal("wrong entries listed")
+ }
+
+ err = ks.Put("foo", k3)
+ if err == nil {
+ t.Fatal("should not be able to overwrite key")
+ }
+
+ exist, err := ks.Has("foo")
+ if !exist {
+ t.Fatal("should know it has a key named foo")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ exist, err = ks.Has("nonexistingkey")
+ if exist {
+ t.Fatal("should know it doesn't have a key named nonexistingkey")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Delete("bar"); err != nil {
+ t.Fatal(err)
+ }
+ if err := ks.Put("beep", k3); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Put("boop", k4); err != nil {
+ t.Fatal(err)
+ }
+ if err := assertGetKey(ks, "foo", k1); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := assertGetKey(ks, "beep", k3); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := assertGetKey(ks, "boop", k4); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Put("..///foo/", k1); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ks.Put("", k1); err == nil {
+ t.Fatal("shouldn't be able to put a key with no name")
+ }
+
+ if err := ks.Put(".foo", k1); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/logo.svg b/logo.svg
new file mode 100644
index 000000000..735f3ab80
--- /dev/null
+++ b/logo.svg
@@ -0,0 +1 @@
+. A "resolved"
+// IPNS path becomes an /ipfs/ path.
+//
+// Traditionally, these paths would be in the form of /ipns/peer_id, which
+// references an IPNS record in a distributed ValueStore (usually the IPFS
+// DHT).
+//
+// Additionally, the /ipns/ namespace can also be used with domain names that
+// use DNSLink (/ipns/, https://docs.ipfs.io/concepts/dnslink/)
+//
+// The package provides implementations for all three resolvers.
+package namesys
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ lru "github.com/hashicorp/golang-lru"
+ iface "github.com/ipfs/boxo/coreiface"
+ opts "github.com/ipfs/boxo/coreiface/options/namesys"
+ "github.com/ipfs/boxo/path"
+ "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/miekg/dns"
+ madns "github.com/multiformats/go-multiaddr-dns"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+)
+
+// mpns (a multi-protocol NameSystem) implements generic IPFS naming.
+//
+// Uses several Resolvers:
+// (a) IPFS routing naming: SFS-like PKI names.
+// (b) dns domains: resolves using links in DNS TXT records
+//
+// It can only publish to: (a) IPFS routing naming.
+type mpns struct {
+ ds ds.Datastore
+
+ dnsResolver, ipnsResolver resolver
+ ipnsPublisher Publisher
+
+ staticMap map[string]path.Path
+ cache *lru.Cache
+}
+
+type Option func(*mpns) error
+
+// WithCache is an option that instructs the name system to use a (LRU) cache of the given size.
+func WithCache(size int) Option {
+ return func(ns *mpns) error {
+ if size <= 0 {
+ return fmt.Errorf("invalid cache size %d; must be > 0", size)
+ }
+
+ cache, err := lru.New(size)
+ if err != nil {
+ return err
+ }
+
+ ns.cache = cache
+ return nil
+ }
+}
+
+// WithDNSResolver is an option that supplies a custom DNS resolver to use instead of the system
+// default.
+func WithDNSResolver(rslv madns.BasicResolver) Option {
+ return func(ns *mpns) error {
+ ns.dnsResolver = NewDNSResolver(rslv.LookupTXT)
+ return nil
+ }
+}
+
+// WithDatastore is an option that supplies a datastore to use instead of an in-memory map datastore. The datastore is used to store published IPNS records and make them available for querying.
+func WithDatastore(ds ds.Datastore) Option {
+ return func(ns *mpns) error {
+ ns.ds = ds
+ return nil
+ }
+}
+
+func loadStaticMap(list string) (map[string]path.Path, error) {
+ staticMap := make(map[string]path.Path)
+ for _, pair := range strings.Split(list, ",") {
+ mapping := strings.SplitN(pair, ":", 2)
+ key := mapping[0]
+ value := path.FromString(mapping[1])
+
+ ipnsKey, err := peer.Decode(key)
+ if err == nil {
+ key = iface.FormatKeyID(ipnsKey)
+ }
+
+ staticMap[key] = value
+ }
+ return staticMap, nil
+}
+
+// NewNameSystem will construct the IPFS naming system based on Routing
+func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) {
+ var staticMap map[string]path.Path
+
+ // Prewarm namesys cache with static records for deterministic tests and debugging.
+ // Useful for testing things like DNSLink without real DNS lookup.
+ // Example:
+ // IPFS_NS_MAP="dnslink-test.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am"
+ if list := os.Getenv("IPFS_NS_MAP"); list != "" {
+ var err error
+ staticMap, err = loadStaticMap(list)
+
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ ns := &mpns{
+ staticMap: staticMap,
+ }
+
+ for _, opt := range opts {
+ err := opt(ns)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ if ns.ds == nil {
+ ns.ds = dssync.MutexWrap(ds.NewMapDatastore())
+ }
+
+ if ns.dnsResolver == nil {
+ ns.dnsResolver = NewDNSResolver(madns.DefaultResolver.LookupTXT)
+ }
+
+ ns.ipnsResolver = NewIpnsResolver(r)
+ ns.ipnsPublisher = NewIpnsPublisher(r, ns.ds)
+
+ return ns, nil
+}
+
+// DefaultResolverCacheTTL defines max ttl of a record placed in namesys cache.
+const DefaultResolverCacheTTL = time.Minute
+
+// Resolve implements Resolver.
+func (ns *mpns) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) {
+ ctx, span := StartSpan(ctx, "MPNS.Resolve", trace.WithAttributes(attribute.String("Name", name)))
+ defer span.End()
+
+ if strings.HasPrefix(name, "/ipfs/") {
+ return path.ParsePath(name)
+ }
+
+ if !strings.HasPrefix(name, "/") {
+ return path.ParsePath("/ipfs/" + name)
+ }
+
+ return resolve(ctx, ns, name, opts.ProcessOpts(options))
+}
+
+func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result {
+ ctx, span := StartSpan(ctx, "MPNS.ResolveAsync", trace.WithAttributes(attribute.String("Name", name)))
+ defer span.End()
+
+ if strings.HasPrefix(name, "/ipfs/") {
+ p, err := path.ParsePath(name)
+ res := make(chan Result, 1)
+ res <- Result{p, err}
+ close(res)
+ return res
+ }
+
+ if !strings.HasPrefix(name, "/") {
+ p, err := path.ParsePath("/ipfs/" + name)
+ res := make(chan Result, 1)
+ res <- Result{p, err}
+ close(res)
+ return res
+ }
+
+ return resolveAsync(ctx, ns, name, opts.ProcessOpts(options))
+}
+
+// resolveOnce implements resolver.
+func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
+ ctx, span := StartSpan(ctx, "MPNS.ResolveOnceAsync")
+ defer span.End()
+
+ out := make(chan onceResult, 1)
+
+ if !strings.HasPrefix(name, ipnsPrefix) {
+ name = ipnsPrefix + name
+ }
+ segments := strings.SplitN(name, "/", 4)
+ if len(segments) < 3 || segments[0] != "" {
+ log.Debugf("invalid name syntax for %s", name)
+ out <- onceResult{err: ErrResolveFailed}
+ close(out)
+ return out
+ }
+
+ key := segments[2]
+
+ // Resolver selection:
+ // 1. if it is a PeerID/CID/multihash resolve through "ipns".
+ // 2. if it is a domain name, resolve through "dns"
+
+ var res resolver
+ ipnsKey, err := peer.Decode(key)
+
+ // CIDs in IPNS are expected to have libp2p-key multicodec
+ // We ease the transition by returning a more meaningful error with a valid CID
+ if err != nil {
+ ipnsCid, cidErr := cid.Decode(key)
+ if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey {
+ fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String()
+ codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid)
+ log.Debugf("RoutingResolver: could not convert public key hash %q to peer ID: %s\n", key, codecErr)
+ out <- onceResult{err: codecErr}
+ close(out)
+ return out
+ }
+ }
+
+ cacheKey := key
+ if err == nil {
+ cacheKey = iface.FormatKeyID(ipnsKey)
+ }
+
+ if p, ok := ns.cacheGet(cacheKey); ok {
+ var err error
+ if len(segments) > 3 {
+ p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3])
+ }
+ span.SetAttributes(attribute.Bool("CacheHit", true))
+ span.RecordError(err)
+
+ out <- onceResult{value: p, err: err}
+ close(out)
+ return out
+ }
+ span.SetAttributes(attribute.Bool("CacheHit", false))
+
+ if err == nil {
+ res = ns.ipnsResolver
+ } else if _, ok := dns.IsDomainName(key); ok {
+ res = ns.dnsResolver
+ } else {
+ out <- onceResult{err: fmt.Errorf("invalid IPNS root: %q", key)}
+ close(out)
+ return out
+ }
+
+ resCh := res.resolveOnceAsync(ctx, key, options)
+ var best onceResult
+ go func() {
+ defer close(out)
+ for {
+ select {
+ case res, ok := <-resCh:
+ if !ok {
+ if best != (onceResult{}) {
+ ns.cacheSet(cacheKey, best.value, best.ttl)
+ }
+ return
+ }
+ if res.err == nil {
+ best = res
+ }
+ p := res.value
+ err := res.err
+ ttl := res.ttl
+
+ // Attach rest of the path
+ if len(segments) > 3 {
+ p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3])
+ }
+
+ emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl, err: err})
+ case <-ctx.Done():
+ return
+ }
+ }
+ }()
+
+ return out
+}
+
+func emitOnceResult(ctx context.Context, outCh chan<- onceResult, r onceResult) {
+ select {
+ case outCh <- r:
+ case <-ctx.Done():
+ }
+}
+
+// Publish implements Publisher
+func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...opts.PublishOption) error {
+ ctx, span := StartSpan(ctx, "MPNS.Publish")
+ defer span.End()
+
+ // This is a bit hacky. We do this because the EOL is based on the current
+ // time, but also needed in the end of the function. Therefore, we parse
+ // the options immediately and add an option PublishWithEOL with the EOL
+ // calculated in this moment.
+ publishOpts := opts.ProcessPublishOptions(options)
+ options = append(options, opts.PublishWithEOL(publishOpts.EOL))
+
+ id, err := peer.IDFromPrivateKey(name)
+ if err != nil {
+ span.RecordError(err)
+ return err
+ }
+ span.SetAttributes(attribute.String("ID", id.String()))
+ if err := ns.ipnsPublisher.Publish(ctx, name, value, options...); err != nil {
+ // Invalidate the cache. Publishing may _partially_ succeed but
+ // still return an error.
+ ns.cacheInvalidate(string(id))
+ span.RecordError(err)
+ return err
+ }
+ ttl := DefaultResolverCacheTTL
+ if publishOpts.TTL >= 0 {
+ ttl = publishOpts.TTL
+ }
+ if ttEOL := time.Until(publishOpts.EOL); ttEOL < ttl {
+ ttl = ttEOL
+ }
+ ns.cacheSet(string(id), value, ttl)
+ return nil
+}
diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go
new file mode 100644
index 000000000..a3f9df01a
--- /dev/null
+++ b/namesys/namesys_test.go
@@ -0,0 +1,210 @@
+package namesys
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "testing"
+ "time"
+
+ opts "github.com/ipfs/boxo/coreiface/options/namesys"
+ "github.com/ipfs/boxo/ipns"
+ "github.com/ipfs/boxo/path"
+ offroute "github.com/ipfs/boxo/routing/offline"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ record "github.com/libp2p/go-libp2p-record"
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem"
+)
+
+type mockResolver struct {
+ entries map[string]string
+}
+
+func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expError error) {
+ t.Helper()
+ p, err := resolver.Resolve(context.Background(), name, opts.Depth(depth))
+ if !errors.Is(err, expError) {
+ t.Fatal(fmt.Errorf(
+ "expected %s with a depth of %d to have a '%s' error, but got '%s'",
+ name, depth, expError, err))
+ }
+ if p.String() != expected {
+ t.Fatal(fmt.Errorf(
+ "%s with depth %d resolved to %s != %s",
+ name, depth, p.String(), expected))
+ }
+}
+
+func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
+ p, err := path.ParsePath(r.entries[name])
+ out := make(chan onceResult, 1)
+ out <- onceResult{value: p, err: err}
+ close(out)
+ return out
+}
+
+func mockResolverOne() *mockResolver {
+ return &mockResolver{
+ entries: map[string]string{
+ "QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj",
+ "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy",
+ "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io",
+ "QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act",
+ "12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash
+ "bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with libp2p-key multicodec
+ },
+ }
+}
+
+func mockResolverTwo() *mockResolver {
+ return &mockResolver{
+ entries: map[string]string{
+ "ipfs.io": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n",
+ },
+ }
+}
+
+func TestNamesysResolution(t *testing.T) {
+ r := &mpns{
+ ipnsResolver: mockResolverOne(),
+ dnsResolver: mockResolverTwo(),
+ }
+
+ testResolution(t, r, "Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+ testResolution(t, r, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+ testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+ testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion)
+ testResolution(t, r, "/ipns/ipfs.io", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+ testResolution(t, r, "/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+ testResolution(t, r, "/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion)
+ testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+ testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", ErrResolveRecursion)
+ testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+ testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion)
+ testResolution(t, r, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+ testResolution(t, r, "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+}
+
+func TestNamesysResolutionWithCache(t *testing.T) {
+ nsMap := "dnslink-test.example.com:/ipfs/bafyaaeykceeaeeqlgiydemzngazc2mrtbimaw,12D3KooWQbpsnyzdBcxw6GUMbijV8WgXE4L8EtfnbcQWLfyxBKho:/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba,k51qzi5uqu5dkwkqm42v9j9kqcam2jiuvloi16g72i4i4amoo2m8u3ol3mqu6s:/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk"
+
+ staticMap, err := loadStaticMap(nsMap)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ r := &mpns{
+ ipnsResolver: mockResolverOne(),
+ dnsResolver: mockResolverTwo(),
+ staticMap: staticMap,
+ }
+
+ testResolution(t, r, "/ipns/dnslink-test.example.com", opts.DefaultDepthLimit, "/ipfs/bafyaaeykceeaeeqlgiydemzngazc2mrtbimaw", nil)
+
+ testResolution(t, r, "/ipns/bafzaajaiaejcbw5i6oyqsktsn36r2vxgl2jzosyao46rybqztxt4rx4tfa3hpogg", opts.DefaultDepthLimit, "/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba", nil)
+ testResolution(t, r, "/ipns/k51qzi5uqu5dlnojhwrggtpty9c0cp5hvnkdozowth4eqb726jvoros8k9niyu", opts.DefaultDepthLimit, "/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba", nil)
+ testResolution(t, r, "/ipns/12D3KooWQbpsnyzdBcxw6GUMbijV8WgXE4L8EtfnbcQWLfyxBKho", opts.DefaultDepthLimit, "/ipfs/bafyaagakcyeaeeqqgiydemzngazc2mrtfvuxa3ttbimba", nil)
+
+ testResolution(t, r, "/ipns/bafzaajaiaejcbpltl72da5f3y7ojrtsa7hsfn5bbnkjbkwyesziqqtdry6vjilku", opts.DefaultDepthLimit, "/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk", nil)
+ testResolution(t, r, "/ipns/k51qzi5uqu5dkwkqm42v9j9kqcam2jiuvloi16g72i4i4amoo2m8u3ol3mqu6s", opts.DefaultDepthLimit, "/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk", nil)
+ testResolution(t, r, "/ipns/12D3KooWNZuG8phqhoNK9KWcUhwfzA3biDKNCUNVWEaJgigr6Acj", opts.DefaultDepthLimit, "/ipfs/bafyaahikdmeaeeqvgiydemzngazc2mrtfvuxa3ttfvsgk5lybimbk", nil)
+
+}
+
+func TestPublishWithCache0(t *testing.T) {
+ dst := dssync.MutexWrap(ds.NewMapDatastore())
+ priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ps, err := pstoremem.NewPeerstore()
+ if err != nil {
+ t.Fatal(err)
+ }
+ pid, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = ps.AddPrivKey(pid, priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{
+ "ipns": ipns.Validator{KeyBook: ps},
+ "pk": record.PublicKeyValidator{},
+ })
+
+ nsys, err := NewNameSystem(routing, WithDatastore(dst))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // CID is arbitrary.
+ p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = nsys.Publish(context.Background(), priv, p)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPublishWithTTL(t *testing.T) {
+ dst := dssync.MutexWrap(ds.NewMapDatastore())
+ priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ps, err := pstoremem.NewPeerstore()
+ if err != nil {
+ t.Fatal(err)
+ }
+ pid, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = ps.AddPrivKey(pid, priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{
+ "ipns": ipns.Validator{KeyBook: ps},
+ "pk": record.PublicKeyValidator{},
+ })
+
+ nsys, err := NewNameSystem(routing, WithDatastore(dst), WithCache(128))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // CID is arbitrary.
+ p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ttl := 1 * time.Second
+ eol := time.Now().Add(2 * time.Second)
+
+ err = nsys.Publish(context.Background(), priv, p, opts.PublishWithEOL(eol), opts.PublishWithTTL(ttl))
+ if err != nil {
+ t.Fatal(err)
+ }
+ ientry, ok := nsys.(*mpns).cache.Get(string(pid))
+ if !ok {
+ t.Fatal("cache get failed")
+ }
+ entry, ok := ientry.(cacheEntry)
+ if !ok {
+ t.Fatal("bad cache item returned")
+ }
+ if entry.eol.Sub(eol) > 10*time.Millisecond {
+ t.Fatalf("bad cache ttl: expected %s, got %s", eol, entry.eol)
+ }
+}
diff --git a/namesys/publisher.go b/namesys/publisher.go
new file mode 100644
index 000000000..24a0b8e4d
--- /dev/null
+++ b/namesys/publisher.go
@@ -0,0 +1,285 @@
+package namesys
+
+import (
+ "context"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/gogo/protobuf/proto"
+ opts "github.com/ipfs/boxo/coreiface/options/namesys"
+ "github.com/ipfs/boxo/ipns"
+ pb "github.com/ipfs/boxo/ipns/pb"
+ "github.com/ipfs/boxo/path"
+ ds "github.com/ipfs/go-datastore"
+ dsquery "github.com/ipfs/go-datastore/query"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/whyrusleeping/base32"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+)
+
+const ipnsPrefix = "/ipns/"
+
+// IpnsPublisher is capable of publishing and resolving names to the IPFS
+// routing system.
+type IpnsPublisher struct {
+ routing routing.ValueStore
+ ds ds.Datastore
+
+ // Used to ensure we assign IPNS records *sequential* sequence numbers.
+ mu sync.Mutex
+}
+
+// NewIpnsPublisher constructs a publisher for the IPFS Routing name system.
+func NewIpnsPublisher(route routing.ValueStore, ds ds.Datastore) *IpnsPublisher {
+ if ds == nil {
+ panic("nil datastore")
+ }
+ return &IpnsPublisher{routing: route, ds: ds}
+}
+
+// Publish implements Publisher. Accepts a keypair and a value,
+// and publishes it out to the routing system
+func (p *IpnsPublisher) Publish(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) error {
+ log.Debugf("Publish %s", value)
+
+ ctx, span := StartSpan(ctx, "IpnsPublisher.Publish", trace.WithAttributes(attribute.String("Value", value.String())))
+ defer span.End()
+
+ record, err := p.updateRecord(ctx, k, value, options...)
+ if err != nil {
+ return err
+ }
+
+ return PutRecordToRouting(ctx, p.routing, k.GetPublic(), record)
+}
+
+// IpnsDsKey returns a datastore key given an IPNS identifier (peer
+// ID). Defines the storage key for IPNS records in the local datastore.
+func IpnsDsKey(id peer.ID) ds.Key {
+ return ds.NewKey("/ipns/" + base32.RawStdEncoding.EncodeToString([]byte(id)))
+}
+
+// ListPublished returns the latest IPNS records published by this node and
+// their expiration times.
+//
+// This method will not search the routing system for records published by other
+// nodes.
+func (p *IpnsPublisher) ListPublished(ctx context.Context) (map[peer.ID]*pb.IpnsEntry, error) {
+ query, err := p.ds.Query(ctx, dsquery.Query{
+ Prefix: ipnsPrefix,
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer query.Close()
+
+ records := make(map[peer.ID]*pb.IpnsEntry)
+ for {
+ select {
+ case result, ok := <-query.Next():
+ if !ok {
+ return records, nil
+ }
+ if result.Error != nil {
+ return nil, result.Error
+ }
+ e := new(pb.IpnsEntry)
+ if err := proto.Unmarshal(result.Value, e); err != nil {
+ // Might as well return what we can.
+ log.Error("found an invalid IPNS entry:", err)
+ continue
+ }
+ if !strings.HasPrefix(result.Key, ipnsPrefix) {
+ log.Errorf("datastore query for keys with prefix %s returned a key: %s", ipnsPrefix, result.Key)
+ continue
+ }
+ k := result.Key[len(ipnsPrefix):]
+ pid, err := base32.RawStdEncoding.DecodeString(k)
+ if err != nil {
+ log.Errorf("ipns ds key invalid: %s", result.Key)
+ continue
+ }
+ records[peer.ID(pid)] = e
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+ }
+}
+
+// GetPublished returns the record this node has published corresponding to the
+// given peer ID.
+//
+// If `checkRouting` is true and we have no existing record, this method will
+// check the routing system for any existing records.
+func (p *IpnsPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouting bool) (*pb.IpnsEntry, error) {
+ ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+ defer cancel()
+
+ value, err := p.ds.Get(ctx, IpnsDsKey(id))
+ switch err {
+ case nil:
+ case ds.ErrNotFound:
+ if !checkRouting {
+ return nil, nil
+ }
+ ipnskey := ipns.RecordKey(id)
+ value, err = p.routing.GetValue(ctx, ipnskey)
+ if err != nil {
+ // Not found or other network issue. Can't really do
+ // anything about this case.
+ if err != routing.ErrNotFound {
+ log.Debugf("error when determining the last published IPNS record for %s: %s", id, err)
+ }
+
+ return nil, nil
+ }
+ default:
+ return nil, err
+ }
+ e := new(pb.IpnsEntry)
+ if err := proto.Unmarshal(value, e); err != nil {
+ return nil, err
+ }
+ return e, nil
+}
+
+func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) (*pb.IpnsEntry, error) {
+ id, err := peer.IDFromPrivateKey(k)
+ if err != nil {
+ return nil, err
+ }
+
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ // get previous records sequence number
+ rec, err := p.GetPublished(ctx, id, true)
+ if err != nil {
+ return nil, err
+ }
+
+ seqno := rec.GetSequence() // returns 0 if rec is nil
+ if rec != nil && value != path.Path(rec.GetValue()) {
+ // Don't bother incrementing the sequence number unless the
+ // value changes.
+ seqno++
+ }
+
+ opts := opts.ProcessPublishOptions(options)
+
+ // Create record
+ entry, err := ipns.Create(k, []byte(value), seqno, opts.EOL, opts.TTL)
+ if err != nil {
+ return nil, err
+ }
+
+ data, err := proto.Marshal(entry)
+ if err != nil {
+ return nil, err
+ }
+
+ // Put the new record.
+ key := IpnsDsKey(id)
+ if err := p.ds.Put(ctx, key, data); err != nil {
+ return nil, err
+ }
+ if err := p.ds.Sync(ctx, key); err != nil {
+ return nil, err
+ }
+ return entry, nil
+}
+
+// PutRecordToRouting publishes the given entry using the provided ValueStore,
+// keyed on the ID associated with the provided public key. The public key is
+// also made available to the routing system so that entries can be verified.
+func PutRecordToRouting(ctx context.Context, r routing.ValueStore, k crypto.PubKey, entry *pb.IpnsEntry) error {
+ ctx, span := StartSpan(ctx, "PutRecordToRouting")
+ defer span.End()
+
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ errs := make(chan error, 2) // At most two errors (IPNS, and public key)
+
+ if err := ipns.EmbedPublicKey(k, entry); err != nil {
+ return err
+ }
+
+ id, err := peer.IDFromPublicKey(k)
+ if err != nil {
+ return err
+ }
+
+ go func() {
+ errs <- PublishEntry(ctx, r, ipns.RecordKey(id), entry)
+ }()
+
+ // Publish the public key if a public key cannot be extracted from the ID
+ // TODO: once v0.4.16 is widespread enough, we can stop doing this
+ // and at that point we can even deprecate the /pk/ namespace in the dht
+ //
+ // NOTE: This check actually checks if the public key has been embedded
+ // in the IPNS entry. This check is sufficient because we embed the
+ // public key in the IPNS entry if it can't be extracted from the ID.
+ if entry.PubKey != nil {
+ go func() {
+ errs <- PublishPublicKey(ctx, r, PkKeyForID(id), k)
+ }()
+
+ if err := waitOnErrChan(ctx, errs); err != nil {
+ return err
+ }
+ }
+
+ return waitOnErrChan(ctx, errs)
+}
+
+func waitOnErrChan(ctx context.Context, errs chan error) error {
+ select {
+ case err := <-errs:
+ return err
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// PublishPublicKey stores the given public key in the ValueStore with the
+// given key.
+func PublishPublicKey(ctx context.Context, r routing.ValueStore, k string, pubk crypto.PubKey) error {
+ ctx, span := StartSpan(ctx, "PublishPublicKey", trace.WithAttributes(attribute.String("Key", k)))
+ defer span.End()
+
+ log.Debugf("Storing pubkey at: %s", k)
+ pkbytes, err := crypto.MarshalPublicKey(pubk)
+ if err != nil {
+ return err
+ }
+
+ // Store associated public key
+ return r.PutValue(ctx, k, pkbytes)
+}
+
+// PublishEntry stores the given IpnsEntry in the ValueStore with the given
+// ipnskey.
+func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec *pb.IpnsEntry) error {
+ ctx, span := StartSpan(ctx, "PublishEntry", trace.WithAttributes(attribute.String("IPNSKey", ipnskey)))
+ defer span.End()
+
+ data, err := proto.Marshal(rec)
+ if err != nil {
+ return err
+ }
+
+ log.Debugf("Storing ipns entry at: %x", ipnskey)
+ // Store ipns entry at "/ipns/"+h(pubkey)
+ return r.PutValue(ctx, ipnskey, data)
+}
+
+// PkKeyForID returns the public key routing key for the given peer ID.
+func PkKeyForID(id peer.ID) string {
+ return "/pk/" + string(id)
+}
diff --git a/namesys/publisher_test.go b/namesys/publisher_test.go
new file mode 100644
index 000000000..b40593c76
--- /dev/null
+++ b/namesys/publisher_test.go
@@ -0,0 +1,156 @@
+package namesys
+
+import (
+ "context"
+ "crypto/rand"
+ "testing"
+ "time"
+
+ "github.com/ipfs/boxo/path"
+
+ dshelp "github.com/ipfs/boxo/datastore/dshelp"
+ "github.com/ipfs/boxo/ipns"
+ mockrouting "github.com/ipfs/boxo/routing/mock"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ testutil "github.com/libp2p/go-libp2p-testing/net"
+ ci "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ ma "github.com/multiformats/go-multiaddr"
+)
+
+type identity struct {
+ testutil.PeerNetParams
+}
+
+func (p *identity) ID() peer.ID {
+ return p.PeerNetParams.ID
+}
+
+func (p *identity) Address() ma.Multiaddr {
+ return p.Addr
+}
+
+func (p *identity) PrivateKey() ci.PrivKey {
+ return p.PrivKey
+}
+
+func (p *identity) PublicKey() ci.PubKey {
+ return p.PubKey
+}
+
+func testNamekeyPublisher(t *testing.T, keyType int, expectedErr error, expectedExistence bool) {
+ // Context
+ ctx := context.Background()
+
+ // Private key
+ privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // ID
+ id, err := peer.IDFromPublicKey(pubKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Value
+ value := []byte("ipfs/TESTING")
+
+ // Seqnum
+ seqnum := uint64(0)
+
+ // Eol
+ eol := time.Now().Add(24 * time.Hour)
+
+ // Routing value store
+ p := testutil.PeerNetParams{
+ ID: id,
+ PrivKey: privKey,
+ PubKey: pubKey,
+ Addr: testutil.ZeroLocalTCPAddress,
+ }
+
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ serv := mockrouting.NewServer()
+ r := serv.ClientWithDatastore(context.Background(), &identity{p}, dstore)
+
+ entry, err := ipns.Create(privKey, value, seqnum, eol, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = PutRecordToRouting(ctx, r, pubKey, entry)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Check for namekey existence in value store
+ namekey := PkKeyForID(id)
+ _, err = r.GetValue(ctx, namekey)
+ if err != expectedErr {
+ t.Fatal(err)
+ }
+
+ // Also check datastore for completeness
+ key := dshelp.NewKeyFromBinary([]byte(namekey))
+ exists, err := dstore.Has(ctx, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if exists != expectedExistence {
+ t.Fatal("Unexpected key existence in datastore")
+ }
+}
+
+func TestRSAPublisher(t *testing.T) {
+ testNamekeyPublisher(t, ci.RSA, nil, true)
+}
+
+func TestEd22519Publisher(t *testing.T) {
+ testNamekeyPublisher(t, ci.Ed25519, ds.ErrNotFound, false)
+}
+
+func TestAsyncDS(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ rt := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t))
+ ds := &checkSyncDS{
+ Datastore: ds.NewMapDatastore(),
+ syncKeys: make(map[ds.Key]struct{}),
+ }
+ publisher := NewIpnsPublisher(rt, ds)
+
+ ipnsFakeID := testutil.RandIdentityOrFatal(t)
+ ipnsVal, err := path.ParsePath("/ipns/foo.bar")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := publisher.Publish(ctx, ipnsFakeID.PrivateKey(), ipnsVal); err != nil {
+ t.Fatal(err)
+ }
+
+ ipnsKey := IpnsDsKey(ipnsFakeID.ID())
+
+ for k := range ds.syncKeys {
+ if k.IsAncestorOf(ipnsKey) || k.Equal(ipnsKey) {
+ return
+ }
+ }
+
+ t.Fatal("ipns key not synced")
+}
+
+type checkSyncDS struct {
+ ds.Datastore
+ syncKeys map[ds.Key]struct{}
+}
+
+func (d *checkSyncDS) Sync(ctx context.Context, prefix ds.Key) error {
+ d.syncKeys[prefix] = struct{}{}
+ return d.Datastore.Sync(ctx, prefix)
+}
diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go
new file mode 100644
index 000000000..03426843b
--- /dev/null
+++ b/namesys/republisher/repub.go
@@ -0,0 +1,186 @@
+// Package republisher provides a utility to automatically re-publish IPNS
+// records related to the keys in a Keystore.
+package republisher
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ keystore "github.com/ipfs/boxo/keystore"
+ "github.com/ipfs/boxo/namesys"
+ "github.com/ipfs/boxo/path"
+ "go.opentelemetry.io/otel/attribute"
+
+ "github.com/gogo/protobuf/proto"
+ opts "github.com/ipfs/boxo/coreiface/options/namesys"
+ "github.com/ipfs/boxo/ipns"
+ pb "github.com/ipfs/boxo/ipns/pb"
+ ds "github.com/ipfs/go-datastore"
+ logging "github.com/ipfs/go-log"
+ "github.com/jbenet/goprocess"
+ gpctx "github.com/jbenet/goprocess/context"
+ ic "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+var errNoEntry = errors.New("no previous entry")
+
+var log = logging.Logger("ipns-repub")
+
+// DefaultRebroadcastInterval is the default interval at which we rebroadcast IPNS records
+var DefaultRebroadcastInterval = time.Hour * 4
+
+// InitialRebroadcastDelay is the delay before first broadcasting IPNS records on start
+var InitialRebroadcastDelay = time.Minute * 1
+
+// FailureRetryInterval is the interval at which we retry IPNS records broadcasts (when they fail)
+var FailureRetryInterval = time.Minute * 5
+
+// DefaultRecordLifetime is the default lifetime for IPNS records
+const DefaultRecordLifetime = time.Hour * 24
+
+// Republisher facilitates the regular publishing of all the IPNS records
+// associated to keys in a Keystore.
+type Republisher struct {
+ ns namesys.Publisher
+ ds ds.Datastore
+ self ic.PrivKey
+ ks keystore.Keystore
+
+ Interval time.Duration
+
+ // how long records that are republished should be valid for
+ RecordLifetime time.Duration
+}
+
+// NewRepublisher creates a new Republisher
+func NewRepublisher(ns namesys.Publisher, ds ds.Datastore, self ic.PrivKey, ks keystore.Keystore) *Republisher {
+ return &Republisher{
+ ns: ns,
+ ds: ds,
+ self: self,
+ ks: ks,
+ Interval: DefaultRebroadcastInterval,
+ RecordLifetime: DefaultRecordLifetime,
+ }
+}
+
+// Run starts the republisher facility. It can be stopped by stopping the
+// provided proc.
+func (rp *Republisher) Run(proc goprocess.Process) {
+ timer := time.NewTimer(InitialRebroadcastDelay)
+ defer timer.Stop()
+ if rp.Interval < InitialRebroadcastDelay {
+ timer.Reset(rp.Interval)
+ }
+
+ for {
+ select {
+ case <-timer.C:
+ timer.Reset(rp.Interval)
+ err := rp.republishEntries(proc)
+ if err != nil {
+ log.Info("republisher failed to republish: ", err)
+ if FailureRetryInterval < rp.Interval {
+ timer.Reset(FailureRetryInterval)
+ }
+ }
+ case <-proc.Closing():
+ return
+ }
+ }
+}
+
+func (rp *Republisher) republishEntries(p goprocess.Process) error {
+ ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p))
+ defer cancel()
+ ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntries")
+ defer span.End()
+
+ // TODO: Use rp.ipns.ListPublished(). We can't currently *do* that
+ // because:
+ // 1. There's no way to get keys from the keystore by ID.
+ // 2. We don't actually have access to the IPNS publisher.
+ err := rp.republishEntry(ctx, rp.self)
+ if err != nil {
+ return err
+ }
+
+ if rp.ks != nil {
+ keyNames, err := rp.ks.List()
+ if err != nil {
+ return err
+ }
+ for _, name := range keyNames {
+ priv, err := rp.ks.Get(name)
+ if err != nil {
+ return err
+ }
+ err = rp.republishEntry(ctx, priv)
+ if err != nil {
+ return err
+ }
+
+ }
+ }
+
+ return nil
+}
+
+func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) error {
+ ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntry")
+ defer span.End()
+ id, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ span.RecordError(err)
+ return err
+ }
+
+ log.Debugf("republishing ipns entry for %s", id)
+
+ // Look for it locally only
+ e, err := rp.getLastIPNSEntry(ctx, id)
+ if err != nil {
+ if err == errNoEntry {
+ span.SetAttributes(attribute.Bool("NoEntry", true))
+ return nil
+ }
+ span.RecordError(err)
+ return err
+ }
+
+ p := path.Path(e.GetValue())
+ prevEol, err := ipns.GetEOL(e)
+ if err != nil {
+ span.RecordError(err)
+ return err
+ }
+
+ // update record with same sequence number
+ eol := time.Now().Add(rp.RecordLifetime)
+ if prevEol.After(eol) {
+ eol = prevEol
+ }
+ err = rp.ns.Publish(ctx, priv, p, opts.PublishWithEOL(eol))
+ span.RecordError(err)
+ return err
+}
+
+func (rp *Republisher) getLastIPNSEntry(ctx context.Context, id peer.ID) (*pb.IpnsEntry, error) {
+ // Look for it locally only
+ val, err := rp.ds.Get(ctx, namesys.IpnsDsKey(id))
+ switch err {
+ case nil:
+ case ds.ErrNotFound:
+ return nil, errNoEntry
+ default:
+ return nil, err
+ }
+
+ e := new(pb.IpnsEntry)
+ if err := proto.Unmarshal(val, e); err != nil {
+ return nil, err
+ }
+ return e, nil
+}
diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go
new file mode 100644
index 000000000..bfd8730c0
--- /dev/null
+++ b/namesys/republisher/repub_test.go
@@ -0,0 +1,266 @@
+package republisher_test
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/gogo/protobuf/proto"
+
+ "github.com/jbenet/goprocess"
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ ic "github.com/libp2p/go-libp2p/core/crypto"
+ host "github.com/libp2p/go-libp2p/core/host"
+ peer "github.com/libp2p/go-libp2p/core/peer"
+ routing "github.com/libp2p/go-libp2p/core/routing"
+
+ opts "github.com/ipfs/boxo/coreiface/options/namesys"
+ "github.com/ipfs/boxo/ipns"
+ ipns_pb "github.com/ipfs/boxo/ipns/pb"
+ "github.com/ipfs/boxo/path"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+
+ keystore "github.com/ipfs/boxo/keystore"
+ "github.com/ipfs/boxo/namesys"
+ . "github.com/ipfs/boxo/namesys/republisher"
+)
+
+type mockNode struct {
+ h host.Host
+ id string
+ privKey ic.PrivKey
+ store ds.Batching
+ dht *dht.IpfsDHT
+ keystore keystore.Keystore
+}
+
+func getMockNode(t *testing.T, ctx context.Context) *mockNode {
+ t.Helper()
+
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ var idht *dht.IpfsDHT
+ h, err := libp2p.New(
+ libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"),
+ libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) {
+ rt, err := dht.New(ctx, h, dht.Mode(dht.ModeServer))
+ idht = rt
+ return rt, err
+ }),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return &mockNode{
+ h: h,
+ id: h.ID().Pretty(),
+ privKey: h.Peerstore().PrivKey(h.ID()),
+ store: dstore,
+ dht: idht,
+ keystore: keystore.NewMemKeystore(),
+ }
+}
+
+func TestRepublish(t *testing.T) {
+ // set cache life to zero for testing low-period repubs
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ var nsystems []namesys.NameSystem
+ var nodes []*mockNode
+ for i := 0; i < 10; i++ {
+ n := getMockNode(t, ctx)
+ ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nsystems = append(nsystems, ns)
+ nodes = append(nodes, n)
+ }
+
+ pinfo := host.InfoFromHost(nodes[0].h)
+
+ for _, n := range nodes[1:] {
+ if err := n.h.Connect(ctx, *pinfo); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // have one node publish a record that is valid for 1 second
+ publisher := nodes[3]
+
+ p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid
+ rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store)
+ name := "/ipns/" + publisher.id
+
+ // Retry in case the record expires before we can fetch it. This can
+ // happen when running the test on a slow machine.
+ var expiration time.Time
+ timeout := time.Second
+ for {
+ expiration = time.Now().Add(time.Second)
+ err := rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = verifyResolution(nsystems, name, p)
+ if err == nil {
+ break
+ }
+
+ if time.Now().After(expiration) {
+ timeout *= 2
+ continue
+ }
+ t.Fatal(err)
+ }
+
+ // Now wait a second, the records will be invalid and we should fail to resolve
+ time.Sleep(timeout)
+ if err := verifyResolutionFails(nsystems, name); err != nil {
+ t.Fatal(err)
+ }
+
+ // The republishers that are contained within the nodes have their timeout set
+ // to 12 hours. Instead of trying to tweak those, we're just going to pretend
+ // they don't exist and make our own.
+ repub := NewRepublisher(rp, publisher.store, publisher.privKey, publisher.keystore)
+ repub.Interval = time.Second
+ repub.RecordLifetime = time.Second * 5
+
+ proc := goprocess.Go(repub.Run)
+ defer proc.Close()
+
+ // now wait a couple seconds for it to fire
+ time.Sleep(time.Second * 2)
+
+ // we should be able to resolve them now
+ if err := verifyResolution(nsystems, name, p); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestLongEOLRepublish(t *testing.T) {
+ // set cache life to zero for testing low-period repubs
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ var nsystems []namesys.NameSystem
+ var nodes []*mockNode
+ for i := 0; i < 10; i++ {
+ n := getMockNode(t, ctx)
+ ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nsystems = append(nsystems, ns)
+ nodes = append(nodes, n)
+ }
+
+ pinfo := host.InfoFromHost(nodes[0].h)
+
+ for _, n := range nodes[1:] {
+ if err := n.h.Connect(ctx, *pinfo); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // have one node publish a record that is valid for 1 second
+ publisher := nodes[3]
+ p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid
+ rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store)
+ name := "/ipns/" + publisher.id
+
+ expiration := time.Now().Add(time.Hour)
+ err := rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = verifyResolution(nsystems, name, p)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // The republishers that are contained within the nodes have their timeout set
+ // to 12 hours. Instead of trying to tweak those, we're just going to pretend
+ // they don't exist and make our own.
+ repub := NewRepublisher(rp, publisher.store, publisher.privKey, publisher.keystore)
+ repub.Interval = time.Millisecond * 500
+ repub.RecordLifetime = time.Second
+
+ proc := goprocess.Go(repub.Run)
+ defer proc.Close()
+
+ // now wait a couple seconds for it to fire a few times
+ time.Sleep(time.Second * 2)
+
+ err = verifyResolution(nsystems, name, p)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ entry, err := getLastIPNSEntry(ctx, publisher.store, publisher.h.ID())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ finalEol, err := ipns.GetEOL(entry)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !finalEol.Equal(expiration) {
+ t.Fatal("expiration time modified")
+ }
+}
+
+func getLastIPNSEntry(ctx context.Context, dstore ds.Datastore, id peer.ID) (*ipns_pb.IpnsEntry, error) {
+ // Look for it locally only
+ val, err := dstore.Get(ctx, namesys.IpnsDsKey(id))
+ if err != nil {
+ return nil, err
+ }
+
+ e := new(ipns_pb.IpnsEntry)
+ if err := proto.Unmarshal(val, e); err != nil {
+ return nil, err
+ }
+ return e, nil
+}
+
+func verifyResolution(nsystems []namesys.NameSystem, key string, exp path.Path) error {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ for _, n := range nsystems {
+ val, err := n.Resolve(ctx, key)
+ if err != nil {
+ return err
+ }
+
+ if val != exp {
+ return errors.New("resolved wrong record")
+ }
+ }
+ return nil
+}
+
+func verifyResolutionFails(nsystems []namesys.NameSystem, key string) error {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ for _, n := range nsystems {
+ _, err := n.Resolve(ctx, key)
+ if err == nil {
+ return errors.New("expected resolution to fail")
+ }
+ }
+ return nil
+}
diff --git a/namesys/resolve/resolve.go b/namesys/resolve/resolve.go
new file mode 100644
index 000000000..b2acf0602
--- /dev/null
+++ b/namesys/resolve/resolve.go
@@ -0,0 +1,56 @@
+package resolve
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+
+ "github.com/ipfs/boxo/path"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+
+ "github.com/ipfs/boxo/namesys"
+)
+
+// ErrNoNamesys is an explicit error for when an IPFS node doesn't
+// (yet) have a name system
+var ErrNoNamesys = errors.New(
+ "core/resolve: no Namesys on IpfsNode - can't resolve ipns entry")
+
+// ResolveIPNS resolves /ipns paths
+func ResolveIPNS(ctx context.Context, nsys namesys.NameSystem, p path.Path) (path.Path, error) {
+ ctx, span := namesys.StartSpan(ctx, "ResolveIPNS", trace.WithAttributes(attribute.String("Path", p.String())))
+ defer span.End()
+ if strings.HasPrefix(p.String(), "/ipns/") {
+ // TODO(cryptix): we should be able to query the local cache for the path
+ if nsys == nil {
+ return "", ErrNoNamesys
+ }
+
+ seg := p.Segments()
+
+ if len(seg) < 2 || seg[1] == "" { // just "//path/to/file
+// - /ipfs/
+// - /ipns//path/to/folder
+// - etc
+type Path string
+
+// ^^^
+// TODO: debate making this a private struct wrapped in a public interface
+// would allow us to control creation, and cache segments.
+
+// FromString safely converts a string type to a Path type.
+func FromString(s string) Path {
+ return Path(s)
+}
+
+// FromCid safely converts a cid.Cid type to a Path type.
+func FromCid(c cid.Cid) Path {
+ return Path("/ipfs/" + c.String())
+}
+
+// Segments returns the different elements of a path
+// (elements are delimited by a /).
+func (p Path) Segments() []string {
+ cleaned := path.Clean(string(p))
+ segments := strings.Split(cleaned, "/")
+
+ // Ignore leading slash
+ if len(segments[0]) == 0 {
+ segments = segments[1:]
+ }
+
+ return segments
+}
+
+// String converts a path to string.
+func (p Path) String() string {
+ return string(p)
+}
+
+// IsJustAKey returns true if the path is of the form or /ipfs/, or
+// /ipld/
+func (p Path) IsJustAKey() bool {
+ parts := p.Segments()
+ return len(parts) == 2 && (parts[0] == "ipfs" || parts[0] == "ipld")
+}
+
+// PopLastSegment returns a new Path without its final segment, and the final
+// segment, separately. If there is no more to pop (the path is just a key),
+// the original path is returned.
+func (p Path) PopLastSegment() (Path, string, error) {
+
+ if p.IsJustAKey() {
+ return p, "", nil
+ }
+
+ segs := p.Segments()
+ newPath, err := ParsePath("/" + strings.Join(segs[:len(segs)-1], "/"))
+ if err != nil {
+ return "", "", err
+ }
+
+ return newPath, segs[len(segs)-1], nil
+}
+
+// FromSegments returns a path given its different segments.
+func FromSegments(prefix string, seg ...string) (Path, error) {
+ return ParsePath(prefix + strings.Join(seg, "/"))
+}
+
+// ParsePath returns a well-formed ipfs Path.
+// The returned path will always be prefixed with /ipfs/ or /ipns/.
+// The prefix will be added if not present in the given string.
+// This function will return an error when the given string is
+// not a valid ipfs path.
+func ParsePath(txt string) (Path, error) {
+ parts := strings.Split(txt, "/")
+ if len(parts) == 1 {
+ kp, err := ParseCidToPath(txt)
+ if err == nil {
+ return kp, nil
+ }
+ }
+
+ // if the path doesnt begin with a '/'
+ // we expect this to start with a hash, and be an 'ipfs' path
+ if parts[0] != "" {
+ if _, err := decodeCid(parts[0]); err != nil {
+ return "", &ErrInvalidPath{error: err, path: txt}
+ }
+ // The case when the path starts with hash without a protocol prefix
+ return Path("/ipfs/" + txt), nil
+ }
+
+ if len(parts) < 3 {
+ return "", &ErrInvalidPath{error: fmt.Errorf("invalid ipfs path"), path: txt}
+ }
+
+ //TODO: make this smarter
+ switch parts[1] {
+ case "ipfs", "ipld":
+ if parts[2] == "" {
+ return "", &ErrInvalidPath{error: fmt.Errorf("not enough path components"), path: txt}
+ }
+ // Validate Cid.
+ _, err := decodeCid(parts[2])
+ if err != nil {
+ return "", &ErrInvalidPath{error: fmt.Errorf("invalid CID: %w", err), path: txt}
+ }
+ case "ipns":
+ if parts[2] == "" {
+ return "", &ErrInvalidPath{error: fmt.Errorf("not enough path components"), path: txt}
+ }
+ default:
+ return "", &ErrInvalidPath{error: fmt.Errorf("unknown namespace %q", parts[1]), path: txt}
+ }
+
+ return Path(txt), nil
+}
+
+// ParseCidToPath takes a CID in string form and returns a valid ipfs Path.
+func ParseCidToPath(txt string) (Path, error) {
+ if txt == "" {
+ return "", &ErrInvalidPath{error: fmt.Errorf("empty"), path: txt}
+ }
+
+ c, err := decodeCid(txt)
+ if err != nil {
+ return "", &ErrInvalidPath{error: err, path: txt}
+ }
+
+ return FromCid(c), nil
+}
+
+// IsValid checks if a path is a valid ipfs Path.
+func (p *Path) IsValid() error {
+ _, err := ParsePath(p.String())
+ return err
+}
+
+// Join joins strings slices using /
+func Join(pths []string) string {
+ return strings.Join(pths, "/")
+}
+
+// SplitList splits strings usings /
+func SplitList(pth string) []string {
+ return strings.Split(pth, "/")
+}
+
+// SplitAbsPath clean up and split fpath. It extracts the first component (which
+// must be a Multihash) and return it separately.
+func SplitAbsPath(fpath Path) (cid.Cid, []string, error) {
+ parts := fpath.Segments()
+ if parts[0] == "ipfs" || parts[0] == "ipld" {
+ parts = parts[1:]
+ }
+
+ // if nothing, bail.
+ if len(parts) == 0 {
+ return cid.Cid{}, nil, &ErrInvalidPath{error: fmt.Errorf("empty"), path: string(fpath)}
+ }
+
+ c, err := decodeCid(parts[0])
+ // first element in the path is a cid
+ if err != nil {
+ return cid.Cid{}, nil, &ErrInvalidPath{error: fmt.Errorf("invalid CID: %w", err), path: string(fpath)}
+ }
+
+ return c, parts[1:], nil
+}
+
+func decodeCid(cstr string) (cid.Cid, error) {
+ c, err := cid.Decode(cstr)
+ if err != nil && len(cstr) == 46 && cstr[:2] == "qm" { // https://github.com/ipfs/go-ipfs/issues/7792
+ return cid.Cid{}, fmt.Errorf("%v (possible lowercased CIDv0; consider converting to a case-agnostic CIDv1, such as base32)", err)
+ }
+ return c, err
+}
diff --git a/path/path_test.go b/path/path_test.go
new file mode 100644
index 000000000..2b26a5678
--- /dev/null
+++ b/path/path_test.go
@@ -0,0 +1,128 @@
+package path
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestPathParsing(t *testing.T) {
+ cases := map[string]bool{
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true,
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": true,
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true,
+ "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true,
+ "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": true,
+ "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true,
+ "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true,
+ "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true,
+ "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true,
+ "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true,
+ "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false,
+ "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false,
+ "/ipfs/foo": false,
+ "/ipfs/": false,
+ "ipfs/": false,
+ "ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false,
+ "/ipld/foo": false,
+ "/ipld/": false,
+ "ipld/": false,
+ "ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false,
+ }
+
+ for p, expected := range cases {
+ _, err := ParsePath(p)
+ valid := err == nil
+ if valid != expected {
+ t.Fatalf("expected %s to have valid == %t", p, expected)
+ }
+ }
+}
+
+func TestNoComponents(t *testing.T) {
+ for _, s := range []string{
+ "/ipfs/",
+ "/ipns/",
+ "/ipld/",
+ } {
+ _, err := ParsePath(s)
+ if err == nil || !strings.Contains(err.Error(), "not enough path components") || !strings.Contains(err.Error(), s) {
+ t.Error("wrong error")
+ }
+ }
+}
+
+func TestInvalidPaths(t *testing.T) {
+ for _, s := range []string{
+ "/ipfs",
+ "/testfs",
+ "/",
+ } {
+ _, err := ParsePath(s)
+ if err == nil || !strings.Contains(err.Error(), "invalid ipfs path") || !strings.Contains(err.Error(), s) {
+ t.Error("wrong error")
+ }
+ }
+}
+
+func TestIsJustAKey(t *testing.T) {
+ cases := map[string]bool{
+ "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true,
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true,
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false,
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": false,
+ "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false,
+ "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": false,
+ "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true,
+ }
+
+ for p, expected := range cases {
+ path, err := ParsePath(p)
+ if err != nil {
+ t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p)
+ }
+ result := path.IsJustAKey()
+ if result != expected {
+ t.Fatalf("expected IsJustAKey(%s) to return %v, not %v", p, expected, result)
+ }
+ }
+}
+
+func TestPopLastSegment(t *testing.T) {
+ cases := map[string][]string{
+ "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""},
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""},
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", "a"},
+ "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", "b"},
+ "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y/z": {"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y", "z"},
+ "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y/z": {"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y", "z"},
+ }
+
+ for p, expected := range cases {
+ path, err := ParsePath(p)
+ if err != nil {
+ t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p)
+ }
+ head, tail, err := path.PopLastSegment()
+ if err != nil {
+ t.Fatalf("PopLastSegment failed, but should have succeeded: %s", err)
+ }
+ headStr := head.String()
+ if headStr != expected[0] {
+ t.Fatalf("expected head of PopLastSegment(%s) to return %v, not %v", p, expected[0], headStr)
+ }
+ if tail != expected[1] {
+ t.Fatalf("expected tail of PopLastSegment(%s) to return %v, not %v", p, expected[1], tail)
+ }
+ }
+}
+
+func TestV0ErrorDueToLowercase(t *testing.T) {
+ badb58 := "/ipfs/qmbwqxbekc3p8tqskc98xmwnzrzdtrlmimpl8wbutgsmnr"
+ _, err := ParsePath(badb58)
+ if err == nil {
+ t.Fatal("should have failed to decode")
+ }
+ if !strings.HasSuffix(err.Error(), "(possible lowercased CIDv0; consider converting to a case-agnostic CIDv1, such as base32)") {
+ t.Fatal("should have meaningful info about case-insensitive fix")
+ }
+}
diff --git a/path/resolver/resolver.go b/path/resolver/resolver.go
new file mode 100644
index 000000000..8192a9eb0
--- /dev/null
+++ b/path/resolver/resolver.go
@@ -0,0 +1,322 @@
+// Package resolver implements utilities for resolving paths within ipfs.
+package resolver
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "time"
+
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
+
+ "github.com/ipfs/boxo/fetcher"
+ fetcherhelpers "github.com/ipfs/boxo/fetcher/helpers"
+ path "github.com/ipfs/boxo/path"
+ "github.com/ipfs/boxo/path/internal"
+ cid "github.com/ipfs/go-cid"
+ format "github.com/ipfs/go-ipld-format"
+ logging "github.com/ipfs/go-log"
+ "github.com/ipld/go-ipld-prime"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+ "github.com/ipld/go-ipld-prime/schema"
+ "github.com/ipld/go-ipld-prime/traversal/selector/builder"
+)
+
+var log = logging.Logger("pathresolv")
+
+// ErrNoComponents is used when Paths after a protocol
+// do not contain at least one component
+var ErrNoComponents = errors.New(
+ "path must contain at least one component")
+
+// ErrNoLink is returned when a link is not found in a path
+type ErrNoLink struct {
+ Name string
+ Node cid.Cid
+}
+
+// Error implements the Error interface for ErrNoLink with a useful
+// human readable message.
+func (e ErrNoLink) Error() string {
+ return fmt.Sprintf("no link named %q under %s", e.Name, e.Node.String())
+}
+
+// Resolver provides path resolution to IPFS.
+type Resolver interface {
+ // ResolveToLastNode walks the given path and returns the cid of the
+ // last block referenced by the path, and the path segments to
+ // traverse from the final block boundary to the final node within the
+ // block.
+ ResolveToLastNode(ctx context.Context, fpath path.Path) (cid.Cid, []string, error)
+ // ResolvePath fetches the node for given path. It returns the last
+ // item returned by ResolvePathComponents and the last link traversed
+ // which can be used to recover the block.
+ ResolvePath(ctx context.Context, fpath path.Path) (ipld.Node, ipld.Link, error)
+ // ResolvePathComponents fetches the nodes for each segment of the given path.
+ // It uses the first path component as a hash (key) of the first node, then
+ // resolves all other components walking the links via a selector traversal
+ ResolvePathComponents(ctx context.Context, fpath path.Path) ([]ipld.Node, error)
+}
+
+// basicResolver implements the Resolver interface.
+// It references a FetcherFactory, which is uses to resolve nodes.
+// TODO: now that this is more modular, try to unify this code with the
+//
+// the resolvers in namesys.
+type basicResolver struct {
+ FetcherFactory fetcher.Factory
+}
+
+// NewBasicResolver constructs a new basic resolver.
+func NewBasicResolver(fetcherFactory fetcher.Factory) Resolver {
+ return &basicResolver{
+ FetcherFactory: fetcherFactory,
+ }
+}
+
+// ResolveToLastNode walks the given path and returns the cid of the last
+// block referenced by the path, and the path segments to traverse from the
+// final block boundary to the final node within the block.
+func (r *basicResolver) ResolveToLastNode(ctx context.Context, fpath path.Path) (cid.Cid, []string, error) {
+ ctx, span := internal.StartSpan(ctx, "basicResolver.ResolveToLastNode", trace.WithAttributes(attribute.Stringer("Path", fpath)))
+ defer span.End()
+
+ c, p, err := path.SplitAbsPath(fpath)
+ if err != nil {
+ return cid.Cid{}, nil, err
+ }
+
+ if len(p) == 0 {
+ return c, nil, nil
+ }
+
+ // create a selector to traverse and match all path segments
+ pathSelector := pathAllSelector(p[:len(p)-1])
+
+ // create a new cancellable session
+ ctx, cancel := context.WithTimeout(ctx, time.Minute)
+ defer cancel()
+
+ // resolve node before last path segment
+ nodes, lastCid, depth, err := r.resolveNodes(ctx, c, pathSelector)
+ if err != nil {
+ return cid.Cid{}, nil, err
+ }
+
+ if len(nodes) < 1 {
+ return cid.Cid{}, nil, fmt.Errorf("path %v did not resolve to a node", fpath)
+ } else if len(nodes) < len(p) {
+ return cid.Undef, nil, ErrNoLink{Name: p[len(nodes)-1], Node: lastCid}
+ }
+
+ parent := nodes[len(nodes)-1]
+ lastSegment := p[len(p)-1]
+
+ // find final path segment within node
+ nd, err := parent.LookupBySegment(ipld.ParsePathSegment(lastSegment))
+ switch err.(type) {
+ case nil:
+ case schema.ErrNoSuchField:
+ return cid.Undef, nil, ErrNoLink{Name: lastSegment, Node: lastCid}
+ default:
+ return cid.Cid{}, nil, err
+ }
+
+ // if last node is not a link, just return it's cid, add path to remainder and return
+ if nd.Kind() != ipld.Kind_Link {
+ // return the cid and the remainder of the path
+ return lastCid, p[len(p)-depth-1:], nil
+ }
+
+ lnk, err := nd.AsLink()
+ if err != nil {
+ return cid.Cid{}, nil, err
+ }
+
+ clnk, ok := lnk.(cidlink.Link)
+ if !ok {
+ return cid.Cid{}, nil, fmt.Errorf("path %v resolves to a link that is not a cid link: %v", fpath, lnk)
+ }
+
+ return clnk.Cid, []string{}, nil
+}
+
+// ResolvePath fetches the node for given path. It returns the last item
+// returned by ResolvePathComponents and the last link traversed which can be used to recover the block.
+//
+// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be
+// possible to load certain values.
+func (r *basicResolver) ResolvePath(ctx context.Context, fpath path.Path) (ipld.Node, ipld.Link, error) {
+ ctx, span := internal.StartSpan(ctx, "basicResolver.ResolvePath", trace.WithAttributes(attribute.Stringer("Path", fpath)))
+ defer span.End()
+
+ // validate path
+ if err := fpath.IsValid(); err != nil {
+ return nil, nil, err
+ }
+
+ c, p, err := path.SplitAbsPath(fpath)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // create a selector to traverse all path segments but only match the last
+ pathSelector := pathLeafSelector(p)
+
+ nodes, c, _, err := r.resolveNodes(ctx, c, pathSelector)
+ if err != nil {
+ return nil, nil, err
+ }
+ if len(nodes) < 1 {
+ return nil, nil, fmt.Errorf("path %v did not resolve to a node", fpath)
+ }
+ return nodes[len(nodes)-1], cidlink.Link{Cid: c}, nil
+}
+
+// ResolveSingle simply resolves one hop of a path through a graph with no
+// extra context (does not opaquely resolve through sharded nodes)
+// Deprecated: fetch node as ipld-prime or convert it and then use a selector to traverse through it.
+func ResolveSingle(ctx context.Context, ds format.NodeGetter, nd format.Node, names []string) (*format.Link, []string, error) {
+ _, span := internal.StartSpan(ctx, "ResolveSingle", trace.WithAttributes(attribute.Stringer("CID", nd.Cid())))
+ defer span.End()
+ return nd.ResolveLink(names)
+}
+
+// ResolvePathComponents fetches the nodes for each segment of the given path.
+// It uses the first path component as a hash (key) of the first node, then
+// resolves all other components walking the links via a selector traversal
+//
+// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be
+// possible to load certain values.
+func (r *basicResolver) ResolvePathComponents(ctx context.Context, fpath path.Path) ([]ipld.Node, error) {
+ ctx, span := internal.StartSpan(ctx, "basicResolver.ResolvePathComponents", trace.WithAttributes(attribute.Stringer("Path", fpath)))
+ defer span.End()
+
+ //lint:ignore SA1019 TODO: replace EventBegin
+ evt := log.EventBegin(ctx, "resolvePathComponents", logging.LoggableMap{"fpath": fpath})
+ defer evt.Done()
+
+ // validate path
+ if err := fpath.IsValid(); err != nil {
+ evt.Append(logging.LoggableMap{"error": err.Error()})
+ return nil, err
+ }
+
+ c, p, err := path.SplitAbsPath(fpath)
+ if err != nil {
+ evt.Append(logging.LoggableMap{"error": err.Error()})
+ return nil, err
+ }
+
+ // create a selector to traverse and match all path segments
+ pathSelector := pathAllSelector(p)
+
+ nodes, _, _, err := r.resolveNodes(ctx, c, pathSelector)
+ if err != nil {
+ evt.Append(logging.LoggableMap{"error": err.Error()})
+ }
+
+ return nodes, err
+}
+
+// ResolveLinks iteratively resolves names by walking the link hierarchy.
+// Every node is fetched from the Fetcher, resolving the next name.
+// Returns the list of nodes forming the path, starting with ndd. This list is
+// guaranteed never to be empty.
+//
+// ResolveLinks(nd, []string{"foo", "bar", "baz"})
+// would retrieve "baz" in ("bar" in ("foo" in nd.Links).Links).Links
+//
+// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be
+// possible to load certain values.
+func (r *basicResolver) ResolveLinks(ctx context.Context, ndd ipld.Node, names []string) ([]ipld.Node, error) {
+ ctx, span := internal.StartSpan(ctx, "basicResolver.ResolveLinks")
+ defer span.End()
+
+ //lint:ignore SA1019 TODO: replace EventBegin
+ evt := log.EventBegin(ctx, "resolveLinks", logging.LoggableMap{"names": names})
+ defer evt.Done()
+
+ // create a selector to traverse and match all path segments
+ pathSelector := pathAllSelector(names)
+
+ session := r.FetcherFactory.NewSession(ctx)
+
+ // traverse selector
+ nodes := []ipld.Node{ndd}
+ err := session.NodeMatching(ctx, ndd, pathSelector, func(res fetcher.FetchResult) error {
+ nodes = append(nodes, res.Node)
+ return nil
+ })
+ if err != nil {
+ evt.Append(logging.LoggableMap{"error": err.Error()})
+ return nil, err
+ }
+
+ return nodes, err
+}
+
+// Finds nodes matching the selector starting with a cid. Returns the matched nodes, the cid of the block containing
+// the last node, and the depth of the last node within its block (root is depth 0).
+func (r *basicResolver) resolveNodes(ctx context.Context, c cid.Cid, sel ipld.Node) ([]ipld.Node, cid.Cid, int, error) {
+ ctx, span := internal.StartSpan(ctx, "basicResolver.resolveNodes", trace.WithAttributes(attribute.Stringer("CID", c)))
+ defer span.End()
+ session := r.FetcherFactory.NewSession(ctx)
+
+ // traverse selector
+ lastLink := cid.Undef
+ depth := 0
+ nodes := []ipld.Node{}
+ err := fetcherhelpers.BlockMatching(ctx, session, cidlink.Link{Cid: c}, sel, func(res fetcher.FetchResult) error {
+ if res.LastBlockLink == nil {
+ res.LastBlockLink = cidlink.Link{Cid: c}
+ }
+ cidLnk, ok := res.LastBlockLink.(cidlink.Link)
+ if !ok {
+ return fmt.Errorf("link is not a cidlink: %v", cidLnk)
+ }
+
+ // if we hit a block boundary
+ if !lastLink.Equals(cidLnk.Cid) {
+ depth = 0
+ lastLink = cidLnk.Cid
+ } else {
+ depth++
+ }
+
+ nodes = append(nodes, res.Node)
+ return nil
+ })
+ if err != nil {
+ return nil, cid.Undef, 0, err
+ }
+
+ return nodes, lastLink, depth, nil
+}
+
+func pathLeafSelector(path []string) ipld.Node {
+ ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
+ return pathSelector(path, ssb, func(p string, s builder.SelectorSpec) builder.SelectorSpec {
+ return ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert(p, s) })
+ })
+}
+
+func pathAllSelector(path []string) ipld.Node {
+ ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
+ return pathSelector(path, ssb, func(p string, s builder.SelectorSpec) builder.SelectorSpec {
+ return ssb.ExploreUnion(
+ ssb.Matcher(),
+ ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) { efsb.Insert(p, s) }),
+ )
+ })
+}
+
+func pathSelector(path []string, ssb builder.SelectorSpecBuilder, reduce func(string, builder.SelectorSpec) builder.SelectorSpec) ipld.Node {
+ spec := ssb.Matcher()
+ for i := len(path) - 1; i >= 0; i-- {
+ spec = reduce(path[i], spec)
+ }
+ return spec.Node()
+}
diff --git a/path/resolver/resolver_test.go b/path/resolver/resolver_test.go
new file mode 100644
index 000000000..9779de58d
--- /dev/null
+++ b/path/resolver/resolver_test.go
@@ -0,0 +1,293 @@
+package resolver_test
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "math/rand"
+ "strings"
+ "testing"
+ "time"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
+ "github.com/ipfs/go-cid"
+ dagpb "github.com/ipld/go-codec-dagpb"
+ "github.com/ipld/go-ipld-prime"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+ "github.com/ipld/go-ipld-prime/schema"
+ "github.com/multiformats/go-multihash"
+
+ merkledag "github.com/ipfs/boxo/ipld/merkledag"
+ dagmock "github.com/ipfs/boxo/ipld/merkledag/test"
+ path "github.com/ipfs/boxo/path"
+ "github.com/ipfs/boxo/path/resolver"
+ "github.com/ipfs/go-unixfsnode"
+ dagcbor "github.com/ipld/go-ipld-prime/codec/dagcbor"
+ dagjson "github.com/ipld/go-ipld-prime/codec/dagjson"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func randNode() *merkledag.ProtoNode {
+ node := new(merkledag.ProtoNode)
+ node.SetData(make([]byte, 32))
+ r := rand.New(rand.NewSource(time.Now().UnixNano()))
+ r.Read(node.Data())
+ return node
+}
+
+func TestRecurivePathResolution(t *testing.T) {
+ ctx := context.Background()
+ bsrv := dagmock.Bserv()
+
+ a := randNode()
+ b := randNode()
+ c := randNode()
+
+ err := b.AddNodeLink("grandchild", c)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = a.AddNodeLink("child", b)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, n := range []*merkledag.ProtoNode{a, b, c} {
+ err = bsrv.AddBlock(ctx, n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ aKey := a.Cid()
+
+ segments := []string{aKey.String(), "child", "grandchild"}
+ p, err := path.FromSegments("/ipfs/", segments...)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fetcherFactory := bsfetcher.NewFetcherConfig(bsrv)
+ fetcherFactory.NodeReifier = unixfsnode.Reify
+ fetcherFactory.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+ if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
+ return tlnkNd.LinkTargetNodePrototype(), nil
+ }
+ return basicnode.Prototype.Any, nil
+ })
+ resolver := resolver.NewBasicResolver(fetcherFactory)
+
+ node, lnk, err := resolver.ResolvePath(ctx, p)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ uNode, ok := node.(unixfsnode.PathedPBNode)
+ require.True(t, ok)
+ fd := uNode.FieldData()
+ byts, err := fd.Must().AsBytes()
+ require.NoError(t, err)
+
+ assert.Equal(t, cidlink.Link{Cid: c.Cid()}, lnk)
+
+ assert.Equal(t, c.Data(), byts)
+ cKey := c.Cid()
+
+ rCid, rest, err := resolver.ResolveToLastNode(ctx, p)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(rest) != 0 {
+ t.Error("expected rest to be empty")
+ }
+
+ if rCid.String() != cKey.String() {
+ t.Fatal(fmt.Errorf(
+ "ResolveToLastNode failed for %s: %s != %s",
+ p.String(), rCid.String(), cKey.String()))
+ }
+
+ p2, err := path.FromSegments("/ipfs/", aKey.String())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rCid, rest, err = resolver.ResolveToLastNode(ctx, p2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(rest) != 0 {
+ t.Error("expected rest to be empty")
+ }
+
+ if rCid.String() != aKey.String() {
+ t.Fatal(fmt.Errorf(
+ "ResolveToLastNode failed for %s: %s != %s",
+ p.String(), rCid.String(), cKey.String()))
+ }
+}
+func TestResolveToLastNode_ErrNoLink(t *testing.T) {
+ ctx := context.Background()
+ bsrv := dagmock.Bserv()
+
+ a := randNode()
+ b := randNode()
+ c := randNode()
+
+ err := b.AddNodeLink("grandchild", c)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = a.AddNodeLink("child", b)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, n := range []*merkledag.ProtoNode{a, b, c} {
+ err = bsrv.AddBlock(ctx, n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ aKey := a.Cid()
+
+ fetcherFactory := bsfetcher.NewFetcherConfig(bsrv)
+ fetcherFactory.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+ if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
+ return tlnkNd.LinkTargetNodePrototype(), nil
+ }
+ return basicnode.Prototype.Any, nil
+ })
+ fetcherFactory.NodeReifier = unixfsnode.Reify
+ r := resolver.NewBasicResolver(fetcherFactory)
+
+ // test missing link intermediate segment
+ segments := []string{aKey.String(), "cheese", "time"}
+ p, err := path.FromSegments("/ipfs/", segments...)
+ require.NoError(t, err)
+
+ _, _, err = r.ResolveToLastNode(ctx, p)
+ require.EqualError(t, err, resolver.ErrNoLink{Name: "cheese", Node: aKey}.Error())
+
+ // test missing link at end
+ bKey := b.Cid()
+ segments = []string{aKey.String(), "child", "apples"}
+ p, err = path.FromSegments("/ipfs/", segments...)
+ require.NoError(t, err)
+
+ _, _, err = r.ResolveToLastNode(ctx, p)
+ require.EqualError(t, err, resolver.ErrNoLink{Name: "apples", Node: bKey}.Error())
+}
+
+func TestResolveToLastNode_NoUnnecessaryFetching(t *testing.T) {
+ ctx := context.Background()
+ bsrv := dagmock.Bserv()
+
+ a := randNode()
+ b := randNode()
+
+ err := a.AddNodeLink("child", b)
+ require.NoError(t, err)
+
+ err = bsrv.AddBlock(ctx, a)
+ require.NoError(t, err)
+
+ aKey := a.Cid()
+
+ segments := []string{aKey.String(), "child"}
+ p, err := path.FromSegments("/ipfs/", segments...)
+ require.NoError(t, err)
+
+ fetcherFactory := bsfetcher.NewFetcherConfig(bsrv)
+ fetcherFactory.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+ if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
+ return tlnkNd.LinkTargetNodePrototype(), nil
+ }
+ return basicnode.Prototype.Any, nil
+ })
+ fetcherFactory.NodeReifier = unixfsnode.Reify
+ resolver := resolver.NewBasicResolver(fetcherFactory)
+
+ resolvedCID, remainingPath, err := resolver.ResolveToLastNode(ctx, p)
+ require.NoError(t, err)
+
+ require.Equal(t, len(remainingPath), 0, "cannot have remaining path")
+ require.Equal(t, b.Cid(), resolvedCID)
+}
+
+func TestPathRemainder(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ bsrv := dagmock.Bserv()
+
+ nb := basicnode.Prototype.Any.NewBuilder()
+ err := dagjson.Decode(nb, strings.NewReader(`{"foo": {"bar": "baz"}}`))
+ require.NoError(t, err)
+ out := new(bytes.Buffer)
+ err = dagcbor.Encode(nb.Build(), out)
+ require.NoError(t, err)
+ lnk, err := cid.Prefix{
+ Version: 1,
+ Codec: cid.DagCBOR,
+ MhType: multihash.SHA2_256,
+ MhLength: 32,
+ }.Sum(out.Bytes())
+ require.NoError(t, err)
+ blk, err := blocks.NewBlockWithCid(out.Bytes(), lnk)
+ require.NoError(t, err)
+ bsrv.AddBlock(ctx, blk)
+ fetcherFactory := bsfetcher.NewFetcherConfig(bsrv)
+ resolver := resolver.NewBasicResolver(fetcherFactory)
+
+ rp1, remainder, err := resolver.ResolveToLastNode(ctx, path.FromString(lnk.String()+"/foo/bar"))
+ require.NoError(t, err)
+
+ assert.Equal(t, lnk, rp1)
+ require.Equal(t, "foo/bar", path.Join(remainder))
+}
+
+func TestResolveToLastNode_MixedSegmentTypes(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ bsrv := dagmock.Bserv()
+ a := randNode()
+ err := bsrv.AddBlock(ctx, a)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nb := basicnode.Prototype.Any.NewBuilder()
+ json := `{"foo":{"bar":[0,{"boom":["baz",1,2,{"/":"CID"},"blop"]}]}}`
+ json = strings.ReplaceAll(json, "CID", a.Cid().String())
+ err = dagjson.Decode(nb, strings.NewReader(json))
+ require.NoError(t, err)
+ out := new(bytes.Buffer)
+ err = dagcbor.Encode(nb.Build(), out)
+ require.NoError(t, err)
+ lnk, err := cid.Prefix{
+ Version: 1,
+ Codec: cid.DagCBOR,
+ MhType: multihash.SHA2_256,
+ MhLength: 32,
+ }.Sum(out.Bytes())
+ require.NoError(t, err)
+ blk, err := blocks.NewBlockWithCid(out.Bytes(), lnk)
+ require.NoError(t, err)
+ bsrv.AddBlock(ctx, blk)
+ fetcherFactory := bsfetcher.NewFetcherConfig(bsrv)
+ resolver := resolver.NewBasicResolver(fetcherFactory)
+
+ cid, remainder, err := resolver.ResolveToLastNode(ctx, path.FromString(lnk.String()+"/foo/bar/1/boom/3"))
+ require.NoError(t, err)
+
+ assert.Equal(t, 0, len(remainder))
+ assert.True(t, cid.Equals(a.Cid()))
+}
diff --git a/pinning/pinner/.gitignore b/pinning/pinner/.gitignore
new file mode 100644
index 000000000..3c342889d
--- /dev/null
+++ b/pinning/pinner/.gitignore
@@ -0,0 +1,8 @@
+*~
+*.log
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool
+*.out
diff --git a/pinning/pinner/dsindex/error.go b/pinning/pinner/dsindex/error.go
new file mode 100644
index 000000000..f3b685bb9
--- /dev/null
+++ b/pinning/pinner/dsindex/error.go
@@ -0,0 +1,8 @@
+package dsindex
+
+import "errors"
+
+var (
+ ErrEmptyKey = errors.New("key is empty")
+ ErrEmptyValue = errors.New("value is empty")
+)
diff --git a/pinning/pinner/dsindex/indexer.go b/pinning/pinner/dsindex/indexer.go
new file mode 100644
index 000000000..8384ad5d5
--- /dev/null
+++ b/pinning/pinner/dsindex/indexer.go
@@ -0,0 +1,277 @@
+// Package dsindex provides secondary indexing functionality for a datastore.
+package dsindex
+
+import (
+ "context"
+ "fmt"
+ "path"
+
+ ds "github.com/ipfs/go-datastore"
+ "github.com/ipfs/go-datastore/namespace"
+ "github.com/ipfs/go-datastore/query"
+ "github.com/multiformats/go-multibase"
+)
+
+// Indexer maintains a secondary index. An index is a collection of key-value
+// mappings where the key is the secondary index that maps to one or more
+// values, where each value is a unique key being indexed.
+type Indexer interface {
+ // Add adds the specified value to the key
+ Add(ctx context.Context, key, value string) error
+
+ // Delete deletes the specified value from the key. If the value is not in
+ // the datastore, this method returns no error.
+ Delete(ctx context.Context, key, value string) error
+
+ // DeleteKey deletes all values in the given key. If a key is not in the
+ // datastore, this method returns no error. Returns a count of values that
+ // were deleted.
+ DeleteKey(ctx context.Context, key string) (count int, err error)
+
+ // DeleteAll deletes all keys managed by this Indexer. Returns a count of
+ // the values that were deleted.
+ DeleteAll(ctx context.Context) (count int, err error)
+
+ // ForEach calls the function for each value in the specified key, until
+ // there are no more values, or until the function returns false. If key
+ // is empty string, then all keys are iterated.
+ ForEach(ctx context.Context, key string, fn func(key, value string) bool) error
+
+ // HasValue determines if the key contains the specified value
+ HasValue(ctx context.Context, key, value string) (bool, error)
+
+ // HasAny determines if any value is in the specified key. If key is
+ // empty string, then all values are searched.
+ HasAny(ctx context.Context, key string) (bool, error)
+
+ // Search returns all values for the given key
+ Search(ctx context.Context, key string) (values []string, err error)
+}
+
+// indexer is a simple implementation of Indexer. This implementation relies
+// on the underlying data store to support efficient querying by prefix.
+//
+// TODO: Consider adding caching
+type indexer struct {
+ dstore ds.Datastore
+}
+
+// New creates a new datastore index. All indexes are stored under the
+// specified index name.
+//
+// To persist the actions of calling Indexer functions, it is necessary to call
+// dstore.Sync.
+func New(dstore ds.Datastore, name ds.Key) Indexer {
+ return &indexer{
+ dstore: namespace.Wrap(dstore, name),
+ }
+}
+
+func (x *indexer) Add(ctx context.Context, key, value string) error {
+ if key == "" {
+ return ErrEmptyKey
+ }
+ if value == "" {
+ return ErrEmptyValue
+ }
+ dsKey := ds.NewKey(encode(key)).ChildString(encode(value))
+ return x.dstore.Put(ctx, dsKey, []byte{})
+}
+
+func (x *indexer) Delete(ctx context.Context, key, value string) error {
+ if key == "" {
+ return ErrEmptyKey
+ }
+ if value == "" {
+ return ErrEmptyValue
+ }
+ return x.dstore.Delete(ctx, ds.NewKey(encode(key)).ChildString(encode(value)))
+}
+
+func (x *indexer) DeleteKey(ctx context.Context, key string) (int, error) {
+ if key == "" {
+ return 0, ErrEmptyKey
+ }
+ return x.deletePrefix(ctx, encode(key))
+}
+
+func (x *indexer) DeleteAll(ctx context.Context) (int, error) {
+ return x.deletePrefix(ctx, "")
+}
+
+func (x *indexer) ForEach(ctx context.Context, key string, fn func(key, value string) bool) error {
+ if key != "" {
+ key = encode(key)
+ }
+
+ q := query.Query{
+ Prefix: key,
+ KeysOnly: true,
+ }
+ results, err := x.dstore.Query(ctx, q)
+ if err != nil {
+ return err
+ }
+ defer results.Close()
+
+ for r := range results.Next() {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if r.Error != nil {
+ return fmt.Errorf("cannot read index: %v", r.Error)
+ }
+ ent := r.Entry
+ decIdx, err := decode(path.Base(path.Dir(ent.Key)))
+ if err != nil {
+ return fmt.Errorf("cannot decode index: %v", err)
+ }
+ decKey, err := decode(path.Base(ent.Key))
+ if err != nil {
+ return fmt.Errorf("cannot decode key: %v", err)
+ }
+ if !fn(decIdx, decKey) {
+ return nil
+ }
+ }
+
+ return nil
+}
+
+func (x *indexer) HasValue(ctx context.Context, key, value string) (bool, error) {
+ if key == "" {
+ return false, ErrEmptyKey
+ }
+ if value == "" {
+ return false, ErrEmptyValue
+ }
+ return x.dstore.Has(ctx, ds.NewKey(encode(key)).ChildString(encode(value)))
+}
+
+func (x *indexer) HasAny(ctx context.Context, key string) (bool, error) {
+ var any bool
+ err := x.ForEach(ctx, key, func(key, value string) bool {
+ any = true
+ return false
+ })
+ return any, err
+}
+
+func (x *indexer) Search(ctx context.Context, key string) ([]string, error) {
+ if key == "" {
+ return nil, ErrEmptyKey
+ }
+ ents, err := x.queryPrefix(ctx, encode(key))
+ if err != nil {
+ return nil, err
+ }
+ if len(ents) == 0 {
+ return nil, nil
+ }
+
+ values := make([]string, len(ents))
+ for i := range ents {
+ values[i], err = decode(path.Base(ents[i].Key))
+ if err != nil {
+ return nil, fmt.Errorf("cannot decode value: %v", err)
+ }
+ }
+ return values, nil
+}
+
+// SyncIndex synchronizes the keys in the target Indexer to match those of the
+// ref Indexer. This function does not change this indexer's key root (name
+// passed into New).
+func SyncIndex(ctx context.Context, ref, target Indexer) (bool, error) {
+ // Build reference index map
+ refs := map[string]string{}
+ err := ref.ForEach(ctx, "", func(key, value string) bool {
+ refs[value] = key
+ return true
+ })
+ if err != nil {
+ return false, err
+ }
+ if len(refs) == 0 {
+ return false, nil
+ }
+
+ // Compare current indexes
+ dels := map[string]string{}
+ err = target.ForEach(ctx, "", func(key, value string) bool {
+ refKey, ok := refs[value]
+ if ok && refKey == key {
+ // same in both; delete from refs, do not add to dels
+ delete(refs, value)
+ } else {
+ dels[value] = key
+ }
+ return true
+ })
+ if err != nil {
+ return false, err
+ }
+
+ // Items in dels are keys that no longer exist
+ for value, key := range dels {
+ err = target.Delete(ctx, key, value)
+ if err != nil {
+ return false, err
+ }
+ }
+
+ // What remains in refs are keys that need to be added
+ for value, key := range refs {
+ err = target.Add(ctx, key, value)
+ if err != nil {
+ return false, err
+ }
+ }
+
+ return len(refs) != 0 || len(dels) != 0, nil
+}
+
+func (x *indexer) deletePrefix(ctx context.Context, prefix string) (int, error) {
+ ents, err := x.queryPrefix(ctx, prefix)
+ if err != nil {
+ return 0, err
+ }
+
+ for i := range ents {
+ err = x.dstore.Delete(ctx, ds.NewKey(ents[i].Key))
+ if err != nil {
+ return 0, err
+ }
+ }
+
+ return len(ents), nil
+}
+
+func (x *indexer) queryPrefix(ctx context.Context, prefix string) ([]query.Entry, error) {
+ q := query.Query{
+ Prefix: prefix,
+ KeysOnly: true,
+ }
+ results, err := x.dstore.Query(ctx, q)
+ if err != nil {
+ return nil, err
+ }
+ return results.Rest()
+}
+
+func encode(data string) string {
+ encData, err := multibase.Encode(multibase.Base64url, []byte(data))
+ if err != nil {
+ // programming error; using unsupported encoding
+ panic(err.Error())
+ }
+ return encData
+}
+
+func decode(data string) (string, error) {
+ _, b, err := multibase.Decode(data)
+ if err != nil {
+ return "", err
+ }
+ return string(b), nil
+}
diff --git a/pinning/pinner/dsindex/indexer_test.go b/pinning/pinner/dsindex/indexer_test.go
new file mode 100644
index 000000000..45372c605
--- /dev/null
+++ b/pinning/pinner/dsindex/indexer_test.go
@@ -0,0 +1,286 @@
+package dsindex
+
+import (
+ "context"
+ "testing"
+
+ ds "github.com/ipfs/go-datastore"
+)
+
+func createIndexer() Indexer {
+ dstore := ds.NewMapDatastore()
+ nameIndex := New(dstore, ds.NewKey("/data/nameindex"))
+
+ ctx := context.Background()
+ nameIndex.Add(ctx, "alice", "a1")
+ nameIndex.Add(ctx, "bob", "b1")
+ nameIndex.Add(ctx, "bob", "b2")
+ nameIndex.Add(ctx, "cathy", "c1")
+
+ return nameIndex
+}
+
+func TestAdd(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nameIndex := createIndexer()
+ err := nameIndex.Add(ctx, "someone", "s1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = nameIndex.Add(ctx, "someone", "s1")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = nameIndex.Add(ctx, "", "noindex")
+ if err != ErrEmptyKey {
+ t.Fatal("unexpected error:", err)
+ }
+
+ err = nameIndex.Add(ctx, "nokey", "")
+ if err != ErrEmptyValue {
+ t.Fatal("unexpected error:", err)
+ }
+}
+
+func TestHasValue(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nameIndex := createIndexer()
+
+ ok, err := nameIndex.HasValue(ctx, "bob", "b1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ok {
+ t.Fatal("missing index")
+ }
+
+ ok, err = nameIndex.HasValue(ctx, "bob", "b3")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ok {
+ t.Fatal("should not have index")
+ }
+
+ _, err = nameIndex.HasValue(ctx, "", "b1")
+ if err != ErrEmptyKey {
+ t.Fatal("unexpected error:", err)
+ }
+
+ _, err = nameIndex.HasValue(ctx, "bob", "")
+ if err != ErrEmptyValue {
+ t.Fatal("unexpected error:", err)
+ }
+}
+
+func TestHasAny(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nameIndex := createIndexer()
+
+ ok, err := nameIndex.HasAny(ctx, "nothere")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ok {
+ t.Fatal("should return false")
+ }
+
+ for _, idx := range []string{"alice", "bob", ""} {
+ ok, err = nameIndex.HasAny(ctx, idx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ok {
+ t.Fatal("missing index", idx)
+ }
+ }
+
+ count, err := nameIndex.DeleteAll(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if count != 4 {
+ t.Fatal("expected 4 deletions")
+ }
+
+ ok, err = nameIndex.HasAny(ctx, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ok {
+ t.Fatal("should return false")
+ }
+}
+
+func TestForEach(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nameIndex := createIndexer()
+
+ found := make(map[string]struct{})
+ err := nameIndex.ForEach(ctx, "bob", func(key, value string) bool {
+ found[value] = struct{}{}
+ return true
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, value := range []string{"b1", "b2"} {
+ _, ok := found[value]
+ if !ok {
+ t.Fatal("missing key for value", value)
+ }
+ }
+
+ values := map[string]string{}
+ err = nameIndex.ForEach(ctx, "", func(key, value string) bool {
+ values[value] = key
+ return true
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(values) != 4 {
+ t.Fatal("expected 4 keys")
+ }
+
+ if values["a1"] != "alice" {
+ t.Error("expected a1: alice")
+ }
+ if values["b1"] != "bob" {
+ t.Error("expected b1: bob")
+ }
+ if values["b2"] != "bob" {
+ t.Error("expected b2: bob")
+ }
+ if values["c1"] != "cathy" {
+ t.Error("expected c1: cathy")
+ }
+}
+
+func TestSearch(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nameIndex := createIndexer()
+
+ ids, err := nameIndex.Search(ctx, "bob")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ids) != 2 {
+ t.Fatal("wrong number of ids - expected 2 got", ids)
+ }
+ for _, id := range ids {
+ if id != "b1" && id != "b2" {
+ t.Fatal("wrong value in id set")
+ }
+ }
+ if ids[0] == ids[1] {
+ t.Fatal("duplicate id")
+ }
+
+ ids, err = nameIndex.Search(ctx, "cathy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ids) != 1 || ids[0] != "c1" {
+ t.Fatal("wrong ids")
+ }
+
+ ids, err = nameIndex.Search(ctx, "amit")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ids) != 0 {
+ t.Fatal("unexpected ids returned")
+ }
+}
+
+func TestDelete(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nameIndex := createIndexer()
+
+ err := nameIndex.Delete(ctx, "bob", "b3")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = nameIndex.Delete(ctx, "alice", "a1")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ok, err := nameIndex.HasValue(ctx, "alice", "a1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ok {
+ t.Fatal("index key should have been deleted")
+ }
+
+ count, err := nameIndex.DeleteKey(ctx, "bob")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if count != 2 {
+ t.Fatal("wrong deleted count")
+ }
+ ok, _ = nameIndex.HasValue(ctx, "bob", "b1")
+ if ok {
+ t.Fatal("index not deleted")
+ }
+}
+
+func TestSyncIndex(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nameIndex := createIndexer()
+
+ dstore := ds.NewMapDatastore()
+ refIndex := New(dstore, ds.NewKey("/ref"))
+ refIndex.Add(ctx, "alice", "a1")
+ refIndex.Add(ctx, "cathy", "zz")
+ refIndex.Add(ctx, "dennis", "d1")
+
+ changed, err := SyncIndex(ctx, refIndex, nameIndex)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !changed {
+ t.Error("change was not indicated")
+ }
+
+ // Create map of id->index in sync target
+ syncs := map[string]string{}
+ err = nameIndex.ForEach(ctx, "", func(key, value string) bool {
+ syncs[value] = key
+ return true
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Iterate items in sync source and make sure they appear in target
+ var itemCount int
+ err = refIndex.ForEach(ctx, "", func(key, value string) bool {
+ itemCount++
+ syncKey, ok := syncs[value]
+ if !ok || key != syncKey {
+ t.Fatal("key", key, "-->", value, "was not synced")
+ }
+ return true
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if itemCount != len(syncs) {
+ t.Fatal("different number of items in sync source and target")
+ }
+}
diff --git a/pinning/pinner/dspinner/pin.go b/pinning/pinner/dspinner/pin.go
new file mode 100644
index 000000000..7441ca65b
--- /dev/null
+++ b/pinning/pinner/dspinner/pin.go
@@ -0,0 +1,1031 @@
+// Package dspinner implements structures and methods to keep track of
+// which objects a user wants to keep stored locally. This implementation
+// stores pin data in a datastore.
+package dspinner
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "path"
+ "sync"
+
+ "github.com/ipfs/boxo/ipld/merkledag"
+ "github.com/ipfs/boxo/ipld/merkledag/dagutils"
+ "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ "github.com/ipfs/go-datastore/query"
+ ipld "github.com/ipfs/go-ipld-format"
+ logging "github.com/ipfs/go-log"
+ "github.com/polydawn/refmt/cbor"
+ "github.com/polydawn/refmt/obj/atlas"
+
+ ipfspinner "github.com/ipfs/boxo/pinning/pinner"
+ "github.com/ipfs/boxo/pinning/pinner/dsindex"
+)
+
+const (
+ basePath = "/pins"
+ pinKeyPath = "/pins/pin"
+ indexKeyPath = "/pins/index"
+ dirtyKeyPath = "/pins/state/dirty"
+)
+
+var (
+ log logging.StandardLogger = logging.Logger("pin")
+
+ linkDirect, linkRecursive string
+
+ pinCidDIndexPath string
+ pinCidRIndexPath string
+ pinNameIndexPath string
+
+ dirtyKey = ds.NewKey(dirtyKeyPath)
+
+ pinAtl atlas.Atlas
+)
+
+func init() {
+ directStr, ok := ipfspinner.ModeToString(ipfspinner.Direct)
+ if !ok {
+ panic("could not find Direct pin enum")
+ }
+ linkDirect = directStr
+
+ recursiveStr, ok := ipfspinner.ModeToString(ipfspinner.Recursive)
+ if !ok {
+ panic("could not find Recursive pin enum")
+ }
+ linkRecursive = recursiveStr
+
+ pinCidRIndexPath = path.Join(indexKeyPath, "cidRindex")
+ pinCidDIndexPath = path.Join(indexKeyPath, "cidDindex")
+ pinNameIndexPath = path.Join(indexKeyPath, "nameIndex")
+
+ pinAtl = atlas.MustBuild(
+ atlas.BuildEntry(pin{}).StructMap().
+ AddField("Cid", atlas.StructMapEntry{SerialName: "cid"}).
+ AddField("Metadata", atlas.StructMapEntry{SerialName: "metadata", OmitEmpty: true}).
+ AddField("Mode", atlas.StructMapEntry{SerialName: "mode"}).
+ AddField("Name", atlas.StructMapEntry{SerialName: "name", OmitEmpty: true}).
+ Complete(),
+ atlas.BuildEntry(cid.Cid{}).Transform().
+ TransformMarshal(atlas.MakeMarshalTransformFunc(func(live cid.Cid) ([]byte, error) { return live.MarshalBinary() })).
+ TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(func(serializable []byte) (cid.Cid, error) {
+ c := cid.Cid{}
+ err := c.UnmarshalBinary(serializable)
+ if err != nil {
+ return cid.Cid{}, err
+ }
+ return c, nil
+ })).Complete(),
+ )
+ pinAtl = pinAtl.WithMapMorphism(atlas.MapMorphism{KeySortMode: atlas.KeySortMode_Strings})
+}
+
+// pinner implements the Pinner interface
+type pinner struct {
+ autoSync bool
+ lock sync.RWMutex
+
+ dserv ipld.DAGService
+ dstore ds.Datastore
+
+ cidDIndex dsindex.Indexer
+ cidRIndex dsindex.Indexer
+ nameIndex dsindex.Indexer
+
+ clean int64
+ dirty int64
+}
+
+var _ ipfspinner.Pinner = (*pinner)(nil)
+
+type pin struct {
+ Id string
+ Cid cid.Cid
+ Metadata map[string]interface{}
+ Mode ipfspinner.Mode
+ Name string
+}
+
+func (p *pin) dsKey() ds.Key {
+ return ds.NewKey(path.Join(pinKeyPath, p.Id))
+}
+
+func newPin(c cid.Cid, mode ipfspinner.Mode, name string) *pin {
+ return &pin{
+ Id: path.Base(ds.RandomKey().String()),
+ Cid: c,
+ Name: name,
+ Mode: mode,
+ }
+}
+
+type syncDAGService interface {
+ ipld.DAGService
+ Sync() error
+}
+
+// New creates a new pinner and loads its keysets from the given datastore. If
+// there is no data present in the datastore, then an empty pinner is returned.
+//
+// By default, changes are automatically flushed to the datastore. This can be
+// disabled by calling SetAutosync(false), which will require that Flush be
+// called explicitly.
+func New(ctx context.Context, dstore ds.Datastore, dserv ipld.DAGService) (*pinner, error) {
+ p := &pinner{
+ autoSync: true,
+ cidDIndex: dsindex.New(dstore, ds.NewKey(pinCidDIndexPath)),
+ cidRIndex: dsindex.New(dstore, ds.NewKey(pinCidRIndexPath)),
+ nameIndex: dsindex.New(dstore, ds.NewKey(pinNameIndexPath)),
+ dserv: dserv,
+ dstore: dstore,
+ }
+
+ data, err := dstore.Get(ctx, dirtyKey)
+ if err != nil {
+ if err == ds.ErrNotFound {
+ return p, nil
+ }
+ return nil, fmt.Errorf("cannot load dirty flag: %v", err)
+ }
+ if data[0] == 1 {
+ p.dirty = 1
+
+ err = p.rebuildIndexes(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("cannot rebuild indexes: %v", err)
+ }
+ }
+
+ return p, nil
+}
+
+// SetAutosync allows auto-syncing to be enabled or disabled during runtime.
+// This may be used to turn off autosync before doing many repeated pinning
+// operations, and then turn it on after. Returns the previous value.
+func (p *pinner) SetAutosync(auto bool) bool {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ p.autoSync, auto = auto, p.autoSync
+ return auto
+}
+
+// Pin the given node, optionally recursive
+func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error {
+ err := p.dserv.Add(ctx, node)
+ if err != nil {
+ return err
+ }
+
+ if recurse {
+ return p.doPinRecursive(ctx, node.Cid(), true)
+ } else {
+ return p.doPinDirect(ctx, node.Cid())
+ }
+}
+
+func (p *pinner) doPinRecursive(ctx context.Context, c cid.Cid, fetch bool) error {
+ cidKey := c.KeyString()
+
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ found, err := p.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+ if found {
+ return nil
+ }
+
+ dirtyBefore := p.dirty
+
+ if fetch {
+ // temporary unlock to fetch the entire graph
+ p.lock.Unlock()
+ // Fetch graph starting at node identified by cid
+ err = merkledag.FetchGraph(ctx, c, p.dserv)
+ p.lock.Lock()
+ if err != nil {
+ return err
+ }
+ }
+
+ // If autosyncing, sync dag service before making any change to pins
+ err = p.flushDagService(ctx, false)
+ if err != nil {
+ return err
+ }
+
+ // Only look again if something has changed.
+ if p.dirty != dirtyBefore {
+ found, err = p.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+ if found {
+ return nil
+ }
+ }
+
+ // TODO: remove this to support multiple pins per CID
+ found, err = p.cidDIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+ if found {
+ _, err = p.removePinsForCid(ctx, c, ipfspinner.Direct)
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = p.addPin(ctx, c, ipfspinner.Recursive, "")
+ if err != nil {
+ return err
+ }
+ return p.flushPins(ctx, false)
+}
+
+func (p *pinner) doPinDirect(ctx context.Context, c cid.Cid) error {
+ cidKey := c.KeyString()
+
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ found, err := p.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+ if found {
+ return fmt.Errorf("%s already pinned recursively", c.String())
+ }
+
+ _, err = p.addPin(ctx, c, ipfspinner.Direct, "")
+ if err != nil {
+ return err
+ }
+
+ return p.flushPins(ctx, false)
+}
+
+func (p *pinner) addPin(ctx context.Context, c cid.Cid, mode ipfspinner.Mode, name string) (string, error) {
+ // Create new pin and store in datastore
+ pp := newPin(c, mode, name)
+
+ // Serialize pin
+ pinData, err := encodePin(pp)
+ if err != nil {
+ return "", fmt.Errorf("could not encode pin: %v", err)
+ }
+
+ p.setDirty(ctx)
+
+ // Store the pin
+ err = p.dstore.Put(ctx, pp.dsKey(), pinData)
+ if err != nil {
+ return "", err
+ }
+
+ // Store CID index
+ switch mode {
+ case ipfspinner.Recursive:
+ err = p.cidRIndex.Add(ctx, c.KeyString(), pp.Id)
+ case ipfspinner.Direct:
+ err = p.cidDIndex.Add(ctx, c.KeyString(), pp.Id)
+ default:
+ panic("pin mode must be recursive or direct")
+ }
+ if err != nil {
+ return "", fmt.Errorf("could not add pin cid index: %v", err)
+ }
+
+ if name != "" {
+ // Store name index
+ err = p.nameIndex.Add(ctx, name, pp.Id)
+ if err != nil {
+ if mode == ipfspinner.Recursive {
+ e := p.cidRIndex.Delete(ctx, c.KeyString(), pp.Id)
+ if e != nil {
+ log.Errorf("error deleting index: %s", e)
+ }
+ } else {
+ e := p.cidDIndex.Delete(ctx, c.KeyString(), pp.Id)
+ if e != nil {
+ log.Errorf("error deleting index: %s", e)
+ }
+ }
+ return "", fmt.Errorf("could not add pin name index: %v", err)
+ }
+ }
+
+ return pp.Id, nil
+}
+
+func (p *pinner) removePin(ctx context.Context, pp *pin) error {
+ p.setDirty(ctx)
+ var err error
+
+ // Remove cid index from datastore
+ if pp.Mode == ipfspinner.Recursive {
+ err = p.cidRIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id)
+ } else {
+ err = p.cidDIndex.Delete(ctx, pp.Cid.KeyString(), pp.Id)
+ }
+ if err != nil {
+ return err
+ }
+
+ if pp.Name != "" {
+ // Remove name index from datastore
+ err = p.nameIndex.Delete(ctx, pp.Name, pp.Id)
+ if err != nil {
+ return err
+ }
+ }
+
+ // The pin is removed last so that an incomplete remove is detected by a
+ // pin that has a missing index.
+ err = p.dstore.Delete(ctx, pp.dsKey())
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// Unpin a given key
+func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error {
+ cidKey := c.KeyString()
+
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ // TODO: use Ls() to lookup pins when new pinning API available
+ /*
+ matchSpec := map[string][]string {
+ "cid": []string{c.String}
+ }
+ matches := p.Ls(matchSpec)
+ */
+ has, err := p.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+
+ if has {
+ if !recursive {
+ return fmt.Errorf("%s is pinned recursively", c.String())
+ }
+ } else {
+ has, err = p.cidDIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+ if !has {
+ return ipfspinner.ErrNotPinned
+ }
+ }
+
+ removed, err := p.removePinsForCid(ctx, c, ipfspinner.Any)
+ if err != nil {
+ return err
+ }
+ if !removed {
+ return nil
+ }
+
+ return p.flushPins(ctx, false)
+}
+
+// IsPinned returns whether or not the given key is pinned
+// and an explanation of why its pinned
+func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+ return p.isPinnedWithType(ctx, c, ipfspinner.Any)
+}
+
+// IsPinnedWithType returns whether or not the given cid is pinned with the
+// given pin type, as well as returning the type of pin its pinned with.
+func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+ return p.isPinnedWithType(ctx, c, mode)
+}
+
+func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (string, bool, error) {
+ cidKey := c.KeyString()
+ switch mode {
+ case ipfspinner.Recursive:
+ has, err := p.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return "", false, err
+ }
+ if has {
+ return linkRecursive, true, nil
+ }
+ return "", false, nil
+ case ipfspinner.Direct:
+ has, err := p.cidDIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return "", false, err
+ }
+ if has {
+ return linkDirect, true, nil
+ }
+ return "", false, nil
+ case ipfspinner.Internal:
+ return "", false, nil
+ case ipfspinner.Indirect:
+ case ipfspinner.Any:
+ has, err := p.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return "", false, err
+ }
+ if has {
+ return linkRecursive, true, nil
+ }
+ has, err = p.cidDIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return "", false, err
+ }
+ if has {
+ return linkDirect, true, nil
+ }
+ default:
+ err := fmt.Errorf(
+ "invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}",
+ mode, ipfspinner.Direct, ipfspinner.Indirect, ipfspinner.Recursive,
+ ipfspinner.Internal, ipfspinner.Any)
+ return "", false, err
+ }
+
+ // Default is Indirect
+ visitedSet := cid.NewSet()
+
+ // No index for given CID, so search children of all recursive pinned CIDs
+ var has bool
+ var rc cid.Cid
+ var e error
+ err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool {
+ rc, e = cid.Cast([]byte(key))
+ if e != nil {
+ return false
+ }
+ has, e = hasChild(ctx, p.dserv, rc, c, visitedSet.Visit)
+ if e != nil {
+ return false
+ }
+ if has {
+ return false
+ }
+ return true
+ })
+ if err != nil {
+ return "", false, err
+ }
+ if e != nil {
+ return "", false, e
+ }
+
+ if has {
+ return rc.String(), true, nil
+ }
+
+ return "", false, nil
+}
+
+// CheckIfPinned checks if a set of keys are pinned, more efficient than
+// calling IsPinned for each key, returns the pinned status of cid(s)
+//
+// TODO: If a CID is pinned by multiple pins, should they all be reported?
+func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]ipfspinner.Pinned, error) {
+ pinned := make([]ipfspinner.Pinned, 0, len(cids))
+ toCheck := cid.NewSet()
+
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+
+ // First check for non-Indirect pins directly
+ for _, c := range cids {
+ cidKey := c.KeyString()
+ has, err := p.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return nil, err
+ }
+ if has {
+ pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Recursive})
+ } else {
+ has, err = p.cidDIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return nil, err
+ }
+ if has {
+ pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Direct})
+ } else {
+ toCheck.Add(c)
+ }
+ }
+ }
+
+ var e error
+ visited := cid.NewSet()
+ err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool {
+ var rk cid.Cid
+ rk, e = cid.Cast([]byte(key))
+ if e != nil {
+ return false
+ }
+ e = merkledag.Walk(ctx, merkledag.GetLinksWithDAG(p.dserv), rk, func(c cid.Cid) bool {
+ if toCheck.Len() == 0 || !visited.Visit(c) {
+ return false
+ }
+
+ if toCheck.Has(c) {
+ pinned = append(pinned, ipfspinner.Pinned{Key: c, Mode: ipfspinner.Indirect, Via: rk})
+ toCheck.Remove(c)
+ }
+
+ return true
+ }, merkledag.Concurrent())
+ if e != nil {
+ return false
+ }
+ return toCheck.Len() > 0
+ })
+ if err != nil {
+ return nil, err
+ }
+ if e != nil {
+ return nil, e
+ }
+
+ // Anything left in toCheck is not pinned
+ for _, k := range toCheck.Keys() {
+ pinned = append(pinned, ipfspinner.Pinned{Key: k, Mode: ipfspinner.NotPinned})
+ }
+
+ return pinned, nil
+}
+
+// removePinsForCid removes all pins for a cid that has the specified mode.
+// Returns true if any pins, and all corresponding CID index entries, were
+// removed. Otherwise, returns false.
+func (p *pinner) removePinsForCid(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) (bool, error) {
+ // Search for pins by CID
+ var ids []string
+ var err error
+ cidKey := c.KeyString()
+ switch mode {
+ case ipfspinner.Recursive:
+ ids, err = p.cidRIndex.Search(ctx, cidKey)
+ case ipfspinner.Direct:
+ ids, err = p.cidDIndex.Search(ctx, cidKey)
+ case ipfspinner.Any:
+ ids, err = p.cidRIndex.Search(ctx, cidKey)
+ if err != nil {
+ return false, err
+ }
+ dIds, err := p.cidDIndex.Search(ctx, cidKey)
+ if err != nil {
+ return false, err
+ }
+ if len(dIds) != 0 {
+ ids = append(ids, dIds...)
+ }
+ }
+ if err != nil {
+ return false, err
+ }
+
+ var removed bool
+
+ // Remove the pin with the requested mode
+ for _, pid := range ids {
+ var pp *pin
+ pp, err = p.loadPin(ctx, pid)
+ if err != nil {
+ if err == ds.ErrNotFound {
+ p.setDirty(ctx)
+ // Fix index; remove index for pin that does not exist
+ switch mode {
+ case ipfspinner.Recursive:
+ _, err = p.cidRIndex.DeleteKey(ctx, cidKey)
+ if err != nil {
+ return false, fmt.Errorf("error deleting index: %s", err)
+ }
+ case ipfspinner.Direct:
+ _, err = p.cidDIndex.DeleteKey(ctx, cidKey)
+ if err != nil {
+ return false, fmt.Errorf("error deleting index: %s", err)
+ }
+ case ipfspinner.Any:
+ _, err = p.cidRIndex.DeleteKey(ctx, cidKey)
+ if err != nil {
+ return false, fmt.Errorf("error deleting index: %s", err)
+ }
+ _, err = p.cidDIndex.DeleteKey(ctx, cidKey)
+ if err != nil {
+ return false, fmt.Errorf("error deleting index: %s", err)
+ }
+ }
+ if err = p.flushPins(ctx, true); err != nil {
+ return false, err
+ }
+ // Mark this as removed since it removed an index, which is
+ // what prevents determines if an item is pinned.
+ removed = true
+ log.Error("found CID index with missing pin")
+ continue
+ }
+ return false, err
+ }
+ if mode == ipfspinner.Any || pp.Mode == mode {
+ err = p.removePin(ctx, pp)
+ if err != nil {
+ return false, err
+ }
+ removed = true
+ }
+ }
+ return removed, nil
+}
+
+// loadPin loads a single pin from the datastore.
+func (p *pinner) loadPin(ctx context.Context, pid string) (*pin, error) {
+ pinData, err := p.dstore.Get(ctx, ds.NewKey(path.Join(pinKeyPath, pid)))
+ if err != nil {
+ return nil, err
+ }
+ return decodePin(pid, pinData)
+}
+
+// DirectKeys returns a slice containing the directly pinned keys
+func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+
+ cidSet := cid.NewSet()
+ var e error
+ err := p.cidDIndex.ForEach(ctx, "", func(key, value string) bool {
+ var c cid.Cid
+ c, e = cid.Cast([]byte(key))
+ if e != nil {
+ return false
+ }
+ cidSet.Add(c)
+ return true
+ })
+ if err != nil {
+ return nil, err
+ }
+ if e != nil {
+ return nil, e
+ }
+
+ return cidSet.Keys(), nil
+}
+
+// RecursiveKeys returns a slice containing the recursively pinned keys
+func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+
+ cidSet := cid.NewSet()
+ var e error
+ err := p.cidRIndex.ForEach(ctx, "", func(key, value string) bool {
+ var c cid.Cid
+ c, e = cid.Cast([]byte(key))
+ if e != nil {
+ return false
+ }
+ cidSet.Add(c)
+ return true
+ })
+ if err != nil {
+ return nil, err
+ }
+ if e != nil {
+ return nil, e
+ }
+
+ return cidSet.Keys(), nil
+}
+
+// InternalPins returns all cids kept pinned for the internal state of the
+// pinner
+func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) {
+ return nil, nil
+}
+
+// Update updates a recursive pin from one cid to another. This is equivalent
+// to pinning the new one and unpinning the old one.
+//
+// TODO: This will not work when multiple pins are supported
+func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ found, err := p.cidRIndex.HasAny(ctx, from.KeyString())
+ if err != nil {
+ return err
+ }
+ if !found {
+ return errors.New("'from' cid was not recursively pinned already")
+ }
+
+ // If `from` already recursively pinned and `to` is the same, then all done
+ if from == to {
+ return nil
+ }
+
+ // Check if the `to` cid is already recursively pinned
+ found, err = p.cidRIndex.HasAny(ctx, to.KeyString())
+ if err != nil {
+ return err
+ }
+ if found {
+ return errors.New("'to' cid was already recursively pinned")
+ }
+
+ // Temporarily unlock while we fetch the differences.
+ p.lock.Unlock()
+ err = dagutils.DiffEnumerate(ctx, p.dserv, from, to)
+ p.lock.Lock()
+
+ if err != nil {
+ return err
+ }
+
+ _, err = p.addPin(ctx, to, ipfspinner.Recursive, "")
+ if err != nil {
+ return err
+ }
+
+ if unpin {
+ _, err = p.removePinsForCid(ctx, from, ipfspinner.Recursive)
+ if err != nil {
+ return err
+ }
+ }
+
+ return p.flushPins(ctx, false)
+}
+
+func (p *pinner) flushDagService(ctx context.Context, force bool) error {
+ if !p.autoSync && !force {
+ return nil
+ }
+ if syncDServ, ok := p.dserv.(syncDAGService); ok {
+ if err := syncDServ.Sync(); err != nil {
+ return fmt.Errorf("cannot sync pinned data: %v", err)
+ }
+ }
+ return nil
+}
+
+func (p *pinner) flushPins(ctx context.Context, force bool) error {
+ if !p.autoSync && !force {
+ return nil
+ }
+ if err := p.dstore.Sync(ctx, ds.NewKey(basePath)); err != nil {
+ return fmt.Errorf("cannot sync pin state: %v", err)
+ }
+ p.setClean(ctx)
+ return nil
+}
+
+// Flush encodes and writes pinner keysets to the datastore
+func (p *pinner) Flush(ctx context.Context) error {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ err := p.flushDagService(ctx, true)
+ if err != nil {
+ return err
+ }
+
+ return p.flushPins(ctx, true)
+}
+
+// PinWithMode allows the user to have fine grained control over pin
+// counts
+func (p *pinner) PinWithMode(ctx context.Context, c cid.Cid, mode ipfspinner.Mode) error {
+ // TODO: remove his to support multiple pins per CID
+ switch mode {
+ case ipfspinner.Recursive:
+ return p.doPinRecursive(ctx, c, false)
+ case ipfspinner.Direct:
+ return p.doPinDirect(ctx, c)
+ default:
+ return fmt.Errorf("unrecognized pin mode")
+ }
+}
+
+// hasChild recursively looks for a Cid among the children of a root Cid.
+// The visit function can be used to shortcut already-visited branches.
+func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) {
+ links, err := ipld.GetLinks(ctx, ng, root)
+ if err != nil {
+ return false, err
+ }
+ for _, lnk := range links {
+ c := lnk.Cid
+ if lnk.Cid.Equals(child) {
+ return true, nil
+ }
+ if visit(c) {
+ has, err := hasChild(ctx, ng, c, child, visit)
+ if err != nil {
+ return false, err
+ }
+
+ if has {
+ return has, nil
+ }
+ }
+ }
+ return false, nil
+}
+
+func encodePin(p *pin) ([]byte, error) {
+ b, err := cbor.MarshalAtlased(p, pinAtl)
+ if err != nil {
+ return nil, err
+ }
+ return b, nil
+}
+
+func decodePin(pid string, data []byte) (*pin, error) {
+ p := &pin{Id: pid}
+ err := cbor.UnmarshalAtlased(cbor.DecodeOptions{}, data, p, pinAtl)
+ if err != nil {
+ return nil, err
+ }
+ return p, nil
+}
+
+// setDirty updates the dirty counter and saves a dirty state in the datastore
+// if the state was previously clean
+func (p *pinner) setDirty(ctx context.Context) {
+ wasClean := p.dirty == p.clean
+ p.dirty++
+
+ if !wasClean {
+ return // do not save; was already dirty
+ }
+
+ data := []byte{1}
+ err := p.dstore.Put(ctx, dirtyKey, data)
+ if err != nil {
+ log.Errorf("failed to set pin dirty flag: %s", err)
+ return
+ }
+ err = p.dstore.Sync(ctx, dirtyKey)
+ if err != nil {
+ log.Errorf("failed to sync pin dirty flag: %s", err)
+ }
+}
+
+// setClean saves a clean state value in the datastore if the state was
+// previously dirty
+func (p *pinner) setClean(ctx context.Context) {
+ if p.dirty == p.clean {
+ return // already clean
+ }
+
+ data := []byte{0}
+ err := p.dstore.Put(ctx, dirtyKey, data)
+ if err != nil {
+ log.Errorf("failed to set clear dirty flag: %s", err)
+ return
+ }
+ if err = p.dstore.Sync(ctx, dirtyKey); err != nil {
+ log.Errorf("failed to sync cleared pin dirty flag: %s", err)
+ return
+ }
+ p.clean = p.dirty // set clean
+}
+
+// sync datastore after every 50 cid repairs
+const syncRepairFrequency = 50
+
+// rebuildIndexes uses the stored pins to rebuild secondary indexes. This
+// resolves any discrepancy between secondary indexes and pins that could
+// result from a program termination between saving the two.
+func (p *pinner) rebuildIndexes(ctx context.Context) error {
+ // Load all pins from the datastore.
+ q := query.Query{
+ Prefix: pinKeyPath,
+ }
+ results, err := p.dstore.Query(ctx, q)
+ if err != nil {
+ return err
+ }
+ defer results.Close()
+
+ var checkedCount, repairedCount int
+
+ // Iterate all pins and check if the corresponding recursive or direct
+ // index is missing. If the index is missing then create the index.
+ for r := range results.Next() {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if r.Error != nil {
+ return fmt.Errorf("cannot read index: %v", r.Error)
+ }
+ ent := r.Entry
+ pp, err := decodePin(path.Base(ent.Key), ent.Value)
+ if err != nil {
+ return err
+ }
+
+ indexKey := pp.Cid.KeyString()
+
+ var indexer, staleIndexer dsindex.Indexer
+ var idxrName, staleIdxrName string
+ if pp.Mode == ipfspinner.Recursive {
+ indexer = p.cidRIndex
+ staleIndexer = p.cidDIndex
+ idxrName = linkRecursive
+ staleIdxrName = linkDirect
+ } else if pp.Mode == ipfspinner.Direct {
+ indexer = p.cidDIndex
+ staleIndexer = p.cidRIndex
+ idxrName = linkDirect
+ staleIdxrName = linkRecursive
+ } else {
+ log.Error("unrecognized pin mode:", pp.Mode)
+ continue
+ }
+
+ // Remove any stale index from unused indexer
+ ok, err := staleIndexer.HasValue(ctx, indexKey, pp.Id)
+ if err != nil {
+ return err
+ }
+ if ok {
+ // Delete any stale index
+ log.Errorf("deleting stale %s pin index for cid %v", staleIdxrName, pp.Cid.String())
+ if err = staleIndexer.Delete(ctx, indexKey, pp.Id); err != nil {
+ return err
+ }
+ }
+
+ // Check that the indexer indexes this pin
+ ok, err = indexer.HasValue(ctx, indexKey, pp.Id)
+ if err != nil {
+ return err
+ }
+
+ var repaired bool
+ if !ok {
+ // Do not rebuild if index has an old value with leading slash
+ ok, err = indexer.HasValue(ctx, indexKey, "/"+pp.Id)
+ if err != nil {
+ return err
+ }
+ if !ok {
+ log.Errorf("repairing %s pin index for cid: %s", idxrName, pp.Cid.String())
+ // There was no index found for this pin. This was either an
+ // incomplete add or and incomplete delete of a pin. Either
+ // way, restore the index to complete the add or to undo the
+ // incomplete delete.
+ if err = indexer.Add(ctx, indexKey, pp.Id); err != nil {
+ return err
+ }
+ repaired = true
+ }
+ }
+ // Check for missing name index
+ if pp.Name != "" {
+ ok, err = p.nameIndex.HasValue(ctx, pp.Name, pp.Id)
+ if err != nil {
+ return err
+ }
+ if !ok {
+ log.Errorf("repairing name pin index for cid: %s", pp.Cid.String())
+ if err = p.nameIndex.Add(ctx, pp.Name, pp.Id); err != nil {
+ return err
+ }
+ }
+ repaired = true
+ }
+
+ if repaired {
+ repairedCount++
+ }
+ checkedCount++
+ if checkedCount%syncRepairFrequency == 0 {
+ p.flushPins(ctx, true)
+ }
+ }
+
+ log.Errorf("checked %d pins for invalid indexes, repaired %d pins", checkedCount, repairedCount)
+ return p.flushPins(ctx, true)
+}
diff --git a/pinning/pinner/dspinner/pin_test.go b/pinning/pinner/dspinner/pin_test.go
new file mode 100644
index 000000000..edf315fcb
--- /dev/null
+++ b/pinning/pinner/dspinner/pin_test.go
@@ -0,0 +1,1347 @@
+package dspinner
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "testing"
+ "time"
+
+ bs "github.com/ipfs/boxo/blockservice"
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+
+ blockstore "github.com/ipfs/boxo/blockstore"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ util "github.com/ipfs/boxo/util"
+ cid "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ "github.com/ipfs/go-datastore/query"
+ dssync "github.com/ipfs/go-datastore/sync"
+ lds "github.com/ipfs/go-ds-leveldb"
+ ipld "github.com/ipfs/go-ipld-format"
+ logging "github.com/ipfs/go-log"
+
+ ipfspin "github.com/ipfs/boxo/pinning/pinner"
+)
+
+var rand = util.NewTimeSeededRand()
+
+type fakeLogger struct {
+ logging.StandardLogger
+ lastError error
+}
+
+func (f *fakeLogger) Error(args ...interface{}) {
+ f.lastError = errors.New(fmt.Sprint(args...))
+}
+
+func (f *fakeLogger) Errorf(format string, args ...interface{}) {
+ f.lastError = fmt.Errorf(format, args...)
+}
+
+func randNode() (*mdag.ProtoNode, cid.Cid) {
+ nd := new(mdag.ProtoNode)
+ nd.SetData(make([]byte, 32))
+ _, err := io.ReadFull(rand, nd.Data())
+ if err != nil {
+ panic(err)
+ }
+ k := nd.Cid()
+ return nd, k
+}
+
+func assertPinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) {
+ _, pinned, err := p.IsPinned(context.Background(), c)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !pinned {
+ t.Fatal(failmsg)
+ }
+}
+
+func assertPinnedWithType(t *testing.T, p ipfspin.Pinner, c cid.Cid, mode ipfspin.Mode, failmsg string) {
+ modeText, pinned, err := p.IsPinnedWithType(context.Background(), c, mode)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ expect, ok := ipfspin.ModeToString(mode)
+ if !ok {
+ t.Fatal("unrecognized pin mode")
+ }
+
+ if !pinned {
+ t.Fatal(failmsg)
+ }
+
+ if mode == ipfspin.Any {
+ return
+ }
+
+ if expect != modeText {
+ t.Fatal("expected", expect, "pin, got", modeText)
+ }
+}
+
+func assertUnpinned(t *testing.T, p ipfspin.Pinner, c cid.Cid, failmsg string) {
+ _, pinned, err := p.IsPinned(context.Background(), c)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if pinned {
+ t.Fatal(failmsg)
+ }
+}
+
+func TestPinnerBasic(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+
+ dserv := mdag.NewDAGService(bserv)
+
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ a, ak := randNode()
+ err = dserv.Add(ctx, a)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Pin A{}
+ err = p.Pin(ctx, a, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assertPinned(t, p, ak, "Failed to find key")
+ assertPinnedWithType(t, p, ak, ipfspin.Direct, "Expected direct pin")
+
+ // create new node c, to be indirectly pinned through b
+ c, _ := randNode()
+ err = dserv.Add(ctx, c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ck := c.Cid()
+
+ // Create new node b, to be parent to a and c
+ b, _ := randNode()
+ err = b.AddNodeLink("child", a)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = b.AddNodeLink("otherchild", c)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = dserv.Add(ctx, b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ bk := b.Cid()
+
+ // recursively pin B{A,C}
+ err = p.Pin(ctx, b, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assertPinned(t, p, ck, "child of recursively pinned node not found")
+
+ assertPinned(t, p, bk, "Pinned node not found")
+ assertPinnedWithType(t, p, bk, ipfspin.Recursive, "Recursively pinned node not found")
+
+ d, _ := randNode()
+ err = d.AddNodeLink("a", a)
+ if err != nil {
+ panic(err)
+ }
+ err = d.AddNodeLink("c", c)
+ if err != nil {
+ panic(err)
+ }
+
+ e, _ := randNode()
+ err = d.AddNodeLink("e", e)
+ if err != nil {
+ panic(err)
+ }
+
+ // Must be in dagserv for unpin to work
+ err = dserv.Add(ctx, e)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = dserv.Add(ctx, d)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Add D{A,C,E}
+ err = p.Pin(ctx, d, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ dk := d.Cid()
+ assertPinned(t, p, dk, "pinned node not found.")
+
+ cids, err := p.RecursiveKeys(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(cids) != 2 {
+ t.Error("expected 2 recursive pins")
+ }
+ if !(bk == cids[0] || bk == cids[1]) {
+ t.Error("expected recursive pin of B")
+ }
+ if !(dk == cids[0] || dk == cids[1]) {
+ t.Error("expected recursive pin of D")
+ }
+
+ pinned, err := p.CheckIfPinned(ctx, ak, bk, ck, dk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(pinned) != 4 {
+ t.Error("incorrect number of results")
+ }
+ for _, pn := range pinned {
+ switch pn.Key {
+ case ak:
+ if pn.Mode != ipfspin.Direct {
+ t.Error("A pinned with wrong mode")
+ }
+ case bk:
+ if pn.Mode != ipfspin.Recursive {
+ t.Error("B pinned with wrong mode")
+ }
+ case ck:
+ if pn.Mode != ipfspin.Indirect {
+ t.Error("C should be pinned indirectly")
+ }
+ if pn.Via != dk && pn.Via != bk {
+ t.Error("C should be pinned via D or B")
+ }
+ case dk:
+ if pn.Mode != ipfspin.Recursive {
+ t.Error("D pinned with wrong mode")
+ }
+ }
+ }
+
+ cids, err = p.DirectKeys(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(cids) != 1 {
+ t.Error("expected 1 direct pin")
+ }
+ if cids[0] != ak {
+ t.Error("wrong direct pin")
+ }
+
+ cids, _ = p.InternalPins(ctx)
+ if len(cids) != 0 {
+ t.Error("shound not have internal keys")
+ }
+
+ err = p.Unpin(ctx, dk, false)
+ if err == nil {
+ t.Fatal("expected error unpinning recursive pin without specifying recursive")
+ }
+
+ // Test recursive unpin
+ err = p.Unpin(ctx, dk, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = p.Unpin(ctx, dk, true)
+ if err != ipfspin.ErrNotPinned {
+ t.Fatal("expected error:", ipfspin.ErrNotPinned)
+ }
+
+ err = p.Flush(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ p, err = New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Test directly pinned
+ assertPinned(t, p, ak, "Could not find pinned node!")
+
+ // Test recursively pinned
+ assertPinned(t, p, bk, "could not find recursively pinned node")
+
+ // Remove the pin but not the index to simulate corruption
+ ids, err := p.cidDIndex.Search(ctx, ak.KeyString())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ids) == 0 {
+ t.Fatal("did not find pin for cid", ak.String())
+ }
+ pp, err := p.loadPin(ctx, ids[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if pp.Mode != ipfspin.Direct {
+ t.Error("loaded pin has wrong mode")
+ }
+ if pp.Cid != ak {
+ t.Error("loaded pin has wrong cid")
+ }
+ err = p.dstore.Delete(ctx, pp.dsKey())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ realLog := log
+ fakeLog := &fakeLogger{}
+ fakeLog.StandardLogger = log
+ log = fakeLog
+ err = p.Pin(ctx, a, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fakeLog.lastError == nil {
+ t.Error("expected error to be logged")
+ } else if fakeLog.lastError.Error() != "found CID index with missing pin" {
+ t.Error("did not get expected log message")
+ }
+
+ log = realLog
+}
+
+func TestAddLoadPin(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+
+ dserv := mdag.NewDAGService(bserv)
+
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ a, ak := randNode()
+ err = dserv.Add(ctx, a)
+ if err != nil {
+ panic(err)
+ }
+
+ mode := ipfspin.Recursive
+ name := "my-pin"
+ pid, err := p.addPin(ctx, ak, mode, name)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Load pin and check that data decoded correctly
+ pinData, err := p.loadPin(ctx, pid)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if pinData.Mode != mode {
+ t.Error("worng pin mode")
+ }
+ if pinData.Cid != ak {
+ t.Error("wrong pin cid")
+ }
+ if pinData.Name != name {
+ t.Error("wrong pin name; expected", name, "got", pinData.Name)
+ }
+}
+
+func TestIsPinnedLookup(t *testing.T) {
+ // Test that lookups work in pins which share
+ // the same branches. For that construct this tree:
+ //
+ // A5->A4->A3->A2->A1->A0
+ // / /
+ // B------- /
+ // \ /
+ // C---------------
+ //
+ // This ensures that IsPinned works for all objects both when they
+ // are pinned and once they have been unpinned.
+ aBranchLen := 6
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+
+ dserv := mdag.NewDAGService(bserv)
+
+ // Create new pinner. New will not load anything since there are
+ // no pins saved in the datastore yet.
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ aKeys, bk, ck, err := makeTree(ctx, aBranchLen, dserv, p)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assertPinned(t, p, aKeys[0], "A0 should be pinned")
+ assertPinned(t, p, aKeys[1], "A1 should be pinned")
+ assertPinned(t, p, ck, "C should be pinned")
+ assertPinned(t, p, bk, "B should be pinned")
+
+ // Unpin A5 recursively
+ if err = p.Unpin(ctx, aKeys[5], true); err != nil {
+ t.Fatal(err)
+ }
+
+ assertPinned(t, p, aKeys[0], "A0 should still be pinned through B")
+ assertUnpinned(t, p, aKeys[4], "A4 should be unpinned")
+
+ // Unpin B recursively
+ if err = p.Unpin(ctx, bk, true); err != nil {
+ t.Fatal(err)
+ }
+ assertUnpinned(t, p, bk, "B should be unpinned")
+ assertUnpinned(t, p, aKeys[1], "A1 should be unpinned")
+ assertPinned(t, p, aKeys[0], "A0 should still be pinned through C")
+}
+
+func TestDuplicateSemantics(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+
+ dserv := mdag.NewDAGService(bserv)
+
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ a, _ := randNode()
+ err = dserv.Add(ctx, a)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // pin is recursively
+ err = p.Pin(ctx, a, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // pinning directly should fail
+ err = p.Pin(ctx, a, false)
+ if err == nil {
+ t.Fatal("expected direct pin to fail")
+ }
+
+ // pinning recursively again should succeed
+ err = p.Pin(ctx, a, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestFlush(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+
+ dserv := mdag.NewDAGService(bserv)
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, k := randNode()
+
+ p.PinWithMode(ctx, k, ipfspin.Recursive)
+ if err = p.Flush(ctx); err != nil {
+ t.Fatal(err)
+ }
+ assertPinned(t, p, k, "expected key to still be pinned")
+}
+
+func TestPinRecursiveFail(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+ dserv := mdag.NewDAGService(bserv)
+
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ a, _ := randNode()
+ b, _ := randNode()
+ err = a.AddNodeLink("child", b)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // NOTE: This isnt a time based test, we expect the pin to fail
+ mctx, cancel := context.WithTimeout(ctx, time.Millisecond)
+ defer cancel()
+
+ err = p.Pin(mctx, a, true)
+ if err == nil {
+ t.Fatal("should have failed to pin here")
+ }
+
+ err = dserv.Add(ctx, b)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = dserv.Add(ctx, a)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // this one is time based... but shouldnt cause any issues
+ mctx, cancel = context.WithTimeout(ctx, time.Second)
+ defer cancel()
+ err = p.Pin(mctx, a, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPinUpdate(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+
+ dserv := mdag.NewDAGService(bserv)
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n1, c1 := randNode()
+ n2, c2 := randNode()
+ _, c3 := randNode()
+
+ if err = dserv.Add(ctx, n1); err != nil {
+ t.Fatal(err)
+ }
+ if err = dserv.Add(ctx, n2); err != nil {
+ t.Fatal(err)
+ }
+
+ if err = p.Pin(ctx, n1, true); err != nil {
+ t.Fatal(err)
+ }
+
+ if err = p.Update(ctx, c1, c2, true); err != nil {
+ t.Fatal(err)
+ }
+
+ assertPinned(t, p, c2, "c2 should be pinned now")
+ assertUnpinned(t, p, c1, "c1 should no longer be pinned")
+
+ if err = p.Update(ctx, c2, c1, false); err != nil {
+ t.Fatal(err)
+ }
+
+ // Test updating same pin that is already pinned.
+ if err = p.Update(ctx, c2, c2, true); err != nil {
+ t.Fatal(err)
+ }
+ // Check that pin is still pinned.
+ _, ok, err := p.IsPinned(ctx, c2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ok {
+ t.Fatal("c2 should still be pinned")
+ }
+
+ // Test updating same pin that is not pinned.
+ if err = p.Update(ctx, c3, c3, false); err == nil {
+ t.Fatal("expected error updating unpinned cid")
+ }
+ _, ok, err = p.IsPinned(ctx, c3)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ok {
+ t.Fatal("c3 should not be pinned")
+ }
+
+ assertPinned(t, p, c2, "c2 should be pinned still")
+ assertPinned(t, p, c1, "c1 should be pinned now")
+}
+
+func TestLoadDirty(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore := dssync.MutexWrap(ds.NewMapDatastore())
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+ dserv := mdag.NewDAGService(bserv)
+
+ p, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ prev := p.SetAutosync(false)
+ if !prev {
+ t.Fatal("expected previous autosync to be true")
+ }
+ prev = p.SetAutosync(false)
+ if prev {
+ t.Fatal("expected previous autosync to be false")
+ }
+ prev = p.SetAutosync(true)
+ if prev {
+ t.Fatal("expected previous autosync to be false")
+ }
+
+ a, ak := randNode()
+ err = dserv.Add(ctx, a)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, bk := randNode()
+
+ err = p.Pin(ctx, a, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cidAKey := ak.KeyString()
+ cidBKey := bk.KeyString()
+
+ // Corrupt index
+ cidRIndex := p.cidRIndex
+ _, err = cidRIndex.DeleteKey(ctx, cidAKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = cidRIndex.Add(ctx, cidBKey, "not-a-pin-id")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Force dirty, since Pin syncs automatically
+ p.setDirty(ctx)
+
+ // Verify dirty
+ data, err := dstore.Get(ctx, dirtyKey)
+ if err != nil {
+ t.Fatalf("could not read dirty flag: %v", err)
+ }
+ if data[0] != 1 {
+ t.Fatal("dirty flag not set")
+ }
+
+ has, err := cidRIndex.HasAny(ctx, cidAKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if has {
+ t.Fatal("index should be deleted")
+ }
+
+ // Create new pinner on same datastore that was never flushed. This should
+ // detect the dirty flag and repair the indexes.
+ p, err = New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify not dirty
+ data, err = dstore.Get(ctx, dirtyKey)
+ if err != nil {
+ t.Fatalf("could not read dirty flag: %v", err)
+ }
+ if data[0] != 0 {
+ t.Fatal("dirty flag is set")
+ }
+
+ // Verify index rebuilt
+ cidRIndex = p.cidRIndex
+ has, err = cidRIndex.HasAny(ctx, cidAKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !has {
+ t.Fatal("index should have been rebuilt")
+ }
+
+ has, err = p.removePinsForCid(ctx, bk, ipfspin.Any)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !has {
+ t.Fatal("expected Unpin to return true since index removed")
+ }
+}
+
+func TestEncodeDecodePin(t *testing.T) {
+ _, c := randNode()
+
+ pin := newPin(c, ipfspin.Recursive, "testpin")
+ pin.Metadata = make(map[string]interface{}, 2)
+ pin.Metadata["hello"] = "world"
+ pin.Metadata["foo"] = "bar"
+
+ encBytes, err := encodePin(pin)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ decPin, err := decodePin(pin.Id, encBytes)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if decPin.Id != pin.Id {
+ t.Errorf("wrong pin id: expect %q got %q", pin.Id, decPin.Id)
+ }
+ if decPin.Cid != pin.Cid {
+ t.Errorf("wrong pin cid: expect %q got %q", pin.Cid.String(), decPin.Cid.String())
+ }
+ if decPin.Mode != pin.Mode {
+ expect, _ := ipfspin.ModeToString(pin.Mode)
+ got, _ := ipfspin.ModeToString(decPin.Mode)
+ t.Errorf("wrong pin mode: expect %s got %s", expect, got)
+ }
+ if decPin.Name != pin.Name {
+ t.Errorf("wrong pin name: expect %q got %q", pin.Name, decPin.Name)
+ }
+ for key, val := range pin.Metadata {
+ dval, ok := decPin.Metadata[key]
+ if !ok {
+ t.Errorf("decoded pin missing metadata key %q", key)
+ }
+ if dval != val {
+ t.Errorf("wrong metadata value: expected %q got %q", val, dval)
+ }
+ }
+}
+
+func makeTree(ctx context.Context, aBranchLen int, dserv ipld.DAGService, p ipfspin.Pinner) (aKeys []cid.Cid, bk cid.Cid, ck cid.Cid, err error) {
+ if aBranchLen < 3 {
+ err = errors.New("set aBranchLen to at least 3")
+ return
+ }
+
+ aNodes := make([]*mdag.ProtoNode, aBranchLen)
+ aKeys = make([]cid.Cid, aBranchLen)
+ for i := 0; i < aBranchLen; i++ {
+ a, _ := randNode()
+ if i >= 1 {
+ if err = a.AddNodeLink("child", aNodes[i-1]); err != nil {
+ return
+ }
+ }
+
+ if err = dserv.Add(ctx, a); err != nil {
+ return
+ }
+ aNodes[i] = a
+ aKeys[i] = a.Cid()
+ }
+
+ // Pin last A recursively
+ if err = p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil {
+ return
+ }
+
+ // Create node B and add A3 as child
+ b, _ := randNode()
+ if err = b.AddNodeLink("mychild", aNodes[3]); err != nil {
+ return
+ }
+
+ // Create C node
+ c, _ := randNode()
+ // Add A0 as child of C
+ if err = c.AddNodeLink("child", aNodes[0]); err != nil {
+ return
+ }
+
+ // Add C
+ if err = dserv.Add(ctx, c); err != nil {
+ return
+ }
+ ck = c.Cid()
+
+ // Add C to B and Add B
+ if err = b.AddNodeLink("myotherchild", c); err != nil {
+ return
+ }
+ if err = dserv.Add(ctx, b); err != nil {
+ return
+ }
+ bk = b.Cid()
+
+ // Pin C recursively
+ if err = p.Pin(ctx, c, true); err != nil {
+ return
+ }
+
+ // Pin B recursively
+ if err = p.Pin(ctx, b, true); err != nil {
+ return
+ }
+
+ if err = p.Flush(ctx); err != nil {
+ return
+ }
+
+ return
+}
+
+func makeNodes(count int, dserv ipld.DAGService) []ipld.Node {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nodes := make([]ipld.Node, count)
+ for i := 0; i < count; i++ {
+ n, _ := randNode()
+ err := dserv.Add(ctx, n)
+ if err != nil {
+ panic(err)
+ }
+ nodes[i] = n
+ }
+ return nodes
+}
+
+func pinNodes(nodes []ipld.Node, p ipfspin.Pinner, recursive bool) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var err error
+
+ for i := range nodes {
+ err = p.Pin(ctx, nodes[i], recursive)
+ if err != nil {
+ panic(err)
+ }
+ }
+ err = p.Flush(ctx)
+ if err != nil {
+ panic(err)
+ }
+}
+
+func unpinNodes(nodes []ipld.Node, p ipfspin.Pinner) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var err error
+
+ for i := range nodes {
+ err = p.Unpin(ctx, nodes[i].Cid(), true)
+ if err != nil {
+ panic(err)
+ }
+ }
+ err = p.Flush(ctx)
+ if err != nil {
+ panic(err)
+ }
+}
+
+type batchWrap struct {
+ ds.Datastore
+}
+
+func (d *batchWrap) Batch(_ context.Context) (ds.Batch, error) {
+ return ds.NewBasicBatch(d), nil
+}
+
+func makeStore() (ds.Datastore, ipld.DAGService) {
+ ldstore, err := lds.NewDatastore("", nil)
+ if err != nil {
+ panic(err)
+ }
+ dstore := &batchWrap{ldstore}
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+ dserv := mdag.NewDAGService(bserv)
+ return dstore, dserv
+}
+
+// BenchmarkLoadRebuild loads a pinner that has some number of saved pins, and
+// compares the load time when rebuilding indexes to loading without rebuilding
+// indexes.
+func BenchmarkLoad(b *testing.B) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore, dserv := makeStore()
+ pinner, err := New(ctx, dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+
+ nodes := makeNodes(4096, dserv)
+ pinNodes(nodes, pinner, true)
+
+ b.Run("RebuildTrue", func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ err = dstore.Put(ctx, dirtyKey, []byte{1})
+ if err != nil {
+ panic(err.Error())
+ }
+
+ _, err = New(ctx, dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+ }
+ })
+
+ b.Run("RebuildFalse", func(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ err = dstore.Put(ctx, dirtyKey, []byte{0})
+ if err != nil {
+ panic(err.Error())
+ }
+
+ _, err = New(ctx, dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+ }
+ })
+}
+
+// BenchmarkNthPins shows the time it takes to create/save 1 pin when a number
+// of other pins already exist. Each run in the series shows performance for
+// creating a pin in a larger number of existing pins.
+func BenchmarkNthPin(b *testing.B) {
+ dstore, dserv := makeStore()
+ pinner, err := New(context.Background(), dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+
+ for count := 1000; count <= 10000; count += 1000 {
+ b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) {
+ benchmarkNthPin(b, count, pinner, dserv)
+ })
+ }
+}
+
+func benchmarkNthPin(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nodes := makeNodes(count, dserv)
+ pinNodes(nodes[:count-1], pinner, true)
+ b.ResetTimer()
+
+ which := count - 1
+ for i := 0; i < b.N; i++ {
+ // Pin the Nth node and Flush
+ err := pinner.Pin(ctx, nodes[which], true)
+ if err != nil {
+ panic(err)
+ }
+ err = pinner.Flush(ctx)
+ if err != nil {
+ panic(err)
+ }
+ // Unpin the nodes so that it can pinned next iter.
+ b.StopTimer()
+ err = pinner.Unpin(ctx, nodes[which].Cid(), true)
+ if err != nil {
+ panic(err)
+ }
+ err = pinner.Flush(ctx)
+ if err != nil {
+ panic(err)
+ }
+ b.StartTimer()
+ }
+}
+
+// BenchmarkNPins demonstrates creating individual pins. Each run in the
+// series shows performance for a larger number of individual pins.
+func BenchmarkNPins(b *testing.B) {
+ for count := 128; count < 16386; count <<= 1 {
+ b.Run(fmt.Sprint("PinDS-", count), func(b *testing.B) {
+ dstore, dserv := makeStore()
+ pinner, err := New(context.Background(), dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+ benchmarkNPins(b, count, pinner, dserv)
+ })
+ }
+}
+
+func benchmarkNPins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nodes := makeNodes(count, dserv)
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ // Pin all the nodes one at a time.
+ for j := range nodes {
+ err := pinner.Pin(ctx, nodes[j], true)
+ if err != nil {
+ panic(err)
+ }
+ err = pinner.Flush(ctx)
+ if err != nil {
+ panic(err)
+ }
+ }
+
+ // Unpin all nodes so that they can be pinned next iter.
+ b.StopTimer()
+ unpinNodes(nodes, pinner)
+ b.StartTimer()
+ }
+}
+
+// BenchmarkNUnpins demonstrates unpinning individual pins. Each run in the
+// series shows performance for a larger number of individual unpins.
+func BenchmarkNUnpins(b *testing.B) {
+ for count := 128; count < 16386; count <<= 1 {
+ b.Run(fmt.Sprint("UnpinDS-", count), func(b *testing.B) {
+ dstore, dserv := makeStore()
+ pinner, err := New(context.Background(), dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+ benchmarkNUnpins(b, count, pinner, dserv)
+ })
+ }
+}
+
+func benchmarkNUnpins(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nodes := makeNodes(count, dserv)
+ pinNodes(nodes, pinner, true)
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ for j := range nodes {
+ // Unpin nodes one at a time.
+ err := pinner.Unpin(ctx, nodes[j].Cid(), true)
+ if err != nil {
+ panic(err)
+ }
+ err = pinner.Flush(ctx)
+ if err != nil {
+ panic(err)
+ }
+ }
+ // Pin all nodes so that they can be unpinned next iter.
+ b.StopTimer()
+ pinNodes(nodes, pinner, true)
+ b.StartTimer()
+ }
+}
+
+// BenchmarkPinAllSeries shows times to pin all nodes with only one Flush at
+// the end.
+func BenchmarkPinAll(b *testing.B) {
+ for count := 128; count < 16386; count <<= 1 {
+ b.Run(fmt.Sprint("PinAllDS-", count), func(b *testing.B) {
+ dstore, dserv := makeStore()
+ pinner, err := New(context.Background(), dstore, dserv)
+ if err != nil {
+ panic(err)
+ }
+ benchmarkPinAll(b, count, pinner, dserv)
+ })
+ }
+}
+
+func benchmarkPinAll(b *testing.B, count int, pinner ipfspin.Pinner, dserv ipld.DAGService) {
+ nodes := makeNodes(count, dserv)
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ pinNodes(nodes, pinner, true)
+
+ b.StopTimer()
+ unpinNodes(nodes, pinner)
+ b.StartTimer()
+ }
+}
+
+func BenchmarkRebuild(b *testing.B) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore, dserv := makeStore()
+ pinIncr := 32768
+
+ for pins := pinIncr; pins <= pinIncr*5; pins += pinIncr {
+ pinner, err := New(ctx, dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+ nodes := makeNodes(pinIncr, dserv)
+ pinNodes(nodes, pinner, true)
+
+ b.Run(fmt.Sprintf("Rebuild %d", pins), func(b *testing.B) {
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ err = dstore.Put(ctx, dirtyKey, []byte{1})
+ if err != nil {
+ panic(err.Error())
+ }
+
+ _, err = New(ctx, dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+ }
+ })
+ }
+}
+
+func TestCidIndex(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore, dserv := makeStore()
+ pinner, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes := makeNodes(1, dserv)
+ node := nodes[0]
+
+ c := node.Cid()
+ cidKey := c.KeyString()
+
+ // Pin the cid
+ pid, err := pinner.addPin(ctx, c, ipfspin.Recursive, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Log("Added pin:", pid)
+ t.Log("CID index:", c.String(), "-->", pid)
+
+ // Check that the index exists
+ ok, err := pinner.cidRIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ok {
+ t.Fatal("R-index has no value for", cidKey)
+ }
+
+ // Check that searching for the cid returns a value
+ values, err := pinner.cidRIndex.Search(ctx, cidKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(values) != 1 {
+ t.Fatal("expect index to return one value")
+ }
+ if values[0] != pid {
+ t.Fatal("indexer should have has value", cidKey, "-->", pid)
+ }
+
+ // Check that index has specific value
+ ok, err = pinner.cidRIndex.HasValue(ctx, cidKey, pid)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ok {
+ t.Fatal("indexer should have has value", cidKey, "-->", pid)
+ }
+
+ // Iterate values of index
+ var seen bool
+ err = pinner.cidRIndex.ForEach(ctx, "", func(key, value string) bool {
+ if seen {
+ t.Fatal("expected one key-value pair")
+ }
+ if key != cidKey {
+ t.Fatal("unexpected key:", key)
+ }
+ if value != pid {
+ t.Fatal("unexpected value:", value)
+ }
+ seen = true
+ return true
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Load all pins from the datastore.
+ q := query.Query{
+ Prefix: pinKeyPath,
+ }
+ results, err := pinner.dstore.Query(ctx, q)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer results.Close()
+
+ // Iterate all pins and check if the corresponding recursive or direct
+ // index is missing. If the index is missing then create the index.
+ seen = false
+ for r := range results.Next() {
+ if seen {
+ t.Fatal("has more than one pin")
+ }
+ if r.Error != nil {
+ t.Fatal(fmt.Errorf("cannot read index: %v", r.Error))
+ }
+ ent := r.Entry
+ pp, err := decodePin(path.Base(ent.Key), ent.Value)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log("Found pin:", pp.Id)
+ if pp.Id != pid {
+ t.Fatal("ID of loaded pin is not the same known to indexer")
+ }
+ seen = true
+ }
+}
+
+func TestRebuild(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dstore, dserv := makeStore()
+ pinner, err := New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes := makeNodes(3, dserv)
+ pinNodes(nodes, pinner, true)
+
+ c1 := nodes[0].Cid()
+ cid1Key := c1.KeyString()
+ c2 := nodes[1].Cid()
+ cid2Key := c2.KeyString()
+ c3 := nodes[2].Cid()
+ cid3Key := c3.KeyString()
+
+ // Get pin IDs
+ values, err := pinner.cidRIndex.Search(ctx, cid1Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pid1 := values[0]
+ values, err = pinner.cidRIndex.Search(ctx, cid2Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pid2 := values[0]
+ values, err = pinner.cidRIndex.Search(ctx, cid3Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pid3 := values[0]
+
+ // Corrupt by adding direct index when there is already a recursive index
+ err = pinner.cidDIndex.Add(ctx, cid1Key, pid1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Corrupt index by deleting cid index 2 to simulate an incomplete add or delete
+ _, err = pinner.cidRIndex.DeleteKey(ctx, cid2Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Corrupt index by deleting pin to simulate corruption
+ var pp *pin
+ pp, err = pinner.loadPin(ctx, pid3)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = pinner.dstore.Delete(ctx, pp.dsKey())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ pinner.setDirty(ctx)
+
+ // Rebuild indexes
+ pinner, err = New(ctx, dstore, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify that indexes have same values as before
+ err = verifyIndexValue(ctx, pinner, cid1Key, pid1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = verifyIndexValue(ctx, pinner, cid2Key, pid2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = verifyIndexValue(ctx, pinner, cid3Key, pid3)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func verifyIndexValue(ctx context.Context, pinner *pinner, cidKey, expectedPid string) error {
+ values, err := pinner.cidRIndex.Search(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+ if len(values) != 1 {
+ return errors.New("expected 1 value")
+ }
+ if expectedPid != values[0] {
+ return errors.New("index has wrong value")
+ }
+ ok, err := pinner.cidDIndex.HasAny(ctx, cidKey)
+ if err != nil {
+ return err
+ }
+ if ok {
+ return errors.New("should not have a direct index")
+ }
+ return nil
+}
diff --git a/pinning/pinner/dspinner/sync_test.go b/pinning/pinner/dspinner/sync_test.go
new file mode 100644
index 000000000..61c4d790e
--- /dev/null
+++ b/pinning/pinner/dspinner/sync_test.go
@@ -0,0 +1,88 @@
+package dspinner
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ bs "github.com/ipfs/boxo/blockservice"
+ blockstore "github.com/ipfs/boxo/blockstore"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ ds "github.com/ipfs/go-datastore"
+ bds "github.com/ipfs/go-ds-badger"
+ lds "github.com/ipfs/go-ds-leveldb"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+func makeStoreLevelDB(dir string) (ds.Datastore, ipld.DAGService) {
+ ldstore, err := lds.NewDatastore(dir, nil)
+ if err != nil {
+ panic(err)
+ }
+ // dstore := &batchWrap{ldstore}
+ dstore := ldstore
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+ dserv := mdag.NewDAGService(bserv)
+ return dstore, dserv
+}
+
+func makeStoreBadger(dir string) (ds.Datastore, ipld.DAGService) {
+ bdstore, err := bds.NewDatastore(dir, nil)
+ if err != nil {
+ panic(err)
+ }
+ dstore := &batchWrap{bdstore}
+ bstore := blockstore.NewBlockstore(dstore)
+ bserv := bs.New(bstore, offline.Exchange(bstore))
+ dserv := mdag.NewDAGService(bserv)
+ return dstore, dserv
+}
+
+func benchAutoSync(b *testing.B, N int, auto bool, dstore ds.Datastore, dserv ipld.DAGService) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ pinner, err := New(ctx, dstore, dserv)
+ if err != nil {
+ panic(err.Error())
+ }
+
+ nodes := makeNodes(N, dserv)
+
+ pinner.SetAutosync(auto)
+ pinNodes(nodes, pinner, true)
+}
+
+func BenchmarkSyncOnceBadger(b *testing.B) {
+ const dsDir = "b-once"
+ dstoreB1, dservB1 := makeStoreBadger(dsDir)
+ defer os.RemoveAll(dsDir)
+ benchAutoSync(b, b.N, false, dstoreB1, dservB1)
+ dstoreB1.Close()
+}
+
+func BenchmarkSyncEveryBadger(b *testing.B) {
+ const dsDir = "b-every"
+ dstoreB2, dservB2 := makeStoreBadger(dsDir)
+ defer os.RemoveAll(dsDir)
+ benchAutoSync(b, b.N, true, dstoreB2, dservB2)
+ dstoreB2.Close()
+}
+
+func BenchmarkSyncOnceLevelDB(b *testing.B) {
+ const dsDir = "l-once"
+ dstoreL1, dservL1 := makeStoreLevelDB(dsDir)
+ defer os.RemoveAll(dsDir)
+ benchAutoSync(b, b.N, false, dstoreL1, dservL1)
+ dstoreL1.Close()
+}
+
+func BenchmarkSyncEveryLevelDB(b *testing.B) {
+ const dsDir = "l-every"
+ dstoreL2, dservL2 := makeStoreLevelDB(dsDir)
+ defer os.RemoveAll(dsDir)
+ benchAutoSync(b, b.N, true, dstoreL2, dservL2)
+ dstoreL2.Close()
+}
diff --git a/pinning/pinner/pin.go b/pinning/pinner/pin.go
new file mode 100644
index 000000000..fcf7d764a
--- /dev/null
+++ b/pinning/pinner/pin.go
@@ -0,0 +1,158 @@
+// Package pin implements structures and methods to keep track of
+// which objects a user wants to keep stored locally.
+package pin
+
+import (
+ "context"
+ "fmt"
+
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+const (
+ linkRecursive = "recursive"
+ linkDirect = "direct"
+ linkIndirect = "indirect"
+ linkInternal = "internal"
+ linkNotPinned = "not pinned"
+ linkAny = "any"
+ linkAll = "all"
+)
+
+// Mode allows to specify different types of pin (recursive, direct etc.).
+// See the Pin Modes constants for a full list.
+type Mode int
+
+// Pin Modes
+const (
+ // Recursive pins pin the target cids along with any reachable children.
+ Recursive Mode = iota
+
+ // Direct pins pin just the target cid.
+ Direct
+
+ // Indirect pins are cids who have some ancestor pinned recursively.
+ Indirect
+
+ // Internal pins are cids used to keep the internal state of the pinner.
+ Internal
+
+ // NotPinned
+ NotPinned
+
+ // Any refers to any pinned cid
+ Any
+)
+
+// ModeToString returns a human-readable name for the Mode.
+func ModeToString(mode Mode) (string, bool) {
+ m := map[Mode]string{
+ Recursive: linkRecursive,
+ Direct: linkDirect,
+ Indirect: linkIndirect,
+ Internal: linkInternal,
+ NotPinned: linkNotPinned,
+ Any: linkAny,
+ }
+ s, ok := m[mode]
+ return s, ok
+}
+
+// StringToMode parses the result of ModeToString() back to a Mode.
+// It returns a boolean which is set to false if the mode is unknown.
+func StringToMode(s string) (Mode, bool) {
+ m := map[string]Mode{
+ linkRecursive: Recursive,
+ linkDirect: Direct,
+ linkIndirect: Indirect,
+ linkInternal: Internal,
+ linkNotPinned: NotPinned,
+ linkAny: Any,
+ linkAll: Any, // "all" and "any" means the same thing
+ }
+ mode, ok := m[s]
+ return mode, ok
+}
+
+// ErrNotPinned is returned when trying to unpin items that are not pinned.
+var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly")
+
+// A Pinner provides the necessary methods to keep track of Nodes which are
+// to be kept locally, according to a pin mode. In practice, a Pinner is in
+// in charge of keeping the list of items from the local storage that should
+// not be garbage-collected.
+type Pinner interface {
+ // IsPinned returns whether or not the given cid is pinned
+ // and an explanation of why its pinned
+ IsPinned(ctx context.Context, c cid.Cid) (string, bool, error)
+
+ // IsPinnedWithType returns whether or not the given cid is pinned with the
+ // given pin type, as well as returning the type of pin its pinned with.
+ IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error)
+
+ // Pin the given node, optionally recursively.
+ // Pin will make sure that the given node and its children if recursive is set
+ // are stored locally.
+ Pin(ctx context.Context, node ipld.Node, recursive bool) error
+
+ // Unpin the given cid. If recursive is true, removes either a recursive or
+ // a direct pin. If recursive is false, only removes a direct pin.
+ // If the pin doesn't exist, return ErrNotPinned
+ Unpin(ctx context.Context, cid cid.Cid, recursive bool) error
+
+ // Update updates a recursive pin from one cid to another
+ // this is more efficient than simply pinning the new one and unpinning the
+ // old one
+ Update(ctx context.Context, from, to cid.Cid, unpin bool) error
+
+ // Check if a set of keys are pinned, more efficient than
+ // calling IsPinned for each key
+ CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error)
+
+ // PinWithMode is for manually editing the pin structure. Use with
+ // care! If used improperly, garbage collection may not be
+ // successful.
+ PinWithMode(context.Context, cid.Cid, Mode) error
+
+ // Flush writes the pin state to the backing datastore
+ Flush(ctx context.Context) error
+
+ // DirectKeys returns all directly pinned cids
+ DirectKeys(ctx context.Context) ([]cid.Cid, error)
+
+ // RecursiveKeys returns all recursively pinned cids
+ RecursiveKeys(ctx context.Context) ([]cid.Cid, error)
+
+ // InternalPins returns all cids kept pinned for the internal state of the
+ // pinner
+ InternalPins(ctx context.Context) ([]cid.Cid, error)
+}
+
+// Pinned represents CID which has been pinned with a pinning strategy.
+// The Via field allows to identify the pinning parent of this CID, in the
+// case that the item is not pinned directly (but rather pinned recursively
+// by some ascendant).
+type Pinned struct {
+ Key cid.Cid
+ Mode Mode
+ Via cid.Cid
+}
+
+// Pinned returns whether or not the given cid is pinned
+func (p Pinned) Pinned() bool {
+ return p.Mode != NotPinned
+}
+
+// String Returns pin status as string
+func (p Pinned) String() string {
+ switch p.Mode {
+ case NotPinned:
+ return "not pinned"
+ case Indirect:
+ return fmt.Sprintf("pinned via %s", p.Via)
+ default:
+ modeStr, _ := ModeToString(p.Mode)
+ return fmt.Sprintf("pinned: %s", modeStr)
+ }
+}
diff --git a/pinning/remote/client/.gitignore b/pinning/remote/client/.gitignore
new file mode 100644
index 000000000..66fd13c90
--- /dev/null
+++ b/pinning/remote/client/.gitignore
@@ -0,0 +1,15 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
diff --git a/pinning/remote/client/README.md b/pinning/remote/client/README.md
new file mode 100644
index 000000000..717d9538b
--- /dev/null
+++ b/pinning/remote/client/README.md
@@ -0,0 +1,16 @@
+## Updating Pinning Service Spec
+
+Download the openapi-generator from https://github.com/OpenAPITools/openapi-generator and generate the code using:
+
+Current code generated with: openapi-generator 5.0.0-beta
+
+```
+openapi-generator generate -g go-experimental -i https://raw.githubusercontent.com/ipfs/pinning-services-api-spec/master/ipfs-pinning-service.yaml -o openapi
+rm openapi/go.mod openapi/go.sum
+```
+
+Notes:
+Due to https://github.com/OpenAPITools/openapi-generator/issues/7473 the code generator the http error codes processing
+may need some manual editing.
+
+`go-experimental` is becoming mainstream and so in later versions will be replaced with `go`
diff --git a/pinning/remote/client/client.go b/pinning/remote/client/client.go
new file mode 100644
index 000000000..6c869d40f
--- /dev/null
+++ b/pinning/remote/client/client.go
@@ -0,0 +1,428 @@
+package go_pinning_service_http_client
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/ipfs/boxo/pinning/remote/client/openapi"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multiaddr"
+ "github.com/multiformats/go-multibase"
+
+ logging "github.com/ipfs/go-log/v2"
+)
+
+var logger = logging.Logger("pinning-service-http-client")
+
+const UserAgent = "go-pinning-service-http-client"
+
+type Client struct {
+ client *openapi.APIClient
+}
+
+func NewClient(url, bearerToken string) *Client {
+ config := openapi.NewConfiguration()
+ config.UserAgent = UserAgent
+ config.AddDefaultHeader("Authorization", "Bearer "+bearerToken)
+ config.Servers = openapi.ServerConfigurations{
+ openapi.ServerConfiguration{
+ URL: url,
+ },
+ }
+
+ return &Client{client: openapi.NewAPIClient(config)}
+}
+
+// TODO: We should probably make sure there are no duplicates sent
+type lsSettings struct {
+ cids []string
+ name string
+ status []Status
+ before *time.Time
+ after *time.Time
+ limit *int32
+ meta map[string]string
+}
+
+type LsOption func(options *lsSettings) error
+
+var PinOpts = pinOpts{}
+
+type pinOpts struct {
+ pinLsOpts
+ pinAddOpts
+}
+
+type pinLsOpts struct{}
+
+func (pinLsOpts) FilterCIDs(cids ...cid.Cid) LsOption {
+ return func(options *lsSettings) error {
+ enc := getCIDEncoder()
+ for _, c := range cids {
+ options.cids = append(options.cids, c.Encode(enc))
+ }
+ return nil
+ }
+}
+
+const maxNameSize = 255
+
+func (pinLsOpts) FilterName(name string) LsOption {
+ return func(options *lsSettings) error {
+ if len(name) > maxNameSize {
+ return fmt.Errorf("name cannot be longer than %d", maxNameSize)
+ }
+ options.name = name
+ return nil
+ }
+}
+
+func (pinLsOpts) FilterStatus(statuses ...Status) LsOption {
+ return func(options *lsSettings) error {
+ for _, s := range statuses {
+ valid := false
+ for _, existing := range validStatuses {
+ if existing == s {
+ valid = true
+ break
+ }
+ }
+ if !valid {
+ return fmt.Errorf("invalid status %s", s)
+ }
+ }
+ options.status = append(options.status, statuses...)
+ return nil
+ }
+}
+
+func (pinLsOpts) FilterBefore(t time.Time) LsOption {
+ return func(options *lsSettings) error {
+ options.before = &t
+ return nil
+ }
+}
+
+func (pinLsOpts) FilterAfter(t time.Time) LsOption {
+ return func(options *lsSettings) error {
+ options.after = &t
+ return nil
+ }
+}
+
+const recordLimit = 1000
+const defaultLimit = 10
+
+func (pinLsOpts) Limit(limit int) LsOption {
+ return func(options *lsSettings) error {
+ if limit > recordLimit {
+ return fmt.Errorf("limit exceeded maximum record limit of %d", recordLimit)
+ }
+ limitCasted := int32(limit)
+ options.limit = &limitCasted
+ return nil
+ }
+}
+
+func (pinLsOpts) LsMeta(meta map[string]string) LsOption {
+ return func(options *lsSettings) error {
+ options.meta = meta
+ return nil
+ }
+}
+
+type pinResults = openapi.PinResults
+
+func (c *Client) Ls(ctx context.Context, opts ...LsOption) (chan PinStatusGetter, chan error) {
+ res := make(chan PinStatusGetter, 1)
+ errs := make(chan error, 1)
+
+ settings := new(lsSettings)
+ for _, o := range opts {
+ if err := o(settings); err != nil {
+ close(res)
+ errs <- err
+ close(errs)
+ return res, errs
+ }
+ }
+
+ go func() {
+ defer func() {
+ if r := recover(); r != nil {
+ var err error
+ switch x := r.(type) {
+ case string:
+ err = fmt.Errorf("unexpected error while listing remote pins: %s", x)
+ case error:
+ err = fmt.Errorf("unexpected error while listing remote pins: %w", x)
+ default:
+ err = errors.New("unknown panic while listing remote pins")
+ }
+ errs <- err
+ }
+ close(errs)
+ close(res)
+ }()
+
+ for {
+ pinRes, err := c.lsInternal(ctx, settings)
+ if err != nil {
+ errs <- err
+ return
+ }
+
+ results := pinRes.GetResults()
+ for _, r := range results {
+ select {
+ case res <- &pinStatusObject{r}:
+ case <-ctx.Done():
+ errs <- ctx.Err()
+ return
+ }
+ }
+
+ batchSize := len(results)
+ if int(pinRes.Count) == batchSize {
+ // no more batches
+ return
+ }
+
+ // Better DX/UX for cases like https://github.com/application-research/estuary/issues/124
+ if batchSize == 0 && int(pinRes.Count) != 0 {
+ errs <- fmt.Errorf("invalid pinning service response: PinResults.count=%d but no PinResults.results", int(pinRes.Count))
+ return
+ }
+
+ oldestResult := results[batchSize-1]
+ settings.before = &oldestResult.Created
+ }
+ }()
+
+ return res, errs
+}
+
+func (c *Client) LsSync(ctx context.Context, opts ...LsOption) ([]PinStatusGetter, error) {
+ resCh, errCh := c.Ls(ctx, opts...)
+
+ var res []PinStatusGetter
+ for r := range resCh {
+ res = append(res, r)
+ }
+
+ return res, <-errCh
+}
+
+// Manual version of Ls that returns a single batch of results and int with total count
+func (c *Client) LsBatchSync(ctx context.Context, opts ...LsOption) ([]PinStatusGetter, int, error) {
+ var res []PinStatusGetter
+
+ settings := new(lsSettings)
+ for _, o := range opts {
+ if err := o(settings); err != nil {
+ return nil, 0, err
+ }
+ }
+
+ pinRes, err := c.lsInternal(ctx, settings)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ results := pinRes.GetResults()
+ for _, r := range results {
+ res = append(res, &pinStatusObject{r})
+ }
+
+ return res, int(pinRes.Count), nil
+}
+
+func (c *Client) lsInternal(ctx context.Context, settings *lsSettings) (pinResults, error) {
+ getter := c.client.PinsApi.PinsGet(ctx)
+ if len(settings.cids) > 0 {
+ getter = getter.Cid(settings.cids)
+ }
+ if len(settings.status) > 0 {
+ statuses := make([]openapi.Status, len(settings.status))
+ for i := 0; i < len(statuses); i++ {
+ statuses[i] = openapi.Status(settings.status[i])
+ }
+ getter = getter.Status(statuses)
+ }
+ if settings.limit == nil {
+ getter = getter.Limit(defaultLimit)
+ } else {
+ getter = getter.Limit(*settings.limit)
+ }
+ if len(settings.name) > 0 {
+ getter = getter.Name(settings.name)
+ }
+ if settings.before != nil {
+ getter = getter.Before(*settings.before)
+ }
+ if settings.after != nil {
+ getter = getter.After(*settings.after)
+ }
+ if settings.meta != nil {
+ getter = getter.Meta(settings.meta)
+ }
+
+ // TODO: Ignoring HTTP Response OK?
+ results, httpresp, err := getter.Execute()
+ if err != nil {
+ err := httperr(httpresp, err)
+ return pinResults{}, err
+ }
+
+ return results, nil
+}
+
+// TODO: We should probably make sure there are no duplicates sent
+type addSettings struct {
+ name string
+ origins []string
+ meta map[string]string
+}
+
+type AddOption func(options *addSettings) error
+
+type pinAddOpts struct{}
+
+func (pinAddOpts) WithName(name string) AddOption {
+ return func(options *addSettings) error {
+ if len(name) > maxNameSize {
+ return fmt.Errorf("name cannot be longer than %d", maxNameSize)
+ }
+ options.name = name
+ return nil
+ }
+}
+
+func (pinAddOpts) WithOrigins(origins ...multiaddr.Multiaddr) AddOption {
+ return func(options *addSettings) error {
+ for _, o := range origins {
+ options.origins = append(options.origins, o.String())
+ }
+ return nil
+ }
+}
+
+func (pinAddOpts) AddMeta(meta map[string]string) AddOption {
+ return func(options *addSettings) error {
+ options.meta = meta
+ return nil
+ }
+}
+
+func (c *Client) Add(ctx context.Context, cid cid.Cid, opts ...AddOption) (PinStatusGetter, error) {
+ settings := new(addSettings)
+ for _, o := range opts {
+ if err := o(settings); err != nil {
+ return nil, err
+ }
+ }
+
+ adder := c.client.PinsApi.PinsPost(ctx)
+ p := openapi.Pin{
+ Cid: cid.Encode(getCIDEncoder()),
+ }
+
+ if len(settings.origins) > 0 {
+ p.SetOrigins(settings.origins)
+ }
+ if settings.meta != nil {
+ p.SetMeta(settings.meta)
+ }
+ if len(settings.name) > 0 {
+ p.SetName(settings.name)
+ }
+
+ result, httpresp, err := adder.Pin(p).Execute()
+ if err != nil {
+ err := httperr(httpresp, err)
+ return nil, err
+ }
+
+ return &pinStatusObject{result}, nil
+}
+
+func (c *Client) GetStatusByID(ctx context.Context, pinID string) (PinStatusGetter, error) {
+ getter := c.client.PinsApi.PinsRequestidGet(ctx, pinID)
+ result, httpresp, err := getter.Execute()
+ if err != nil {
+ err := httperr(httpresp, err)
+ return nil, err
+ }
+
+ return &pinStatusObject{result}, nil
+}
+
+func (c *Client) DeleteByID(ctx context.Context, pinID string) error {
+ deleter := c.client.PinsApi.PinsRequestidDelete(ctx, pinID)
+ httpresp, err := deleter.Execute()
+ if err != nil {
+ err := httperr(httpresp, err)
+ return err
+ }
+ return nil
+}
+
+func (c *Client) Replace(ctx context.Context, pinID string, cid cid.Cid, opts ...AddOption) (PinStatusGetter, error) {
+ settings := new(addSettings)
+ for _, o := range opts {
+ if err := o(settings); err != nil {
+ return nil, err
+ }
+ }
+
+ adder := c.client.PinsApi.PinsRequestidPost(ctx, pinID)
+ p := openapi.Pin{
+ Cid: cid.Encode(getCIDEncoder()),
+ }
+
+ if len(settings.origins) > 0 {
+ p.SetOrigins(settings.origins)
+ }
+ if settings.meta != nil {
+ p.SetMeta(settings.meta)
+ }
+ if len(settings.name) > 0 {
+ p.SetName(settings.name)
+ }
+
+ result, httpresp, err := adder.Pin(p).Execute()
+ if err != nil {
+ err := httperr(httpresp, err)
+ return nil, err
+ }
+
+ return &pinStatusObject{result}, nil
+}
+
+func getCIDEncoder() multibase.Encoder {
+ enc, err := multibase.NewEncoder(multibase.Base32)
+ if err != nil {
+ panic(err)
+ }
+ return enc
+}
+
+func httperr(resp *http.Response, e error) error {
+ oerr, ok := e.(openapi.GenericOpenAPIError)
+ if ok {
+ ferr, ok := oerr.Model().(openapi.Failure)
+ if ok {
+ return errors.Wrapf(e, "reason: %q, details: %q", ferr.Error.GetReason(), ferr.Error.GetDetails())
+ }
+ }
+
+ if resp == nil {
+ return errors.Wrapf(e, "empty response from remote pinning service")
+ }
+
+ return errors.Wrapf(e, "remote pinning service returned http error %d", resp.StatusCode)
+}
diff --git a/pinning/remote/client/cmd/main.go b/pinning/remote/client/cmd/main.go
new file mode 100644
index 000000000..c85f51eff
--- /dev/null
+++ b/pinning/remote/client/cmd/main.go
@@ -0,0 +1,92 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ pinclient "github.com/ipfs/boxo/pinning/remote/client"
+ "github.com/ipfs/go-cid"
+)
+
+func main() {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ url, ok := os.LookupEnv("PS_URL")
+ if !ok {
+ panic("No Pinning Service URL found")
+ }
+
+ key, ok := os.LookupEnv("PS_KEY")
+ if !ok {
+ panic("No Pinning Service API Key found")
+ }
+
+ c := pinclient.NewClient(url, key)
+
+ ipfsPgCid, err := cid.Parse("bafybeiayvrj27f65vbecspbnuavehcb3znvnt2strop2rfbczupudoizya")
+ if err != nil {
+ panic(err)
+ }
+
+ libp2pCid, err := cid.Parse("bafybeiejgrxo4p4uofgfzvlg5twrg5w7tfwpf7aciiswfacfbdpevg2xfy")
+ if err != nil {
+ panic(err)
+ }
+ _ = ipfsPgCid
+
+ listPins(ctx, c)
+
+ fmt.Println("Adding libp2p home page")
+ ps, err := c.Add(ctx, libp2pCid, pinclient.PinOpts.WithName("libp2p"))
+ if err == nil {
+ fmt.Printf("PinStatus: %v \n", ps)
+ } else {
+ fmt.Println(err)
+ }
+
+ listPins(ctx, c)
+
+ fmt.Println("Check on pin status")
+ if ps == nil {
+ panic("Skipping pin status check because the pin is null")
+ }
+
+ var pinned bool
+ for !pinned {
+ status, err := c.GetStatusByID(ctx, ps.GetRequestId())
+ if err == nil {
+ fmt.Println(status.GetStatus())
+ pinned = status.GetStatus() == pinclient.StatusPinned
+ } else {
+ fmt.Println(err)
+ }
+ time.Sleep(time.Millisecond * 500)
+ }
+
+ listPins(ctx, c)
+
+ fmt.Println("Delete pin")
+ err = c.DeleteByID(ctx, ps.GetRequestId())
+ if err == nil {
+ fmt.Println("Successfully deleted pin")
+ } else {
+ fmt.Println(err)
+ }
+
+ listPins(ctx, c)
+}
+
+func listPins(ctx context.Context, c *pinclient.Client) {
+ fmt.Println("List all pins")
+ pins, err := c.LsSync(ctx)
+ if err != nil {
+ fmt.Println(err)
+ } else {
+ for _, p := range pins {
+ fmt.Printf("Pin: %v \n", p)
+ }
+ }
+}
diff --git a/pinning/remote/client/model.go b/pinning/remote/client/model.go
new file mode 100644
index 000000000..60d614477
--- /dev/null
+++ b/pinning/remote/client/model.go
@@ -0,0 +1,167 @@
+package go_pinning_service_http_client
+
+import (
+ "encoding/json"
+ "fmt"
+ "time"
+
+ "github.com/ipfs/boxo/pinning/remote/client/openapi"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multiaddr"
+)
+
+// PinGetter Getter for Pin object
+type PinGetter interface {
+ fmt.Stringer
+ json.Marshaler
+ // CID to be pinned recursively
+ GetCid() cid.Cid
+ // Optional name for pinned data; can be used for lookups later
+ GetName() string
+ // Optional list of multiaddrs known to provide the data
+ GetOrigins() []string
+ // Optional metadata for pin object
+ GetMeta() map[string]string
+}
+
+type pinObject struct {
+ openapi.Pin
+}
+
+func (p *pinObject) MarshalJSON() ([]byte, error) {
+ var originsStr string
+ if o := p.GetOrigins(); o != nil {
+ originsBytes, err := json.Marshal(o)
+ if err == nil {
+ originsStr = string(originsBytes)
+ }
+ }
+
+ metaStr := "{}"
+ if meta := p.GetMeta(); meta != nil {
+ metaBytes, err := json.Marshal(meta)
+ if err == nil {
+ metaStr = string(metaBytes)
+ }
+ }
+
+ str := fmt.Sprintf("{ \"Cid\" : \"%v\", \"Name\" : \"%s\", \"Origins\" : %v, \"Meta\" : %v }",
+ p.GetCid(), p.GetName(), originsStr, metaStr)
+ return []byte(str), nil
+}
+
+func (p *pinObject) String() string {
+ marshalled, err := json.MarshalIndent(p, "", "\t")
+ if err != nil {
+ return ""
+ }
+
+ return string(marshalled)
+}
+
+func (p *pinObject) GetCid() cid.Cid {
+ c, err := cid.Parse(p.Pin.Cid)
+ if err != nil {
+ return cid.Undef
+ }
+ return c
+}
+
+type Status string
+
+const (
+ StatusUnknown Status = ""
+ StatusQueued Status = Status(openapi.QUEUED)
+ StatusPinning Status = Status(openapi.PINNING)
+ StatusPinned Status = Status(openapi.PINNED)
+ StatusFailed Status = Status(openapi.FAILED)
+)
+
+func (s Status) String() string {
+ switch s {
+ case StatusQueued, StatusPinning, StatusPinned, StatusFailed:
+ return string(s)
+ default:
+ return string(StatusUnknown)
+ }
+}
+
+var validStatuses = []Status{"queued", "pinning", "pinned", "failed"}
+
+// PinStatusGetter Getter for Pin object with status
+type PinStatusGetter interface {
+ fmt.Stringer
+ json.Marshaler
+ // Globally unique ID of the pin request; can be used to check the status of ongoing pinning, modification of pin object, or pin removal
+ GetRequestId() string
+ GetStatus() Status
+ // Immutable timestamp indicating when a pin request entered a pinning service; can be used for filtering results and pagination
+ GetCreated() time.Time
+ GetPin() PinGetter
+ // List of multiaddrs designated by pinning service for transferring any new data from external peers
+ GetDelegates() []multiaddr.Multiaddr
+ // Optional info for PinStatus response
+ GetInfo() map[string]string
+}
+
+type pinStatusObject struct {
+ openapi.PinStatus
+}
+
+func (p *pinStatusObject) GetDelegates() []multiaddr.Multiaddr {
+ delegates := p.PinStatus.GetDelegates()
+ addrs := make([]multiaddr.Multiaddr, 0, len(delegates))
+ for _, d := range delegates {
+ a, err := multiaddr.NewMultiaddr(d)
+ if err != nil {
+ logger.Errorf("returned delegate is an invalid multiaddr: %w", err)
+ continue
+ }
+ addrs = append(addrs, a)
+ }
+ return addrs
+}
+
+func (p *pinStatusObject) GetPin() PinGetter {
+ return &pinObject{p.Pin}
+}
+
+func (p *pinStatusObject) GetStatus() Status {
+ return Status(p.PinStatus.GetStatus())
+}
+
+func (p *pinStatusObject) GetRequestId() string {
+ return p.GetRequestid()
+}
+
+func (p *pinStatusObject) MarshalJSON() ([]byte, error) {
+ var delegatesStr string
+ if d := p.GetDelegates(); d != nil {
+ delegatesBytes, err := json.Marshal(d)
+ if err == nil {
+ delegatesStr = string(delegatesBytes)
+ }
+ }
+
+ infoStr := "{}"
+ if info := p.GetInfo(); info != nil {
+ infoBytes, err := json.Marshal(info)
+ if err == nil {
+ infoStr = string(infoBytes)
+ }
+ }
+
+ str := fmt.Sprintf("{\"Pin\" : %v, \"RequestID\" : \"%s\", \"Status\" : \"%s\", \"Created\" : \"%v\", \"Delegates\" : %v, \"Info\" : %v }",
+ p.GetPin(), p.GetRequestId(), p.GetStatus(), p.GetCreated(), delegatesStr, infoStr)
+
+ return []byte(str), nil
+}
+
+func (p *pinStatusObject) String() string {
+ marshalled, err := json.MarshalIndent(p, "", "\t")
+ if err != nil {
+ return ""
+ }
+
+ return string(marshalled)
+}
diff --git a/pinning/remote/client/openapi/README.md b/pinning/remote/client/openapi/README.md
new file mode 100644
index 000000000..fe21b9a00
--- /dev/null
+++ b/pinning/remote/client/openapi/README.md
@@ -0,0 +1,217 @@
+# Go API client for openapi
+
+
+
+## About this spec
+The IPFS Pinning Service API is intended to be an implementation-agnostic API:
+- For use and implementation by pinning service providers
+- For use in client mode by IPFS nodes and GUI-based applications
+
+> **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).**
+
+# Schemas
+This section describes the most important object types and conventions.
+
+A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml).
+
+## Identifiers
+### cid
+[Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively.
+### requestid
+Unique identifier of a pin request.
+
+When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests.
+
+Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions.
+
+## Objects
+### Pin object
+
+
+
+The `Pin` object is a representation of a pin request.
+
+It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`.
+
+### Pin status response
+
+
+
+The `PinStatus` object is a representation of the current state of a pinning operation.
+It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`.
+
+## The pin lifecycle
+
+
+
+### Creating a new pin object
+The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response:
+- `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future
+- `status` in `PinStatus` indicates the current state of a pin
+
+### Checking status of in-progress pinning
+`status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time.
+
+In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin.
+
+### Replacing an existing pin object
+The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically.
+
+### Removing a pin object
+A pin object can be removed via `DELETE /pins/{requestid}`.
+
+
+## Provider hints
+Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`.
+
+The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer.
+
+This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs.
+
+## Custom metadata
+Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API.
+### Pin metadata
+String keys and values passed in `Pin.meta` are persisted with the pin object.
+
+Potential uses:
+- `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}`
+- `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep)
+
+Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality.
+
+### Pin status info
+Additional `PinStatus.info` can be returned by pinning service.
+
+Potential uses:
+- `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.)
+- `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead
+- `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs)
+- `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire
+
+# Pagination and filtering
+Pin objects can be listed by executing `GET /pins` with optional parameters:
+
+- When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest.
+- The number of returned items can be adjusted with the `limit` parameter (implicit default is 10).
+- If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried.
+- To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results.
+- Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters.
+
+> **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+
+
+
+## Overview
+This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the [OpenAPI-spec](https://www.openapis.org/) from a remote server, you can easily generate an API client.
+
+- API version: 0.1.1
+- Package version: 1.0.0
+- Build package: org.openapitools.codegen.languages.GoClientExperimentalCodegen
+
+## Installation
+
+Install the following dependencies:
+
+```shell
+go get github.com/stretchr/testify/assert
+go get golang.org/x/oauth2
+go get golang.org/x/net/context
+```
+
+Put the package under your project folder and add the following in import:
+
+```golang
+import sw "./openapi"
+```
+
+## Configuration of Server URL
+
+Default configuration comes with `Servers` field that contains server objects as defined in the OpenAPI specification.
+
+### Select Server Configuration
+
+For using other server than the one defined on index 0 set context value `sw.ContextServerIndex` of type `int`.
+
+```golang
+ctx := context.WithValue(context.Background(), sw.ContextServerIndex, 1)
+```
+
+### Templated Server URL
+
+Templated server URL is formatted using default variables from configuration or from context value `sw.ContextServerVariables` of type `map[string]string`.
+
+```golang
+ctx := context.WithValue(context.Background(), sw.ContextServerVariables, map[string]string{
+ "basePath": "v2",
+})
+```
+
+Note, enum values are always validated and all unused variables are silently ignored.
+
+### URLs Configuration per Operation
+
+Each operation can use different server URL defined using `OperationServers` map in the `Configuration`.
+An operation is uniquely identifield by `"{classname}Service.{nickname}"` string.
+Similar rules for overriding default operation server index and variables applies by using `sw.ContextOperationServerIndices` and `sw.ContextOperationServerVariables` context maps.
+
+```
+ctx := context.WithValue(context.Background(), sw.ContextOperationServerIndices, map[string]int{
+ "{classname}Service.{nickname}": 2,
+})
+ctx = context.WithValue(context.Background(), sw.ContextOperationServerVariables, map[string]map[string]string{
+ "{classname}Service.{nickname}": {
+ "port": "8443",
+ },
+})
+```
+
+## Documentation for API Endpoints
+
+All URIs are relative to *https://pinning-service.example.com*
+
+Class | Method | HTTP request | Description
+------------ | ------------- | ------------- | -------------
+*PinsApi* | [**PinsGet**](docs/PinsApi.md#pinsget) | **Get** /pins | List pin objects
+*PinsApi* | [**PinsPost**](docs/PinsApi.md#pinspost) | **Post** /pins | Add pin object
+*PinsApi* | [**PinsRequestidDelete**](docs/PinsApi.md#pinsrequestiddelete) | **Delete** /pins/{requestid} | Remove pin object
+*PinsApi* | [**PinsRequestidGet**](docs/PinsApi.md#pinsrequestidget) | **Get** /pins/{requestid} | Get pin object
+*PinsApi* | [**PinsRequestidPost**](docs/PinsApi.md#pinsrequestidpost) | **Post** /pins/{requestid} | Replace pin object
+
+
+## Documentation For Models
+
+ - [Failure](docs/Failure.md)
+ - [FailureError](docs/FailureError.md)
+ - [Pin](docs/Pin.md)
+ - [PinResults](docs/PinResults.md)
+ - [PinStatus](docs/PinStatus.md)
+ - [Status](docs/Status.md)
+
+
+## Documentation For Authorization
+
+
+
+### accessToken
+
+
+## Documentation for Utility Methods
+
+Due to the fact that model structure members are all pointers, this package contains
+a number of utility functions to easily obtain pointers to values of basic types.
+Each of these functions takes a value of the given basic type and returns a pointer to it:
+
+* `PtrBool`
+* `PtrInt`
+* `PtrInt32`
+* `PtrInt64`
+* `PtrFloat`
+* `PtrFloat32`
+* `PtrFloat64`
+* `PtrString`
+* `PtrTime`
+
+## Author
+
+
+
diff --git a/pinning/remote/client/openapi/api_pins.go b/pinning/remote/client/openapi/api_pins.go
new file mode 100644
index 000000000..b2858c5ee
--- /dev/null
+++ b/pinning/remote/client/openapi/api_pins.go
@@ -0,0 +1,654 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ _context "context"
+ _io "io"
+ _nethttp "net/http"
+ _neturl "net/url"
+ "strings"
+ "time"
+)
+
+// Linger please
+var (
+ _ _context.Context
+)
+
+// PinsApiService PinsApi service
+type PinsApiService service
+
+type apiPinsGetRequest struct {
+ ctx _context.Context
+ apiService *PinsApiService
+ cid *[]string
+ name *string
+ status *[]Status
+ before *time.Time
+ after *time.Time
+ limit *int32
+ meta *map[string]string
+}
+
+func (r apiPinsGetRequest) Cid(cid []string) apiPinsGetRequest {
+ r.cid = &cid
+ return r
+}
+
+func (r apiPinsGetRequest) Name(name string) apiPinsGetRequest {
+ r.name = &name
+ return r
+}
+
+func (r apiPinsGetRequest) Status(status []Status) apiPinsGetRequest {
+ r.status = &status
+ return r
+}
+
+func (r apiPinsGetRequest) Before(before time.Time) apiPinsGetRequest {
+ r.before = &before
+ return r
+}
+
+func (r apiPinsGetRequest) After(after time.Time) apiPinsGetRequest {
+ r.after = &after
+ return r
+}
+
+func (r apiPinsGetRequest) Limit(limit int32) apiPinsGetRequest {
+ r.limit = &limit
+ return r
+}
+
+func (r apiPinsGetRequest) Meta(meta map[string]string) apiPinsGetRequest {
+ r.meta = &meta
+ return r
+}
+
+/*
+PinsGet List pin objects
+List all the pin objects, matching optional filters; when no filter is provided, only successful pins are returned
+ - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+
+@return apiPinsGetRequest
+*/
+func (a *PinsApiService) PinsGet(ctx _context.Context) apiPinsGetRequest {
+ return apiPinsGetRequest{
+ apiService: a,
+ ctx: ctx,
+ }
+}
+
+/*
+Execute executes the request
+
+ @return PinResults
+*/
+func (r apiPinsGetRequest) Execute() (PinResults, *_nethttp.Response, error) {
+ var (
+ localVarHTTPMethod = _nethttp.MethodGet
+ localVarPostBody interface{}
+ localVarFormFileName string
+ localVarFileName string
+ localVarFileBytes []byte
+ localVarReturnValue PinResults
+ )
+
+ localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsGet")
+ if err != nil {
+ return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()}
+ }
+
+ localVarPath := localBasePath + "/pins"
+
+ localVarHeaderParams := make(map[string]string)
+ localVarQueryParams := _neturl.Values{}
+ localVarFormParams := _neturl.Values{}
+
+ if r.cid != nil {
+ localVarQueryParams.Add("cid", parameterToString(*r.cid, "csv"))
+ }
+ if r.name != nil {
+ localVarQueryParams.Add("name", parameterToString(*r.name, ""))
+ }
+ if r.status != nil {
+ localVarQueryParams.Add("status", parameterToString(*r.status, "csv"))
+ }
+ if r.before != nil {
+ localVarQueryParams.Add("before", parameterToString(*r.before, ""))
+ }
+ if r.after != nil {
+ localVarQueryParams.Add("after", parameterToString(*r.after, ""))
+ }
+ if r.limit != nil {
+ localVarQueryParams.Add("limit", parameterToString(*r.limit, ""))
+ }
+ if r.meta != nil {
+ localVarQueryParams.Add("meta", parameterToString(*r.meta, ""))
+ }
+ // to determine the Content-Type header
+ localVarHTTPContentTypes := []string{}
+
+ // set Content-Type header
+ localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+ if localVarHTTPContentType != "" {
+ localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+ }
+
+ // to determine the Accept header
+ localVarHTTPHeaderAccepts := []string{"application/json"}
+
+ // set Accept header
+ localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+ if localVarHTTPHeaderAccept != "" {
+ localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+ }
+ req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+ if err != nil {
+ return localVarReturnValue, nil, err
+ }
+
+ localVarHTTPResponse, err := r.apiService.client.callAPI(req)
+ if err != nil || localVarHTTPResponse == nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body)
+ localVarHTTPResponse.Body.Close()
+ if err != nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ if localVarHTTPResponse.StatusCode >= 300 {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: localVarHTTPResponse.Status,
+ }
+ if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 {
+ var v Failure
+ err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr.error = err.Error()
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ newErr.model = v
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: err.Error(),
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ return localVarReturnValue, localVarHTTPResponse, nil
+}
+
+type apiPinsPostRequest struct {
+ ctx _context.Context
+ apiService *PinsApiService
+ pin *Pin
+}
+
+func (r apiPinsPostRequest) Pin(pin Pin) apiPinsPostRequest {
+ r.pin = &pin
+ return r
+}
+
+/*
+PinsPost Add pin object
+Add a new pin object for the current access token
+ - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+
+@return apiPinsPostRequest
+*/
+func (a *PinsApiService) PinsPost(ctx _context.Context) apiPinsPostRequest {
+ return apiPinsPostRequest{
+ apiService: a,
+ ctx: ctx,
+ }
+}
+
+/*
+Execute executes the request
+
+ @return PinStatus
+*/
+func (r apiPinsPostRequest) Execute() (PinStatus, *_nethttp.Response, error) {
+ var (
+ localVarHTTPMethod = _nethttp.MethodPost
+ localVarPostBody interface{}
+ localVarFormFileName string
+ localVarFileName string
+ localVarFileBytes []byte
+ localVarReturnValue PinStatus
+ )
+
+ localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsPost")
+ if err != nil {
+ return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()}
+ }
+
+ localVarPath := localBasePath + "/pins"
+
+ localVarHeaderParams := make(map[string]string)
+ localVarQueryParams := _neturl.Values{}
+ localVarFormParams := _neturl.Values{}
+
+ if r.pin == nil {
+ return localVarReturnValue, nil, reportError("pin is required and must be specified")
+ }
+
+ // to determine the Content-Type header
+ localVarHTTPContentTypes := []string{"application/json"}
+
+ // set Content-Type header
+ localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+ if localVarHTTPContentType != "" {
+ localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+ }
+
+ // to determine the Accept header
+ localVarHTTPHeaderAccepts := []string{"application/json"}
+
+ // set Accept header
+ localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+ if localVarHTTPHeaderAccept != "" {
+ localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+ }
+ // body params
+ localVarPostBody = r.pin
+ req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+ if err != nil {
+ return localVarReturnValue, nil, err
+ }
+
+ localVarHTTPResponse, err := r.apiService.client.callAPI(req)
+ if err != nil || localVarHTTPResponse == nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body)
+ localVarHTTPResponse.Body.Close()
+ if err != nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ if localVarHTTPResponse.StatusCode >= 300 {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: localVarHTTPResponse.Status,
+ }
+ if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 {
+ var v Failure
+ err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr.error = err.Error()
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ newErr.model = v
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: err.Error(),
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ return localVarReturnValue, localVarHTTPResponse, nil
+}
+
+type apiPinsRequestidDeleteRequest struct {
+ ctx _context.Context
+ apiService *PinsApiService
+ requestid string
+}
+
+/*
+PinsRequestidDelete Remove pin object
+Remove a pin object
+ - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ - @param requestid
+
+@return apiPinsRequestidDeleteRequest
+*/
+func (a *PinsApiService) PinsRequestidDelete(ctx _context.Context, requestid string) apiPinsRequestidDeleteRequest {
+ return apiPinsRequestidDeleteRequest{
+ apiService: a,
+ ctx: ctx,
+ requestid: requestid,
+ }
+}
+
+/*
+Execute executes the request
+*/
+func (r apiPinsRequestidDeleteRequest) Execute() (*_nethttp.Response, error) {
+ var (
+ localVarHTTPMethod = _nethttp.MethodDelete
+ localVarPostBody interface{}
+ localVarFormFileName string
+ localVarFileName string
+ localVarFileBytes []byte
+ )
+
+ localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsRequestidDelete")
+ if err != nil {
+ return nil, GenericOpenAPIError{error: err.Error()}
+ }
+
+ localVarPath := localBasePath + "/pins/{requestid}"
+ localVarPath = strings.Replace(localVarPath, "{"+"requestid"+"}", _neturl.PathEscape(parameterToString(r.requestid, "")), -1)
+
+ localVarHeaderParams := make(map[string]string)
+ localVarQueryParams := _neturl.Values{}
+ localVarFormParams := _neturl.Values{}
+
+ // to determine the Content-Type header
+ localVarHTTPContentTypes := []string{}
+
+ // set Content-Type header
+ localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+ if localVarHTTPContentType != "" {
+ localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+ }
+
+ // to determine the Accept header
+ localVarHTTPHeaderAccepts := []string{"application/json"}
+
+ // set Accept header
+ localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+ if localVarHTTPHeaderAccept != "" {
+ localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+ }
+ req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+ if err != nil {
+ return nil, err
+ }
+
+ localVarHTTPResponse, err := r.apiService.client.callAPI(req)
+ if err != nil || localVarHTTPResponse == nil {
+ return localVarHTTPResponse, err
+ }
+
+ localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body)
+ localVarHTTPResponse.Body.Close()
+ if err != nil {
+ return localVarHTTPResponse, err
+ }
+
+ if localVarHTTPResponse.StatusCode >= 300 {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: localVarHTTPResponse.Status,
+ }
+ if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 {
+ var v Failure
+ err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr.error = err.Error()
+ return localVarHTTPResponse, newErr
+ }
+ newErr.model = v
+ return localVarHTTPResponse, newErr
+ }
+ return localVarHTTPResponse, newErr
+ }
+
+ return localVarHTTPResponse, nil
+}
+
+type apiPinsRequestidGetRequest struct {
+ ctx _context.Context
+ apiService *PinsApiService
+ requestid string
+}
+
+/*
+PinsRequestidGet Get pin object
+Get a pin object and its status
+ - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ - @param requestid
+
+@return apiPinsRequestidGetRequest
+*/
+func (a *PinsApiService) PinsRequestidGet(ctx _context.Context, requestid string) apiPinsRequestidGetRequest {
+ return apiPinsRequestidGetRequest{
+ apiService: a,
+ ctx: ctx,
+ requestid: requestid,
+ }
+}
+
+/*
+Execute executes the request
+
+ @return PinStatus
+*/
+func (r apiPinsRequestidGetRequest) Execute() (PinStatus, *_nethttp.Response, error) {
+ var (
+ localVarHTTPMethod = _nethttp.MethodGet
+ localVarPostBody interface{}
+ localVarFormFileName string
+ localVarFileName string
+ localVarFileBytes []byte
+ localVarReturnValue PinStatus
+ )
+
+ localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsRequestidGet")
+ if err != nil {
+ return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()}
+ }
+
+ localVarPath := localBasePath + "/pins/{requestid}"
+ localVarPath = strings.Replace(localVarPath, "{"+"requestid"+"}", _neturl.PathEscape(parameterToString(r.requestid, "")), -1)
+
+ localVarHeaderParams := make(map[string]string)
+ localVarQueryParams := _neturl.Values{}
+ localVarFormParams := _neturl.Values{}
+
+ // to determine the Content-Type header
+ localVarHTTPContentTypes := []string{}
+
+ // set Content-Type header
+ localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+ if localVarHTTPContentType != "" {
+ localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+ }
+
+ // to determine the Accept header
+ localVarHTTPHeaderAccepts := []string{"application/json"}
+
+ // set Accept header
+ localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+ if localVarHTTPHeaderAccept != "" {
+ localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+ }
+ req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+ if err != nil {
+ return localVarReturnValue, nil, err
+ }
+
+ localVarHTTPResponse, err := r.apiService.client.callAPI(req)
+ if err != nil || localVarHTTPResponse == nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body)
+ localVarHTTPResponse.Body.Close()
+ if err != nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ if localVarHTTPResponse.StatusCode >= 300 {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: localVarHTTPResponse.Status,
+ }
+ if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 {
+ var v Failure
+ err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr.error = err.Error()
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ newErr.model = v
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: err.Error(),
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ return localVarReturnValue, localVarHTTPResponse, nil
+}
+
+type apiPinsRequestidPostRequest struct {
+ ctx _context.Context
+ apiService *PinsApiService
+ requestid string
+ pin *Pin
+}
+
+func (r apiPinsRequestidPostRequest) Pin(pin Pin) apiPinsRequestidPostRequest {
+ r.pin = &pin
+ return r
+}
+
+/*
+PinsRequestidPost Replace pin object
+Replace an existing pin object (shortcut for executing remove and add operations in one step to avoid unnecessary garbage collection of blocks present in both recursive pins)
+ - @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
+ - @param requestid
+
+@return apiPinsRequestidPostRequest
+*/
+func (a *PinsApiService) PinsRequestidPost(ctx _context.Context, requestid string) apiPinsRequestidPostRequest {
+ return apiPinsRequestidPostRequest{
+ apiService: a,
+ ctx: ctx,
+ requestid: requestid,
+ }
+}
+
+/*
+Execute executes the request
+
+ @return PinStatus
+*/
+func (r apiPinsRequestidPostRequest) Execute() (PinStatus, *_nethttp.Response, error) {
+ var (
+ localVarHTTPMethod = _nethttp.MethodPost
+ localVarPostBody interface{}
+ localVarFormFileName string
+ localVarFileName string
+ localVarFileBytes []byte
+ localVarReturnValue PinStatus
+ )
+
+ localBasePath, err := r.apiService.client.cfg.ServerURLWithContext(r.ctx, "PinsApiService.PinsRequestidPost")
+ if err != nil {
+ return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()}
+ }
+
+ localVarPath := localBasePath + "/pins/{requestid}"
+ localVarPath = strings.Replace(localVarPath, "{"+"requestid"+"}", _neturl.PathEscape(parameterToString(r.requestid, "")), -1)
+
+ localVarHeaderParams := make(map[string]string)
+ localVarQueryParams := _neturl.Values{}
+ localVarFormParams := _neturl.Values{}
+
+ if r.pin == nil {
+ return localVarReturnValue, nil, reportError("pin is required and must be specified")
+ }
+
+ // to determine the Content-Type header
+ localVarHTTPContentTypes := []string{"application/json"}
+
+ // set Content-Type header
+ localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
+ if localVarHTTPContentType != "" {
+ localVarHeaderParams["Content-Type"] = localVarHTTPContentType
+ }
+
+ // to determine the Accept header
+ localVarHTTPHeaderAccepts := []string{"application/json"}
+
+ // set Accept header
+ localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
+ if localVarHTTPHeaderAccept != "" {
+ localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
+ }
+ // body params
+ localVarPostBody = r.pin
+ req, err := r.apiService.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes)
+ if err != nil {
+ return localVarReturnValue, nil, err
+ }
+
+ localVarHTTPResponse, err := r.apiService.client.callAPI(req)
+ if err != nil || localVarHTTPResponse == nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ localVarBody, err := _io.ReadAll(localVarHTTPResponse.Body)
+ localVarHTTPResponse.Body.Close()
+ if err != nil {
+ return localVarReturnValue, localVarHTTPResponse, err
+ }
+
+ if localVarHTTPResponse.StatusCode >= 300 {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: localVarHTTPResponse.Status,
+ }
+ if localVarHTTPResponse.StatusCode >= 400 && localVarHTTPResponse.StatusCode <= 600 {
+ var v Failure
+ err = r.apiService.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr.error = err.Error()
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ newErr.model = v
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ err = r.apiService.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
+ if err != nil {
+ newErr := GenericOpenAPIError{
+ body: localVarBody,
+ error: err.Error(),
+ }
+ return localVarReturnValue, localVarHTTPResponse, newErr
+ }
+
+ return localVarReturnValue, localVarHTTPResponse, nil
+}
diff --git a/pinning/remote/client/openapi/client.go b/pinning/remote/client/openapi/client.go
new file mode 100644
index 000000000..b3cea998b
--- /dev/null
+++ b/pinning/remote/client/openapi/client.go
@@ -0,0 +1,498 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "mime/multipart"
+ "net/http"
+ "net/http/httputil"
+ "net/url"
+ "os"
+ "path/filepath"
+ "reflect"
+ "regexp"
+ "strings"
+ "time"
+
+ "golang.org/x/oauth2"
+)
+
+var (
+ jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`)
+ xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`)
+)
+
+// APIClient manages communication with the IPFS Pinning Service API API v0.1.1
+// In most cases there should be only one, shared, APIClient.
+type APIClient struct {
+ cfg *Configuration
+ common service // Reuse a single struct instead of allocating one for each service on the heap.
+
+ // API Services
+
+ PinsApi *PinsApiService
+}
+
+type service struct {
+ client *APIClient
+}
+
+// NewAPIClient creates a new API client. Requires a userAgent string describing your application.
+// optionally a custom http.Client to allow for advanced features such as caching.
+func NewAPIClient(cfg *Configuration) *APIClient {
+ if cfg.HTTPClient == nil {
+ cfg.HTTPClient = http.DefaultClient
+ }
+
+ c := &APIClient{}
+ c.cfg = cfg
+ c.common.client = c
+
+ // API Services
+ c.PinsApi = (*PinsApiService)(&c.common)
+
+ return c
+}
+
+// selectHeaderContentType select a content type from the available list.
+func selectHeaderContentType(contentTypes []string) string {
+ if len(contentTypes) == 0 {
+ return ""
+ }
+ if contains(contentTypes, "application/json") {
+ return "application/json"
+ }
+ return contentTypes[0] // use the first content type specified in 'consumes'
+}
+
+// selectHeaderAccept join all accept types and return
+func selectHeaderAccept(accepts []string) string {
+ if len(accepts) == 0 {
+ return ""
+ }
+
+ if contains(accepts, "application/json") {
+ return "application/json"
+ }
+
+ return strings.Join(accepts, ",")
+}
+
+// contains is a case insenstive match, finding needle in a haystack
+func contains(haystack []string, needle string) bool {
+ for _, a := range haystack {
+ if strings.EqualFold(a, needle) {
+ return true
+ }
+ }
+ return false
+}
+
+// parameterToString convert interface{} parameters to string, using a delimiter if format is provided.
+func parameterToString(obj interface{}, collectionFormat string) string {
+ var delimiter string
+
+ switch collectionFormat {
+ case "pipes":
+ delimiter = "|"
+ case "ssv":
+ delimiter = " "
+ case "tsv":
+ delimiter = "\t"
+ case "csv":
+ delimiter = ","
+ }
+
+ if reflect.TypeOf(obj).Kind() == reflect.Slice {
+ return strings.Trim(strings.Replace(fmt.Sprint(obj), " ", delimiter, -1), "[]")
+ } else if t, ok := obj.(time.Time); ok {
+ return t.Format(time.RFC3339Nano)
+ }
+
+ return fmt.Sprintf("%v", obj)
+}
+
+// callAPI do the request.
+func (c *APIClient) callAPI(request *http.Request) (*http.Response, error) {
+ if c.cfg.Debug {
+ dump, err := httputil.DumpRequestOut(request, true)
+ if err != nil {
+ return nil, err
+ }
+ log.Printf("\n%s\n", string(dump))
+ }
+
+ resp, err := c.cfg.HTTPClient.Do(request)
+ if err != nil {
+ return resp, err
+ }
+
+ if c.cfg.Debug {
+ dump, err := httputil.DumpResponse(resp, true)
+ if err != nil {
+ return resp, err
+ }
+ log.Printf("\n%s\n", string(dump))
+ }
+ return resp, err
+}
+
+// Allow modification of underlying config for alternate implementations and testing
+// Caution: modifying the configuration while live can cause data races and potentially unwanted behavior
+func (c *APIClient) GetConfig() *Configuration {
+ return c.cfg
+}
+
+// prepareRequest build the request
+func (c *APIClient) prepareRequest(
+ ctx context.Context,
+ path string, method string,
+ postBody interface{},
+ headerParams map[string]string,
+ queryParams url.Values,
+ formParams url.Values,
+ formFileName string,
+ fileName string,
+ fileBytes []byte) (localVarRequest *http.Request, err error) {
+
+ var body *bytes.Buffer
+
+ // Detect postBody type and post.
+ if postBody != nil {
+ contentType := headerParams["Content-Type"]
+ if contentType == "" {
+ contentType = detectContentType(postBody)
+ headerParams["Content-Type"] = contentType
+ }
+
+ body, err = setBody(postBody, contentType)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // add form parameters and file if available.
+ if strings.HasPrefix(headerParams["Content-Type"], "multipart/form-data") && len(formParams) > 0 || (len(fileBytes) > 0 && fileName != "") {
+ if body != nil {
+ return nil, errors.New("cannot specify postBody and multipart form at the same time")
+ }
+ body = &bytes.Buffer{}
+ w := multipart.NewWriter(body)
+
+ for k, v := range formParams {
+ for _, iv := range v {
+ if strings.HasPrefix(k, "@") { // file
+ err = addFile(w, k[1:], iv)
+ if err != nil {
+ return nil, err
+ }
+ } else { // form value
+ w.WriteField(k, iv)
+ }
+ }
+ }
+ if len(fileBytes) > 0 && fileName != "" {
+ w.Boundary()
+ //_, fileNm := filepath.Split(fileName)
+ part, err := w.CreateFormFile(formFileName, filepath.Base(fileName))
+ if err != nil {
+ return nil, err
+ }
+ _, err = part.Write(fileBytes)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Set the Boundary in the Content-Type
+ headerParams["Content-Type"] = w.FormDataContentType()
+
+ // Set Content-Length
+ headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len())
+ w.Close()
+ }
+
+ if strings.HasPrefix(headerParams["Content-Type"], "application/x-www-form-urlencoded") && len(formParams) > 0 {
+ if body != nil {
+ return nil, errors.New("cannot specify postBody and x-www-form-urlencoded form at the same time")
+ }
+ body = &bytes.Buffer{}
+ body.WriteString(formParams.Encode())
+ // Set Content-Length
+ headerParams["Content-Length"] = fmt.Sprintf("%d", body.Len())
+ }
+
+ // Setup path and query parameters
+ url, err := url.Parse(path)
+ if err != nil {
+ return nil, err
+ }
+
+ // Override request host, if applicable
+ if c.cfg.Host != "" {
+ url.Host = c.cfg.Host
+ }
+
+ // Override request scheme, if applicable
+ if c.cfg.Scheme != "" {
+ url.Scheme = c.cfg.Scheme
+ }
+
+ // Adding Query Param
+ query := url.Query()
+ for k, v := range queryParams {
+ for _, iv := range v {
+ query.Add(k, iv)
+ }
+ }
+
+ // Encode the parameters.
+ url.RawQuery = query.Encode()
+
+ // Generate a new request
+ if body != nil {
+ localVarRequest, err = http.NewRequest(method, url.String(), body)
+ } else {
+ localVarRequest, err = http.NewRequest(method, url.String(), nil)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ // add header parameters, if any
+ if len(headerParams) > 0 {
+ headers := http.Header{}
+ for h, v := range headerParams {
+ headers.Set(h, v)
+ }
+ localVarRequest.Header = headers
+ }
+
+ // Add the user agent to the request.
+ localVarRequest.Header.Add("User-Agent", c.cfg.UserAgent)
+
+ if ctx != nil {
+ // add context to the request
+ localVarRequest = localVarRequest.WithContext(ctx)
+
+ // Walk through any authentication.
+
+ // OAuth2 authentication
+ if tok, ok := ctx.Value(ContextOAuth2).(oauth2.TokenSource); ok {
+ // We were able to grab an oauth2 token from the context
+ var latestToken *oauth2.Token
+ if latestToken, err = tok.Token(); err != nil {
+ return nil, err
+ }
+
+ latestToken.SetAuthHeader(localVarRequest)
+ }
+
+ // Basic HTTP Authentication
+ if auth, ok := ctx.Value(ContextBasicAuth).(BasicAuth); ok {
+ localVarRequest.SetBasicAuth(auth.UserName, auth.Password)
+ }
+
+ // AccessToken Authentication
+ if auth, ok := ctx.Value(ContextAccessToken).(string); ok {
+ localVarRequest.Header.Add("Authorization", "Bearer "+auth)
+ }
+ }
+
+ for header, value := range c.cfg.DefaultHeader {
+ localVarRequest.Header.Add(header, value)
+ }
+ return localVarRequest, nil
+}
+
+func (c *APIClient) decode(v interface{}, b []byte, contentType string) (err error) {
+ if len(b) == 0 {
+ return nil
+ }
+ if s, ok := v.(*string); ok {
+ *s = string(b)
+ return nil
+ }
+ if xmlCheck.MatchString(contentType) {
+ if err = xml.Unmarshal(b, v); err != nil {
+ return err
+ }
+ return nil
+ }
+ if jsonCheck.MatchString(contentType) {
+ if actualObj, ok := v.(interface{ GetActualInstance() interface{} }); ok { // oneOf, anyOf schemas
+ if unmarshalObj, ok := actualObj.(interface{ UnmarshalJSON([]byte) error }); ok { // make sure it has UnmarshalJSON defined
+ if err = unmarshalObj.UnmarshalJSON(b); err != nil {
+ return err
+ }
+ } else {
+ return errors.New("unknown type with GetActualInstance but no unmarshalObj.UnmarshalJSON defined")
+ }
+ } else if err = json.Unmarshal(b, v); err != nil { // simple model
+ return err
+ }
+ return nil
+ }
+ return errors.New("undefined response type")
+}
+
+// Add a file to the multipart request
+func addFile(w *multipart.Writer, fieldName, path string) error {
+ file, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ part, err := w.CreateFormFile(fieldName, filepath.Base(path))
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(part, file)
+
+ return err
+}
+
+// Prevent trying to import "fmt"
+func reportError(format string, a ...interface{}) error {
+ return fmt.Errorf(format, a...)
+}
+
+// Set request body from an interface{}
+func setBody(body interface{}, contentType string) (bodyBuf *bytes.Buffer, err error) {
+ if bodyBuf == nil {
+ bodyBuf = &bytes.Buffer{}
+ }
+
+ if reader, ok := body.(io.Reader); ok {
+ _, err = bodyBuf.ReadFrom(reader)
+ } else if b, ok := body.([]byte); ok {
+ _, err = bodyBuf.Write(b)
+ } else if s, ok := body.(string); ok {
+ _, err = bodyBuf.WriteString(s)
+ } else if s, ok := body.(*string); ok {
+ _, err = bodyBuf.WriteString(*s)
+ } else if jsonCheck.MatchString(contentType) {
+ err = json.NewEncoder(bodyBuf).Encode(body)
+ } else if xmlCheck.MatchString(contentType) {
+ err = xml.NewEncoder(bodyBuf).Encode(body)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ if bodyBuf.Len() == 0 {
+ err = fmt.Errorf("invalid body type %s", contentType)
+ return nil, err
+ }
+ return bodyBuf, nil
+}
+
+// detectContentType method is used to figure out `Request.Body` content type for request header
+func detectContentType(body interface{}) string {
+ contentType := "text/plain; charset=utf-8"
+ kind := reflect.TypeOf(body).Kind()
+
+ switch kind {
+ case reflect.Struct, reflect.Map, reflect.Ptr:
+ contentType = "application/json; charset=utf-8"
+ case reflect.String:
+ contentType = "text/plain; charset=utf-8"
+ default:
+ if b, ok := body.([]byte); ok {
+ contentType = http.DetectContentType(b)
+ } else if kind == reflect.Slice {
+ contentType = "application/json; charset=utf-8"
+ }
+ }
+
+ return contentType
+}
+
+// Ripped from https://github.com/gregjones/httpcache/blob/master/httpcache.go
+type cacheControl map[string]string
+
+func parseCacheControl(headers http.Header) cacheControl {
+ cc := cacheControl{}
+ ccHeader := headers.Get("Cache-Control")
+ for _, part := range strings.Split(ccHeader, ",") {
+ part = strings.Trim(part, " ")
+ if part == "" {
+ continue
+ }
+ if strings.ContainsRune(part, '=') {
+ keyval := strings.Split(part, "=")
+ cc[strings.Trim(keyval[0], " ")] = strings.Trim(keyval[1], ",")
+ } else {
+ cc[part] = ""
+ }
+ }
+ return cc
+}
+
+// CacheExpires helper function to determine remaining time before repeating a request.
+func CacheExpires(r *http.Response) time.Time {
+ // Figure out when the cache expires.
+ var expires time.Time
+ now, err := time.Parse(time.RFC1123, r.Header.Get("date"))
+ if err != nil {
+ return time.Now()
+ }
+ respCacheControl := parseCacheControl(r.Header)
+
+ if maxAge, ok := respCacheControl["max-age"]; ok {
+ lifetime, err := time.ParseDuration(maxAge + "s")
+ if err != nil {
+ expires = now
+ } else {
+ expires = now.Add(lifetime)
+ }
+ } else {
+ expiresHeader := r.Header.Get("Expires")
+ if expiresHeader != "" {
+ expires, err = time.Parse(time.RFC1123, expiresHeader)
+ if err != nil {
+ expires = now
+ }
+ }
+ }
+ return expires
+}
+
+// GenericOpenAPIError Provides access to the body, error and model on returned errors.
+type GenericOpenAPIError struct {
+ body []byte
+ error string
+ model interface{}
+}
+
+// Error returns non-empty string if there was an error.
+func (e GenericOpenAPIError) Error() string {
+ return e.error
+}
+
+// Body returns the raw bytes of the response
+func (e GenericOpenAPIError) Body() []byte {
+ return e.body
+}
+
+// Model returns the unpacked model of the error
+func (e GenericOpenAPIError) Model() interface{} {
+ return e.model
+}
diff --git a/pinning/remote/client/openapi/configuration.go b/pinning/remote/client/openapi/configuration.go
new file mode 100644
index 000000000..bb0a8e507
--- /dev/null
+++ b/pinning/remote/client/openapi/configuration.go
@@ -0,0 +1,228 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strings"
+)
+
+// contextKeys are used to identify the type of value in the context.
+// Since these are string, it is possible to get a short description of the
+// context key for logging and debugging using key.String().
+
+type contextKey string
+
+func (c contextKey) String() string {
+ return "auth " + string(c)
+}
+
+var (
+ // ContextOAuth2 takes an oauth2.TokenSource as authentication for the request.
+ ContextOAuth2 = contextKey("token")
+
+ // ContextBasicAuth takes BasicAuth as authentication for the request.
+ ContextBasicAuth = contextKey("basic")
+
+ // ContextAccessToken takes a string oauth2 access token as authentication for the request.
+ ContextAccessToken = contextKey("accesstoken")
+
+ // ContextAPIKeys takes a string apikey as authentication for the request
+ ContextAPIKeys = contextKey("apiKeys")
+
+ // ContextHttpSignatureAuth takes HttpSignatureAuth as authentication for the request.
+ ContextHttpSignatureAuth = contextKey("httpsignature")
+
+ // ContextServerIndex uses a server configuration from the index.
+ ContextServerIndex = contextKey("serverIndex")
+
+ // ContextOperationServerIndices uses a server configuration from the index mapping.
+ ContextOperationServerIndices = contextKey("serverOperationIndices")
+
+ // ContextServerVariables overrides a server configuration variables.
+ ContextServerVariables = contextKey("serverVariables")
+
+ // ContextOperationServerVariables overrides a server configuration variables using operation specific values.
+ ContextOperationServerVariables = contextKey("serverOperationVariables")
+)
+
+// BasicAuth provides basic http authentication to a request passed via context using ContextBasicAuth
+type BasicAuth struct {
+ UserName string `json:"userName,omitempty"`
+ Password string `json:"password,omitempty"`
+}
+
+// APIKey provides API key based authentication to a request passed via context using ContextAPIKey
+type APIKey struct {
+ Key string
+ Prefix string
+}
+
+// ServerVariable stores the information about a server variable
+type ServerVariable struct {
+ Description string
+ DefaultValue string
+ EnumValues []string
+}
+
+// ServerConfiguration stores the information about a server
+type ServerConfiguration struct {
+ URL string
+ Description string
+ Variables map[string]ServerVariable
+}
+
+// ServerConfigurations stores multiple ServerConfiguration items
+type ServerConfigurations []ServerConfiguration
+
+// Configuration stores the configuration of the API client
+type Configuration struct {
+ Host string `json:"host,omitempty"`
+ Scheme string `json:"scheme,omitempty"`
+ DefaultHeader map[string]string `json:"defaultHeader,omitempty"`
+ UserAgent string `json:"userAgent,omitempty"`
+ Debug bool `json:"debug,omitempty"`
+ Servers ServerConfigurations
+ OperationServers map[string]ServerConfigurations
+ HTTPClient *http.Client
+}
+
+// NewConfiguration returns a new Configuration object
+func NewConfiguration() *Configuration {
+ cfg := &Configuration{
+ DefaultHeader: make(map[string]string),
+ UserAgent: "OpenAPI-Generator/1.0.0/go",
+ Debug: false,
+ Servers: ServerConfigurations{
+ {
+ URL: "https://pinning-service.example.com",
+ Description: "No description provided",
+ },
+ },
+ OperationServers: map[string]ServerConfigurations{},
+ }
+ return cfg
+}
+
+// AddDefaultHeader adds a new HTTP header to the default header in the request
+func (c *Configuration) AddDefaultHeader(key string, value string) {
+ c.DefaultHeader[key] = value
+}
+
+// URL formats template on a index using given variables
+func (sc ServerConfigurations) URL(index int, variables map[string]string) (string, error) {
+ if index < 0 || len(sc) <= index {
+ return "", fmt.Errorf("index %v out of range %v", index, len(sc)-1)
+ }
+ server := sc[index]
+ url := server.URL
+
+ // go through variables and replace placeholders
+ for name, variable := range server.Variables {
+ if value, ok := variables[name]; ok {
+ found := bool(len(variable.EnumValues) == 0)
+ for _, enumValue := range variable.EnumValues {
+ if value == enumValue {
+ found = true
+ }
+ }
+ if !found {
+ return "", fmt.Errorf("the variable %s in the server URL has invalid value %v. Must be %v", name, value, variable.EnumValues)
+ }
+ url = strings.Replace(url, "{"+name+"}", value, -1)
+ } else {
+ url = strings.Replace(url, "{"+name+"}", variable.DefaultValue, -1)
+ }
+ }
+ return url, nil
+}
+
+// ServerURL returns URL based on server settings
+func (c *Configuration) ServerURL(index int, variables map[string]string) (string, error) {
+ return c.Servers.URL(index, variables)
+}
+
+func getServerIndex(ctx context.Context) (int, error) {
+ si := ctx.Value(ContextServerIndex)
+ if si != nil {
+ if index, ok := si.(int); ok {
+ return index, nil
+ }
+ return 0, reportError("Invalid type %T should be int", si)
+ }
+ return 0, nil
+}
+
+func getServerOperationIndex(ctx context.Context, endpoint string) (int, error) {
+ osi := ctx.Value(ContextOperationServerIndices)
+ if osi != nil {
+ if operationIndices, ok := osi.(map[string]int); !ok {
+ return 0, reportError("Invalid type %T should be map[string]int", osi)
+ } else {
+ index, ok := operationIndices[endpoint]
+ if ok {
+ return index, nil
+ }
+ }
+ }
+ return getServerIndex(ctx)
+}
+
+func getServerVariables(ctx context.Context) (map[string]string, error) {
+ sv := ctx.Value(ContextServerVariables)
+ if sv != nil {
+ if variables, ok := sv.(map[string]string); ok {
+ return variables, nil
+ }
+ return nil, reportError("ctx value of ContextServerVariables has invalid type %T should be map[string]string", sv)
+ }
+ return nil, nil
+}
+
+func getServerOperationVariables(ctx context.Context, endpoint string) (map[string]string, error) {
+ osv := ctx.Value(ContextOperationServerVariables)
+ if osv != nil {
+ if operationVariables, ok := osv.(map[string]map[string]string); !ok {
+ return nil, reportError("ctx value of ContextOperationServerVariables has invalid type %T should be map[string]map[string]string", osv)
+ } else {
+ variables, ok := operationVariables[endpoint]
+ if ok {
+ return variables, nil
+ }
+ }
+ }
+ return getServerVariables(ctx)
+}
+
+// ServerURLWithContext returns a new server URL given an endpoint
+func (c *Configuration) ServerURLWithContext(ctx context.Context, endpoint string) (string, error) {
+ sc, ok := c.OperationServers[endpoint]
+ if !ok {
+ sc = c.Servers
+ }
+
+ if ctx == nil {
+ return sc.URL(0, nil)
+ }
+
+ index, err := getServerOperationIndex(ctx, endpoint)
+ if err != nil {
+ return "", err
+ }
+
+ variables, err := getServerOperationVariables(ctx, endpoint)
+ if err != nil {
+ return "", err
+ }
+
+ return sc.URL(index, variables)
+}
diff --git a/pinning/remote/client/openapi/docs/Failure.md b/pinning/remote/client/openapi/docs/Failure.md
new file mode 100644
index 000000000..c899f7138
--- /dev/null
+++ b/pinning/remote/client/openapi/docs/Failure.md
@@ -0,0 +1,51 @@
+# Failure
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Error** | [**FailureError**](Failure_error.md) | |
+
+## Methods
+
+### NewFailure
+
+`func NewFailure(error_ FailureError, ) *Failure`
+
+NewFailure instantiates a new Failure object
+This constructor will assign default values to properties that have it defined,
+and makes sure properties required by API are set, but the set of arguments
+will change when the set of required properties is changed
+
+### NewFailureWithDefaults
+
+`func NewFailureWithDefaults() *Failure`
+
+NewFailureWithDefaults instantiates a new Failure object
+This constructor will only assign default values to properties that have it defined,
+but it doesn't guarantee that properties required by API are set
+
+### GetError
+
+`func (o *Failure) GetError() FailureError`
+
+GetError returns the Error field if non-nil, zero value otherwise.
+
+### GetErrorOk
+
+`func (o *Failure) GetErrorOk() (*FailureError, bool)`
+
+GetErrorOk returns a tuple with the Error field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetError
+
+`func (o *Failure) SetError(v FailureError)`
+
+SetError sets Error field to given value.
+
+
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+
diff --git a/pinning/remote/client/openapi/docs/FailureError.md b/pinning/remote/client/openapi/docs/FailureError.md
new file mode 100644
index 000000000..478f1b942
--- /dev/null
+++ b/pinning/remote/client/openapi/docs/FailureError.md
@@ -0,0 +1,77 @@
+# FailureError
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Reason** | **string** | Mandatory string identifying the type of error |
+**Details** | Pointer to **string** | Optional, longer description of the error; may include UUID of transaction for support, links to documentation etc | [optional]
+
+## Methods
+
+### NewFailureError
+
+`func NewFailureError(reason string, ) *FailureError`
+
+NewFailureError instantiates a new FailureError object
+This constructor will assign default values to properties that have it defined,
+and makes sure properties required by API are set, but the set of arguments
+will change when the set of required properties is changed
+
+### NewFailureErrorWithDefaults
+
+`func NewFailureErrorWithDefaults() *FailureError`
+
+NewFailureErrorWithDefaults instantiates a new FailureError object
+This constructor will only assign default values to properties that have it defined,
+but it doesn't guarantee that properties required by API are set
+
+### GetReason
+
+`func (o *FailureError) GetReason() string`
+
+GetReason returns the Reason field if non-nil, zero value otherwise.
+
+### GetReasonOk
+
+`func (o *FailureError) GetReasonOk() (*string, bool)`
+
+GetReasonOk returns a tuple with the Reason field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetReason
+
+`func (o *FailureError) SetReason(v string)`
+
+SetReason sets Reason field to given value.
+
+
+### GetDetails
+
+`func (o *FailureError) GetDetails() string`
+
+GetDetails returns the Details field if non-nil, zero value otherwise.
+
+### GetDetailsOk
+
+`func (o *FailureError) GetDetailsOk() (*string, bool)`
+
+GetDetailsOk returns a tuple with the Details field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetDetails
+
+`func (o *FailureError) SetDetails(v string)`
+
+SetDetails sets Details field to given value.
+
+### HasDetails
+
+`func (o *FailureError) HasDetails() bool`
+
+HasDetails returns a boolean if a field has been set.
+
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+
diff --git a/pinning/remote/client/openapi/docs/Pin.md b/pinning/remote/client/openapi/docs/Pin.md
new file mode 100644
index 000000000..e5d3e0f18
--- /dev/null
+++ b/pinning/remote/client/openapi/docs/Pin.md
@@ -0,0 +1,129 @@
+# Pin
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Cid** | **string** | Content Identifier (CID) to be pinned recursively |
+**Name** | Pointer to **string** | Optional name for pinned data; can be used for lookups later | [optional]
+**Origins** | Pointer to **[]string** | Optional list of multiaddrs known to provide the data | [optional]
+**Meta** | Pointer to **map[string]string** | Optional metadata for pin object | [optional]
+
+## Methods
+
+### NewPin
+
+`func NewPin(cid string, ) *Pin`
+
+NewPin instantiates a new Pin object
+This constructor will assign default values to properties that have it defined,
+and makes sure properties required by API are set, but the set of arguments
+will change when the set of required properties is changed
+
+### NewPinWithDefaults
+
+`func NewPinWithDefaults() *Pin`
+
+NewPinWithDefaults instantiates a new Pin object
+This constructor will only assign default values to properties that have it defined,
+but it doesn't guarantee that properties required by API are set
+
+### GetCid
+
+`func (o *Pin) GetCid() string`
+
+GetCid returns the Cid field if non-nil, zero value otherwise.
+
+### GetCidOk
+
+`func (o *Pin) GetCidOk() (*string, bool)`
+
+GetCidOk returns a tuple with the Cid field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetCid
+
+`func (o *Pin) SetCid(v string)`
+
+SetCid sets Cid field to given value.
+
+
+### GetName
+
+`func (o *Pin) GetName() string`
+
+GetName returns the Name field if non-nil, zero value otherwise.
+
+### GetNameOk
+
+`func (o *Pin) GetNameOk() (*string, bool)`
+
+GetNameOk returns a tuple with the Name field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetName
+
+`func (o *Pin) SetName(v string)`
+
+SetName sets Name field to given value.
+
+### HasName
+
+`func (o *Pin) HasName() bool`
+
+HasName returns a boolean if a field has been set.
+
+### GetOrigins
+
+`func (o *Pin) GetOrigins() []string`
+
+GetOrigins returns the Origins field if non-nil, zero value otherwise.
+
+### GetOriginsOk
+
+`func (o *Pin) GetOriginsOk() (*[]string, bool)`
+
+GetOriginsOk returns a tuple with the Origins field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetOrigins
+
+`func (o *Pin) SetOrigins(v []string)`
+
+SetOrigins sets Origins field to given value.
+
+### HasOrigins
+
+`func (o *Pin) HasOrigins() bool`
+
+HasOrigins returns a boolean if a field has been set.
+
+### GetMeta
+
+`func (o *Pin) GetMeta() map[string]string`
+
+GetMeta returns the Meta field if non-nil, zero value otherwise.
+
+### GetMetaOk
+
+`func (o *Pin) GetMetaOk() (*map[string]string, bool)`
+
+GetMetaOk returns a tuple with the Meta field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetMeta
+
+`func (o *Pin) SetMeta(v map[string]string)`
+
+SetMeta sets Meta field to given value.
+
+### HasMeta
+
+`func (o *Pin) HasMeta() bool`
+
+HasMeta returns a boolean if a field has been set.
+
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+
diff --git a/pinning/remote/client/openapi/docs/PinResults.md b/pinning/remote/client/openapi/docs/PinResults.md
new file mode 100644
index 000000000..1982bfddb
--- /dev/null
+++ b/pinning/remote/client/openapi/docs/PinResults.md
@@ -0,0 +1,72 @@
+# PinResults
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Count** | **int32** | The total number of pin objects that exist for passed query filters |
+**Results** | [**[]PinStatus**](PinStatus.md) | An array of PinStatus results |
+
+## Methods
+
+### NewPinResults
+
+`func NewPinResults(count int32, results []PinStatus, ) *PinResults`
+
+NewPinResults instantiates a new PinResults object
+This constructor will assign default values to properties that have it defined,
+and makes sure properties required by API are set, but the set of arguments
+will change when the set of required properties is changed
+
+### NewPinResultsWithDefaults
+
+`func NewPinResultsWithDefaults() *PinResults`
+
+NewPinResultsWithDefaults instantiates a new PinResults object
+This constructor will only assign default values to properties that have it defined,
+but it doesn't guarantee that properties required by API are set
+
+### GetCount
+
+`func (o *PinResults) GetCount() int32`
+
+GetCount returns the Count field if non-nil, zero value otherwise.
+
+### GetCountOk
+
+`func (o *PinResults) GetCountOk() (*int32, bool)`
+
+GetCountOk returns a tuple with the Count field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetCount
+
+`func (o *PinResults) SetCount(v int32)`
+
+SetCount sets Count field to given value.
+
+
+### GetResults
+
+`func (o *PinResults) GetResults() []PinStatus`
+
+GetResults returns the Results field if non-nil, zero value otherwise.
+
+### GetResultsOk
+
+`func (o *PinResults) GetResultsOk() (*[]PinStatus, bool)`
+
+GetResultsOk returns a tuple with the Results field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetResults
+
+`func (o *PinResults) SetResults(v []PinStatus)`
+
+SetResults sets Results field to given value.
+
+
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+
diff --git a/pinning/remote/client/openapi/docs/PinStatus.md b/pinning/remote/client/openapi/docs/PinStatus.md
new file mode 100644
index 000000000..40ae992ab
--- /dev/null
+++ b/pinning/remote/client/openapi/docs/PinStatus.md
@@ -0,0 +1,161 @@
+# PinStatus
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**Requestid** | **string** | Globally unique identifier of the pin request; can be used to check the status of ongoing pinning, or pin removal |
+**Status** | [**Status**](Status.md) | |
+**Created** | [**time.Time**](time.Time.md) | Immutable timestamp indicating when a pin request entered a pinning service; can be used for filtering results and pagination |
+**Pin** | [**Pin**](Pin.md) | |
+**Delegates** | **[]string** | List of multiaddrs designated by pinning service for transferring any new data from external peers |
+**Info** | Pointer to **map[string]string** | Optional info for PinStatus response | [optional]
+
+## Methods
+
+### NewPinStatus
+
+`func NewPinStatus(requestid string, status Status, created time.Time, pin Pin, delegates []string, ) *PinStatus`
+
+NewPinStatus instantiates a new PinStatus object
+This constructor will assign default values to properties that have it defined,
+and makes sure properties required by API are set, but the set of arguments
+will change when the set of required properties is changed
+
+### NewPinStatusWithDefaults
+
+`func NewPinStatusWithDefaults() *PinStatus`
+
+NewPinStatusWithDefaults instantiates a new PinStatus object
+This constructor will only assign default values to properties that have it defined,
+but it doesn't guarantee that properties required by API are set
+
+### GetRequestid
+
+`func (o *PinStatus) GetRequestid() string`
+
+GetRequestid returns the Requestid field if non-nil, zero value otherwise.
+
+### GetRequestidOk
+
+`func (o *PinStatus) GetRequestidOk() (*string, bool)`
+
+GetRequestidOk returns a tuple with the Requestid field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetRequestid
+
+`func (o *PinStatus) SetRequestid(v string)`
+
+SetRequestid sets Requestid field to given value.
+
+
+### GetStatus
+
+`func (o *PinStatus) GetStatus() Status`
+
+GetStatus returns the Status field if non-nil, zero value otherwise.
+
+### GetStatusOk
+
+`func (o *PinStatus) GetStatusOk() (*Status, bool)`
+
+GetStatusOk returns a tuple with the Status field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetStatus
+
+`func (o *PinStatus) SetStatus(v Status)`
+
+SetStatus sets Status field to given value.
+
+
+### GetCreated
+
+`func (o *PinStatus) GetCreated() time.Time`
+
+GetCreated returns the Created field if non-nil, zero value otherwise.
+
+### GetCreatedOk
+
+`func (o *PinStatus) GetCreatedOk() (*time.Time, bool)`
+
+GetCreatedOk returns a tuple with the Created field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetCreated
+
+`func (o *PinStatus) SetCreated(v time.Time)`
+
+SetCreated sets Created field to given value.
+
+
+### GetPin
+
+`func (o *PinStatus) GetPin() Pin`
+
+GetPin returns the Pin field if non-nil, zero value otherwise.
+
+### GetPinOk
+
+`func (o *PinStatus) GetPinOk() (*Pin, bool)`
+
+GetPinOk returns a tuple with the Pin field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetPin
+
+`func (o *PinStatus) SetPin(v Pin)`
+
+SetPin sets Pin field to given value.
+
+
+### GetDelegates
+
+`func (o *PinStatus) GetDelegates() []string`
+
+GetDelegates returns the Delegates field if non-nil, zero value otherwise.
+
+### GetDelegatesOk
+
+`func (o *PinStatus) GetDelegatesOk() (*[]string, bool)`
+
+GetDelegatesOk returns a tuple with the Delegates field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetDelegates
+
+`func (o *PinStatus) SetDelegates(v []string)`
+
+SetDelegates sets Delegates field to given value.
+
+
+### GetInfo
+
+`func (o *PinStatus) GetInfo() map[string]string`
+
+GetInfo returns the Info field if non-nil, zero value otherwise.
+
+### GetInfoOk
+
+`func (o *PinStatus) GetInfoOk() (*map[string]string, bool)`
+
+GetInfoOk returns a tuple with the Info field if it's non-nil, zero value otherwise
+and a boolean to check if the value has been set.
+
+### SetInfo
+
+`func (o *PinStatus) SetInfo(v map[string]string)`
+
+SetInfo sets Info field to given value.
+
+### HasInfo
+
+`func (o *PinStatus) HasInfo() bool`
+
+HasInfo returns a boolean if a field has been set.
+
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+
diff --git a/pinning/remote/client/openapi/docs/PinsApi.md b/pinning/remote/client/openapi/docs/PinsApi.md
new file mode 100644
index 000000000..00cff667b
--- /dev/null
+++ b/pinning/remote/client/openapi/docs/PinsApi.md
@@ -0,0 +1,367 @@
+# \PinsApi
+
+All URIs are relative to *https://pinning-service.example.com*
+
+Method | HTTP request | Description
+------------- | ------------- | -------------
+[**PinsGet**](PinsApi.md#PinsGet) | **Get** /pins | List pin objects
+[**PinsPost**](PinsApi.md#PinsPost) | **Post** /pins | Add pin object
+[**PinsRequestidDelete**](PinsApi.md#PinsRequestidDelete) | **Delete** /pins/{requestid} | Remove pin object
+[**PinsRequestidGet**](PinsApi.md#PinsRequestidGet) | **Get** /pins/{requestid} | Get pin object
+[**PinsRequestidPost**](PinsApi.md#PinsRequestidPost) | **Post** /pins/{requestid} | Replace pin object
+
+
+
+## PinsGet
+
+> PinResults PinsGet(ctx).Cid(cid).Name(name).Status(status).Before(before).After(after).Limit(limit).Meta(meta).Execute()
+
+List pin objects
+
+
+
+### Example
+
+```go
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ openapiclient "./openapi"
+)
+
+func main() {
+ cid := []string{"Inner_example"} // []string | Return pin objects responsible for pinning the specified CID(s); be aware that using longer hash functions introduces further constraints on the number of CIDs that will fit under the limit of 2000 characters per URL in browser contexts (optional)
+ name := "name_example" // string | Return pin objects with names that contain provided value (case-insensitive, partial or full match) (optional)
+ status := []Status{openapiclient.Status{}} // []Status | Return pin objects for pins with the specified status (optional)
+ before := Get-Date // time.Time | Return results created (queued) before provided timestamp (optional)
+ after := Get-Date // time.Time | Return results created (queued) after provided timestamp (optional)
+ limit := 987 // int32 | Max records to return (optional) (default to 10)
+ meta := map[string]string{ "Key" = "Value" } // map[string]string | Return pin objects that match specified metadata (optional)
+
+ configuration := openapiclient.NewConfiguration()
+ api_client := openapiclient.NewAPIClient(configuration)
+ resp, r, err := api_client.PinsApi.PinsGet(context.Background(), ).Cid(cid).Name(name).Status(status).Before(before).After(after).Limit(limit).Meta(meta).Execute()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsGet``: %v\n", err)
+ fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
+ }
+ // response from `PinsGet`: PinResults
+ fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsGet`: %v\n", resp)
+}
+```
+
+### Path Parameters
+
+
+
+### Other Parameters
+
+Other parameters are passed through a pointer to a apiPinsGetRequest struct via the builder pattern
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+ **cid** | [**[]string**](string.md) | Return pin objects responsible for pinning the specified CID(s); be aware that using longer hash functions introduces further constraints on the number of CIDs that will fit under the limit of 2000 characters per URL in browser contexts |
+ **name** | **string** | Return pin objects with names that contain provided value (case-insensitive, partial or full match) |
+ **status** | [**[]Status**](Status.md) | Return pin objects for pins with the specified status |
+ **before** | **time.Time** | Return results created (queued) before provided timestamp |
+ **after** | **time.Time** | Return results created (queued) after provided timestamp |
+ **limit** | **int32** | Max records to return | [default to 10]
+ **meta** | [**map[string]string**](string.md) | Return pin objects that match specified metadata |
+
+### Return type
+
+[**PinResults**](PinResults.md)
+
+### Authorization
+
+[accessToken](../README.md#accessToken)
+
+### HTTP request headers
+
+- **Content-Type**: Not defined
+- **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
+
+## PinsPost
+
+> PinStatus PinsPost(ctx).Pin(pin).Execute()
+
+Add pin object
+
+
+
+### Example
+
+```go
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ openapiclient "./openapi"
+)
+
+func main() {
+ pin := openapiclient.Pin{Cid: "Cid_example", Name: "Name_example", Origins: []string{"Origins_example"), Meta: map[string]string{ "Key" = "Value" }} // Pin |
+
+ configuration := openapiclient.NewConfiguration()
+ api_client := openapiclient.NewAPIClient(configuration)
+ resp, r, err := api_client.PinsApi.PinsPost(context.Background(), pin).Execute()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsPost``: %v\n", err)
+ fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
+ }
+ // response from `PinsPost`: PinStatus
+ fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsPost`: %v\n", resp)
+}
+```
+
+### Path Parameters
+
+
+
+### Other Parameters
+
+Other parameters are passed through a pointer to a apiPinsPostRequest struct via the builder pattern
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+ **pin** | [**Pin**](Pin.md) | |
+
+### Return type
+
+[**PinStatus**](PinStatus.md)
+
+### Authorization
+
+[accessToken](../README.md#accessToken)
+
+### HTTP request headers
+
+- **Content-Type**: application/json
+- **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
+
+## PinsRequestidDelete
+
+> PinsRequestidDelete(ctx, requestid).Execute()
+
+Remove pin object
+
+
+
+### Example
+
+```go
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ openapiclient "./openapi"
+)
+
+func main() {
+ requestid := "requestid_example" // string |
+
+ configuration := openapiclient.NewConfiguration()
+ api_client := openapiclient.NewAPIClient(configuration)
+ resp, r, err := api_client.PinsApi.PinsRequestidDelete(context.Background(), requestid).Execute()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsRequestidDelete``: %v\n", err)
+ fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
+ }
+}
+```
+
+### Path Parameters
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc.
+**requestid** | **string** | |
+
+### Other Parameters
+
+Other parameters are passed through a pointer to a apiPinsRequestidDeleteRequest struct via the builder pattern
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+
+
+### Return type
+
+ (empty response body)
+
+### Authorization
+
+[accessToken](../README.md#accessToken)
+
+### HTTP request headers
+
+- **Content-Type**: Not defined
+- **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
+
+## PinsRequestidGet
+
+> PinStatus PinsRequestidGet(ctx, requestid).Execute()
+
+Get pin object
+
+
+
+### Example
+
+```go
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ openapiclient "./openapi"
+)
+
+func main() {
+ requestid := "requestid_example" // string |
+
+ configuration := openapiclient.NewConfiguration()
+ api_client := openapiclient.NewAPIClient(configuration)
+ resp, r, err := api_client.PinsApi.PinsRequestidGet(context.Background(), requestid).Execute()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsRequestidGet``: %v\n", err)
+ fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
+ }
+ // response from `PinsRequestidGet`: PinStatus
+ fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsRequestidGet`: %v\n", resp)
+}
+```
+
+### Path Parameters
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc.
+**requestid** | **string** | |
+
+### Other Parameters
+
+Other parameters are passed through a pointer to a apiPinsRequestidGetRequest struct via the builder pattern
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+
+
+### Return type
+
+[**PinStatus**](PinStatus.md)
+
+### Authorization
+
+[accessToken](../README.md#accessToken)
+
+### HTTP request headers
+
+- **Content-Type**: Not defined
+- **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
+
+## PinsRequestidPost
+
+> PinStatus PinsRequestidPost(ctx, requestid).Pin(pin).Execute()
+
+Replace pin object
+
+
+
+### Example
+
+```go
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ openapiclient "./openapi"
+)
+
+func main() {
+ requestid := "requestid_example" // string |
+ pin := openapiclient.Pin{Cid: "Cid_example", Name: "Name_example", Origins: []string{"Origins_example"), Meta: map[string]string{ "Key" = "Value" }} // Pin |
+
+ configuration := openapiclient.NewConfiguration()
+ api_client := openapiclient.NewAPIClient(configuration)
+ resp, r, err := api_client.PinsApi.PinsRequestidPost(context.Background(), requestid, pin).Execute()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error when calling `PinsApi.PinsRequestidPost``: %v\n", err)
+ fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
+ }
+ // response from `PinsRequestidPost`: PinStatus
+ fmt.Fprintf(os.Stdout, "Response from `PinsApi.PinsRequestidPost`: %v\n", resp)
+}
+```
+
+### Path Parameters
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+**ctx** | **context.Context** | context for authentication, logging, cancellation, deadlines, tracing, etc.
+**requestid** | **string** | |
+
+### Other Parameters
+
+Other parameters are passed through a pointer to a apiPinsRequestidPostRequest struct via the builder pattern
+
+
+Name | Type | Description | Notes
+------------- | ------------- | ------------- | -------------
+
+ **pin** | [**Pin**](Pin.md) | |
+
+### Return type
+
+[**PinStatus**](PinStatus.md)
+
+### Authorization
+
+[accessToken](../README.md#accessToken)
+
+### HTTP request headers
+
+- **Content-Type**: application/json
+- **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints)
+[[Back to Model list]](../README.md#documentation-for-models)
+[[Back to README]](../README.md)
+
diff --git a/pinning/remote/client/openapi/docs/Status.md b/pinning/remote/client/openapi/docs/Status.md
new file mode 100644
index 000000000..01176af11
--- /dev/null
+++ b/pinning/remote/client/openapi/docs/Status.md
@@ -0,0 +1,11 @@
+# Status
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+
diff --git a/pinning/remote/client/openapi/model_failure.go b/pinning/remote/client/openapi/model_failure.go
new file mode 100644
index 000000000..ec1fc8268
--- /dev/null
+++ b/pinning/remote/client/openapi/model_failure.go
@@ -0,0 +1,105 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "encoding/json"
+)
+
+// Failure Response for a failed request
+type Failure struct {
+ Error FailureError `json:"error"`
+}
+
+// NewFailure instantiates a new Failure object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewFailure(error_ FailureError) *Failure {
+ this := Failure{}
+ this.Error = error_
+ return &this
+}
+
+// NewFailureWithDefaults instantiates a new Failure object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewFailureWithDefaults() *Failure {
+ this := Failure{}
+ return &this
+}
+
+// GetError returns the Error field value
+func (o *Failure) GetError() FailureError {
+ if o == nil {
+ var ret FailureError
+ return ret
+ }
+
+ return o.Error
+}
+
+// GetErrorOk returns a tuple with the Error field value
+// and a boolean to check if the value has been set.
+func (o *Failure) GetErrorOk() (*FailureError, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Error, true
+}
+
+// SetError sets field value
+func (o *Failure) SetError(v FailureError) {
+ o.Error = v
+}
+
+func (o Failure) MarshalJSON() ([]byte, error) {
+ toSerialize := map[string]interface{}{}
+ if true {
+ toSerialize["error"] = o.Error
+ }
+ return json.Marshal(toSerialize)
+}
+
+type NullableFailure struct {
+ value *Failure
+ isSet bool
+}
+
+func (v NullableFailure) Get() *Failure {
+ return v.value
+}
+
+func (v *NullableFailure) Set(val *Failure) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableFailure) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableFailure) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableFailure(val *Failure) *NullableFailure {
+ return &NullableFailure{value: val, isSet: true}
+}
+
+func (v NullableFailure) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableFailure) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
diff --git a/pinning/remote/client/openapi/model_failure_error.go b/pinning/remote/client/openapi/model_failure_error.go
new file mode 100644
index 000000000..4f38acdf9
--- /dev/null
+++ b/pinning/remote/client/openapi/model_failure_error.go
@@ -0,0 +1,143 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "encoding/json"
+)
+
+// FailureError struct for FailureError
+type FailureError struct {
+ // Mandatory string identifying the type of error
+ Reason string `json:"reason"`
+ // Optional, longer description of the error; may include UUID of transaction for support, links to documentation etc
+ Details *string `json:"details,omitempty"`
+}
+
+// NewFailureError instantiates a new FailureError object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewFailureError(reason string) *FailureError {
+ this := FailureError{}
+ this.Reason = reason
+ return &this
+}
+
+// NewFailureErrorWithDefaults instantiates a new FailureError object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewFailureErrorWithDefaults() *FailureError {
+ this := FailureError{}
+ return &this
+}
+
+// GetReason returns the Reason field value
+func (o *FailureError) GetReason() string {
+ if o == nil {
+ var ret string
+ return ret
+ }
+
+ return o.Reason
+}
+
+// GetReasonOk returns a tuple with the Reason field value
+// and a boolean to check if the value has been set.
+func (o *FailureError) GetReasonOk() (*string, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Reason, true
+}
+
+// SetReason sets field value
+func (o *FailureError) SetReason(v string) {
+ o.Reason = v
+}
+
+// GetDetails returns the Details field value if set, zero value otherwise.
+func (o *FailureError) GetDetails() string {
+ if o == nil || o.Details == nil {
+ var ret string
+ return ret
+ }
+ return *o.Details
+}
+
+// GetDetailsOk returns a tuple with the Details field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *FailureError) GetDetailsOk() (*string, bool) {
+ if o == nil || o.Details == nil {
+ return nil, false
+ }
+ return o.Details, true
+}
+
+// HasDetails returns a boolean if a field has been set.
+func (o *FailureError) HasDetails() bool {
+ if o != nil && o.Details != nil {
+ return true
+ }
+
+ return false
+}
+
+// SetDetails gets a reference to the given string and assigns it to the Details field.
+func (o *FailureError) SetDetails(v string) {
+ o.Details = &v
+}
+
+func (o FailureError) MarshalJSON() ([]byte, error) {
+ toSerialize := map[string]interface{}{}
+ if true {
+ toSerialize["reason"] = o.Reason
+ }
+ if o.Details != nil {
+ toSerialize["details"] = o.Details
+ }
+ return json.Marshal(toSerialize)
+}
+
+type NullableFailureError struct {
+ value *FailureError
+ isSet bool
+}
+
+func (v NullableFailureError) Get() *FailureError {
+ return v.value
+}
+
+func (v *NullableFailureError) Set(val *FailureError) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableFailureError) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableFailureError) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableFailureError(val *FailureError) *NullableFailureError {
+ return &NullableFailureError{value: val, isSet: true}
+}
+
+func (v NullableFailureError) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableFailureError) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
diff --git a/pinning/remote/client/openapi/model_pin.go b/pinning/remote/client/openapi/model_pin.go
new file mode 100644
index 000000000..0152d43bb
--- /dev/null
+++ b/pinning/remote/client/openapi/model_pin.go
@@ -0,0 +1,217 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "encoding/json"
+)
+
+// Pin Pin object
+type Pin struct {
+ // Content Identifier (CID) to be pinned recursively
+ Cid string `json:"cid"`
+ // Optional name for pinned data; can be used for lookups later
+ Name *string `json:"name,omitempty"`
+ // Optional list of multiaddrs known to provide the data
+ Origins *[]string `json:"origins,omitempty"`
+ // Optional metadata for pin object
+ Meta *map[string]string `json:"meta,omitempty"`
+}
+
+// NewPin instantiates a new Pin object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewPin(cid string) *Pin {
+ this := Pin{}
+ this.Cid = cid
+ return &this
+}
+
+// NewPinWithDefaults instantiates a new Pin object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewPinWithDefaults() *Pin {
+ this := Pin{}
+ return &this
+}
+
+// GetCid returns the Cid field value
+func (o *Pin) GetCid() string {
+ if o == nil {
+ var ret string
+ return ret
+ }
+
+ return o.Cid
+}
+
+// GetCidOk returns a tuple with the Cid field value
+// and a boolean to check if the value has been set.
+func (o *Pin) GetCidOk() (*string, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Cid, true
+}
+
+// SetCid sets field value
+func (o *Pin) SetCid(v string) {
+ o.Cid = v
+}
+
+// GetName returns the Name field value if set, zero value otherwise.
+func (o *Pin) GetName() string {
+ if o == nil || o.Name == nil {
+ var ret string
+ return ret
+ }
+ return *o.Name
+}
+
+// GetNameOk returns a tuple with the Name field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *Pin) GetNameOk() (*string, bool) {
+ if o == nil || o.Name == nil {
+ return nil, false
+ }
+ return o.Name, true
+}
+
+// HasName returns a boolean if a field has been set.
+func (o *Pin) HasName() bool {
+ if o != nil && o.Name != nil {
+ return true
+ }
+
+ return false
+}
+
+// SetName gets a reference to the given string and assigns it to the Name field.
+func (o *Pin) SetName(v string) {
+ o.Name = &v
+}
+
+// GetOrigins returns the Origins field value if set, zero value otherwise.
+func (o *Pin) GetOrigins() []string {
+ if o == nil || o.Origins == nil {
+ var ret []string
+ return ret
+ }
+ return *o.Origins
+}
+
+// GetOriginsOk returns a tuple with the Origins field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *Pin) GetOriginsOk() (*[]string, bool) {
+ if o == nil || o.Origins == nil {
+ return nil, false
+ }
+ return o.Origins, true
+}
+
+// HasOrigins returns a boolean if a field has been set.
+func (o *Pin) HasOrigins() bool {
+ if o != nil && o.Origins != nil {
+ return true
+ }
+
+ return false
+}
+
+// SetOrigins gets a reference to the given []string and assigns it to the Origins field.
+func (o *Pin) SetOrigins(v []string) {
+ o.Origins = &v
+}
+
+// GetMeta returns the Meta field value if set, zero value otherwise.
+func (o *Pin) GetMeta() map[string]string {
+ if o == nil || o.Meta == nil {
+ var ret map[string]string
+ return ret
+ }
+ return *o.Meta
+}
+
+// GetMetaOk returns a tuple with the Meta field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *Pin) GetMetaOk() (*map[string]string, bool) {
+ if o == nil || o.Meta == nil {
+ return nil, false
+ }
+ return o.Meta, true
+}
+
+// HasMeta returns a boolean if a field has been set.
+func (o *Pin) HasMeta() bool {
+ if o != nil && o.Meta != nil {
+ return true
+ }
+
+ return false
+}
+
+// SetMeta gets a reference to the given map[string]string and assigns it to the Meta field.
+func (o *Pin) SetMeta(v map[string]string) {
+ o.Meta = &v
+}
+
+func (o Pin) MarshalJSON() ([]byte, error) {
+ toSerialize := map[string]interface{}{}
+ if true {
+ toSerialize["cid"] = o.Cid
+ }
+ if o.Name != nil {
+ toSerialize["name"] = o.Name
+ }
+ if o.Origins != nil {
+ toSerialize["origins"] = o.Origins
+ }
+ if o.Meta != nil {
+ toSerialize["meta"] = o.Meta
+ }
+ return json.Marshal(toSerialize)
+}
+
+type NullablePin struct {
+ value *Pin
+ isSet bool
+}
+
+func (v NullablePin) Get() *Pin {
+ return v.value
+}
+
+func (v *NullablePin) Set(val *Pin) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullablePin) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullablePin) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullablePin(val *Pin) *NullablePin {
+ return &NullablePin{value: val, isSet: true}
+}
+
+func (v NullablePin) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullablePin) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
diff --git a/pinning/remote/client/openapi/model_pin_results.go b/pinning/remote/client/openapi/model_pin_results.go
new file mode 100644
index 000000000..eacb5e021
--- /dev/null
+++ b/pinning/remote/client/openapi/model_pin_results.go
@@ -0,0 +1,136 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "encoding/json"
+)
+
+// PinResults Response used for listing pin objects matching request
+type PinResults struct {
+ // The total number of pin objects that exist for passed query filters
+ Count int32 `json:"count"`
+ // An array of PinStatus results
+ Results []PinStatus `json:"results"`
+}
+
+// NewPinResults instantiates a new PinResults object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewPinResults(count int32, results []PinStatus) *PinResults {
+ this := PinResults{}
+ this.Count = count
+ this.Results = results
+ return &this
+}
+
+// NewPinResultsWithDefaults instantiates a new PinResults object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewPinResultsWithDefaults() *PinResults {
+ this := PinResults{}
+ return &this
+}
+
+// GetCount returns the Count field value
+func (o *PinResults) GetCount() int32 {
+ if o == nil {
+ var ret int32
+ return ret
+ }
+
+ return o.Count
+}
+
+// GetCountOk returns a tuple with the Count field value
+// and a boolean to check if the value has been set.
+func (o *PinResults) GetCountOk() (*int32, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Count, true
+}
+
+// SetCount sets field value
+func (o *PinResults) SetCount(v int32) {
+ o.Count = v
+}
+
+// GetResults returns the Results field value
+func (o *PinResults) GetResults() []PinStatus {
+ if o == nil {
+ var ret []PinStatus
+ return ret
+ }
+
+ return o.Results
+}
+
+// GetResultsOk returns a tuple with the Results field value
+// and a boolean to check if the value has been set.
+func (o *PinResults) GetResultsOk() (*[]PinStatus, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Results, true
+}
+
+// SetResults sets field value
+func (o *PinResults) SetResults(v []PinStatus) {
+ o.Results = v
+}
+
+func (o PinResults) MarshalJSON() ([]byte, error) {
+ toSerialize := map[string]interface{}{}
+ if true {
+ toSerialize["count"] = o.Count
+ }
+ if true {
+ toSerialize["results"] = o.Results
+ }
+ return json.Marshal(toSerialize)
+}
+
+type NullablePinResults struct {
+ value *PinResults
+ isSet bool
+}
+
+func (v NullablePinResults) Get() *PinResults {
+ return v.value
+}
+
+func (v *NullablePinResults) Set(val *PinResults) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullablePinResults) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullablePinResults) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullablePinResults(val *PinResults) *NullablePinResults {
+ return &NullablePinResults{value: val, isSet: true}
+}
+
+func (v NullablePinResults) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullablePinResults) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
diff --git a/pinning/remote/client/openapi/model_pin_status.go b/pinning/remote/client/openapi/model_pin_status.go
new file mode 100644
index 000000000..0f44e62c2
--- /dev/null
+++ b/pinning/remote/client/openapi/model_pin_status.go
@@ -0,0 +1,262 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "encoding/json"
+ "time"
+)
+
+// PinStatus Pin object with status
+type PinStatus struct {
+ // Globally unique identifier of the pin request; can be used to check the status of ongoing pinning, or pin removal
+ Requestid string `json:"requestid"`
+ Status Status `json:"status"`
+ // Immutable timestamp indicating when a pin request entered a pinning service; can be used for filtering results and pagination
+ Created time.Time `json:"created"`
+ Pin Pin `json:"pin"`
+ // List of multiaddrs designated by pinning service for transferring any new data from external peers
+ Delegates []string `json:"delegates"`
+ // Optional info for PinStatus response
+ Info *map[string]string `json:"info,omitempty"`
+}
+
+// NewPinStatus instantiates a new PinStatus object
+// This constructor will assign default values to properties that have it defined,
+// and makes sure properties required by API are set, but the set of arguments
+// will change when the set of required properties is changed
+func NewPinStatus(requestid string, status Status, created time.Time, pin Pin, delegates []string) *PinStatus {
+ this := PinStatus{}
+ this.Requestid = requestid
+ this.Status = status
+ this.Created = created
+ this.Pin = pin
+ this.Delegates = delegates
+ return &this
+}
+
+// NewPinStatusWithDefaults instantiates a new PinStatus object
+// This constructor will only assign default values to properties that have it defined,
+// but it doesn't guarantee that properties required by API are set
+func NewPinStatusWithDefaults() *PinStatus {
+ this := PinStatus{}
+ return &this
+}
+
+// GetRequestid returns the Requestid field value
+func (o *PinStatus) GetRequestid() string {
+ if o == nil {
+ var ret string
+ return ret
+ }
+
+ return o.Requestid
+}
+
+// GetRequestidOk returns a tuple with the Requestid field value
+// and a boolean to check if the value has been set.
+func (o *PinStatus) GetRequestidOk() (*string, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Requestid, true
+}
+
+// SetRequestid sets field value
+func (o *PinStatus) SetRequestid(v string) {
+ o.Requestid = v
+}
+
+// GetStatus returns the Status field value
+func (o *PinStatus) GetStatus() Status {
+ if o == nil {
+ var ret Status
+ return ret
+ }
+
+ return o.Status
+}
+
+// GetStatusOk returns a tuple with the Status field value
+// and a boolean to check if the value has been set.
+func (o *PinStatus) GetStatusOk() (*Status, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Status, true
+}
+
+// SetStatus sets field value
+func (o *PinStatus) SetStatus(v Status) {
+ o.Status = v
+}
+
+// GetCreated returns the Created field value
+func (o *PinStatus) GetCreated() time.Time {
+ if o == nil {
+ var ret time.Time
+ return ret
+ }
+
+ return o.Created
+}
+
+// GetCreatedOk returns a tuple with the Created field value
+// and a boolean to check if the value has been set.
+func (o *PinStatus) GetCreatedOk() (*time.Time, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Created, true
+}
+
+// SetCreated sets field value
+func (o *PinStatus) SetCreated(v time.Time) {
+ o.Created = v
+}
+
+// GetPin returns the Pin field value
+func (o *PinStatus) GetPin() Pin {
+ if o == nil {
+ var ret Pin
+ return ret
+ }
+
+ return o.Pin
+}
+
+// GetPinOk returns a tuple with the Pin field value
+// and a boolean to check if the value has been set.
+func (o *PinStatus) GetPinOk() (*Pin, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Pin, true
+}
+
+// SetPin sets field value
+func (o *PinStatus) SetPin(v Pin) {
+ o.Pin = v
+}
+
+// GetDelegates returns the Delegates field value
+func (o *PinStatus) GetDelegates() []string {
+ if o == nil {
+ var ret []string
+ return ret
+ }
+
+ return o.Delegates
+}
+
+// GetDelegatesOk returns a tuple with the Delegates field value
+// and a boolean to check if the value has been set.
+func (o *PinStatus) GetDelegatesOk() (*[]string, bool) {
+ if o == nil {
+ return nil, false
+ }
+ return &o.Delegates, true
+}
+
+// SetDelegates sets field value
+func (o *PinStatus) SetDelegates(v []string) {
+ o.Delegates = v
+}
+
+// GetInfo returns the Info field value if set, zero value otherwise.
+func (o *PinStatus) GetInfo() map[string]string {
+ if o == nil || o.Info == nil {
+ var ret map[string]string
+ return ret
+ }
+ return *o.Info
+}
+
+// GetInfoOk returns a tuple with the Info field value if set, nil otherwise
+// and a boolean to check if the value has been set.
+func (o *PinStatus) GetInfoOk() (*map[string]string, bool) {
+ if o == nil || o.Info == nil {
+ return nil, false
+ }
+ return o.Info, true
+}
+
+// HasInfo returns a boolean if a field has been set.
+func (o *PinStatus) HasInfo() bool {
+ if o != nil && o.Info != nil {
+ return true
+ }
+
+ return false
+}
+
+// SetInfo gets a reference to the given map[string]string and assigns it to the Info field.
+func (o *PinStatus) SetInfo(v map[string]string) {
+ o.Info = &v
+}
+
+func (o PinStatus) MarshalJSON() ([]byte, error) {
+ toSerialize := map[string]interface{}{}
+ if true {
+ toSerialize["requestid"] = o.Requestid
+ }
+ if true {
+ toSerialize["status"] = o.Status
+ }
+ if true {
+ toSerialize["created"] = o.Created
+ }
+ if true {
+ toSerialize["pin"] = o.Pin
+ }
+ if true {
+ toSerialize["delegates"] = o.Delegates
+ }
+ if o.Info != nil {
+ toSerialize["info"] = o.Info
+ }
+ return json.Marshal(toSerialize)
+}
+
+type NullablePinStatus struct {
+ value *PinStatus
+ isSet bool
+}
+
+func (v NullablePinStatus) Get() *PinStatus {
+ return v.value
+}
+
+func (v *NullablePinStatus) Set(val *PinStatus) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullablePinStatus) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullablePinStatus) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullablePinStatus(val *PinStatus) *NullablePinStatus {
+ return &NullablePinStatus{value: val, isSet: true}
+}
+
+func (v NullablePinStatus) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullablePinStatus) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
diff --git a/pinning/remote/client/openapi/model_status.go b/pinning/remote/client/openapi/model_status.go
new file mode 100644
index 000000000..56944819f
--- /dev/null
+++ b/pinning/remote/client/openapi/model_status.go
@@ -0,0 +1,84 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "encoding/json"
+ "fmt"
+)
+
+// Status Status a pin object can have at a pinning service
+type Status string
+
+// List of Status
+const (
+ QUEUED Status = "queued"
+ PINNING Status = "pinning"
+ PINNED Status = "pinned"
+ FAILED Status = "failed"
+)
+
+func (v *Status) UnmarshalJSON(src []byte) error {
+ var value string
+ err := json.Unmarshal(src, &value)
+ if err != nil {
+ return err
+ }
+ enumTypeValue := Status(value)
+ for _, existing := range []Status{"queued", "pinning", "pinned", "failed"} {
+ if existing == enumTypeValue {
+ *v = enumTypeValue
+ return nil
+ }
+ }
+
+ return fmt.Errorf("%+v is not a valid Status", value)
+}
+
+// Ptr returns reference to Status value
+func (v Status) Ptr() *Status {
+ return &v
+}
+
+type NullableStatus struct {
+ value *Status
+ isSet bool
+}
+
+func (v NullableStatus) Get() *Status {
+ return v.value
+}
+
+func (v *NullableStatus) Set(val *Status) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableStatus) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableStatus) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableStatus(val *Status) *NullableStatus {
+ return &NullableStatus{value: val, isSet: true}
+}
+
+func (v NullableStatus) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableStatus) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
diff --git a/pinning/remote/client/openapi/response.go b/pinning/remote/client/openapi/response.go
new file mode 100644
index 000000000..8f9fb0b08
--- /dev/null
+++ b/pinning/remote/client/openapi/response.go
@@ -0,0 +1,46 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "net/http"
+)
+
+// APIResponse stores the API response returned by the server.
+type APIResponse struct {
+ *http.Response `json:"-"`
+ Message string `json:"message,omitempty"`
+ // Operation is the name of the OpenAPI operation.
+ Operation string `json:"operation,omitempty"`
+ // RequestURL is the request URL. This value is always available, even if the
+ // embedded *http.Response is nil.
+ RequestURL string `json:"url,omitempty"`
+ // Method is the HTTP method used for the request. This value is always
+ // available, even if the embedded *http.Response is nil.
+ Method string `json:"method,omitempty"`
+ // Payload holds the contents of the response body (which may be nil or empty).
+ // This is provided here as the raw response.Body() reader will have already
+ // been drained.
+ Payload []byte `json:"-"`
+}
+
+// NewAPIResponse returns a new APIResonse object.
+func NewAPIResponse(r *http.Response) *APIResponse {
+
+ response := &APIResponse{Response: r}
+ return response
+}
+
+// NewAPIResponseWithError returns a new APIResponse object with the provided error message.
+func NewAPIResponseWithError(errorMessage string) *APIResponse {
+
+ response := &APIResponse{Message: errorMessage}
+ return response
+}
diff --git a/pinning/remote/client/openapi/utils.go b/pinning/remote/client/openapi/utils.go
new file mode 100644
index 000000000..25d36f11b
--- /dev/null
+++ b/pinning/remote/client/openapi/utils.go
@@ -0,0 +1,327 @@
+/*
+ * IPFS Pinning Service API
+ *
+ * ## About this spec The IPFS Pinning Service API is intended to be an implementation-agnostic API: - For use and implementation by pinning service providers - For use in client mode by IPFS nodes and GUI-based applications > **Note**: while ready for implementation, this spec is still a work in progress! 🏗️ **Your input and feedback are welcome and valuable as we develop this API spec. Please join the design discussion at [github.com/ipfs/pinning-services-api-spec](https://github.com/ipfs/pinning-services-api-spec).** # Schemas This section describes the most important object types and conventions. A full list of fields and schemas can be found in the `schemas` section of the [YAML file](https://github.com/ipfs/pinning-services-api-spec/blob/master/ipfs-pinning-service.yaml). ## Identifiers ### cid [Content Identifier (CID)](https://docs.ipfs.io/concepts/content-addressing/) points at the root of a DAG that is pinned recursively. ### requestid Unique identifier of a pin request. When a pin is created, the service responds with unique `requestid` that can be later used for pin removal. When the same `cid` is pinned again, a different `requestid` is returned to differentiate between those pin requests. Service implementation should use UUID, `hash(accessToken,Pin,PinStatus.created)`, or any other opaque identifier that provides equally strong protection against race conditions. ## Objects ### Pin object  The `Pin` object is a representation of a pin request. It includes the `cid` of data to be pinned, as well as optional metadata in `name`, `origins`, and `meta`. ### Pin status response  The `PinStatus` object is a representation of the current state of a pinning operation. It includes the original `pin` object, along with the current `status` and globally unique `requestid` of the entire pinning request, which can be used for future status checks and management. Addresses in the `delegates` array are peers delegated by the pinning service for facilitating direct file transfers (more details in the provider hints section). Any additional vendor-specific information is returned in optional `info`. ## The pin lifecycle  ### Creating a new pin object The user sends a `Pin` object to `POST /pins` and receives a `PinStatus` response: - `requestid` in `PinStatus` is the identifier of the pin operation, which can can be used for checking status, and removing the pin in the future - `status` in `PinStatus` indicates the current state of a pin ### Checking status of in-progress pinning `status` (in `PinStatus`) may indicate a pending state (`queued` or `pinning`). This means the data behind `Pin.cid` was not found on the pinning service and is being fetched from the IPFS network at large, which may take time. In this case, the user can periodically check pinning progress via `GET /pins/{requestid}` until pinning is successful, or the user decides to remove the pending pin. ### Replacing an existing pin object The user can replace an existing pin object via `POST /pins/{requestid}`. This is a shortcut for removing a pin object identified by `requestid` and creating a new one in a single API call that protects against undesired garbage collection of blocks common to both pins. Useful when updating a pin representing a huge dataset where most of blocks did not change. The new pin object `requestid` is returned in the `PinStatus` response. The old pin object is deleted automatically. ### Removing a pin object A pin object can be removed via `DELETE /pins/{requestid}`. ## Provider hints Pinning of new data can be accelerated by providing a list of known data sources in `Pin.origins`, and connecting at least one of them to pinning service nodes at `PinStatus.delegates`. The most common scenario is a client putting its own IPFS node's multiaddrs in `Pin.origins`, and then directly connecting to every multiaddr returned by a pinning service in `PinStatus.delegates` to initiate transfer. This ensures data transfer starts immediately (without waiting for provider discovery over DHT), and direct dial from a client works around peer routing issues in restrictive network topologies such as NATs. ## Custom metadata Pinning services are encouraged to add support for additional features by leveraging the optional `Pin.meta` and `PinStatus.info` fields. While these attributes can be application- or vendor-specific, we encourage the community at large to leverage these attributes as a sandbox to come up with conventions that could become part of future revisions of this API. ### Pin metadata String keys and values passed in `Pin.meta` are persisted with the pin object. Potential uses: - `Pin.meta[app_id]`: Attaching a unique identifier to pins created by an app enables filtering pins per app via `?meta={\"app_id\":}` - `Pin.meta[vendor_policy]`: Vendor-specific policy (for example: which region to use, how many copies to keep) Note that it is OK for a client to omit or ignore these optional attributes; doing so should not impact the basic pinning functionality. ### Pin status info Additional `PinStatus.info` can be returned by pinning service. Potential uses: - `PinStatus.info[status_details]`: more info about the current status (queue position, percentage of transferred data, summary of where data is stored, etc); when `PinStatus.status=failed`, it could provide a reason why a pin operation failed (e.g. lack of funds, DAG too big, etc.) - `PinStatus.info[dag_size]`: the size of pinned data, along with DAG overhead - `PinStatus.info[raw_size]`: the size of data without DAG overhead (eg. unixfs) - `PinStatus.info[pinned_until]`: if vendor supports time-bound pins, this could indicate when the pin will expire # Pagination and filtering Pin objects can be listed by executing `GET /pins` with optional parameters: - When no filters are provided, the endpoint will return a small batch of the 10 most recently created items, from the latest to the oldest. - The number of returned items can be adjusted with the `limit` parameter (implicit default is 10). - If the value in `PinResults.count` is bigger than the length of `PinResults.results`, the client can infer there are more results that can be queried. - To read more items, pass the `before` filter with the timestamp from `PinStatus.created` found in the oldest item in the current batch of results. Repeat to read all results. - Returned results can be fine-tuned by applying optional `after`, `cid`, `name`, `status`, or `meta` filters. > **Note**: pagination by the `created` timestamp requires each value to be globally unique. Any future considerations to add support for bulk creation must account for this.
+ *
+ * API version: 0.1.1
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+package openapi
+
+import (
+ "encoding/json"
+ "time"
+)
+
+// PtrBool is a helper routine that returns a pointer to given integer value.
+func PtrBool(v bool) *bool { return &v }
+
+// PtrInt is a helper routine that returns a pointer to given integer value.
+func PtrInt(v int) *int { return &v }
+
+// PtrInt32 is a helper routine that returns a pointer to given integer value.
+func PtrInt32(v int32) *int32 { return &v }
+
+// PtrInt64 is a helper routine that returns a pointer to given integer value.
+func PtrInt64(v int64) *int64 { return &v }
+
+// PtrFloat32 is a helper routine that returns a pointer to given float value.
+func PtrFloat32(v float32) *float32 { return &v }
+
+// PtrFloat64 is a helper routine that returns a pointer to given float value.
+func PtrFloat64(v float64) *float64 { return &v }
+
+// PtrString is a helper routine that returns a pointer to given string value.
+func PtrString(v string) *string { return &v }
+
+// PtrTime is helper routine that returns a pointer to given Time value.
+func PtrTime(v time.Time) *time.Time { return &v }
+
+type NullableBool struct {
+ value *bool
+ isSet bool
+}
+
+func (v NullableBool) Get() *bool {
+ return v.value
+}
+
+func (v *NullableBool) Set(val *bool) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableBool) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableBool) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableBool(val *bool) *NullableBool {
+ return &NullableBool{value: val, isSet: true}
+}
+
+func (v NullableBool) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableBool) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
+
+type NullableInt struct {
+ value *int
+ isSet bool
+}
+
+func (v NullableInt) Get() *int {
+ return v.value
+}
+
+func (v *NullableInt) Set(val *int) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableInt) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableInt) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableInt(val *int) *NullableInt {
+ return &NullableInt{value: val, isSet: true}
+}
+
+func (v NullableInt) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableInt) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
+
+type NullableInt32 struct {
+ value *int32
+ isSet bool
+}
+
+func (v NullableInt32) Get() *int32 {
+ return v.value
+}
+
+func (v *NullableInt32) Set(val *int32) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableInt32) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableInt32) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableInt32(val *int32) *NullableInt32 {
+ return &NullableInt32{value: val, isSet: true}
+}
+
+func (v NullableInt32) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableInt32) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
+
+type NullableInt64 struct {
+ value *int64
+ isSet bool
+}
+
+func (v NullableInt64) Get() *int64 {
+ return v.value
+}
+
+func (v *NullableInt64) Set(val *int64) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableInt64) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableInt64) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableInt64(val *int64) *NullableInt64 {
+ return &NullableInt64{value: val, isSet: true}
+}
+
+func (v NullableInt64) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableInt64) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
+
+type NullableFloat32 struct {
+ value *float32
+ isSet bool
+}
+
+func (v NullableFloat32) Get() *float32 {
+ return v.value
+}
+
+func (v *NullableFloat32) Set(val *float32) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableFloat32) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableFloat32) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableFloat32(val *float32) *NullableFloat32 {
+ return &NullableFloat32{value: val, isSet: true}
+}
+
+func (v NullableFloat32) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableFloat32) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
+
+type NullableFloat64 struct {
+ value *float64
+ isSet bool
+}
+
+func (v NullableFloat64) Get() *float64 {
+ return v.value
+}
+
+func (v *NullableFloat64) Set(val *float64) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableFloat64) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableFloat64) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableFloat64(val *float64) *NullableFloat64 {
+ return &NullableFloat64{value: val, isSet: true}
+}
+
+func (v NullableFloat64) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableFloat64) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
+
+type NullableString struct {
+ value *string
+ isSet bool
+}
+
+func (v NullableString) Get() *string {
+ return v.value
+}
+
+func (v *NullableString) Set(val *string) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableString) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableString) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableString(val *string) *NullableString {
+ return &NullableString{value: val, isSet: true}
+}
+
+func (v NullableString) MarshalJSON() ([]byte, error) {
+ return json.Marshal(v.value)
+}
+
+func (v *NullableString) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
+
+type NullableTime struct {
+ value *time.Time
+ isSet bool
+}
+
+func (v NullableTime) Get() *time.Time {
+ return v.value
+}
+
+func (v *NullableTime) Set(val *time.Time) {
+ v.value = val
+ v.isSet = true
+}
+
+func (v NullableTime) IsSet() bool {
+ return v.isSet
+}
+
+func (v *NullableTime) Unset() {
+ v.value = nil
+ v.isSet = false
+}
+
+func NewNullableTime(val *time.Time) *NullableTime {
+ return &NullableTime{value: val, isSet: true}
+}
+
+func (v NullableTime) MarshalJSON() ([]byte, error) {
+ return v.value.MarshalJSON()
+}
+
+func (v *NullableTime) UnmarshalJSON(src []byte) error {
+ v.isSet = true
+ return json.Unmarshal(src, &v.value)
+}
diff --git a/provider/README.md b/provider/README.md
new file mode 100644
index 000000000..0e4f4650d
--- /dev/null
+++ b/provider/README.md
@@ -0,0 +1,30 @@
+## Usage
+
+Here's how you create, start, interact with, and stop the provider system:
+
+```golang
+import (
+ "context"
+ "time"
+
+ "github.com/ipfs/boxo/provider"
+ "github.com/ipfs/boxo/provider/queue"
+ "github.com/ipfs/boxo/provider/simple"
+)
+
+rsys := (your routing system here)
+dstore := (your datastore here)
+cid := (your cid to provide here)
+
+q := queue.NewQueue(context.Background(), "example", dstore)
+
+reprov := simple.NewReprovider(context.Background(), time.Hour * 12, rsys, simple.NewBlockstoreProvider(dstore))
+prov := simple.NewProvider(context.Background(), q, rsys)
+sys := provider.NewSystem(prov, reprov)
+
+sys.Run()
+
+sys.Provide(cid)
+
+sys.Close()
+```
diff --git a/provider/batched/system.go b/provider/batched/system.go
new file mode 100644
index 000000000..e3cb0325a
--- /dev/null
+++ b/provider/batched/system.go
@@ -0,0 +1,420 @@
+package batched
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strconv"
+ "sync"
+ "time"
+
+ provider "github.com/ipfs/boxo/provider"
+ "github.com/ipfs/boxo/provider/queue"
+ "github.com/ipfs/boxo/provider/simple"
+ "github.com/ipfs/boxo/verifcid"
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-datastore"
+ logging "github.com/ipfs/go-log"
+ "github.com/multiformats/go-multihash"
+)
+
+var log = logging.Logger("provider.batched")
+
+type BatchProvidingSystem struct {
+ ctx context.Context
+ close context.CancelFunc
+ closewg sync.WaitGroup
+
+ reprovideInterval time.Duration
+ initalReprovideDelay time.Duration
+ initialReprovideDelaySet bool
+
+ rsys provideMany
+ keyProvider simple.KeyChanFunc
+
+ q *queue.Queue
+ ds datastore.Batching
+
+ reprovideCh chan cid.Cid
+
+ totalProvides, lastReprovideBatchSize int
+ avgProvideDuration, lastReprovideDuration time.Duration
+}
+
+var _ provider.System = (*BatchProvidingSystem)(nil)
+
+type provideMany interface {
+ ProvideMany(ctx context.Context, keys []multihash.Multihash) error
+ Ready() bool
+}
+
+// Option defines the functional option type that can be used to configure
+// BatchProvidingSystem instances
+type Option func(system *BatchProvidingSystem) error
+
+var lastReprovideKey = datastore.NewKey("/provider/reprovide/lastreprovide")
+
+func New(provider provideMany, q *queue.Queue, opts ...Option) (*BatchProvidingSystem, error) {
+ s := &BatchProvidingSystem{
+ reprovideInterval: time.Hour * 24,
+ rsys: provider,
+ keyProvider: nil,
+ q: q,
+ ds: datastore.NewMapDatastore(),
+ reprovideCh: make(chan cid.Cid),
+ }
+
+ for _, o := range opts {
+ if err := o(s); err != nil {
+ return nil, err
+ }
+ }
+
+ // Setup default behavior for the initial reprovide delay
+ //
+ // If the reprovide ticker is larger than a minute (likely),
+ // provide once after we've been up a minute.
+ //
+ // Don't provide _immediately_ as we might be just about to stop.
+ if !s.initialReprovideDelaySet && s.reprovideInterval > time.Minute {
+ s.initalReprovideDelay = time.Minute
+ s.initialReprovideDelaySet = true
+ }
+
+ if s.keyProvider == nil {
+ s.keyProvider = func(ctx context.Context) (<-chan cid.Cid, error) {
+ ch := make(chan cid.Cid)
+ close(ch)
+ return ch, nil
+ }
+ }
+
+ // This is after the options processing so we do not have to worry about leaking a context if there is an
+ // initialization error processing the options
+ ctx, cancel := context.WithCancel(context.Background())
+ s.ctx = ctx
+ s.close = cancel
+
+ return s, nil
+}
+
+func Datastore(batching datastore.Batching) Option {
+ return func(system *BatchProvidingSystem) error {
+ system.ds = batching
+ return nil
+ }
+}
+
+func ReproviderInterval(duration time.Duration) Option {
+ return func(system *BatchProvidingSystem) error {
+ system.reprovideInterval = duration
+ return nil
+ }
+}
+
+func KeyProvider(fn simple.KeyChanFunc) Option {
+ return func(system *BatchProvidingSystem) error {
+ system.keyProvider = fn
+ return nil
+ }
+}
+
+func initialReprovideDelay(duration time.Duration) Option {
+ return func(system *BatchProvidingSystem) error {
+ system.initialReprovideDelaySet = true
+ system.initalReprovideDelay = duration
+ return nil
+ }
+}
+
+func (s *BatchProvidingSystem) Run() {
+ // how long we wait between the first provider we hear about and batching up the provides to send out
+ const pauseDetectionThreshold = time.Millisecond * 500
+ // how long we are willing to collect providers for the batch after we receive the first one
+ const maxCollectionDuration = time.Minute * 10
+
+ provCh := s.q.Dequeue()
+
+ s.closewg.Add(1)
+ go func() {
+ defer s.closewg.Done()
+
+ m := make(map[cid.Cid]struct{})
+
+ // setup stopped timers
+ maxCollectionDurationTimer := time.NewTimer(time.Hour)
+ pauseDetectTimer := time.NewTimer(time.Hour)
+ stopAndEmptyTimer(maxCollectionDurationTimer)
+ stopAndEmptyTimer(pauseDetectTimer)
+
+ // make sure timers are cleaned up
+ defer maxCollectionDurationTimer.Stop()
+ defer pauseDetectTimer.Stop()
+
+ resetTimersAfterReceivingProvide := func() {
+ firstProvide := len(m) == 0
+ if firstProvide {
+ // after receiving the first provider start up the timers
+ maxCollectionDurationTimer.Reset(maxCollectionDuration)
+ pauseDetectTimer.Reset(pauseDetectionThreshold)
+ } else {
+ // otherwise just do a full restart of the pause timer
+ stopAndEmptyTimer(pauseDetectTimer)
+ pauseDetectTimer.Reset(pauseDetectionThreshold)
+ }
+ }
+
+ for {
+ performedReprovide := false
+
+ // at the start of every loop the maxCollectionDurationTimer and pauseDetectTimer should be already be
+ // stopped and have empty channels
+ loop:
+ for {
+ select {
+ case <-maxCollectionDurationTimer.C:
+ // if this timer has fired then the pause timer has started so let's stop and empty it
+ stopAndEmptyTimer(pauseDetectTimer)
+ break loop
+ default:
+ }
+
+ select {
+ case c := <-provCh:
+ resetTimersAfterReceivingProvide()
+ m[c] = struct{}{}
+ continue
+ default:
+ }
+
+ select {
+ case c := <-provCh:
+ resetTimersAfterReceivingProvide()
+ m[c] = struct{}{}
+ case c := <-s.reprovideCh:
+ resetTimersAfterReceivingProvide()
+ m[c] = struct{}{}
+ performedReprovide = true
+ case <-pauseDetectTimer.C:
+ // if this timer has fired then the max collection timer has started so let's stop and empty it
+ stopAndEmptyTimer(maxCollectionDurationTimer)
+ break loop
+ case <-maxCollectionDurationTimer.C:
+ // if this timer has fired then the pause timer has started so let's stop and empty it
+ stopAndEmptyTimer(pauseDetectTimer)
+ break loop
+ case <-s.ctx.Done():
+ return
+ }
+ }
+
+ if len(m) == 0 {
+ continue
+ }
+
+ keys := make([]multihash.Multihash, 0, len(m))
+ for c := range m {
+ delete(m, c)
+
+ // hash security
+ if err := verifcid.ValidateCid(c); err != nil {
+ log.Errorf("insecure hash in reprovider, %s (%s)", c, err)
+ continue
+ }
+
+ keys = append(keys, c.Hash())
+ }
+
+ // in case after removing all the invalid CIDs there are no valid ones left
+ if len(keys) == 0 {
+ continue
+ }
+
+ for !s.rsys.Ready() {
+ log.Debugf("reprovider system not ready")
+ select {
+ case <-time.After(time.Minute):
+ case <-s.ctx.Done():
+ return
+ }
+ }
+
+ log.Debugf("starting provide of %d keys", len(keys))
+ start := time.Now()
+ err := s.rsys.ProvideMany(s.ctx, keys)
+ if err != nil {
+ log.Debugf("providing failed %v", err)
+ continue
+ }
+ dur := time.Since(start)
+
+ totalProvideTime := int64(s.totalProvides) * int64(s.avgProvideDuration)
+ recentAvgProvideDuration := time.Duration(int64(dur) / int64(len(keys)))
+ s.avgProvideDuration = time.Duration((totalProvideTime + int64(dur)) / int64(s.totalProvides+len(keys)))
+ s.totalProvides += len(keys)
+
+ log.Debugf("finished providing of %d keys. It took %v with an average of %v per provide", len(keys), dur, recentAvgProvideDuration)
+
+ if performedReprovide {
+ s.lastReprovideBatchSize = len(keys)
+ s.lastReprovideDuration = dur
+
+ if err := s.ds.Put(s.ctx, lastReprovideKey, storeTime(time.Now())); err != nil {
+ log.Errorf("could not store last reprovide time: %v", err)
+ }
+ if err := s.ds.Sync(s.ctx, lastReprovideKey); err != nil {
+ log.Errorf("could not perform sync of last reprovide time: %v", err)
+ }
+ }
+ }
+ }()
+
+ s.closewg.Add(1)
+ go func() {
+ defer s.closewg.Done()
+
+ var initialReprovideCh, reprovideCh <-chan time.Time
+
+ // If reproviding is enabled (non-zero)
+ if s.reprovideInterval > 0 {
+ reprovideTicker := time.NewTicker(s.reprovideInterval)
+ defer reprovideTicker.Stop()
+ reprovideCh = reprovideTicker.C
+
+ // if there is a non-zero initial reprovide time that was set in the initializer or if the fallback has been
+ if s.initialReprovideDelaySet {
+ initialReprovideTimer := time.NewTimer(s.initalReprovideDelay)
+ defer initialReprovideTimer.Stop()
+
+ initialReprovideCh = initialReprovideTimer.C
+ }
+ }
+
+ for s.ctx.Err() == nil {
+ select {
+ case <-initialReprovideCh:
+ case <-reprovideCh:
+ case <-s.ctx.Done():
+ return
+ }
+
+ err := s.reprovide(s.ctx, false)
+
+ // only log if we've hit an actual error, otherwise just tell the client we're shutting down
+ if s.ctx.Err() == nil && err != nil {
+ log.Errorf("failed to reprovide: %s", err)
+ }
+ }
+ }()
+}
+
+func stopAndEmptyTimer(t *time.Timer) {
+ if !t.Stop() {
+ <-t.C
+ }
+}
+
+func storeTime(t time.Time) []byte {
+ val := []byte(fmt.Sprintf("%d", t.UnixNano()))
+ return val
+}
+
+func parseTime(b []byte) (time.Time, error) {
+ tns, err := strconv.ParseInt(string(b), 10, 64)
+ if err != nil {
+ return time.Time{}, err
+ }
+ return time.Unix(0, tns), nil
+}
+
+func (s *BatchProvidingSystem) Close() error {
+ s.close()
+ err := s.q.Close()
+ s.closewg.Wait()
+ return err
+}
+
+func (s *BatchProvidingSystem) Provide(cid cid.Cid) error {
+ return s.q.Enqueue(cid)
+}
+
+func (s *BatchProvidingSystem) Reprovide(ctx context.Context) error {
+ return s.reprovide(ctx, true)
+}
+
+func (s *BatchProvidingSystem) reprovide(ctx context.Context, force bool) error {
+ if !s.shouldReprovide() && !force {
+ return nil
+ }
+
+ kch, err := s.keyProvider(ctx)
+ if err != nil {
+ return err
+ }
+
+reprovideCidLoop:
+ for {
+ select {
+ case c, ok := <-kch:
+ if !ok {
+ break reprovideCidLoop
+ }
+
+ select {
+ case s.reprovideCh <- c:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+
+ return nil
+}
+
+func (s *BatchProvidingSystem) getLastReprovideTime() (time.Time, error) {
+ val, err := s.ds.Get(s.ctx, lastReprovideKey)
+ if errors.Is(err, datastore.ErrNotFound) {
+ return time.Time{}, nil
+ }
+ if err != nil {
+ return time.Time{}, fmt.Errorf("could not get last reprovide time")
+ }
+
+ t, err := parseTime(val)
+ if err != nil {
+ return time.Time{}, fmt.Errorf("could not decode last reprovide time, got %q", string(val))
+ }
+
+ return t, nil
+}
+
+func (s *BatchProvidingSystem) shouldReprovide() bool {
+ t, err := s.getLastReprovideTime()
+ if err != nil {
+ log.Debugf("getting last reprovide time failed: %s", err)
+ return false
+ }
+
+ if time.Since(t) < time.Duration(float64(s.reprovideInterval)*0.5) {
+ return false
+ }
+ return true
+}
+
+type BatchedProviderStats struct {
+ TotalProvides, LastReprovideBatchSize int
+ AvgProvideDuration, LastReprovideDuration time.Duration
+}
+
+// Stat returns various stats about this provider system
+func (s *BatchProvidingSystem) Stat(ctx context.Context) (BatchedProviderStats, error) {
+ // TODO: Does it matter that there is no locking around the total+average values?
+ return BatchedProviderStats{
+ TotalProvides: s.totalProvides,
+ LastReprovideBatchSize: s.lastReprovideBatchSize,
+ AvgProvideDuration: s.avgProvideDuration,
+ LastReprovideDuration: s.lastReprovideDuration,
+ }, nil
+}
diff --git a/provider/batched/system_test.go b/provider/batched/system_test.go
new file mode 100644
index 000000000..4fe0624af
--- /dev/null
+++ b/provider/batched/system_test.go
@@ -0,0 +1,117 @@
+package batched
+
+import (
+ "context"
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ mh "github.com/multiformats/go-multihash"
+
+ q "github.com/ipfs/boxo/provider/queue"
+)
+
+type mockProvideMany struct {
+ lk sync.Mutex
+ keys []mh.Multihash
+}
+
+func (m *mockProvideMany) ProvideMany(ctx context.Context, keys []mh.Multihash) error {
+ m.lk.Lock()
+ defer m.lk.Unlock()
+ m.keys = keys
+ return nil
+}
+
+func (m *mockProvideMany) Ready() bool {
+ return true
+}
+
+func (m *mockProvideMany) GetKeys() []mh.Multihash {
+ m.lk.Lock()
+ defer m.lk.Unlock()
+ return m.keys[:]
+}
+
+var _ provideMany = (*mockProvideMany)(nil)
+
+func TestBatched(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := dssync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := q.NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ provider := &mockProvideMany{}
+
+ ctx, cancel := context.WithTimeout(ctx, time.Second*10)
+ defer cancel()
+
+ const numProvides = 100
+ keysToProvide := make(map[cid.Cid]int)
+ for i := 0; i < numProvides; i++ {
+ h, err := mh.Sum([]byte(strconv.Itoa(i)), mh.SHA2_256, -1)
+ if err != nil {
+ panic(err)
+ }
+ c := cid.NewCidV1(cid.Raw, h)
+ keysToProvide[c] = i
+ }
+
+ batchSystem, err := New(provider, queue, KeyProvider(func(ctx context.Context) (<-chan cid.Cid, error) {
+ ch := make(chan cid.Cid)
+ go func() {
+ for k := range keysToProvide {
+ select {
+ case ch <- k:
+ case <-ctx.Done():
+ return
+ }
+ }
+ }()
+ return ch, nil
+ }), initialReprovideDelay(0))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ batchSystem.Run()
+
+ var keys []mh.Multihash
+ for {
+ if ctx.Err() != nil {
+ t.Fatal("test hung")
+ }
+ keys = provider.GetKeys()
+ if len(keys) != 0 {
+ break
+ }
+ time.Sleep(time.Millisecond * 100)
+ }
+
+ if len(keys) != numProvides {
+ t.Fatalf("expected %d provider keys, got %d", numProvides, len(keys))
+ }
+
+ provMap := make(map[string]struct{})
+ for _, k := range keys {
+ provMap[string(k)] = struct{}{}
+ }
+
+ for i := 0; i < numProvides; i++ {
+ h, err := mh.Sum([]byte(strconv.Itoa(i)), mh.SHA2_256, -1)
+ if err != nil {
+ panic(err)
+ }
+ if _, found := provMap[string(h)]; !found {
+ t.Fatalf("could not find provider with value %d", i)
+ }
+ }
+}
diff --git a/provider/offline.go b/provider/offline.go
new file mode 100644
index 000000000..030a70ab1
--- /dev/null
+++ b/provider/offline.go
@@ -0,0 +1,29 @@
+package provider
+
+import (
+ "context"
+
+ "github.com/ipfs/go-cid"
+)
+
+type offlineProvider struct{}
+
+// NewOfflineProvider creates a ProviderSystem that does nothing
+func NewOfflineProvider() System {
+ return &offlineProvider{}
+}
+
+func (op *offlineProvider) Run() {
+}
+
+func (op *offlineProvider) Close() error {
+ return nil
+}
+
+func (op *offlineProvider) Provide(cid.Cid) error {
+ return nil
+}
+
+func (op *offlineProvider) Reprovide(context.Context) error {
+ return nil
+}
diff --git a/provider/provider.go b/provider/provider.go
new file mode 100644
index 000000000..3b9c6ba3e
--- /dev/null
+++ b/provider/provider.go
@@ -0,0 +1,27 @@
+package provider
+
+import (
+ "context"
+
+ "github.com/ipfs/go-cid"
+)
+
+// Provider announces blocks to the network
+type Provider interface {
+ // Run is used to begin processing the provider work
+ Run()
+ // Provide takes a cid and makes an attempt to announce it to the network
+ Provide(cid.Cid) error
+ // Close stops the provider
+ Close() error
+}
+
+// Reprovider reannounces blocks to the network
+type Reprovider interface {
+ // Run is used to begin processing the reprovider work and waiting for reprovide triggers
+ Run()
+ // Trigger a reprovide
+ Trigger(context.Context) error
+ // Close stops the reprovider
+ Close() error
+}
diff --git a/provider/queue/queue.go b/provider/queue/queue.go
new file mode 100644
index 000000000..618256bbe
--- /dev/null
+++ b/provider/queue/queue.go
@@ -0,0 +1,156 @@
+package queue
+
+import (
+ "context"
+ "fmt"
+ cid "github.com/ipfs/go-cid"
+ datastore "github.com/ipfs/go-datastore"
+ namespace "github.com/ipfs/go-datastore/namespace"
+ query "github.com/ipfs/go-datastore/query"
+ logging "github.com/ipfs/go-log"
+)
+
+var log = logging.Logger("provider.queue")
+
+// Queue provides a best-effort durability, FIFO interface to the datastore for storing cids
+//
+// Best-effort durability just means that cids in the process of being provided when a
+// crash or shutdown occurs may be in the queue when the node is brought back online
+// depending on whether the underlying datastore has synchronous or asynchronous writes.
+type Queue struct {
+ // used to differentiate queues in datastore
+ // e.g. provider vs reprovider
+ name string
+ ctx context.Context
+ ds datastore.Datastore // Must be threadsafe
+ dequeue chan cid.Cid
+ enqueue chan cid.Cid
+ close context.CancelFunc
+ closed chan struct{}
+
+ counter uint64
+}
+
+// NewQueue creates a queue for cids
+func NewQueue(ctx context.Context, name string, ds datastore.Datastore) (*Queue, error) {
+ namespaced := namespace.Wrap(ds, datastore.NewKey("/"+name+"/queue/"))
+ cancelCtx, cancel := context.WithCancel(ctx)
+ q := &Queue{
+ name: name,
+ ctx: cancelCtx,
+ ds: namespaced,
+ dequeue: make(chan cid.Cid),
+ enqueue: make(chan cid.Cid),
+ close: cancel,
+ closed: make(chan struct{}, 1),
+ }
+ q.work()
+ return q, nil
+}
+
+// Close stops the queue
+func (q *Queue) Close() error {
+ q.close()
+ <-q.closed
+ return nil
+}
+
+// Enqueue puts a cid in the queue
+func (q *Queue) Enqueue(cid cid.Cid) error {
+ select {
+ case q.enqueue <- cid:
+ return nil
+ case <-q.ctx.Done():
+ return fmt.Errorf("failed to enqueue CID: shutting down")
+ }
+}
+
+// Dequeue returns a channel that if listened to will remove entries from the queue
+func (q *Queue) Dequeue() <-chan cid.Cid {
+ return q.dequeue
+}
+
+// Run dequeues and enqueues when available.
+func (q *Queue) work() {
+ go func() {
+ var k datastore.Key = datastore.Key{}
+ var c cid.Cid = cid.Undef
+
+ defer func() {
+ // also cancels any in-progess enqueue tasks.
+ q.close()
+ // unblocks anyone waiting
+ close(q.dequeue)
+ // unblocks the close call
+ close(q.closed)
+ }()
+
+ for {
+ if c == cid.Undef {
+ head, err := q.getQueueHead()
+
+ if err != nil {
+ log.Errorf("error querying for head of queue: %s, stopping provider", err)
+ return
+ } else if head != nil {
+ k = datastore.NewKey(head.Key)
+ c, err = cid.Parse(head.Value)
+ if err != nil {
+ log.Warnf("error parsing queue entry cid with key (%s), removing it from queue: %s", head.Key, err)
+ err = q.ds.Delete(q.ctx, k)
+ if err != nil {
+ log.Errorf("error deleting queue entry with key (%s), due to error (%s), stopping provider", head.Key, err)
+ return
+ }
+ continue
+ }
+ } else {
+ c = cid.Undef
+ }
+ }
+
+ // If c != cid.Undef set dequeue and attempt write, otherwise wait for enqueue
+ var dequeue chan cid.Cid
+ if c != cid.Undef {
+ dequeue = q.dequeue
+ }
+
+ select {
+ case toQueue := <-q.enqueue:
+ keyPath := fmt.Sprintf("%020d/%s", q.counter, c.String())
+ q.counter++
+ nextKey := datastore.NewKey(keyPath)
+
+ if err := q.ds.Put(q.ctx, nextKey, toQueue.Bytes()); err != nil {
+ log.Errorf("Failed to enqueue cid: %s", err)
+ continue
+ }
+ case dequeue <- c:
+ err := q.ds.Delete(q.ctx, k)
+
+ if err != nil {
+ log.Errorf("Failed to delete queued cid %s with key %s: %s", c, k, err)
+ continue
+ }
+ c = cid.Undef
+ case <-q.ctx.Done():
+ return
+ }
+ }
+ }()
+}
+
+func (q *Queue) getQueueHead() (*query.Entry, error) {
+ qry := query.Query{Orders: []query.Order{query.OrderByKey{}}, Limit: 1}
+ results, err := q.ds.Query(q.ctx, qry)
+ if err != nil {
+ return nil, err
+ }
+ defer results.Close()
+ r, ok := results.NextSync()
+ if !ok {
+ return nil, nil
+ }
+
+ return &r.Entry, r.Error
+}
diff --git a/provider/queue/queue_test.go b/provider/queue/queue_test.go
new file mode 100644
index 000000000..a0fa36c3a
--- /dev/null
+++ b/provider/queue/queue_test.go
@@ -0,0 +1,133 @@
+package queue
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-datastore"
+ "github.com/ipfs/go-datastore/sync"
+ blocksutil "github.com/ipfs/go-ipfs-blocksutil"
+)
+
+var blockGenerator = blocksutil.NewBlockGenerator()
+
+func makeCids(n int) []cid.Cid {
+ cids := make([]cid.Cid, 0, n)
+ for i := 0; i < n; i++ {
+ c := blockGenerator.Next().Cid()
+ cids = append(cids, c)
+ }
+ return cids
+}
+
+func assertOrdered(cids []cid.Cid, q *Queue, t *testing.T) {
+ for _, c := range cids {
+ select {
+ case dequeued := <-q.dequeue:
+ if c != dequeued {
+ t.Fatalf("Error in ordering of CIDs retrieved from queue. Expected: %s, got: %s", c, dequeued)
+ }
+
+ case <-time.After(time.Second * 1):
+ t.Fatal("Timeout waiting for cids to be provided.")
+ }
+ }
+}
+
+func TestBasicOperation(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cids := makeCids(10)
+
+ for _, c := range cids {
+ queue.Enqueue(c)
+ }
+
+ assertOrdered(cids, queue, t)
+}
+
+func TestMangledData(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cids := makeCids(10)
+ for _, c := range cids {
+ queue.Enqueue(c)
+ }
+
+ // put bad data in the queue
+ queueKey := datastore.NewKey("/test/0")
+ err = queue.ds.Put(ctx, queueKey, []byte("borked"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // expect to only see the valid cids we entered
+ expected := cids
+ assertOrdered(expected, queue, t)
+}
+
+func TestInitialization(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cids := makeCids(10)
+ for _, c := range cids {
+ queue.Enqueue(c)
+ }
+
+ assertOrdered(cids[:5], queue, t)
+
+ // make a new queue, same data
+ queue, err = NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assertOrdered(cids[5:], queue, t)
+}
+
+func TestInitializationWithManyCids(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cids := makeCids(25)
+ for _, c := range cids {
+ queue.Enqueue(c)
+ }
+
+ // make a new queue, same data
+ queue, err = NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ assertOrdered(cids, queue, t)
+}
diff --git a/provider/simple/provider.go b/provider/simple/provider.go
new file mode 100644
index 000000000..63de031ad
--- /dev/null
+++ b/provider/simple/provider.go
@@ -0,0 +1,116 @@
+// Package simple implements structures and methods to provide blocks,
+// keep track of which blocks are provided, and to allow those blocks to
+// be reprovided.
+package simple
+
+import (
+ "context"
+ "time"
+
+ q "github.com/ipfs/boxo/provider/queue"
+ "github.com/ipfs/go-cid"
+ logging "github.com/ipfs/go-log"
+ "github.com/libp2p/go-libp2p/core/routing"
+)
+
+var logP = logging.Logger("provider.simple")
+
+// Provider announces blocks to the network
+type Provider struct {
+ ctx context.Context
+ // the CIDs for which provide announcements should be made
+ queue *q.Queue
+ // used to announce providing to the network
+ contentRouting routing.ContentRouting
+ // how long to wait for announce to complete before giving up
+ timeout time.Duration
+ // how many workers concurrently work through thhe queue
+ workerLimit int
+}
+
+// Option defines the functional option type that can be used to configure
+// provider instances
+type Option func(*Provider)
+
+// WithTimeout is an option to set a timeout on a provider
+func WithTimeout(timeout time.Duration) Option {
+ return func(p *Provider) {
+ p.timeout = timeout
+ }
+}
+
+// MaxWorkers is an option to set the max workers on a provider
+func MaxWorkers(count int) Option {
+ return func(p *Provider) {
+ p.workerLimit = count
+ }
+}
+
+// NewProvider creates a provider that announces blocks to the network using a content router
+func NewProvider(ctx context.Context, queue *q.Queue, contentRouting routing.ContentRouting, options ...Option) *Provider {
+ p := &Provider{
+ ctx: ctx,
+ queue: queue,
+ contentRouting: contentRouting,
+ workerLimit: 8,
+ }
+
+ for _, option := range options {
+ option(p)
+ }
+
+ return p
+}
+
+// Close stops the provider
+func (p *Provider) Close() error {
+ return p.queue.Close()
+}
+
+// Run workers to handle provide requests.
+func (p *Provider) Run() {
+ p.handleAnnouncements()
+}
+
+// Provide the given cid using specified strategy.
+func (p *Provider) Provide(root cid.Cid) error {
+ return p.queue.Enqueue(root)
+}
+
+// Handle all outgoing cids by providing (announcing) them
+func (p *Provider) handleAnnouncements() {
+ for workers := 0; workers < p.workerLimit; workers++ {
+ go func() {
+ for p.ctx.Err() == nil {
+ select {
+ case <-p.ctx.Done():
+ return
+ case c, ok := <-p.queue.Dequeue():
+ if !ok {
+ // queue closed.
+ return
+ }
+
+ p.doProvide(c)
+ }
+ }
+ }()
+ }
+}
+
+func (p *Provider) doProvide(c cid.Cid) {
+ ctx := p.ctx
+ if p.timeout > 0 {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(ctx, p.timeout)
+ defer cancel()
+ } else {
+ ctx = p.ctx
+ }
+
+ logP.Info("announce - start - ", c)
+ if err := p.contentRouting.Provide(ctx, c, true); err != nil {
+ logP.Warnf("Unable to provide entry: %s, %s", c, err)
+ }
+ logP.Info("announce - end - ", c)
+}
diff --git a/provider/simple/provider_test.go b/provider/simple/provider_test.go
new file mode 100644
index 000000000..96f009937
--- /dev/null
+++ b/provider/simple/provider_test.go
@@ -0,0 +1,162 @@
+package simple_test
+
+import (
+ "context"
+ "math/rand"
+ "testing"
+ "time"
+
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-datastore"
+ "github.com/ipfs/go-datastore/sync"
+ blocksutil "github.com/ipfs/go-ipfs-blocksutil"
+ "github.com/libp2p/go-libp2p/core/peer"
+
+ q "github.com/ipfs/boxo/provider/queue"
+
+ . "github.com/ipfs/boxo/provider/simple"
+)
+
+var blockGenerator = blocksutil.NewBlockGenerator()
+
+type mockRouting struct {
+ provided chan cid.Cid
+}
+
+func (r *mockRouting) Provide(ctx context.Context, cid cid.Cid, recursive bool) error {
+ select {
+ case r.provided <- cid:
+ case <-ctx.Done():
+ panic("context cancelled, but shouldn't have")
+ }
+ return nil
+}
+
+func (r *mockRouting) FindProvidersAsync(ctx context.Context, cid cid.Cid, timeout int) <-chan peer.AddrInfo {
+ return nil
+}
+
+func mockContentRouting() *mockRouting {
+ r := mockRouting{}
+ r.provided = make(chan cid.Cid)
+ return &r
+}
+
+func TestAnnouncement(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := q.NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ r := mockContentRouting()
+
+ prov := NewProvider(ctx, queue, r)
+ prov.Run()
+
+ cids := cid.NewSet()
+
+ for i := 0; i < 100; i++ {
+ c := blockGenerator.Next().Cid()
+ cids.Add(c)
+ }
+
+ go func() {
+ for _, c := range cids.Keys() {
+ err = prov.Provide(c)
+ // A little goroutine stirring to exercise some different states
+ r := rand.Intn(10)
+ time.Sleep(time.Microsecond * time.Duration(r))
+ }
+ }()
+
+ for cids.Len() > 0 {
+ select {
+ case cp := <-r.provided:
+ if !cids.Has(cp) {
+ t.Fatal("Wrong CID provided")
+ }
+ cids.Remove(cp)
+ case <-time.After(time.Second * 5):
+ t.Fatal("Timeout waiting for cids to be provided.")
+ }
+ }
+ prov.Close()
+
+ select {
+ case cp := <-r.provided:
+ t.Fatal("did not expect to provide CID: ", cp)
+ case <-time.After(time.Second * 1):
+ }
+}
+
+func TestClose(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := q.NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ r := mockContentRouting()
+
+ prov := NewProvider(ctx, queue, r)
+ prov.Run()
+
+ prov.Close()
+
+ select {
+ case cp := <-r.provided:
+ t.Fatal("did not expect to provide anything, provided: ", cp)
+ case <-time.After(time.Second * 1):
+ }
+}
+
+func TestAnnouncementTimeout(t *testing.T) {
+ ctx := context.Background()
+ defer ctx.Done()
+
+ ds := sync.MutexWrap(datastore.NewMapDatastore())
+ queue, err := q.NewQueue(ctx, "test", ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ r := mockContentRouting()
+
+ prov := NewProvider(ctx, queue, r, WithTimeout(1*time.Second))
+ prov.Run()
+
+ cids := cid.NewSet()
+
+ for i := 0; i < 100; i++ {
+ c := blockGenerator.Next().Cid()
+ cids.Add(c)
+ }
+
+ go func() {
+ for _, c := range cids.Keys() {
+ err = prov.Provide(c)
+ // A little goroutine stirring to exercise some different states
+ r := rand.Intn(10)
+ time.Sleep(time.Microsecond * time.Duration(r))
+ }
+ }()
+
+ for cids.Len() > 0 {
+ select {
+ case cp := <-r.provided:
+ if !cids.Has(cp) {
+ t.Fatal("Wrong CID provided")
+ }
+ cids.Remove(cp)
+ case <-time.After(time.Second * 5):
+ t.Fatal("Timeout waiting for cids to be provided.")
+ }
+ }
+}
diff --git a/provider/simple/reprovide.go b/provider/simple/reprovide.go
new file mode 100644
index 000000000..a29b484fc
--- /dev/null
+++ b/provider/simple/reprovide.go
@@ -0,0 +1,255 @@
+package simple
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/cenkalti/backoff"
+ blocks "github.com/ipfs/boxo/blockstore"
+ "github.com/ipfs/boxo/fetcher"
+ fetcherhelpers "github.com/ipfs/boxo/fetcher/helpers"
+ "github.com/ipfs/boxo/verifcid"
+ "github.com/ipfs/go-cid"
+ "github.com/ipfs/go-cidutil"
+ logging "github.com/ipfs/go-log"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ "github.com/libp2p/go-libp2p/core/routing"
+)
+
+var logR = logging.Logger("reprovider.simple")
+
+// ErrClosed is returned by Trigger when operating on a closed reprovider.
+var ErrClosed = errors.New("reprovider service stopped")
+
+// KeyChanFunc is function streaming CIDs to pass to content routing
+type KeyChanFunc func(context.Context) (<-chan cid.Cid, error)
+
+// Reprovider reannounces blocks to the network
+type Reprovider struct {
+ // Reprovider context. Cancel to stop, then wait on closedCh.
+ ctx context.Context
+ cancel context.CancelFunc
+ closedCh chan struct{}
+
+ // Trigger triggers a reprovide.
+ trigger chan chan<- error
+
+ // The routing system to provide values through
+ rsys routing.ContentRouting
+
+ keyProvider KeyChanFunc
+
+ tick time.Duration
+}
+
+// NewReprovider creates new Reprovider instance.
+func NewReprovider(ctx context.Context, reprovideInterval time.Duration, rsys routing.ContentRouting, keyProvider KeyChanFunc) *Reprovider {
+ ctx, cancel := context.WithCancel(ctx)
+ return &Reprovider{
+ ctx: ctx,
+ cancel: cancel,
+ closedCh: make(chan struct{}),
+ trigger: make(chan chan<- error),
+
+ rsys: rsys,
+ keyProvider: keyProvider,
+ tick: reprovideInterval,
+ }
+}
+
+// Close the reprovider
+func (rp *Reprovider) Close() error {
+ rp.cancel()
+ <-rp.closedCh
+ return nil
+}
+
+// Run re-provides keys with 'tick' interval or when triggered
+func (rp *Reprovider) Run() {
+ defer close(rp.closedCh)
+
+ var initialReprovideCh, reprovideCh <-chan time.Time
+
+ // If reproviding is enabled (non-zero)
+ if rp.tick > 0 {
+ reprovideTicker := time.NewTicker(rp.tick)
+ defer reprovideTicker.Stop()
+ reprovideCh = reprovideTicker.C
+
+ // If the reprovide ticker is larger than a minute (likely),
+ // provide once after we've been up a minute.
+ //
+ // Don't provide _immediately_ as we might be just about to stop.
+ if rp.tick > time.Minute {
+ initialReprovideTimer := time.NewTimer(time.Minute)
+ defer initialReprovideTimer.Stop()
+
+ initialReprovideCh = initialReprovideTimer.C
+ }
+ }
+
+ var done chan<- error
+ for rp.ctx.Err() == nil {
+ select {
+ case <-initialReprovideCh:
+ case <-reprovideCh:
+ case done = <-rp.trigger:
+ case <-rp.ctx.Done():
+ return
+ }
+
+ err := rp.Reprovide()
+
+ // only log if we've hit an actual error, otherwise just tell the client we're shutting down
+ if rp.ctx.Err() != nil {
+ err = ErrClosed
+ } else if err != nil {
+ logR.Errorf("failed to reprovide: %s", err)
+ }
+
+ if done != nil {
+ if err != nil {
+ done <- err
+ }
+ close(done)
+ }
+ }
+}
+
+// Reprovide registers all keys given by rp.keyProvider to libp2p content routing
+func (rp *Reprovider) Reprovide() error {
+ keychan, err := rp.keyProvider(rp.ctx)
+ if err != nil {
+ return fmt.Errorf("failed to get key chan: %s", err)
+ }
+ for c := range keychan {
+ // hash security
+ if err := verifcid.ValidateCid(c); err != nil {
+ logR.Errorf("insecure hash in reprovider, %s (%s)", c, err)
+ continue
+ }
+ op := func() error {
+ err := rp.rsys.Provide(rp.ctx, c, true)
+ if err != nil {
+ logR.Debugf("Failed to provide key: %s", err)
+ }
+ return err
+ }
+
+ err := backoff.Retry(op, backoff.WithContext(backoff.NewExponentialBackOff(), rp.ctx))
+ if err != nil {
+ logR.Debugf("Providing failed after number of retries: %s", err)
+ return err
+ }
+ }
+ return nil
+}
+
+// Trigger starts the reprovision process in rp.Run and waits for it to finish.
+//
+// Returns an error if a reprovide is already in progress.
+func (rp *Reprovider) Trigger(ctx context.Context) error {
+ resultCh := make(chan error, 1)
+ select {
+ case rp.trigger <- resultCh:
+ default:
+ return fmt.Errorf("reprovider is already running")
+ }
+
+ select {
+ case err := <-resultCh:
+ return err
+ case <-rp.ctx.Done():
+ return ErrClosed
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// Strategies
+
+// NewBlockstoreProvider returns key provider using bstore.AllKeysChan
+func NewBlockstoreProvider(bstore blocks.Blockstore) KeyChanFunc {
+ return func(ctx context.Context) (<-chan cid.Cid, error) {
+ return bstore.AllKeysChan(ctx)
+ }
+}
+
+// Pinner interface defines how the simple.Reprovider wants to interact
+// with a Pinning service
+type Pinner interface {
+ DirectKeys(ctx context.Context) ([]cid.Cid, error)
+ RecursiveKeys(ctx context.Context) ([]cid.Cid, error)
+}
+
+// NewPinnedProvider returns provider supplying pinned keys
+func NewPinnedProvider(onlyRoots bool, pinning Pinner, fetchConfig fetcher.Factory) KeyChanFunc {
+ return func(ctx context.Context) (<-chan cid.Cid, error) {
+ set, err := pinSet(ctx, pinning, fetchConfig, onlyRoots)
+ if err != nil {
+ return nil, err
+ }
+
+ outCh := make(chan cid.Cid)
+ go func() {
+ defer close(outCh)
+ for c := range set.New {
+ select {
+ case <-ctx.Done():
+ return
+ case outCh <- c:
+ }
+ }
+
+ }()
+
+ return outCh, nil
+ }
+}
+
+func pinSet(ctx context.Context, pinning Pinner, fetchConfig fetcher.Factory, onlyRoots bool) (*cidutil.StreamingSet, error) {
+ set := cidutil.NewStreamingSet()
+
+ go func() {
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ defer close(set.New)
+
+ dkeys, err := pinning.DirectKeys(ctx)
+ if err != nil {
+ logR.Errorf("reprovide direct pins: %s", err)
+ return
+ }
+ for _, key := range dkeys {
+ set.Visitor(ctx)(key)
+ }
+
+ rkeys, err := pinning.RecursiveKeys(ctx)
+ if err != nil {
+ logR.Errorf("reprovide indirect pins: %s", err)
+ return
+ }
+
+ session := fetchConfig.NewSession(ctx)
+ for _, key := range rkeys {
+ set.Visitor(ctx)(key)
+ if !onlyRoots {
+ err := fetcherhelpers.BlockAll(ctx, session, cidlink.Link{Cid: key}, func(res fetcher.FetchResult) error {
+ clink, ok := res.LastBlockLink.(cidlink.Link)
+ if ok {
+ set.Visitor(ctx)(clink.Cid)
+ }
+ return nil
+ })
+ if err != nil {
+ logR.Errorf("reprovide indirect pins: %s", err)
+ return
+ }
+ }
+ }
+ }()
+
+ return set, nil
+}
diff --git a/provider/simple/reprovide_test.go b/provider/simple/reprovide_test.go
new file mode 100644
index 000000000..26127aadd
--- /dev/null
+++ b/provider/simple/reprovide_test.go
@@ -0,0 +1,284 @@
+package simple_test
+
+import (
+ "bytes"
+ "context"
+ "testing"
+ "time"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ bsrv "github.com/ipfs/boxo/blockservice"
+ blockstore "github.com/ipfs/boxo/blockstore"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
+ mock "github.com/ipfs/boxo/routing/mock"
+ "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/codec/dagcbor"
+ "github.com/ipld/go-ipld-prime/fluent/qp"
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ basicnode "github.com/ipld/go-ipld-prime/node/basic"
+ testutil "github.com/libp2p/go-libp2p-testing/net"
+ "github.com/libp2p/go-libp2p/core/peer"
+ mh "github.com/multiformats/go-multihash"
+
+ . "github.com/ipfs/boxo/provider/simple"
+)
+
+func setupRouting(t *testing.T) (clA, clB mock.Client, idA, idB peer.ID) {
+ mrserv := mock.NewServer()
+
+ iidA := testutil.RandIdentityOrFatal(t)
+ iidB := testutil.RandIdentityOrFatal(t)
+
+ clA = mrserv.Client(iidA)
+ clB = mrserv.Client(iidB)
+
+ return clA, clB, iidA.ID(), iidB.ID()
+}
+
+func setupDag(t *testing.T) (nodes []cid.Cid, bstore blockstore.Blockstore) {
+ bstore = blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
+ for _, data := range []string{"foo", "bar"} {
+ nb := basicnode.Prototype.Any.NewBuilder()
+ err := nb.AssignString(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ blk := toBlock(t, nb.Build())
+ err = bstore.Put(context.Background(), blk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes = append(nodes, blk.Cid())
+ nd, err := qp.BuildMap(basicnode.Prototype.Map, 1, func(ma ipld.MapAssembler) {
+ qp.MapEntry(ma, "child", qp.Link(cidlink.Link{Cid: blk.Cid()}))
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ blk = toBlock(t, nd)
+ err = bstore.Put(context.Background(), blk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodes = append(nodes, blk.Cid())
+ }
+
+ return nodes, bstore
+}
+
+func toBlock(t *testing.T, nd ipld.Node) blocks.Block {
+ buf := new(bytes.Buffer)
+ err := dagcbor.Encode(nd, buf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, err := cid.Prefix{
+ Version: 1,
+ Codec: cid.DagCBOR,
+ MhType: mh.SHA2_256,
+ MhLength: -1,
+ }.Sum(buf.Bytes())
+ if err != nil {
+ t.Fatal(err)
+ }
+ blk, err := blocks.NewBlockWithCid(buf.Bytes(), c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return blk
+}
+
+func TestReprovide(t *testing.T) {
+ testReprovide(t, func(r *Reprovider, ctx context.Context) error {
+ return r.Reprovide()
+ })
+}
+
+func TestTrigger(t *testing.T) {
+ testReprovide(t, func(r *Reprovider, ctx context.Context) error {
+ go r.Run()
+ time.Sleep(1 * time.Second)
+ defer r.Close()
+ err := r.Trigger(ctx)
+ return err
+ })
+}
+
+func testReprovide(t *testing.T, trigger func(r *Reprovider, ctx context.Context) error) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ clA, clB, idA, _ := setupRouting(t)
+ nodes, bstore := setupDag(t)
+
+ keyProvider := NewBlockstoreProvider(bstore)
+ reprov := NewReprovider(ctx, time.Hour, clA, keyProvider)
+ reprov.Trigger(context.Background())
+ err := trigger(reprov, ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var providers []peer.AddrInfo
+ maxProvs := 100
+
+ for _, c := range nodes {
+ // We provide raw cids because of the multihash keying
+ // FIXME(@Jorropo): I think this change should be done in the DHT layer, probably an issue with our routing mock.
+ b := c.Bytes()
+ b[1] = 0x55 // rewrite the cid to raw
+ _, c, err := cid.CidFromBytes(b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ provChan := clB.FindProvidersAsync(ctx, c, maxProvs)
+ for p := range provChan {
+ providers = append(providers, p)
+ }
+
+ if len(providers) == 0 {
+ t.Fatal("Should have gotten a provider")
+ }
+
+ if providers[0].ID != idA {
+ t.Fatal("Somehow got the wrong peer back as a provider.")
+ }
+ }
+}
+
+func TestTriggerTwice(t *testing.T) {
+ // Ensure we can only trigger once at a time.
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ clA, _, _, _ := setupRouting(t)
+
+ keyCh := make(chan cid.Cid)
+ startCh := make(chan struct{})
+ keyFunc := func(ctx context.Context) (<-chan cid.Cid, error) {
+ <-startCh
+ return keyCh, nil
+ }
+
+ reprov := NewReprovider(ctx, time.Hour, clA, keyFunc)
+ go reprov.Run()
+ defer reprov.Close()
+
+ // Wait for the reprovider to start, otherwise, the reprovider will
+ // think a concurrent reprovide is running.
+ //
+ // We _could_ fix this race... but that would be complexity for nothing.
+ // 1. We start a reprovide 1 minute after startup anyways.
+ // 2. The window is really narrow.
+ time.Sleep(1 * time.Second)
+
+ errCh := make(chan error, 2)
+
+ // Trigger in the background
+ go func() {
+ errCh <- reprov.Trigger(ctx)
+ }()
+
+ // Wait for the trigger to really start.
+ startCh <- struct{}{}
+
+ start := time.Now()
+ // Try to trigger again, this should fail immediately.
+ if err := reprov.Trigger(ctx); err == nil {
+ t.Fatal("expected an error")
+ }
+ if time.Since(start) > 10*time.Millisecond {
+ t.Fatal("expected reprovide to fail instantly")
+ }
+
+ // Let the trigger progress.
+ close(keyCh)
+
+ // Check the result.
+ err := <-errCh
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Try to trigger again, this should work.
+ go func() {
+ errCh <- reprov.Trigger(ctx)
+ }()
+ startCh <- struct{}{}
+ err = <-errCh
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+type mockPinner struct {
+ recursive []cid.Cid
+ direct []cid.Cid
+}
+
+func (mp *mockPinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) {
+ return mp.direct, nil
+}
+
+func (mp *mockPinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) {
+ return mp.recursive, nil
+}
+
+func TestReprovidePinned(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ nodes, bstore := setupDag(t)
+
+ fetchConfig := bsfetcher.NewFetcherConfig(bsrv.New(bstore, offline.Exchange(bstore)))
+
+ for i := 0; i < 2; i++ {
+ clA, clB, idA, _ := setupRouting(t)
+
+ onlyRoots := i == 0
+ t.Logf("only roots: %v", onlyRoots)
+
+ var provide, dont []cid.Cid
+ if onlyRoots {
+ provide = []cid.Cid{nodes[1], nodes[3]}
+ dont = []cid.Cid{nodes[0], nodes[2]}
+ } else {
+ provide = []cid.Cid{nodes[0], nodes[1], nodes[3]}
+ dont = []cid.Cid{nodes[2]}
+ }
+
+ keyProvider := NewPinnedProvider(onlyRoots, &mockPinner{
+ recursive: []cid.Cid{nodes[1]},
+ direct: []cid.Cid{nodes[3]},
+ }, fetchConfig)
+
+ reprov := NewReprovider(ctx, time.Hour, clA, keyProvider)
+ err := reprov.Reprovide()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i, c := range provide {
+ prov, ok := <-clB.FindProvidersAsync(ctx, c, 1)
+ if !ok {
+ t.Errorf("Should have gotten a provider for %d", i)
+ continue
+ }
+
+ if prov.ID != idA {
+ t.Errorf("Somehow got the wrong peer back as a provider.")
+ continue
+ }
+ }
+ for i, c := range dont {
+ prov, ok := <-clB.FindProvidersAsync(ctx, c, 1)
+ if ok {
+ t.Fatalf("found provider %s for %d, expected none", prov.ID, i)
+ }
+ }
+ }
+}
diff --git a/provider/system.go b/provider/system.go
new file mode 100644
index 000000000..9fc3e8879
--- /dev/null
+++ b/provider/system.go
@@ -0,0 +1,60 @@
+package provider
+
+import (
+ "context"
+
+ "github.com/ipfs/go-cid"
+)
+
+// System defines the interface for interacting with the value
+// provider system
+type System interface {
+ Run()
+ Close() error
+ Provide(cid.Cid) error
+ Reprovide(context.Context) error
+}
+
+type system struct {
+ provider Provider
+ reprovider Reprovider
+}
+
+// NewSystem constructs a new provider system from a provider and reprovider
+func NewSystem(provider Provider, reprovider Reprovider) System {
+ return &system{provider, reprovider}
+}
+
+// Run the provider system by running the provider and reprovider
+func (s *system) Run() {
+ go s.provider.Run()
+ go s.reprovider.Run()
+}
+
+// Close the provider and reprovider
+func (s *system) Close() error {
+ var errs []error
+
+ if err := s.provider.Close(); err != nil {
+ errs = append(errs, err)
+ }
+
+ if err := s.reprovider.Close(); err != nil {
+ errs = append(errs, err)
+ }
+
+ if len(errs) > 0 {
+ return errs[0]
+ }
+ return nil
+}
+
+// Provide a value
+func (s *system) Provide(cid cid.Cid) error {
+ return s.provider.Provide(cid)
+}
+
+// Reprovide all the previously provided values
+func (s *system) Reprovide(ctx context.Context) error {
+ return s.reprovider.Trigger(ctx)
+}
diff --git a/routing/http/README.md b/routing/http/README.md
new file mode 100644
index 000000000..65650ed50
--- /dev/null
+++ b/routing/http/README.md
@@ -0,0 +1,24 @@
+go-delegated-routing
+=======================
+
+> Delegated routing Client and Server over Reframe RPC
+
+This package provides delegated routing implementation in Go:
+- Client (for IPFS nodes like [Kubo](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingrouters-parameters)),
+- Server (for public indexers such as https://cid.contact)
+
+## Documentation
+
+- Go docs: https://pkg.go.dev/github.com/ipfs/boxo/routing/http/
+
+## Lead Maintainer
+
+🦗🎶
+
+## Contributing
+
+Contributions are welcome! This repository is part of the IPFS project and therefore governed by our [contributing guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md).
+
+## License
+
+[SPDX-License-Identifier: Apache-2.0 OR MIT](LICENSE.md)
\ No newline at end of file
diff --git a/routing/http/client/client.go b/routing/http/client/client.go
new file mode 100644
index 000000000..4e5cfb8f4
--- /dev/null
+++ b/routing/http/client/client.go
@@ -0,0 +1,247 @@
+package client
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/benbjohnson/clock"
+ ipns "github.com/ipfs/boxo/ipns"
+ "github.com/ipfs/boxo/routing/http/contentrouter"
+ "github.com/ipfs/boxo/routing/http/internal/drjson"
+ "github.com/ipfs/boxo/routing/http/server"
+ "github.com/ipfs/boxo/routing/http/types"
+ "github.com/ipfs/go-cid"
+ logging "github.com/ipfs/go-log/v2"
+ record "github.com/libp2p/go-libp2p-record"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+)
+
+var logger = logging.Logger("service/delegatedrouting")
+
+type client struct {
+ baseURL string
+ httpClient httpClient
+ validator record.Validator
+ clock clock.Clock
+
+ peerID peer.ID
+ addrs []types.Multiaddr
+ identity crypto.PrivKey
+
+ // called immeidately after signing a provide req
+ // used for testing, e.g. testing the server with a mangled signature
+ afterSignCallback func(req *types.WriteBitswapProviderRecord)
+}
+
+// defaultUserAgent is used as a fallback to inform HTTP server which library
+// version sent a request
+var defaultUserAgent = moduleVersion()
+
+var _ contentrouter.Client = &client{}
+
+type httpClient interface {
+ Do(req *http.Request) (*http.Response, error)
+}
+
+type option func(*client)
+
+func WithIdentity(identity crypto.PrivKey) option {
+ return func(c *client) {
+ c.identity = identity
+ }
+}
+
+func WithHTTPClient(h httpClient) option {
+ return func(c *client) {
+ c.httpClient = h
+ }
+}
+
+func WithUserAgent(ua string) option {
+ return func(c *client) {
+ if ua == "" {
+ return
+ }
+ httpClient, ok := c.httpClient.(*http.Client)
+ if !ok {
+ return
+ }
+ transport, ok := httpClient.Transport.(*ResponseBodyLimitedTransport)
+ if !ok {
+ return
+ }
+ transport.UserAgent = ua
+ }
+}
+
+func WithProviderInfo(peerID peer.ID, addrs []multiaddr.Multiaddr) option {
+ return func(c *client) {
+ c.peerID = peerID
+ for _, a := range addrs {
+ c.addrs = append(c.addrs, types.Multiaddr{Multiaddr: a})
+ }
+ }
+}
+
+// New creates a content routing API client.
+// The Provider and identity parameters are option. If they are nil, the `Provide` method will not function.
+func New(baseURL string, opts ...option) (*client, error) {
+ defaultHTTPClient := &http.Client{
+ Transport: &ResponseBodyLimitedTransport{
+ RoundTripper: http.DefaultTransport,
+ LimitBytes: 1 << 20,
+ UserAgent: defaultUserAgent,
+ },
+ }
+ client := &client{
+ baseURL: baseURL,
+ httpClient: defaultHTTPClient,
+ validator: ipns.Validator{},
+ clock: clock.New(),
+ }
+
+ for _, opt := range opts {
+ opt(client)
+ }
+
+ if client.identity != nil && client.peerID.Size() != 0 && !client.peerID.MatchesPublicKey(client.identity.GetPublic()) {
+ return nil, errors.New("identity does not match provider")
+ }
+
+ return client, nil
+}
+
+func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs []types.ProviderResponse, err error) {
+ measurement := newMeasurement("FindProviders")
+ defer func() {
+ measurement.length = len(provs)
+ measurement.record(ctx)
+ }()
+
+ url := c.baseURL + server.ProvidePath + key.String()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+ measurement.host = req.Host
+
+ start := c.clock.Now()
+ resp, err := c.httpClient.Do(req)
+
+ measurement.err = err
+ measurement.latency = c.clock.Since(start)
+
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ measurement.statusCode = resp.StatusCode
+ if resp.StatusCode == http.StatusNotFound {
+ return nil, nil
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, httpError(resp.StatusCode, resp.Body)
+ }
+
+ parsedResp := &types.ReadProvidersResponse{}
+ err = json.NewDecoder(resp.Body).Decode(parsedResp)
+ return parsedResp.Providers, err
+}
+
+func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) {
+ if c.identity == nil {
+ return 0, errors.New("cannot provide Bitswap records without an identity")
+ }
+ if c.peerID.Size() == 0 {
+ return 0, errors.New("cannot provide Bitswap records without a peer ID")
+ }
+
+ ks := make([]types.CID, len(keys))
+ for i, c := range keys {
+ ks[i] = types.CID{Cid: c}
+ }
+
+ now := c.clock.Now()
+
+ req := types.WriteBitswapProviderRecord{
+ Protocol: "transport-bitswap",
+ Schema: types.SchemaBitswap,
+ Payload: types.BitswapPayload{
+ Keys: ks,
+ AdvisoryTTL: &types.Duration{Duration: ttl},
+ Timestamp: &types.Time{Time: now},
+ ID: &c.peerID,
+ Addrs: c.addrs,
+ },
+ }
+ err := req.Sign(c.peerID, c.identity)
+ if err != nil {
+ return 0, err
+ }
+
+ if c.afterSignCallback != nil {
+ c.afterSignCallback(&req)
+ }
+
+ advisoryTTL, err := c.provideSignedBitswapRecord(ctx, &req)
+ if err != nil {
+ return 0, err
+ }
+
+ return advisoryTTL, err
+}
+
+// ProvideAsync makes a provide request to a delegated router
+func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.WriteBitswapProviderRecord) (time.Duration, error) {
+ req := types.WriteProvidersRequest{Providers: []types.WriteProviderRecord{bswp}}
+
+ url := c.baseURL + server.ProvidePath
+
+ b, err := drjson.MarshalJSONBytes(req)
+ if err != nil {
+ return 0, err
+ }
+
+ httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewBuffer(b))
+ if err != nil {
+ return 0, err
+ }
+
+ resp, err := c.httpClient.Do(httpReq)
+ if err != nil {
+ return 0, fmt.Errorf("making HTTP req to provide a signed record: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return 0, httpError(resp.StatusCode, resp.Body)
+ }
+ var provideResult types.WriteProvidersResponse
+ err = json.NewDecoder(resp.Body).Decode(&provideResult)
+ if err != nil {
+ return 0, err
+ }
+ if len(provideResult.ProvideResults) != 1 {
+ return 0, fmt.Errorf("expected 1 result but got %d", len(provideResult.ProvideResults))
+ }
+
+ v, ok := provideResult.ProvideResults[0].(*types.WriteBitswapProviderRecordResponse)
+ if !ok {
+ return 0, fmt.Errorf("expected AdvisoryTTL field")
+ }
+
+ if v.AdvisoryTTL != nil {
+ return v.AdvisoryTTL.Duration, nil
+ }
+
+ return 0, nil
+}
diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go
new file mode 100644
index 000000000..d8fc4abac
--- /dev/null
+++ b/routing/http/client/client_test.go
@@ -0,0 +1,342 @@
+package client
+
+import (
+ "context"
+ "crypto/rand"
+ "net/http"
+ "net/http/httptest"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/benbjohnson/clock"
+ "github.com/ipfs/boxo/routing/http/server"
+ "github.com/ipfs/boxo/routing/http/types"
+ "github.com/ipfs/go-cid"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+ "github.com/multiformats/go-multibase"
+ "github.com/multiformats/go-multihash"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+)
+
+type mockContentRouter struct{ mock.Mock }
+
+func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid) ([]types.ProviderResponse, error) {
+ args := m.Called(ctx, key)
+ return args.Get(0).([]types.ProviderResponse), args.Error(1)
+}
+func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) {
+ args := m.Called(ctx, req)
+ return args.Get(0).(time.Duration), args.Error(1)
+}
+
+func (m *mockContentRouter) Provide(ctx context.Context, req *server.WriteProvideRequest) (types.ProviderResponse, error) {
+ args := m.Called(ctx, req)
+ return args.Get(0).(types.ProviderResponse), args.Error(1)
+}
+
+type testDeps struct {
+ router *mockContentRouter
+ server *httptest.Server
+ peerID peer.ID
+ addrs []multiaddr.Multiaddr
+ client *client
+}
+
+func makeTestDeps(t *testing.T) testDeps {
+ const testUserAgent = "testUserAgent"
+ peerID, addrs, identity := makeProviderAndIdentity()
+ router := &mockContentRouter{}
+ server := httptest.NewServer(server.Handler(router))
+ t.Cleanup(server.Close)
+ serverAddr := "http://" + server.Listener.Addr().String()
+ c, err := New(serverAddr, WithProviderInfo(peerID, addrs), WithIdentity(identity), WithUserAgent(testUserAgent))
+ if err != nil {
+ panic(err)
+ }
+ assertUserAgentOverride(t, c, testUserAgent)
+ return testDeps{
+ router: router,
+ server: server,
+ peerID: peerID,
+ addrs: addrs,
+ client: c,
+ }
+}
+
+func assertUserAgentOverride(t *testing.T, c *client, expected string) {
+ httpClient, ok := c.httpClient.(*http.Client)
+ if !ok {
+ t.Error("invalid c.httpClient")
+ }
+ transport, ok := httpClient.Transport.(*ResponseBodyLimitedTransport)
+ if !ok {
+ t.Error("invalid httpClient.Transport")
+ }
+ if transport.UserAgent != expected {
+ t.Error("invalid httpClient.Transport.UserAgent")
+ }
+}
+
+func makeCID() cid.Cid {
+ buf := make([]byte, 63)
+ _, err := rand.Read(buf)
+ if err != nil {
+ panic(err)
+ }
+ mh, err := multihash.Encode(buf, multihash.SHA2_256)
+ if err != nil {
+ panic(err)
+ }
+ c := cid.NewCidV1(0, mh)
+ return c
+}
+
+func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) {
+ for _, a := range addrs {
+ drmas = append(drmas, types.Multiaddr{Multiaddr: a})
+ }
+ return
+}
+
+func drAddrsToAddrs(drmas []types.Multiaddr) (addrs []multiaddr.Multiaddr) {
+ for _, a := range drmas {
+ addrs = append(addrs, a.Multiaddr)
+ }
+ return
+}
+
+func makeBSReadProviderResp() types.ReadBitswapProviderRecord {
+ peerID, addrs, _ := makeProviderAndIdentity()
+ return types.ReadBitswapProviderRecord{
+ Protocol: "transport-bitswap",
+ Schema: types.SchemaBitswap,
+ ID: &peerID,
+ Addrs: addrsToDRAddrs(addrs),
+ }
+}
+
+func makeProviderAndIdentity() (peer.ID, []multiaddr.Multiaddr, crypto.PrivKey) {
+ priv, _, err := crypto.GenerateEd25519Key(rand.Reader)
+ if err != nil {
+ panic(err)
+ }
+ peerID, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ panic(err)
+ }
+ ma1, err := multiaddr.NewMultiaddr("/ip4/0.0.0.0/tcp/4001")
+ if err != nil {
+ panic(err)
+ }
+
+ ma2, err := multiaddr.NewMultiaddr("/ip4/0.0.0.0/tcp/4002")
+ if err != nil {
+ panic(err)
+ }
+
+ return peerID, []multiaddr.Multiaddr{ma1, ma2}, priv
+}
+
+func TestClient_FindProviders(t *testing.T) {
+ bsReadProvResp := makeBSReadProviderResp()
+ bitswapProvs := []types.ProviderResponse{&bsReadProvResp}
+
+ cases := []struct {
+ name string
+ httpStatusCode int
+ stopServer bool
+ routerProvs []types.ProviderResponse
+ routerErr error
+
+ expProvs []types.ProviderResponse
+ expErrContains []string
+ expWinErrContains []string
+ }{
+ {
+ name: "happy case",
+ routerProvs: bitswapProvs,
+ expProvs: bitswapProvs,
+ },
+ {
+ name: "returns an error if there's a non-200 response",
+ httpStatusCode: 500,
+ expErrContains: []string{"HTTP error with StatusCode=500: "},
+ },
+ {
+ name: "returns an error if the HTTP client returns a non-HTTP error",
+ stopServer: true,
+ expErrContains: []string{"connect: connection refused"},
+ expWinErrContains: []string{"connectex: No connection could be made because the target machine actively refused it."},
+ },
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ deps := makeTestDeps(t)
+ client := deps.client
+ router := deps.router
+
+ if c.httpStatusCode != 0 {
+ deps.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(c.httpStatusCode)
+ })
+ }
+
+ if c.stopServer {
+ deps.server.Close()
+ }
+ cid := makeCID()
+
+ router.On("FindProviders", mock.Anything, cid).
+ Return(c.routerProvs, c.routerErr)
+
+ provs, err := client.FindProviders(context.Background(), cid)
+
+ var errList []string
+ if runtime.GOOS == "windows" && len(c.expWinErrContains) != 0 {
+ errList = c.expWinErrContains
+ } else {
+ errList = c.expErrContains
+ }
+
+ for _, exp := range errList {
+ require.ErrorContains(t, err, exp)
+ }
+ if len(errList) == 0 {
+ require.NoError(t, err)
+ }
+
+ assert.Equal(t, c.expProvs, provs)
+ })
+ }
+}
+
+func TestClient_Provide(t *testing.T) {
+ cases := []struct {
+ name string
+ manglePath bool
+ mangleSignature bool
+ stopServer bool
+ noProviderInfo bool
+ noIdentity bool
+
+ cids []cid.Cid
+ ttl time.Duration
+
+ routerAdvisoryTTL time.Duration
+ routerErr error
+
+ expErrContains string
+ expWinErrContains string
+
+ expAdvisoryTTL time.Duration
+ }{
+ {
+ name: "happy case",
+ cids: []cid.Cid{makeCID()},
+ ttl: 1 * time.Hour,
+ routerAdvisoryTTL: 1 * time.Minute,
+
+ expAdvisoryTTL: 1 * time.Minute,
+ },
+ {
+ name: "should return a 403 if the payload signature verification fails",
+ cids: []cid.Cid{},
+ mangleSignature: true,
+
+ expErrContains: "HTTP error with StatusCode=403",
+ },
+ {
+ name: "should return error if identity is not provided",
+ noIdentity: true,
+ expErrContains: "cannot provide Bitswap records without an identity",
+ },
+ {
+ name: "should return error if provider is not provided",
+ noProviderInfo: true,
+ expErrContains: "cannot provide Bitswap records without a peer ID",
+ },
+ {
+ name: "returns an error if there's a non-200 response",
+ manglePath: true,
+ expErrContains: "HTTP error with StatusCode=404: 404 page not found",
+ },
+ {
+ name: "returns an error if the HTTP client returns a non-HTTP error",
+ stopServer: true,
+ expErrContains: "connect: connection refused",
+ expWinErrContains: "connectex: No connection could be made because the target machine actively refused it.",
+ },
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ // deps := makeTestDeps(t)
+ deps := makeTestDeps(t)
+ client := deps.client
+ router := deps.router
+
+ if c.noIdentity {
+ client.identity = nil
+ }
+ if c.noProviderInfo {
+ client.peerID = ""
+ client.addrs = nil
+ }
+
+ clock := clock.NewMock()
+ clock.Set(time.Now())
+ client.clock = clock
+
+ ctx := context.Background()
+
+ if c.manglePath {
+ client.baseURL += "/foo"
+ }
+ if c.stopServer {
+ deps.server.Close()
+ }
+ if c.mangleSignature {
+ client.afterSignCallback = func(req *types.WriteBitswapProviderRecord) {
+ mh, err := multihash.Encode([]byte("boom"), multihash.SHA2_256)
+ require.NoError(t, err)
+ mb, err := multibase.Encode(multibase.Base64, mh)
+ require.NoError(t, err)
+
+ req.Signature = mb
+ }
+ }
+
+ expectedProvReq := &server.BitswapWriteProvideRequest{
+ Keys: c.cids,
+ Timestamp: clock.Now().Truncate(time.Millisecond),
+ AdvisoryTTL: c.ttl,
+ Addrs: drAddrsToAddrs(client.addrs),
+ ID: client.peerID,
+ }
+
+ router.On("ProvideBitswap", mock.Anything, expectedProvReq).
+ Return(c.routerAdvisoryTTL, c.routerErr)
+
+ advisoryTTL, err := client.ProvideBitswap(ctx, c.cids, c.ttl)
+
+ var errorString string
+ if runtime.GOOS == "windows" && c.expWinErrContains != "" {
+ errorString = c.expWinErrContains
+ } else {
+ errorString = c.expErrContains
+ }
+
+ if errorString != "" {
+ require.ErrorContains(t, err, errorString)
+ } else {
+ require.NoError(t, err)
+ }
+
+ assert.Equal(t, c.expAdvisoryTTL, advisoryTTL)
+ })
+ }
+}
diff --git a/routing/http/client/error.go b/routing/http/client/error.go
new file mode 100644
index 000000000..b3b7904d5
--- /dev/null
+++ b/routing/http/client/error.go
@@ -0,0 +1,27 @@
+package client
+
+import (
+ "fmt"
+ "io"
+)
+
+type HTTPError struct {
+ StatusCode int
+ Body string
+}
+
+func (e *HTTPError) Error() string {
+ return fmt.Sprintf("HTTP error with StatusCode=%d: %s", e.StatusCode, e.Body)
+}
+
+func httpError(statusCode int, body io.Reader) error {
+ bodyBytes, err := io.ReadAll(io.LimitReader(body, 1024))
+ if err != nil {
+ logger.Warnw("could not read body bytes from error response", "Error", err)
+ bodyBytes = []byte("unable to read body")
+ }
+ return &HTTPError{
+ StatusCode: statusCode,
+ Body: string(bodyBytes),
+ }
+}
diff --git a/routing/http/client/measures.go b/routing/http/client/measures.go
new file mode 100644
index 000000000..942460518
--- /dev/null
+++ b/routing/http/client/measures.go
@@ -0,0 +1,119 @@
+package client
+
+import (
+ "context"
+ "errors"
+ "net"
+ "strconv"
+ "time"
+
+ "go.opencensus.io/stats"
+ "go.opencensus.io/stats/view"
+ "go.opencensus.io/tag"
+)
+
+var (
+ distMS = view.Distribution(0, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000)
+ distLength = view.Distribution(0, 1, 2, 5, 10, 11, 12, 15, 20, 50, 100, 200, 500)
+
+ measureLatency = stats.Int64("routing_http_client_latency", "the latency of operations by the routing HTTP client", stats.UnitMilliseconds)
+ measureLength = stats.Int64("routing_http_client_length", "the number of elements in a response collection", stats.UnitDimensionless)
+
+ keyOperation = tag.MustNewKey("operation")
+ keyHost = tag.MustNewKey("host")
+ keyStatusCode = tag.MustNewKey("code")
+ keyError = tag.MustNewKey("error")
+
+ ViewLatency = &view.View{
+ Measure: measureLatency,
+ Aggregation: distMS,
+ TagKeys: []tag.Key{keyOperation, keyHost, keyStatusCode, keyError},
+ }
+ ViewLength = &view.View{
+ Measure: measureLength,
+ Aggregation: distLength,
+ TagKeys: []tag.Key{keyOperation, keyHost},
+ }
+
+ OpenCensusViews = []*view.View{
+ ViewLatency,
+ ViewLength,
+ }
+)
+
+type measurement struct {
+ operation string
+ err error
+ latency time.Duration
+ statusCode int
+ host string
+ length int
+}
+
+func (m measurement) record(ctx context.Context) {
+ stats.RecordWithTags(
+ ctx,
+ []tag.Mutator{
+ tag.Upsert(keyHost, m.host),
+ tag.Upsert(keyOperation, m.operation),
+ tag.Upsert(keyStatusCode, strconv.Itoa(m.statusCode)),
+ tag.Upsert(keyError, metricsErrStr(m.err)),
+ },
+ measureLatency.M(m.latency.Milliseconds()),
+ )
+ if m.err == nil {
+ stats.RecordWithTags(
+ ctx,
+ []tag.Mutator{
+ tag.Upsert(keyHost, m.host),
+ tag.Upsert(keyOperation, m.operation),
+ },
+ measureLength.M(int64(m.length)),
+ )
+ }
+}
+
+func newMeasurement(operation string) measurement {
+ return measurement{
+ operation: operation,
+ host: "None",
+ }
+}
+
+// metricsErrStr converts an error into a string that can be used as a metric label.
+// Errs are mapped to strings explicitly to avoid accidental high dimensionality.
+func metricsErrStr(err error) string {
+ if err == nil {
+ return "None"
+ }
+ var httpErr *HTTPError
+ if errors.As(err, &httpErr) {
+ return "HTTP"
+ }
+ if errors.Is(err, context.DeadlineExceeded) {
+ return "DeadlineExceeded"
+ }
+ if errors.Is(err, context.Canceled) {
+ return "Canceled"
+ }
+ var dnsErr *net.DNSError
+ if errors.As(err, &dnsErr) {
+ if dnsErr.IsNotFound {
+ return "DNSNotFound"
+ }
+ if dnsErr.IsTimeout {
+ return "DNSTimeout"
+ }
+ return "DNS"
+ }
+
+ var netErr net.Error
+ if errors.As(err, &netErr) {
+ if netErr.Timeout() {
+ return "NetTimeout"
+ }
+ return "Net"
+ }
+
+ return "Other"
+}
diff --git a/routing/http/client/transport.go b/routing/http/client/transport.go
new file mode 100644
index 000000000..357d25cb2
--- /dev/null
+++ b/routing/http/client/transport.go
@@ -0,0 +1,79 @@
+package client
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "reflect"
+ "runtime/debug"
+ "strings"
+)
+
+type ResponseBodyLimitedTransport struct {
+ http.RoundTripper
+ LimitBytes int64
+ UserAgent string
+}
+
+func (r *ResponseBodyLimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ if r.UserAgent != "" {
+ req.Header.Set("User-Agent", r.UserAgent)
+ }
+ resp, err := r.RoundTripper.RoundTrip(req)
+ if resp != nil && resp.Body != nil {
+ resp.Body = &limitReadCloser{
+ limit: r.LimitBytes,
+ ReadCloser: resp.Body,
+ }
+ }
+ return resp, err
+}
+
+type limitReadCloser struct {
+ limit int64
+ bytesRead int64
+ io.ReadCloser
+}
+
+func (l *limitReadCloser) Read(p []byte) (int, error) {
+ n, err := l.ReadCloser.Read(p)
+ l.bytesRead += int64(n)
+ if l.bytesRead > l.limit {
+ return 0, fmt.Errorf("reached read limit of %d bytes after reading %d bytes", l.limit, l.bytesRead)
+ }
+ return n, err
+}
+
+// ImportPath is the canonical import path that allows us to identify
+// official client builds vs modified forks, and use that info in User-Agent header.
+var ImportPath = importPath()
+
+// importPath returns the path that library consumers would have in go.mod
+func importPath() string {
+ p := reflect.ValueOf(ResponseBodyLimitedTransport{}).Type().PkgPath()
+ // we have monorepo, so stripping the remainder
+ return strings.TrimSuffix(p, "/routing/http/client")
+}
+
+// moduleVersion returns a useful user agent version string allowing us to
+// identify requests coming from official releases of this module vs forks.
+func moduleVersion() (ua string) {
+ ua = ImportPath
+ var module *debug.Module
+ if bi, ok := debug.ReadBuildInfo(); ok {
+ // If debug.ReadBuildInfo was successful, we can read Version by finding
+ // this client in the dependency list of the app that has it in go.mod
+ for _, dep := range bi.Deps {
+ if dep.Path == ImportPath {
+ module = dep
+ break
+ }
+ }
+ if module != nil {
+ ua += "@" + module.Version
+ return
+ }
+ ua += "@unknown"
+ }
+ return
+}
diff --git a/routing/http/client/transport_test.go b/routing/http/client/transport_test.go
new file mode 100644
index 000000000..7da545062
--- /dev/null
+++ b/routing/http/client/transport_test.go
@@ -0,0 +1,84 @@
+package client
+
+import (
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+type testServer struct {
+ bytesToWrite int
+}
+
+func (s *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ bytes := make([]byte, s.bytesToWrite)
+ for i := 0; i < s.bytesToWrite; i++ {
+ bytes[i] = 'a'
+ }
+ _, err := w.Write(bytes)
+ if err != nil {
+ panic(err)
+ }
+}
+
+func TestResponseBodyLimitedTransport(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ limit int64
+ serverSend int
+
+ expErr string
+ }{
+ {
+ name: "under the limit should succeed",
+ limit: 1 << 20,
+ serverSend: 1 << 19,
+ },
+ {
+ name: "over the limit should fail",
+ limit: 1 << 20,
+ serverSend: 1 << 21,
+ expErr: "reached read limit of 1048576 bytes after reading",
+ },
+ {
+ name: "exactly on the limit should succeed",
+ limit: 1 << 20,
+ serverSend: 1 << 20,
+ },
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ server := httptest.NewServer(&testServer{bytesToWrite: c.serverSend})
+ t.Cleanup(server.Close)
+
+ client := server.Client()
+ client.Transport = &ResponseBodyLimitedTransport{
+ LimitBytes: c.limit,
+ RoundTripper: client.Transport,
+ }
+
+ resp, err := client.Get(server.URL)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+
+ _, err = io.ReadAll(resp.Body)
+
+ if c.expErr == "" {
+ assert.NoError(t, err)
+ } else {
+ assert.Contains(t, err.Error(), c.expErr)
+ }
+
+ })
+ }
+}
+
+func TestUserAgentVersionString(t *testing.T) {
+ // forks will have to update below lines to pass test
+ assert.Equal(t, importPath(), "github.com/ipfs/boxo")
+ // @unknown because we run in tests
+ assert.Equal(t, moduleVersion(), "github.com/ipfs/boxo@unknown")
+}
diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go
new file mode 100644
index 000000000..572ac2655
--- /dev/null
+++ b/routing/http/contentrouter/contentrouter.go
@@ -0,0 +1,140 @@
+package contentrouter
+
+import (
+ "context"
+ "reflect"
+ "time"
+
+ "github.com/ipfs/boxo/routing/http/internal"
+ "github.com/ipfs/boxo/routing/http/types"
+ "github.com/ipfs/go-cid"
+ logging "github.com/ipfs/go-log/v2"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/multiformats/go-multiaddr"
+ "github.com/multiformats/go-multihash"
+)
+
+var logger = logging.Logger("service/contentrouting")
+
+const ttl = 24 * time.Hour
+
+type Client interface {
+ ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error)
+ FindProviders(ctx context.Context, key cid.Cid) ([]types.ProviderResponse, error)
+}
+
+type contentRouter struct {
+ client Client
+ maxProvideConcurrency int
+ maxProvideBatchSize int
+}
+
+var _ routing.ContentRouting = (*contentRouter)(nil)
+
+type option func(c *contentRouter)
+
+func WithMaxProvideConcurrency(max int) option {
+ return func(c *contentRouter) {
+ c.maxProvideConcurrency = max
+ }
+}
+
+func WithMaxProvideBatchSize(max int) option {
+ return func(c *contentRouter) {
+ c.maxProvideBatchSize = max
+ }
+}
+
+func NewContentRoutingClient(c Client, opts ...option) *contentRouter {
+ cr := &contentRouter{
+ client: c,
+ maxProvideConcurrency: 5,
+ maxProvideBatchSize: 100,
+ }
+ for _, opt := range opts {
+ opt(cr)
+ }
+ return cr
+}
+
+func (c *contentRouter) Provide(ctx context.Context, key cid.Cid, announce bool) error {
+ // If 'true' is
+ // passed, it also announces it, otherwise it is just kept in the local
+ // accounting of which objects are being provided.
+ if !announce {
+ return nil
+ }
+
+ _, err := c.client.ProvideBitswap(ctx, []cid.Cid{key}, ttl)
+ return err
+}
+
+// ProvideMany provides a set of keys to the remote delegate.
+// Large sets of keys are chunked into multiple requests and sent concurrently, according to the concurrency configuration.
+// TODO: implement retries through transient errors
+func (c *contentRouter) ProvideMany(ctx context.Context, mhKeys []multihash.Multihash) error {
+ keys := make([]cid.Cid, 0, len(mhKeys))
+ for _, m := range mhKeys {
+ keys = append(keys, cid.NewCidV1(cid.Raw, m))
+ }
+
+ if len(keys) <= c.maxProvideBatchSize {
+ _, err := c.client.ProvideBitswap(ctx, keys, ttl)
+ return err
+ }
+
+ return internal.DoBatch(
+ ctx,
+ c.maxProvideBatchSize,
+ c.maxProvideConcurrency,
+ keys,
+ func(ctx context.Context, batch []cid.Cid) error {
+ _, err := c.client.ProvideBitswap(ctx, batch, ttl)
+ return err
+ },
+ )
+}
+
+// Ready is part of the existing `ProvideMany` interface.
+func (c *contentRouter) Ready() bool {
+ return true
+}
+
+func (c *contentRouter) FindProvidersAsync(ctx context.Context, key cid.Cid, numResults int) <-chan peer.AddrInfo {
+ results, err := c.client.FindProviders(ctx, key)
+ if err != nil {
+ logger.Warnw("error finding providers", "CID", key, "Error", err)
+ ch := make(chan peer.AddrInfo)
+ close(ch)
+ return ch
+ }
+
+ ch := make(chan peer.AddrInfo, len(results))
+ for _, r := range results {
+ if r.GetSchema() == types.SchemaBitswap {
+ result, ok := r.(*types.ReadBitswapProviderRecord)
+ if !ok {
+ logger.Errorw(
+ "problem casting find providers result",
+ "Schema", r.GetSchema(),
+ "Type", reflect.TypeOf(r).String(),
+ )
+ continue
+ }
+
+ var addrs []multiaddr.Multiaddr
+ for _, a := range result.Addrs {
+ addrs = append(addrs, a.Multiaddr)
+ }
+
+ ch <- peer.AddrInfo{
+ ID: *result.ID,
+ Addrs: addrs,
+ }
+ }
+
+ }
+ close(ch)
+ return ch
+}
diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go
new file mode 100644
index 000000000..643ad301f
--- /dev/null
+++ b/routing/http/contentrouter/contentrouter_test.go
@@ -0,0 +1,139 @@
+package contentrouter
+
+import (
+ "context"
+ "crypto/rand"
+ "testing"
+ "time"
+
+ "github.com/ipfs/boxo/routing/http/types"
+ "github.com/ipfs/go-cid"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multihash"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+)
+
+type mockClient struct{ mock.Mock }
+
+func (m *mockClient) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) {
+ args := m.Called(ctx, keys, ttl)
+ return args.Get(0).(time.Duration), args.Error(1)
+}
+func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) ([]types.ProviderResponse, error) {
+ args := m.Called(ctx, key)
+ return args.Get(0).([]types.ProviderResponse), args.Error(1)
+}
+func (m *mockClient) Ready(ctx context.Context) (bool, error) {
+ args := m.Called(ctx)
+ return args.Bool(0), args.Error(1)
+}
+func makeCID() cid.Cid {
+ buf := make([]byte, 63)
+ _, err := rand.Read(buf)
+ if err != nil {
+ panic(err)
+ }
+ mh, err := multihash.Encode(buf, multihash.SHA2_256)
+ if err != nil {
+ panic(err)
+ }
+ c := cid.NewCidV1(cid.Raw, mh)
+ return c
+}
+
+func TestProvide(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ announce bool
+
+ expNotProvided bool
+ }{
+ {
+ name: "announce=false results in no client request",
+ announce: false,
+ expNotProvided: true,
+ },
+ {
+ name: "announce=true results in a client req",
+ announce: true,
+ },
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ ctx := context.Background()
+ key := makeCID()
+ client := &mockClient{}
+ crc := NewContentRoutingClient(client)
+
+ if !c.expNotProvided {
+ client.On("ProvideBitswap", ctx, []cid.Cid{key}, ttl).Return(time.Minute, nil)
+ }
+
+ err := crc.Provide(ctx, key, c.announce)
+ assert.NoError(t, err)
+
+ if c.expNotProvided {
+ client.AssertNumberOfCalls(t, "ProvideBitswap", 0)
+ }
+
+ })
+ }
+}
+
+func TestProvideMany(t *testing.T) {
+ cids := []cid.Cid{makeCID(), makeCID()}
+ var mhs []multihash.Multihash
+ for _, c := range cids {
+ mhs = append(mhs, c.Hash())
+ }
+ ctx := context.Background()
+ client := &mockClient{}
+ crc := NewContentRoutingClient(client)
+
+ client.On("ProvideBitswap", ctx, cids, ttl).Return(time.Minute, nil)
+
+ err := crc.ProvideMany(ctx, mhs)
+ require.NoError(t, err)
+}
+
+func TestFindProvidersAsync(t *testing.T) {
+ key := makeCID()
+ ctx := context.Background()
+ client := &mockClient{}
+ crc := NewContentRoutingClient(client)
+
+ p1 := peer.ID("peer1")
+ p2 := peer.ID("peer2")
+ ais := []types.ProviderResponse{
+ &types.ReadBitswapProviderRecord{
+ Protocol: "transport-bitswap",
+ Schema: types.SchemaBitswap,
+ ID: &p1,
+ },
+ &types.ReadBitswapProviderRecord{
+ Protocol: "transport-bitswap",
+ Schema: types.SchemaBitswap,
+ ID: &p2,
+ },
+ &types.UnknownProviderRecord{
+ Protocol: "UNKNOWN",
+ },
+ }
+
+ client.On("FindProviders", ctx, key).Return(ais, nil)
+
+ aiChan := crc.FindProvidersAsync(ctx, key, 2)
+
+ var actualAIs []peer.AddrInfo
+ for ai := range aiChan {
+ actualAIs = append(actualAIs, ai)
+ }
+
+ expected := []peer.AddrInfo{
+ {ID: p1},
+ {ID: p2},
+ }
+
+ require.Equal(t, expected, actualAIs)
+}
diff --git a/routing/http/internal/drjson/json.go b/routing/http/internal/drjson/json.go
new file mode 100644
index 000000000..3bc3ab942
--- /dev/null
+++ b/routing/http/internal/drjson/json.go
@@ -0,0 +1,26 @@
+package drjson
+
+import (
+ "bytes"
+ "encoding/json"
+)
+
+func marshalJSON(val any) (*bytes.Buffer, error) {
+ buf := &bytes.Buffer{}
+ enc := json.NewEncoder(buf)
+ enc.SetEscapeHTML(false)
+ err := enc.Encode(val)
+ return buf, err
+}
+
+// MarshalJSONBytes is needed to avoid changes
+// on the original bytes due to HTML escapes.
+func MarshalJSONBytes(val any) ([]byte, error) {
+ buf, err := marshalJSON(val)
+ if err != nil {
+ return nil, err
+ }
+
+ // remove last \n added by Encode
+ return buf.Bytes()[:buf.Len()-1], nil
+}
diff --git a/routing/http/internal/drjson/json_test.go b/routing/http/internal/drjson/json_test.go
new file mode 100644
index 000000000..0e3bae81b
--- /dev/null
+++ b/routing/http/internal/drjson/json_test.go
@@ -0,0 +1,16 @@
+package drjson
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestMarshalJSON(t *testing.T) {
+ // ensure that < is not escaped, which is the default Go behavior
+ bytes, err := MarshalJSONBytes(map[string]string{"<": "<"})
+ if err != nil {
+ panic(err)
+ }
+ require.Equal(t, "{\"<\":\"<\"}", string(bytes))
+}
diff --git a/routing/http/internal/goroutines.go b/routing/http/internal/goroutines.go
new file mode 100644
index 000000000..4c44428ab
--- /dev/null
+++ b/routing/http/internal/goroutines.go
@@ -0,0 +1,71 @@
+package internal
+
+import (
+ "context"
+ "sync"
+
+ "github.com/samber/lo"
+)
+
+// DoBatch processes a slice of items with concurrency no higher than maxConcurrency by splitting it into batches no larger than maxBatchSize.
+// If an error is returned for any batch, the process is short-circuited and the error is immediately returned.
+func DoBatch[A any](ctx context.Context, maxBatchSize, maxConcurrency int, items []A, f func(context.Context, []A) error) error {
+ if len(items) == 0 {
+ return nil
+ }
+ batches := lo.Chunk(items, maxBatchSize)
+ workerCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ batchChan := make(chan []A)
+ errChan := make(chan error)
+ wg := sync.WaitGroup{}
+ for i := 0; i < maxConcurrency && i < len(batches); i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for {
+ select {
+ case batch := <-batchChan:
+ err := f(workerCtx, batch)
+ if err != nil {
+ select {
+ case errChan <- err:
+ case <-workerCtx.Done():
+ return
+ }
+ }
+ case <-workerCtx.Done():
+ return
+ }
+ }
+ }()
+ }
+
+ // work sender
+ go func() {
+ defer close(errChan)
+ defer wg.Wait()
+ for _, batch := range batches {
+ select {
+ case batchChan <- batch:
+ case <-workerCtx.Done():
+ return
+ }
+ }
+ cancel()
+ }()
+
+ // receive any errors
+ select {
+ case err, ok := <-errChan:
+ if !ok {
+ // we finished without any errors, congratulations
+ return nil
+ }
+ // short circuit on the first error we get
+ // canceling the worker ctx and thus all workers,
+ return err
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
diff --git a/routing/http/internal/goroutines_test.go b/routing/http/internal/goroutines_test.go
new file mode 100644
index 000000000..a2ee48730
--- /dev/null
+++ b/routing/http/internal/goroutines_test.go
@@ -0,0 +1,107 @@
+package internal
+
+import (
+ "context"
+ "errors"
+ "reflect"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func sequence(n int) (items []int) {
+ for i := 0; i < n; i++ {
+ items = append(items, i+1)
+ }
+ return
+}
+
+func singleItemBatches(items []int) (batches [][]int) {
+ for _, item := range items {
+ batches = append(batches, []int{item})
+ }
+ return
+}
+
+func TestDoBatch(t *testing.T) {
+ cases := []struct {
+ name string
+ items []int
+ maxBatchSize int
+ maxConcurrency int
+ shouldErrOnce bool
+
+ expBatches [][]int
+ expErrContains string
+ }{
+ {
+ name: "no items",
+ },
+ {
+ name: "batch size = 1",
+ items: sequence(3),
+ maxBatchSize: 1,
+ maxConcurrency: 1,
+ expBatches: [][]int{{1}, {2}, {3}},
+ },
+ {
+ name: "batch size > 1",
+ items: sequence(6),
+ maxBatchSize: 2,
+ maxConcurrency: 2,
+ expBatches: [][]int{{1, 2}, {3, 4}, {5, 6}},
+ },
+ {
+ name: "a lot of items and concurrency",
+ items: sequence(1000),
+ maxBatchSize: 1,
+ maxConcurrency: 100,
+ expBatches: singleItemBatches(sequence(1000)),
+ },
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+ defer cancel()
+
+ var mut sync.Mutex
+ var batches [][]int
+
+ var onceMut sync.Mutex
+ var errored bool
+
+ DoBatch(ctx, c.maxBatchSize, c.maxConcurrency, c.items, func(ctx context.Context, batch []int) error {
+ if c.shouldErrOnce {
+ onceMut.Lock()
+ if !errored {
+ errored = true
+ defer onceMut.Unlock()
+ return errors.New("boom")
+ }
+ onceMut.Unlock()
+ }
+
+ mut.Lock()
+ batches = append(batches, batch)
+ mut.Unlock()
+ return nil
+ })
+
+ require.Equal(t, len(c.expBatches), len(batches), "expected equal len %v %v", c.expBatches, batches)
+ for _, expBatch := range c.expBatches {
+ requireContainsBatch(t, batches, expBatch)
+ }
+ })
+ }
+}
+
+func requireContainsBatch(t *testing.T, batches [][]int, batch []int) {
+ for _, b := range batches {
+ if reflect.DeepEqual(batch, b) {
+ return
+ }
+ }
+ t.Fatalf("expected batch %v, but not found in batches %v", batch, batches)
+}
diff --git a/routing/http/server/server.go b/routing/http/server/server.go
new file mode 100644
index 000000000..ca650ff28
--- /dev/null
+++ b/routing/http/server/server.go
@@ -0,0 +1,186 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "time"
+
+ "github.com/gorilla/mux"
+ "github.com/ipfs/boxo/routing/http/internal/drjson"
+ "github.com/ipfs/boxo/routing/http/types"
+ "github.com/ipfs/go-cid"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+
+ logging "github.com/ipfs/go-log/v2"
+)
+
+var logger = logging.Logger("service/server/delegatedrouting")
+
+const ProvidePath = "/routing/v1/providers/"
+const FindProvidersPath = "/routing/v1/providers/{cid}"
+
+type ContentRouter interface {
+ FindProviders(ctx context.Context, key cid.Cid) ([]types.ProviderResponse, error)
+ ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error)
+ Provide(ctx context.Context, req *WriteProvideRequest) (types.ProviderResponse, error)
+}
+
+type BitswapWriteProvideRequest struct {
+ Keys []cid.Cid
+ Timestamp time.Time
+ AdvisoryTTL time.Duration
+ ID peer.ID
+ Addrs []multiaddr.Multiaddr
+}
+
+type WriteProvideRequest struct {
+ Protocol string
+ Schema string
+ Bytes []byte
+}
+
+type serverOption func(s *server)
+
+func Handler(svc ContentRouter, opts ...serverOption) http.Handler {
+ server := &server{
+ svc: svc,
+ }
+
+ for _, opt := range opts {
+ opt(server)
+ }
+
+ r := mux.NewRouter()
+ r.HandleFunc(ProvidePath, server.provide).Methods(http.MethodPut)
+ r.HandleFunc(FindProvidersPath, server.findProviders).Methods(http.MethodGet)
+
+ return r
+}
+
+type server struct {
+ svc ContentRouter
+}
+
+func (s *server) provide(w http.ResponseWriter, httpReq *http.Request) {
+ req := types.WriteProvidersRequest{}
+ err := json.NewDecoder(httpReq.Body).Decode(&req)
+ _ = httpReq.Body.Close()
+ if err != nil {
+ writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("invalid request: %w", err))
+ return
+ }
+
+ resp := types.WriteProvidersResponse{}
+
+ for i, prov := range req.Providers {
+ switch v := prov.(type) {
+ case *types.WriteBitswapProviderRecord:
+ err := v.Verify()
+ if err != nil {
+ logErr("Provide", "signature verification failed", err)
+ writeErr(w, "Provide", http.StatusForbidden, errors.New("signature verification failed"))
+ return
+ }
+
+ keys := make([]cid.Cid, len(v.Payload.Keys))
+ for i, k := range v.Payload.Keys {
+ keys[i] = k.Cid
+
+ }
+ addrs := make([]multiaddr.Multiaddr, len(v.Payload.Addrs))
+ for i, a := range v.Payload.Addrs {
+ addrs[i] = a.Multiaddr
+ }
+ advisoryTTL, err := s.svc.ProvideBitswap(httpReq.Context(), &BitswapWriteProvideRequest{
+ Keys: keys,
+ Timestamp: v.Payload.Timestamp.Time,
+ AdvisoryTTL: v.Payload.AdvisoryTTL.Duration,
+ ID: *v.Payload.ID,
+ Addrs: addrs,
+ })
+ if err != nil {
+ writeErr(w, "Provide", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err))
+ return
+ }
+ resp.ProvideResults = append(resp.ProvideResults,
+ &types.WriteBitswapProviderRecordResponse{
+ Protocol: v.Protocol,
+ Schema: v.Schema,
+ AdvisoryTTL: &types.Duration{Duration: advisoryTTL},
+ },
+ )
+ case *types.UnknownProviderRecord:
+ provResp, err := s.svc.Provide(httpReq.Context(), &WriteProvideRequest{
+ Protocol: v.Protocol,
+ Schema: v.Schema,
+ Bytes: v.Bytes,
+ })
+ if err != nil {
+ writeErr(w, "Provide", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err))
+ return
+ }
+ resp.ProvideResults = append(resp.ProvideResults, provResp)
+ default:
+ writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("provider record %d does not contain a protocol", i))
+ return
+ }
+ }
+ writeResult(w, "Provide", resp)
+}
+
+func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) {
+ vars := mux.Vars(httpReq)
+ cidStr := vars["cid"]
+ cid, err := cid.Decode(cidStr)
+ if err != nil {
+ writeErr(w, "FindProviders", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err))
+ return
+ }
+ providers, err := s.svc.FindProviders(httpReq.Context(), cid)
+ if err != nil {
+ writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err))
+ return
+ }
+ response := types.ReadProvidersResponse{Providers: providers}
+ writeResult(w, "FindProviders", response)
+}
+
+func writeResult(w http.ResponseWriter, method string, val any) {
+ w.Header().Add("Content-Type", "application/json")
+
+ // keep the marshaling separate from the writing, so we can distinguish bugs (which surface as 500)
+ // from transient network issues (which surface as transport errors)
+ b, err := drjson.MarshalJSONBytes(val)
+ if err != nil {
+ writeErr(w, method, http.StatusInternalServerError, fmt.Errorf("marshaling response: %w", err))
+ return
+ }
+
+ _, err = io.Copy(w, bytes.NewBuffer(b))
+ if err != nil {
+ logErr("Provide", "writing response body", err)
+ }
+}
+
+func writeErr(w http.ResponseWriter, method string, statusCode int, cause error) {
+ w.WriteHeader(statusCode)
+ causeStr := cause.Error()
+ if len(causeStr) > 1024 {
+ causeStr = causeStr[:1024]
+ }
+ _, err := w.Write([]byte(causeStr))
+ if err != nil {
+ logErr(method, "error writing error cause", err)
+ return
+ }
+}
+
+func logErr(method, msg string, err error) {
+ logger.Infow(msg, "Method", method, "Error", err)
+}
diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go
new file mode 100644
index 000000000..5609b2ed3
--- /dev/null
+++ b/routing/http/server/server_test.go
@@ -0,0 +1,63 @@
+package server
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/ipfs/boxo/routing/http/types"
+ "github.com/ipfs/go-cid"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+)
+
+func TestHeaders(t *testing.T) {
+ router := &mockContentRouter{}
+ server := httptest.NewServer(Handler(router))
+ t.Cleanup(server.Close)
+ serverAddr := "http://" + server.Listener.Addr().String()
+
+ result := []types.ProviderResponse{
+ &types.ReadBitswapProviderRecord{
+ Protocol: "transport-bitswap",
+ Schema: types.SchemaBitswap,
+ },
+ }
+
+ c := "baeabep4vu3ceru7nerjjbk37sxb7wmftteve4hcosmyolsbsiubw2vr6pqzj6mw7kv6tbn6nqkkldnklbjgm5tzbi4hkpkled4xlcr7xz4bq"
+ cb, err := cid.Decode(c)
+ require.NoError(t, err)
+
+ router.On("FindProviders", mock.Anything, cb).
+ Return(result, nil)
+
+ resp, err := http.Get(serverAddr + ProvidePath + c)
+ require.NoError(t, err)
+ require.Equal(t, 200, resp.StatusCode)
+ header := resp.Header.Get("Content-Type")
+ require.Equal(t, "application/json", header)
+
+ resp, err = http.Get(serverAddr + ProvidePath + "BAD_CID")
+ require.NoError(t, err)
+ require.Equal(t, 400, resp.StatusCode)
+ header = resp.Header.Get("Content-Type")
+ require.Equal(t, "text/plain; charset=utf-8", header)
+}
+
+type mockContentRouter struct{ mock.Mock }
+
+func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid) ([]types.ProviderResponse, error) {
+ args := m.Called(ctx, key)
+ return args.Get(0).([]types.ProviderResponse), args.Error(1)
+}
+func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) {
+ args := m.Called(ctx, req)
+ return args.Get(0).(time.Duration), args.Error(1)
+}
+
+func (m *mockContentRouter) Provide(ctx context.Context, req *WriteProvideRequest) (types.ProviderResponse, error) {
+ args := m.Called(ctx, req)
+ return args.Get(0).(types.ProviderResponse), args.Error(1)
+}
diff --git a/routing/http/types/ipfs.go b/routing/http/types/ipfs.go
new file mode 100644
index 000000000..1e139fa22
--- /dev/null
+++ b/routing/http/types/ipfs.go
@@ -0,0 +1,42 @@
+package types
+
+import (
+ "encoding/json"
+
+ "github.com/ipfs/boxo/routing/http/internal/drjson"
+ "github.com/ipfs/go-cid"
+ "github.com/multiformats/go-multiaddr"
+)
+
+type CID struct{ cid.Cid }
+
+func (c *CID) MarshalJSON() ([]byte, error) { return drjson.MarshalJSONBytes(c.String()) }
+func (c *CID) UnmarshalJSON(b []byte) error {
+ var s string
+ err := json.Unmarshal(b, &s)
+ if err != nil {
+ return err
+ }
+ decodedCID, err := cid.Decode(s)
+ if err != nil {
+ return err
+ }
+ c.Cid = decodedCID
+ return nil
+}
+
+type Multiaddr struct{ multiaddr.Multiaddr }
+
+func (m *Multiaddr) UnmarshalJSON(b []byte) error {
+ var s string
+ err := json.Unmarshal(b, &s)
+ if err != nil {
+ return err
+ }
+ ma, err := multiaddr.NewMultiaddr(s)
+ if err != nil {
+ return err
+ }
+ m.Multiaddr = ma
+ return nil
+}
diff --git a/routing/http/types/provider.go b/routing/http/types/provider.go
new file mode 100644
index 000000000..ef9e95ada
--- /dev/null
+++ b/routing/http/types/provider.go
@@ -0,0 +1,142 @@
+package types
+
+import (
+ "encoding/json"
+)
+
+// WriteProviderRecord is a type that enforces structs to imlement it to avoid confusion
+type WriteProviderRecord interface {
+ IsWriteProviderRecord()
+}
+
+// ReadProviderRecord is a type that enforces structs to imlement it to avoid confusion
+type ReadProviderRecord interface {
+ IsReadProviderRecord()
+}
+
+type WriteProvidersRequest struct {
+ Providers []WriteProviderRecord
+}
+
+func (r *WriteProvidersRequest) UnmarshalJSON(b []byte) error {
+ type wpr struct {
+ Providers []json.RawMessage
+ }
+ var tempWPR wpr
+ err := json.Unmarshal(b, &tempWPR)
+ if err != nil {
+ return err
+ }
+
+ for _, provBytes := range tempWPR.Providers {
+ var rawProv UnknownProviderRecord
+ err := json.Unmarshal(provBytes, &rawProv)
+ if err != nil {
+ return err
+ }
+
+ switch rawProv.Schema {
+ case SchemaBitswap:
+ var prov WriteBitswapProviderRecord
+ err := json.Unmarshal(rawProv.Bytes, &prov)
+ if err != nil {
+ return err
+ }
+ r.Providers = append(r.Providers, &prov)
+ default:
+ var prov UnknownProviderRecord
+ err := json.Unmarshal(b, &prov)
+ if err != nil {
+ return err
+ }
+ r.Providers = append(r.Providers, &prov)
+ }
+ }
+ return nil
+}
+
+// ProviderResponse is implemented for any ProviderResponse. It needs to have a Protocol field.
+type ProviderResponse interface {
+ GetProtocol() string
+ GetSchema() string
+}
+
+// WriteProvidersResponse is the result of a Provide operation
+type WriteProvidersResponse struct {
+ ProvideResults []ProviderResponse
+}
+
+// rawWriteProvidersResponse is a helper struct to make possible to parse WriteProvidersResponse's
+type rawWriteProvidersResponse struct {
+ ProvideResults []json.RawMessage
+}
+
+func (r *WriteProvidersResponse) UnmarshalJSON(b []byte) error {
+ var tempWPR rawWriteProvidersResponse
+ err := json.Unmarshal(b, &tempWPR)
+ if err != nil {
+ return err
+ }
+
+ for _, provBytes := range tempWPR.ProvideResults {
+ var rawProv UnknownProviderRecord
+ err := json.Unmarshal(provBytes, &rawProv)
+ if err != nil {
+ return err
+ }
+
+ switch rawProv.Schema {
+ case SchemaBitswap:
+ var prov WriteBitswapProviderRecordResponse
+ err := json.Unmarshal(rawProv.Bytes, &prov)
+ if err != nil {
+ return err
+ }
+ r.ProvideResults = append(r.ProvideResults, &prov)
+ default:
+ r.ProvideResults = append(r.ProvideResults, &rawProv)
+ }
+ }
+
+ return nil
+}
+
+// ReadProvidersResponse is the result of a Provide request
+type ReadProvidersResponse struct {
+ Providers []ProviderResponse
+}
+
+// rawReadProvidersResponse is a helper struct to make possible to parse ReadProvidersResponse's
+type rawReadProvidersResponse struct {
+ Providers []json.RawMessage
+}
+
+func (r *ReadProvidersResponse) UnmarshalJSON(b []byte) error {
+ var tempFPR rawReadProvidersResponse
+ err := json.Unmarshal(b, &tempFPR)
+ if err != nil {
+ return err
+ }
+
+ for _, provBytes := range tempFPR.Providers {
+ var readProv UnknownProviderRecord
+ err := json.Unmarshal(provBytes, &readProv)
+ if err != nil {
+ return err
+ }
+
+ switch readProv.Schema {
+ case SchemaBitswap:
+ var prov ReadBitswapProviderRecord
+ err := json.Unmarshal(readProv.Bytes, &prov)
+ if err != nil {
+ return err
+ }
+ r.Providers = append(r.Providers, &prov)
+ default:
+ r.Providers = append(r.Providers, &readProv)
+ }
+
+ }
+ return nil
+}
diff --git a/routing/http/types/provider_bitswap.go b/routing/http/types/provider_bitswap.go
new file mode 100644
index 000000000..66243dd5d
--- /dev/null
+++ b/routing/http/types/provider_bitswap.go
@@ -0,0 +1,184 @@
+package types
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/ipfs/boxo/routing/http/internal/drjson"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multibase"
+)
+
+const SchemaBitswap = "bitswap"
+
+var _ WriteProviderRecord = &WriteBitswapProviderRecord{}
+
+// WriteBitswapProviderRecord is used when we want to add a new provider record that is using bitswap.
+type WriteBitswapProviderRecord struct {
+ Protocol string
+ Schema string
+ Signature string
+
+ // this content must be untouched because it is signed and we need to verify it
+ RawPayload json.RawMessage `json:"Payload"`
+ Payload BitswapPayload `json:"-"`
+}
+
+type BitswapPayload struct {
+ Keys []CID
+ Timestamp *Time
+ AdvisoryTTL *Duration
+ ID *peer.ID
+ Addrs []Multiaddr
+}
+
+func (*WriteBitswapProviderRecord) IsWriteProviderRecord() {}
+
+type tmpBWPR WriteBitswapProviderRecord
+
+func (p *WriteBitswapProviderRecord) UnmarshalJSON(b []byte) error {
+ var bwp tmpBWPR
+ err := json.Unmarshal(b, &bwp)
+ if err != nil {
+ return err
+ }
+
+ p.Protocol = bwp.Protocol
+ p.Schema = bwp.Schema
+ p.Signature = bwp.Signature
+ p.RawPayload = bwp.RawPayload
+
+ return json.Unmarshal(bwp.RawPayload, &p.Payload)
+}
+
+func (p *WriteBitswapProviderRecord) IsSigned() bool {
+ return p.Signature != ""
+}
+
+func (p *WriteBitswapProviderRecord) setRawPayload() error {
+ payloadBytes, err := drjson.MarshalJSONBytes(p.Payload)
+ if err != nil {
+ return fmt.Errorf("marshaling bitswap write provider payload: %w", err)
+ }
+
+ p.RawPayload = payloadBytes
+
+ return nil
+}
+
+func (p *WriteBitswapProviderRecord) Sign(peerID peer.ID, key crypto.PrivKey) error {
+ if p.IsSigned() {
+ return errors.New("already signed")
+ }
+
+ if key == nil {
+ return errors.New("no key provided")
+ }
+
+ sid, err := peer.IDFromPrivateKey(key)
+ if err != nil {
+ return err
+ }
+ if sid != peerID {
+ return errors.New("not the correct signing key")
+ }
+
+ err = p.setRawPayload()
+ if err != nil {
+ return err
+ }
+ hash := sha256.Sum256([]byte(p.RawPayload))
+ sig, err := key.Sign(hash[:])
+ if err != nil {
+ return err
+ }
+
+ sigStr, err := multibase.Encode(multibase.Base64, sig)
+ if err != nil {
+ return fmt.Errorf("multibase-encoding signature: %w", err)
+ }
+
+ p.Signature = sigStr
+ return nil
+}
+
+func (p *WriteBitswapProviderRecord) Verify() error {
+ if !p.IsSigned() {
+ return errors.New("not signed")
+ }
+
+ if p.Payload.ID == nil {
+ return errors.New("peer ID must be specified")
+ }
+
+ // note that we only generate and set the payload if it hasn't already been set
+ // to allow for passing through the payload untouched if it is already provided
+ if p.RawPayload == nil {
+ err := p.setRawPayload()
+ if err != nil {
+ return err
+ }
+ }
+
+ pk, err := p.Payload.ID.ExtractPublicKey()
+ if err != nil {
+ return fmt.Errorf("extracing public key from peer ID: %w", err)
+ }
+
+ _, sigBytes, err := multibase.Decode(p.Signature)
+ if err != nil {
+ return fmt.Errorf("multibase-decoding signature to verify: %w", err)
+ }
+
+ hash := sha256.Sum256([]byte(p.RawPayload))
+ ok, err := pk.Verify(hash[:], sigBytes)
+ if err != nil {
+ return fmt.Errorf("verifying hash with signature: %w", err)
+ }
+ if !ok {
+ return errors.New("signature failed to verify")
+ }
+
+ return nil
+}
+
+var _ ProviderResponse = &WriteBitswapProviderRecordResponse{}
+
+// WriteBitswapProviderRecordResponse will be returned as a result of WriteBitswapProviderRecord
+type WriteBitswapProviderRecordResponse struct {
+ Protocol string
+ Schema string
+ AdvisoryTTL *Duration
+}
+
+func (wbprr *WriteBitswapProviderRecordResponse) GetProtocol() string {
+ return wbprr.Protocol
+}
+
+func (wbprr *WriteBitswapProviderRecordResponse) GetSchema() string {
+ return wbprr.Schema
+}
+
+var _ ReadProviderRecord = &ReadBitswapProviderRecord{}
+var _ ProviderResponse = &ReadBitswapProviderRecord{}
+
+// ReadBitswapProviderRecord is a provider result with parameters for bitswap providers
+type ReadBitswapProviderRecord struct {
+ Protocol string
+ Schema string
+ ID *peer.ID
+ Addrs []Multiaddr
+}
+
+func (rbpr *ReadBitswapProviderRecord) GetProtocol() string {
+ return rbpr.Protocol
+}
+
+func (rbpr *ReadBitswapProviderRecord) GetSchema() string {
+ return rbpr.Schema
+}
+
+func (*ReadBitswapProviderRecord) IsReadProviderRecord() {}
diff --git a/routing/http/types/provider_unknown.go b/routing/http/types/provider_unknown.go
new file mode 100644
index 000000000..3dadc0e9b
--- /dev/null
+++ b/routing/http/types/provider_unknown.go
@@ -0,0 +1,61 @@
+package types
+
+import (
+ "encoding/json"
+
+ "github.com/ipfs/boxo/routing/http/internal/drjson"
+)
+
+var _ ReadProviderRecord = &UnknownProviderRecord{}
+var _ WriteProviderRecord = &UnknownProviderRecord{}
+var _ ProviderResponse = &UnknownProviderRecord{}
+
+// UnknownProviderRecord is used when we cannot parse the provider record using `GetProtocol`
+type UnknownProviderRecord struct {
+ Protocol string
+ Schema string
+ Bytes []byte
+}
+
+func (u *UnknownProviderRecord) GetProtocol() string {
+ return u.Protocol
+}
+
+func (u *UnknownProviderRecord) GetSchema() string {
+ return u.Schema
+}
+
+func (u *UnknownProviderRecord) IsReadProviderRecord() {}
+func (u UnknownProviderRecord) IsWriteProviderRecord() {}
+
+func (u *UnknownProviderRecord) UnmarshalJSON(b []byte) error {
+ m := map[string]interface{}{}
+ if err := json.Unmarshal(b, &m); err != nil {
+ return err
+ }
+
+ ps, ok := m["Protocol"].(string)
+ if ok {
+ u.Protocol = ps
+ }
+ schema, ok := m["Schema"].(string)
+ if ok {
+ u.Schema = schema
+ }
+
+ u.Bytes = b
+
+ return nil
+}
+
+func (u UnknownProviderRecord) MarshalJSON() ([]byte, error) {
+ m := map[string]interface{}{}
+ err := json.Unmarshal(u.Bytes, &m)
+ if err != nil {
+ return nil, err
+ }
+ m["Protocol"] = u.Protocol
+ m["Schema"] = u.Schema
+
+ return drjson.MarshalJSONBytes(m)
+}
diff --git a/routing/http/types/time.go b/routing/http/types/time.go
new file mode 100644
index 000000000..1d938807e
--- /dev/null
+++ b/routing/http/types/time.go
@@ -0,0 +1,36 @@
+package types
+
+import (
+ "encoding/json"
+ "time"
+
+ "github.com/ipfs/boxo/routing/http/internal/drjson"
+)
+
+type Time struct{ time.Time }
+
+func (t *Time) MarshalJSON() ([]byte, error) {
+ return drjson.MarshalJSONBytes(t.Time.UnixMilli())
+}
+func (t *Time) UnmarshalJSON(b []byte) error {
+ var timestamp int64
+ err := json.Unmarshal(b, ×tamp)
+ if err != nil {
+ return err
+ }
+ t.Time = time.UnixMilli(timestamp)
+ return nil
+}
+
+type Duration struct{ time.Duration }
+
+func (d *Duration) MarshalJSON() ([]byte, error) { return drjson.MarshalJSONBytes(d.Duration) }
+func (d *Duration) UnmarshalJSON(b []byte) error {
+ var dur int64
+ err := json.Unmarshal(b, &dur)
+ if err != nil {
+ return err
+ }
+ d.Duration = time.Duration(dur)
+ return nil
+}
diff --git a/routing/mock/centralized_client.go b/routing/mock/centralized_client.go
new file mode 100644
index 000000000..ac3f938cc
--- /dev/null
+++ b/routing/mock/centralized_client.go
@@ -0,0 +1,88 @@
+package mockrouting
+
+import (
+ "context"
+ "time"
+
+ "github.com/ipfs/go-cid"
+ logging "github.com/ipfs/go-log"
+ tnet "github.com/libp2p/go-libp2p-testing/net"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+ ma "github.com/multiformats/go-multiaddr"
+)
+
+var log = logging.Logger("mockrouter")
+
+type client struct {
+ vs routing.ValueStore
+ server server
+ peer tnet.Identity
+}
+
+// PutValue FIXME(brian): is this method meant to simulate putting a value into the network?
+func (c *client) PutValue(ctx context.Context, key string, val []byte, opts ...routing.Option) error {
+ log.Debugf("PutValue: %s", key)
+ return c.vs.PutValue(ctx, key, val, opts...)
+}
+
+// GetValue FIXME(brian): is this method meant to simulate getting a value from the network?
+func (c *client) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) {
+ log.Debugf("GetValue: %s", key)
+ return c.vs.GetValue(ctx, key, opts...)
+}
+
+func (c *client) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) {
+ log.Debugf("SearchValue: %s", key)
+ return c.vs.SearchValue(ctx, key, opts...)
+}
+
+func (c *client) FindProviders(ctx context.Context, key cid.Cid) ([]peer.AddrInfo, error) {
+ return c.server.Providers(key), nil
+}
+
+func (c *client) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) {
+ log.Debugf("FindPeer: %s", pid)
+ return peer.AddrInfo{}, nil
+}
+
+func (c *client) FindProvidersAsync(ctx context.Context, k cid.Cid, max int) <-chan peer.AddrInfo {
+ out := make(chan peer.AddrInfo)
+ go func() {
+ defer close(out)
+ for i, p := range c.server.Providers(k) {
+ if max <= i {
+ return
+ }
+ select {
+ case out <- p:
+ case <-ctx.Done():
+ return
+ }
+ }
+ }()
+ return out
+}
+
+// Provide returns once the message is on the network. Value is not necessarily
+// visible yet.
+func (c *client) Provide(_ context.Context, key cid.Cid, brd bool) error {
+ if !brd {
+ return nil
+ }
+ info := peer.AddrInfo{
+ ID: c.peer.ID(),
+ Addrs: []ma.Multiaddr{c.peer.Address()},
+ }
+ return c.server.Announce(info, key)
+}
+
+func (c *client) Ping(ctx context.Context, p peer.ID) (time.Duration, error) {
+ return 0, nil
+}
+
+func (c *client) Bootstrap(context.Context) error {
+ return nil
+}
+
+var _ routing.Routing = &client{}
diff --git a/routing/mock/centralized_server.go b/routing/mock/centralized_server.go
new file mode 100644
index 000000000..d55de7081
--- /dev/null
+++ b/routing/mock/centralized_server.go
@@ -0,0 +1,91 @@
+package mockrouting
+
+import (
+ "context"
+ "math/rand"
+ "sync"
+ "time"
+
+ "github.com/ipfs/boxo/routing/offline"
+ "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ tnet "github.com/libp2p/go-libp2p-testing/net"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+// server is the mockrouting.Client's private interface to the routing server
+type server interface {
+ Announce(peer.AddrInfo, cid.Cid) error
+ Providers(cid.Cid) []peer.AddrInfo
+
+ Server
+}
+
+// s is an implementation of the private server interface
+type s struct {
+ delayConf DelayConfig
+
+ lock sync.RWMutex
+ providers map[string]map[peer.ID]providerRecord
+}
+
+type providerRecord struct {
+ Peer peer.AddrInfo
+ Created time.Time
+}
+
+func (rs *s) Announce(p peer.AddrInfo, c cid.Cid) error {
+ rs.lock.Lock()
+ defer rs.lock.Unlock()
+
+ k := c.KeyString()
+
+ _, ok := rs.providers[k]
+ if !ok {
+ rs.providers[k] = make(map[peer.ID]providerRecord)
+ }
+ rs.providers[k][p.ID] = providerRecord{
+ Created: time.Now(),
+ Peer: p,
+ }
+ return nil
+}
+
+func (rs *s) Providers(c cid.Cid) []peer.AddrInfo {
+ rs.delayConf.Query.Wait() // before locking
+
+ rs.lock.RLock()
+ defer rs.lock.RUnlock()
+ k := c.KeyString()
+
+ var ret []peer.AddrInfo
+ records, ok := rs.providers[k]
+ if !ok {
+ return ret
+ }
+ for _, r := range records {
+ if time.Since(r.Created) > rs.delayConf.ValueVisibility.Get() {
+ ret = append(ret, r.Peer)
+ }
+ }
+
+ for i := range ret {
+ j := rand.Intn(i + 1)
+ ret[i], ret[j] = ret[j], ret[i]
+ }
+
+ return ret
+}
+
+func (rs *s) Client(p tnet.Identity) Client {
+ return rs.ClientWithDatastore(context.Background(), p, dssync.MutexWrap(ds.NewMapDatastore()))
+}
+
+func (rs *s) ClientWithDatastore(_ context.Context, p tnet.Identity, datastore ds.Datastore) Client {
+ return &client{
+ peer: p,
+ vs: offline.NewOfflineRouter(datastore, MockValidator{}),
+ server: rs,
+ }
+}
diff --git a/routing/mock/centralized_test.go b/routing/mock/centralized_test.go
new file mode 100644
index 000000000..403e21350
--- /dev/null
+++ b/routing/mock/centralized_test.go
@@ -0,0 +1,176 @@
+package mockrouting
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ u "github.com/ipfs/boxo/util"
+ "github.com/ipfs/go-cid"
+ delay "github.com/ipfs/go-ipfs-delay"
+ tnet "github.com/libp2p/go-libp2p-testing/net"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+func TestKeyNotFound(t *testing.T) {
+
+ var pi = tnet.RandIdentityOrFatal(t)
+ var key = cid.NewCidV0(u.Hash([]byte("mock key")))
+ var ctx = context.Background()
+
+ rs := NewServer()
+ providers := rs.Client(pi).FindProvidersAsync(ctx, key, 10)
+ _, ok := <-providers
+ if ok {
+ t.Fatal("should be closed")
+ }
+}
+
+func TestClientFindProviders(t *testing.T) {
+ pi := tnet.RandIdentityOrFatal(t)
+ rs := NewServer()
+ client := rs.Client(pi)
+
+ k := cid.NewCidV0(u.Hash([]byte("hello")))
+ err := client.Provide(context.Background(), k, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // This is bad... but simulating networks is hard
+ time.Sleep(time.Millisecond * 300)
+ max := 100
+
+ providersFromClient := client.FindProvidersAsync(context.Background(), k, max)
+ isInClient := false
+ for p := range providersFromClient {
+ if p.ID == pi.ID() {
+ isInClient = true
+ }
+ }
+ if !isInClient {
+ t.Fatal("Despite client providing key, client didn't receive peer when finding providers")
+ }
+}
+
+func TestClientOverMax(t *testing.T) {
+ rs := NewServer()
+ k := cid.NewCidV0(u.Hash([]byte("hello")))
+ numProvidersForHelloKey := 100
+ for i := 0; i < numProvidersForHelloKey; i++ {
+ pi := tnet.RandIdentityOrFatal(t)
+ err := rs.Client(pi).Provide(context.Background(), k, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ max := 10
+ pi := tnet.RandIdentityOrFatal(t)
+ client := rs.Client(pi)
+
+ providersFromClient := client.FindProvidersAsync(context.Background(), k, max)
+ i := 0
+ for range providersFromClient {
+ i++
+ }
+ if i != max {
+ t.Fatal("Too many providers returned")
+ }
+}
+
+// TODO does dht ensure won't receive self as a provider? probably not.
+func TestCanceledContext(t *testing.T) {
+ rs := NewServer()
+ k := cid.NewCidV0(u.Hash([]byte("hello")))
+
+ // avoid leaking goroutine, without using the context to signal
+ // (we want the goroutine to keep trying to publish on a
+ // cancelled context until we've tested it doesnt do anything.)
+ done := make(chan struct{})
+ defer func() { done <- struct{}{} }()
+
+ t.Log("async'ly announce infinite stream of providers for key")
+ i := 0
+ go func() { // infinite stream
+ for {
+ select {
+ case <-done:
+ t.Log("exiting async worker")
+ return
+ default:
+ }
+
+ pi, err := tnet.RandIdentity()
+ if err != nil {
+ t.Error(err)
+ }
+ err = rs.Client(pi).Provide(context.Background(), k, true)
+ if err != nil {
+ t.Error(err)
+ }
+ i++
+ }
+ }()
+
+ local := tnet.RandIdentityOrFatal(t)
+ client := rs.Client(local)
+
+ t.Log("warning: max is finite so this test is non-deterministic")
+ t.Log("context cancellation could simply take lower priority")
+ t.Log("and result in receiving the max number of results")
+ max := 1000
+
+ t.Log("cancel the context before consuming")
+ ctx, cancelFunc := context.WithCancel(context.Background())
+ cancelFunc()
+ providers := client.FindProvidersAsync(ctx, k, max)
+
+ numProvidersReturned := 0
+ for range providers {
+ numProvidersReturned++
+ }
+ t.Log(numProvidersReturned)
+
+ if numProvidersReturned == max {
+ t.Fatal("Context cancel had no effect")
+ }
+}
+
+func TestValidAfter(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ pi := tnet.RandIdentityOrFatal(t)
+ key := cid.NewCidV0(u.Hash([]byte("mock key")))
+ conf := DelayConfig{
+ ValueVisibility: delay.Fixed(1 * time.Hour),
+ Query: delay.Fixed(0),
+ }
+
+ rs := NewServerWithDelay(conf)
+
+ rs.Client(pi).Provide(ctx, key, true)
+
+ var providers []peer.AddrInfo
+ max := 100
+ providersChan := rs.Client(pi).FindProvidersAsync(ctx, key, max)
+ for p := range providersChan {
+ providers = append(providers, p)
+ }
+ if len(providers) > 0 {
+ t.Fail()
+ }
+
+ conf.ValueVisibility.Set(0)
+ time.Sleep(100 * time.Millisecond)
+
+ providersChan = rs.Client(pi).FindProvidersAsync(ctx, key, max)
+ t.Log("providers", providers)
+ for p := range providersChan {
+ providers = append(providers, p)
+ }
+ if len(providers) != 1 {
+ t.Fail()
+ }
+}
diff --git a/routing/mock/interface.go b/routing/mock/interface.go
new file mode 100644
index 000000000..35430a72c
--- /dev/null
+++ b/routing/mock/interface.go
@@ -0,0 +1,59 @@
+// Package mockrouting provides a virtual routing server. To use it,
+// create a virtual routing server and use the Client() method to get a
+// routing client (Routing). The server quacks like a DHT but is
+// really a local in-memory hash table.
+package mockrouting
+
+import (
+ "context"
+
+ ds "github.com/ipfs/go-datastore"
+ delay "github.com/ipfs/go-ipfs-delay"
+ tnet "github.com/libp2p/go-libp2p-testing/net"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+)
+
+// MockValidator is a record validator that always returns success.
+type MockValidator struct{}
+
+func (MockValidator) Validate(_ string, _ []byte) error { return nil }
+func (MockValidator) Select(_ string, _ [][]byte) (int, error) { return 0, nil }
+
+// Server provides mockrouting Clients
+type Server interface {
+ Client(p tnet.Identity) Client
+ ClientWithDatastore(context.Context, tnet.Identity, ds.Datastore) Client
+}
+
+// Client implements Routing
+type Client interface {
+ routing.Routing
+}
+
+// NewServer returns a mockrouting Server
+func NewServer() Server {
+ return NewServerWithDelay(DelayConfig{
+ ValueVisibility: delay.Fixed(0),
+ Query: delay.Fixed(0),
+ })
+}
+
+// NewServerWithDelay returns a mockrouting Server with a delay!
+func NewServerWithDelay(conf DelayConfig) Server {
+ return &s{
+ providers: make(map[string]map[peer.ID]providerRecord),
+ delayConf: conf,
+ }
+}
+
+// DelayConfig can be used to configured the fake delays of a mock server.
+// Use with NewServerWithDelay().
+type DelayConfig struct {
+ // ValueVisibility is the time it takes for a value to be visible in the network
+ // FIXME there _must_ be a better term for this
+ ValueVisibility delay.D
+
+ // Query is the time it takes to receive a response from a routing query
+ Query delay.D
+}
diff --git a/routing/none/none_client.go b/routing/none/none_client.go
new file mode 100644
index 000000000..6f400b54a
--- /dev/null
+++ b/routing/none/none_client.go
@@ -0,0 +1,55 @@
+// Package nilrouting implements a routing client that does nothing.
+package nilrouting
+
+import (
+ "context"
+ "errors"
+
+ "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ record "github.com/libp2p/go-libp2p-record"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+)
+
+type nilclient struct {
+}
+
+func (c *nilclient) PutValue(_ context.Context, _ string, _ []byte, _ ...routing.Option) error {
+ return nil
+}
+
+func (c *nilclient) GetValue(_ context.Context, _ string, _ ...routing.Option) ([]byte, error) {
+ return nil, errors.New("tried GetValue from nil routing")
+}
+
+func (c *nilclient) SearchValue(_ context.Context, _ string, _ ...routing.Option) (<-chan []byte, error) {
+ return nil, errors.New("tried SearchValue from nil routing")
+}
+
+func (c *nilclient) FindPeer(_ context.Context, _ peer.ID) (peer.AddrInfo, error) {
+ return peer.AddrInfo{}, nil
+}
+
+func (c *nilclient) FindProvidersAsync(_ context.Context, _ cid.Cid, _ int) <-chan peer.AddrInfo {
+ out := make(chan peer.AddrInfo)
+ defer close(out)
+ return out
+}
+
+func (c *nilclient) Provide(_ context.Context, _ cid.Cid, _ bool) error {
+ return nil
+}
+
+func (c *nilclient) Bootstrap(_ context.Context) error {
+ return nil
+}
+
+// ConstructNilRouting creates an Routing client which does nothing.
+func ConstructNilRouting(_ context.Context, _ host.Host, _ ds.Batching, _ record.Validator) (routing.Routing, error) {
+ return &nilclient{}, nil
+}
+
+// ensure nilclient satisfies interface
+var _ routing.Routing = &nilclient{}
diff --git a/routing/offline/offline.go b/routing/offline/offline.go
new file mode 100644
index 000000000..b9d5281ff
--- /dev/null
+++ b/routing/offline/offline.go
@@ -0,0 +1,126 @@
+// Package offline implements Routing with a client which
+// is only able to perform offline operations.
+package offline
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "time"
+
+ "github.com/gogo/protobuf/proto"
+ dshelp "github.com/ipfs/boxo/datastore/dshelp"
+ "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ record "github.com/libp2p/go-libp2p-record"
+ pb "github.com/libp2p/go-libp2p-record/pb"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+)
+
+// ErrOffline is returned when trying to perform operations that
+// require connectivity.
+var ErrOffline = errors.New("routing system in offline mode")
+
+// NewOfflineRouter returns an Routing implementation which only performs
+// offline operations. It allows to Put and Get signed dht
+// records to and from the local datastore.
+func NewOfflineRouter(dstore ds.Datastore, validator record.Validator) routing.Routing {
+ return &offlineRouting{
+ datastore: dstore,
+ validator: validator,
+ }
+}
+
+// offlineRouting implements the Routing interface,
+// but only provides the capability to Put and Get signed dht
+// records to and from the local datastore.
+type offlineRouting struct {
+ datastore ds.Datastore
+ validator record.Validator
+}
+
+func (c *offlineRouting) PutValue(ctx context.Context, key string, val []byte, _ ...routing.Option) error {
+ if err := c.validator.Validate(key, val); err != nil {
+ return err
+ }
+ if old, err := c.GetValue(ctx, key); err == nil {
+ // be idempotent to be nice.
+ if bytes.Equal(old, val) {
+ return nil
+ }
+ // check to see if the older record is better
+ i, err := c.validator.Select(key, [][]byte{val, old})
+ if err != nil {
+ // this shouldn't happen for validated records.
+ return err
+ }
+ if i != 0 {
+ return errors.New("can't replace a newer record with an older one")
+ }
+ }
+ rec := record.MakePutRecord(key, val)
+ data, err := proto.Marshal(rec)
+ if err != nil {
+ return err
+ }
+
+ return c.datastore.Put(ctx, dshelp.NewKeyFromBinary([]byte(key)), data)
+}
+
+func (c *offlineRouting) GetValue(ctx context.Context, key string, _ ...routing.Option) ([]byte, error) {
+ buf, err := c.datastore.Get(ctx, dshelp.NewKeyFromBinary([]byte(key)))
+ if err != nil {
+ return nil, err
+ }
+
+ rec := new(pb.Record)
+ err = proto.Unmarshal(buf, rec)
+ if err != nil {
+ return nil, err
+ }
+ val := rec.GetValue()
+
+ err = c.validator.Validate(key, val)
+ if err != nil {
+ return nil, err
+ }
+ return val, nil
+}
+
+func (c *offlineRouting) SearchValue(ctx context.Context, key string, _ ...routing.Option) (<-chan []byte, error) {
+ out := make(chan []byte, 1)
+ go func() {
+ defer close(out)
+ v, err := c.GetValue(ctx, key)
+ if err == nil {
+ out <- v
+ }
+ }()
+ return out, nil
+}
+
+func (c *offlineRouting) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) {
+ return peer.AddrInfo{}, ErrOffline
+}
+
+func (c *offlineRouting) FindProvidersAsync(ctx context.Context, k cid.Cid, max int) <-chan peer.AddrInfo {
+ out := make(chan peer.AddrInfo)
+ close(out)
+ return out
+}
+
+func (c *offlineRouting) Provide(_ context.Context, k cid.Cid, _ bool) error {
+ return ErrOffline
+}
+
+func (c *offlineRouting) Ping(ctx context.Context, p peer.ID) (time.Duration, error) {
+ return 0, ErrOffline
+}
+
+func (c *offlineRouting) Bootstrap(context.Context) error {
+ return nil
+}
+
+// ensure offlineRouting matches the Routing interface
+var _ routing.Routing = &offlineRouting{}
diff --git a/routing/offline/offline_test.go b/routing/offline/offline_test.go
new file mode 100644
index 000000000..9a17e8689
--- /dev/null
+++ b/routing/offline/offline_test.go
@@ -0,0 +1,91 @@
+package offline
+
+import (
+ "bytes"
+ "context"
+ "testing"
+
+ cid "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/libp2p/go-libp2p/core/test"
+
+ mh "github.com/multiformats/go-multihash"
+)
+
+type blankValidator struct{}
+
+func (blankValidator) Validate(_ string, _ []byte) error { return nil }
+func (blankValidator) Select(_ string, _ [][]byte) (int, error) { return 0, nil }
+
+func TestOfflineRouterStorage(t *testing.T) {
+ ctx := context.Background()
+
+ nds := ds.NewMapDatastore()
+ offline := NewOfflineRouter(nds, blankValidator{})
+
+ if err := offline.PutValue(ctx, "key", []byte("testing 1 2 3")); err != nil {
+ t.Fatal(err)
+ }
+
+ val, err := offline.GetValue(ctx, "key")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal([]byte("testing 1 2 3"), val) {
+ t.Fatal("OfflineRouter does not properly store")
+ }
+
+ _, err = offline.GetValue(ctx, "notHere")
+ if err == nil {
+ t.Fatal("Router should throw errors for unfound records")
+ }
+
+ local, err := offline.GetValue(ctx, "key", routing.Offline)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = offline.GetValue(ctx, "notHere", routing.Offline)
+ if err == nil {
+ t.Fatal("Router should throw errors for unfound records")
+ }
+
+ if !bytes.Equal([]byte("testing 1 2 3"), local) {
+ t.Fatal("OfflineRouter does not properly store")
+ }
+}
+
+func TestOfflineRouterLocal(t *testing.T) {
+ ctx := context.Background()
+
+ nds := ds.NewMapDatastore()
+ offline := NewOfflineRouter(nds, blankValidator{})
+
+ id, _ := test.RandPeerID()
+ _, err := offline.FindPeer(ctx, id)
+ if err != ErrOffline {
+ t.Fatal("OfflineRouting should alert that its offline")
+ }
+
+ h, _ := mh.Sum([]byte("test data1"), mh.SHA2_256, -1)
+ c1 := cid.NewCidV0(h)
+ pChan := offline.FindProvidersAsync(ctx, c1, 1)
+ p, ok := <-pChan
+ if ok {
+ t.Fatalf("FindProvidersAsync did not return a closed channel. Instead we got %+v !", p)
+ }
+
+ h2, _ := mh.Sum([]byte("test data1"), mh.SHA2_256, -1)
+ c2 := cid.NewCidV0(h2)
+ err = offline.Provide(ctx, c2, true)
+ if err != ErrOffline {
+ t.Fatal("OfflineRouting should alert that its offline")
+ }
+
+ err = offline.Bootstrap(ctx)
+ if err != nil {
+ t.Fatal("You shouldn't be able to bootstrap offline routing.")
+ }
+}
diff --git a/tar/extractor.go b/tar/extractor.go
new file mode 100644
index 000000000..b5377ddca
--- /dev/null
+++ b/tar/extractor.go
@@ -0,0 +1,327 @@
+package tar
+
+import (
+ "archive/tar"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ fp "path/filepath"
+ "strings"
+)
+
+var errTraverseSymlink = errors.New("cannot traverse symlinks")
+var errInvalidRoot = errors.New("tar has invalid root")
+var errInvalidRootMultipleRoots = fmt.Errorf("contains more than one root or the root directory is not the first entry : %w", errInvalidRoot)
+
+// Extractor is used for extracting tar files to a filesystem.
+//
+// The Extractor can only extract tar files containing files, directories and symlinks. Additionally, the tar files must
+// either have a single file, or symlink in them, or must have all of its objects inside of a single root directory
+// object.
+//
+// If the tar file contains a single file/symlink then it will try and extract it with semantics similar to Linux's
+// `cp`. In particular, the name of the extracted file/symlink will match the extraction path. If the extraction path
+// is a directory then it will extract into the directory using its original name.
+//
+// Overwriting: Extraction of files and symlinks will result in overwriting the existing objects with the same name
+// when possible (i.e. other files, symlinks, and empty directories).
+type Extractor struct {
+ Path string
+ Progress func(int64) int64
+}
+
+// Extract extracts a tar file to the file system. See the Extractor for more information on the limitations on the
+// tar files that can be extracted.
+func (te *Extractor) Extract(reader io.Reader) error {
+ if isNullDevice(te.Path) {
+ return nil
+ }
+
+ tarReader := tar.NewReader(reader)
+
+ var firstObjectWasDir bool
+
+ header, err := tarReader.Next()
+ if err != nil && err != io.EOF {
+ return err
+ }
+ if header == nil || err == io.EOF {
+ return fmt.Errorf("empty tar file")
+ }
+
+ // Specially handle the first entry assuming it is a single root object (e.g. root directory, single file,
+ // or single symlink)
+
+ // track what the root tar path is so we can ensure that all other entries are below the root
+ if strings.Contains(header.Name, "/") {
+ return fmt.Errorf("root name contains multiple components : %q : %w", header.Name, errInvalidRoot)
+ }
+ switch header.Name {
+ case "", ".", "..":
+ return fmt.Errorf("invalid root path: %q : %w", header.Name, errInvalidRoot)
+ }
+ rootName := header.Name
+
+ // Get the platform-specific output path
+ rootOutputPath := fp.Clean(te.Path)
+ if err := validatePlatformPath(rootOutputPath); err != nil {
+ return err
+ }
+
+ // If the last element in the rootOutputPath (which is passed by the user) is a symlink do not follow it
+ // this makes it easier for users to reason about where files are getting extracted to even when the tar is not
+ // from a trusted source
+ //
+ // For example, if the user extracts a mutable link to a tar file (http://sometimesbad.tld/t.tar) and situationally
+ // it contains a folder, file, or symlink the outputs could hop around the user's file system. This is especially
+ // annoying since we allow symlinks to point anywhere a user might want them to.
+ switch header.Typeflag {
+ case tar.TypeDir:
+ // if this is the root directory, use it as the output path for remaining files
+ firstObjectWasDir = true
+ if err := te.extractDir(rootOutputPath); err != nil {
+ return err
+ }
+ case tar.TypeReg, tar.TypeSymlink:
+ // Check if the output path already exists, so we know whether we should
+ // create our output with that name, or if we should put the output inside
+ // a preexisting directory
+
+ rootIsExistingDirectory := false
+ // We do not follow links here
+ if stat, err := os.Lstat(rootOutputPath); err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ } else if stat.IsDir() {
+ rootIsExistingDirectory = true
+ }
+
+ outputPath := rootOutputPath
+ // If the root is a directory which already exists then put the file/symlink in the directory
+ if rootIsExistingDirectory {
+ // make sure the root has a valid name
+ if err := validatePathComponent(rootName); err != nil {
+ return err
+ }
+
+ // If the output path directory exists then put the file/symlink into the directory.
+ outputPath = fp.Join(rootOutputPath, rootName)
+ }
+
+ // If an object with the target name already exists overwrite it
+ if header.Typeflag == tar.TypeReg {
+ if err := te.extractFile(outputPath, tarReader); err != nil {
+ return err
+ }
+ } else if err := te.extractSymlink(outputPath, header); err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf("unrecognized tar header type: %d", header.Typeflag)
+ }
+
+ // files come recursively in order
+ for {
+ header, err := tarReader.Next()
+ if err != nil && err != io.EOF {
+ return err
+ }
+ if header == nil || err == io.EOF {
+ break
+ }
+
+ // Make sure that we only have a single root element
+ if !firstObjectWasDir {
+ return fmt.Errorf("the root was not a directory and the tar has multiple entries: %w", errInvalidRoot)
+ }
+
+ // validate the path to remove paths we refuse to work with and make it easier to reason about
+ if err := validateTarPath(header.Name); err != nil {
+ return err
+ }
+ cleanedPath := header.Name
+
+ relPath, err := getRelativePath(rootName, cleanedPath)
+ if err != nil {
+ return err
+ }
+
+ outputPath, err := te.outputPath(rootOutputPath, relPath)
+ if err != nil {
+ return err
+ }
+
+ // This check should already be covered by previous validation, but may catch bugs that slip through.
+ // Checks if the relative path matches or exceeds the root
+ // We check for matching because the outputPath function strips the original root
+ rel, err := fp.Rel(rootOutputPath, outputPath)
+ if err != nil || rel == "." {
+ return errInvalidRootMultipleRoots
+ }
+ for _, e := range strings.Split(fp.ToSlash(rel), "/") {
+ if e == ".." {
+ return fmt.Errorf("relative path contains '..'")
+ }
+ }
+
+ switch header.Typeflag {
+ case tar.TypeDir:
+ if err := te.extractDir(outputPath); err != nil {
+ return err
+ }
+ case tar.TypeReg:
+ if err := te.extractFile(outputPath, tarReader); err != nil {
+ return err
+ }
+ case tar.TypeSymlink:
+ if err := te.extractSymlink(outputPath, header); err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf("unrecognized tar header type: %d", header.Typeflag)
+ }
+ }
+ return nil
+}
+
+// validateTarPath returns an error if the path has problematic characters
+func validateTarPath(tarPath string) error {
+ if len(tarPath) == 0 {
+ return fmt.Errorf("path is empty")
+ }
+
+ if tarPath[0] == '/' {
+ return fmt.Errorf("%q : path starts with '/'", tarPath)
+ }
+
+ elems := strings.Split(tarPath, "/") // break into elems
+ for _, e := range elems {
+ switch e {
+ case "", ".", "..":
+ return fmt.Errorf("%q : path contains %q", tarPath, e)
+ }
+ }
+ return nil
+}
+
+// getRelativePath returns the relative path between rootTarPath and tarPath. Assumes both paths have been cleaned.
+// Will error if the tarPath is not below the rootTarPath.
+func getRelativePath(rootName, tarPath string) (string, error) {
+ if !strings.HasPrefix(tarPath, rootName+"/") {
+ return "", errInvalidRootMultipleRoots
+ }
+ return tarPath[len(rootName)+1:], nil
+}
+
+// outputPath returns the directory path at which to place the file relativeTarPath. Assumes relativeTarPath is cleaned.
+func (te *Extractor) outputPath(basePlatformPath, relativeTarPath string) (string, error) {
+ elems := strings.Split(relativeTarPath, "/")
+
+ platformPath := basePlatformPath
+ for i, e := range elems {
+ if err := validatePathComponent(e); err != nil {
+ return "", err
+ }
+ platformPath = fp.Join(platformPath, e)
+
+ // Last element is not checked since it will be removed (if it exists) by any of the extraction functions.
+ // For more details see:
+ // https://github.com/libarchive/libarchive/blob/0fd2ed25d78e9f4505de5dcb6208c6c0ff8d2edb/libarchive/archive_write_disk_posix.c#L2810
+ if i == len(elems)-1 {
+ break
+ }
+
+ fi, err := os.Lstat(platformPath)
+ if err != nil {
+ return "", err
+ }
+
+ if fi.Mode()&os.ModeSymlink != 0 {
+ return "", errTraverseSymlink
+ }
+ if !fi.Mode().IsDir() {
+ return "", errors.New("cannot traverse non-directory objects")
+ }
+ }
+
+ return platformPath, nil
+}
+
+var errExtractedDirToSymlink = errors.New("cannot extract to symlink")
+
+func (te *Extractor) extractDir(path string) error {
+ err := os.MkdirAll(path, 0755)
+ if err != nil {
+ return err
+ }
+
+ if stat, err := os.Lstat(path); err != nil {
+ return err
+ } else if !stat.IsDir() {
+ return errExtractedDirToSymlink
+ }
+ return nil
+}
+
+func (te *Extractor) extractSymlink(path string, h *tar.Header) error {
+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+
+ return os.Symlink(h.Linkname, path)
+}
+
+func (te *Extractor) extractFile(path string, r *tar.Reader) error {
+ // Attempt removing the target so we can overwrite files, symlinks and empty directories
+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+
+ // Create a temporary file in the target directory and then rename the temporary file to the target to better deal
+ // with races on the file system.
+ base := fp.Dir(path)
+ tmpfile, err := os.CreateTemp(base, "")
+ if err != nil {
+ return err
+ }
+ if err := copyWithProgress(tmpfile, r, te.Progress); err != nil {
+ _ = tmpfile.Close()
+ _ = os.Remove(tmpfile.Name())
+ return err
+ }
+ if err := tmpfile.Close(); err != nil {
+ _ = os.Remove(tmpfile.Name())
+ return err
+ }
+
+ if err := os.Rename(tmpfile.Name(), path); err != nil {
+ _ = os.Remove(tmpfile.Name())
+ return err
+ }
+
+ return nil
+}
+
+func copyWithProgress(to io.Writer, from io.Reader, cb func(int64) int64) error {
+ buf := make([]byte, 4096)
+ for {
+ n, err := from.Read(buf)
+ if n != 0 {
+ if cb != nil {
+ cb(int64(n))
+ }
+ _, err2 := to.Write(buf[:n])
+ if err2 != nil {
+ return err2
+ }
+ }
+ if err != nil {
+ if err == io.EOF {
+ return nil
+ }
+ return err
+ }
+ }
+}
diff --git a/tar/extractor_test.go b/tar/extractor_test.go
new file mode 100644
index 000000000..717c65d19
--- /dev/null
+++ b/tar/extractor_test.go
@@ -0,0 +1,437 @@
+package tar
+
+import (
+ "archive/tar"
+ "bytes"
+ "fmt"
+ "io"
+ "os"
+ fp "path/filepath"
+ "runtime"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+var symlinksEnabled bool
+var symlinksEnabledErr error
+
+func init() {
+ // check if the platform supports symlinks
+ // inspired by https://github.com/golang/go/blob/770f1de8c54256d5b17447028e47b201ba8e62c8/src/internal/testenv/testenv_windows.go#L17
+
+ tmpdir, err := os.MkdirTemp("", "platformsymtest")
+ if err != nil {
+ panic("failed to create temp directory: " + err.Error())
+ }
+ defer os.RemoveAll(tmpdir)
+
+ symlinksEnabledErr = os.Symlink("target", fp.Join(tmpdir, "symlink"))
+ symlinksEnabled = symlinksEnabledErr == nil
+
+ if !symlinksEnabled {
+ // for now assume symlinks only fail on Windows, Android and Plan9
+ // taken from https://github.com/golang/go/blob/770f1de8c54256d5b17447028e47b201ba8e62c8/src/internal/testenv/testenv_notwin.go#L14
+ // and https://github.com/golang/go/blob/770f1de8c54256d5b17447028e47b201ba8e62c8/src/internal/testenv/testenv_windows.go#L34
+ switch runtime.GOOS {
+ case "windows", "android", "plan9":
+ default:
+ panic(fmt.Errorf("attempted symlink creation failed: %w", symlinksEnabledErr))
+ }
+ }
+}
+
+func TestSingleFile(t *testing.T) {
+ fileName := "file..ext"
+ fileData := "file data"
+
+ testTarExtraction(t, nil, []tarEntry{
+ &fileTarEntry{fileName, []byte(fileData)},
+ },
+ func(t *testing.T, extractDir string) {
+ f, err := os.Open(fp.Join(extractDir, fileName))
+ assert.NoError(t, err)
+ data, err := io.ReadAll(f)
+ assert.NoError(t, err)
+ assert.Equal(t, fileData, string(data))
+ assert.NoError(t, f.Close())
+ },
+ nil,
+ )
+}
+
+func TestSingleDirectory(t *testing.T) {
+ dirName := "dir..sfx"
+
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{dirName},
+ },
+ func(t *testing.T, extractDir string) {
+ f, err := os.Open(extractDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ objs, err := f.Readdir(1)
+ if err == io.EOF && len(objs) == 0 {
+ return
+ }
+ t.Fatalf("expected an empty directory")
+ },
+ nil,
+ )
+}
+
+func TestDirectoryFollowSymlinkToNothing(t *testing.T) {
+ dirName := "dir"
+ childName := "child"
+
+ entries := []tarEntry{
+ &dirTarEntry{dirName},
+ &dirTarEntry{dirName + "/" + childName},
+ }
+
+ testTarExtraction(t, func(t *testing.T, rootDir string) {
+ target := fp.Join(rootDir, tarOutRoot)
+ if err := os.Symlink(fp.Join(target, "foo"), fp.Join(target, childName)); err != nil {
+ t.Fatal(err)
+ }
+ }, entries, nil,
+ os.ErrExist,
+ )
+}
+
+func TestDirectoryFollowSymlinkToFile(t *testing.T) {
+ dirName := "dir"
+ childName := "child"
+
+ entries := []tarEntry{
+ &dirTarEntry{dirName},
+ &dirTarEntry{dirName + "/" + childName},
+ }
+
+ testTarExtraction(t, func(t *testing.T, rootDir string) {
+ target := fp.Join(rootDir, tarOutRoot)
+ symlinkTarget := fp.Join(target, "foo")
+ if err := os.WriteFile(symlinkTarget, []byte("original data"), os.ModePerm); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(symlinkTarget, fp.Join(target, childName)); err != nil {
+ t.Fatal(err)
+ }
+ }, entries, nil,
+ syscall.ENOTDIR,
+ )
+}
+
+func TestDirectoryFollowSymlinkToDirectory(t *testing.T) {
+ dirName := "dir"
+ childName := "child"
+
+ entries := []tarEntry{
+ &dirTarEntry{dirName},
+ &dirTarEntry{dirName + "/" + childName},
+ }
+
+ testTarExtraction(t, func(t *testing.T, rootDir string) {
+ target := fp.Join(rootDir, tarOutRoot)
+ symlinkTarget := fp.Join(target, "foo")
+ if err := os.Mkdir(symlinkTarget, os.ModePerm); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(symlinkTarget, fp.Join(target, childName)); err != nil {
+ t.Fatal(err)
+ }
+ }, entries, nil,
+ errExtractedDirToSymlink,
+ )
+}
+
+func TestSingleSymlink(t *testing.T) {
+ if !symlinksEnabled {
+ t.Skip("symlinks disabled on this platform", symlinksEnabledErr)
+ }
+
+ targetName := "file"
+ symlinkName := "symlink"
+
+ testTarExtraction(t, nil, []tarEntry{
+ &symlinkTarEntry{targetName, symlinkName},
+ }, func(t *testing.T, extractDir string) {
+ symlinkPath := fp.Join(extractDir, symlinkName)
+ fi, err := os.Lstat(symlinkPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, fi.Mode()&os.ModeSymlink != 0, true, "expected to be a symlink")
+ targetPath, err := os.Readlink(symlinkPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, targetName, targetPath)
+ }, nil)
+}
+
+func TestMultipleRoots(t *testing.T) {
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{"root"},
+ &dirTarEntry{"sibling"},
+ }, nil, errInvalidRoot)
+}
+
+func TestMultipleRootsNested(t *testing.T) {
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{"root/child1"},
+ &dirTarEntry{"root/child2"},
+ }, nil, errInvalidRoot)
+}
+
+func TestOutOfOrderRoot(t *testing.T) {
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{"root/child"},
+ &dirTarEntry{"root"},
+ }, nil, errInvalidRoot)
+}
+
+func TestOutOfOrder(t *testing.T) {
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{"root/child/grandchild"},
+ &dirTarEntry{"root/child"},
+ }, nil, errInvalidRoot)
+}
+
+func TestNestedDirectories(t *testing.T) {
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{"root"},
+ &dirTarEntry{"root/child"},
+ &dirTarEntry{"root/child/grandchild"},
+ }, func(t *testing.T, extractDir string) {
+ walkIndex := 0
+ err := fp.Walk(extractDir,
+ func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ switch walkIndex {
+ case 0:
+ assert.Equal(t, info.Name(), tarOutRoot)
+ case 1:
+ assert.Equal(t, info.Name(), "child")
+ case 2:
+ assert.Equal(t, info.Name(), "grandchild")
+ default:
+ assert.Fail(t, "has more than 3 entries", path)
+ }
+ walkIndex++
+ return nil
+ })
+ assert.NoError(t, err)
+ }, nil)
+}
+
+func TestRootDirectoryHasSubpath(t *testing.T) {
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{"root/child"},
+ &dirTarEntry{"root/child/grandchild"},
+ }, nil, errInvalidRoot)
+}
+
+func TestFilesAndFolders(t *testing.T) {
+ testTarExtraction(t, nil, []tarEntry{
+ &dirTarEntry{"root"},
+ &dirTarEntry{"root/childdir"},
+ &fileTarEntry{"root/childdir/file1", []byte("some data")},
+ }, nil, nil)
+}
+
+func TestInternalSymlinkTraverse(t *testing.T) {
+ if !symlinksEnabled {
+ t.Skip("symlinks disabled on this platform", symlinksEnabledErr)
+ }
+ testTarExtraction(t, nil, []tarEntry{
+ // FIXME: We are ignoring the first element in the path check so
+ // we add a directory at the start to bypass this.
+ &dirTarEntry{"root"},
+ &dirTarEntry{"root/child"},
+ &symlinkTarEntry{"child", "root/symlink-dir"},
+ &fileTarEntry{"root/symlink-dir/file", []byte("file")},
+ },
+ nil,
+ errTraverseSymlink,
+ )
+}
+
+func TestExternalSymlinkTraverse(t *testing.T) {
+ if !symlinksEnabled {
+ t.Skip("symlinks disabled on this platform", symlinksEnabledErr)
+ }
+ testTarExtraction(t, nil, []tarEntry{
+ // FIXME: We are ignoring the first element in the path check so
+ // we add a directory at the start to bypass this.
+ &dirTarEntry{"inner"},
+ &symlinkTarEntry{"..", "inner/symlink-dir"},
+ &fileTarEntry{"inner/symlink-dir/file", []byte("overwrite content")},
+ },
+ nil,
+ errTraverseSymlink,
+ )
+}
+
+func TestLastElementOverwrite(t *testing.T) {
+ if !symlinksEnabled {
+ t.Skip("symlinks disabled on this platform", symlinksEnabledErr)
+ }
+ const originalData = "original"
+ testTarExtraction(t, func(t *testing.T, rootDir string) {
+ // Create an outside target that will try to be overwritten.
+ // This file will reside outside of the extraction directory root.
+ f, err := os.Create(fp.Join(rootDir, "outside-ref"))
+ assert.NoError(t, err)
+ n, err := f.WriteString(originalData)
+ assert.NoError(t, err)
+ assert.Equal(t, len(originalData), n)
+ },
+ []tarEntry{
+ &dirTarEntry{"root"},
+ &symlinkTarEntry{"../outside-ref", "root/symlink"},
+ &fileTarEntry{"root/symlink", []byte("overwrite content")},
+ },
+ func(t *testing.T, extractDir string) {
+ // Check that outside-ref still exists but has not been
+ // overwritten or truncated (still size the same).
+ info, err := os.Stat(fp.Join(extractDir, "..", "outside-ref"))
+ assert.NoError(t, err)
+
+ assert.Equal(t, len(originalData), int(info.Size()), "outside reference has been overwritten")
+ },
+ nil,
+ )
+}
+
+const tarOutRoot = "tar-out-root"
+
+func testTarExtraction(t *testing.T, setup func(t *testing.T, rootDir string), tarEntries []tarEntry, check func(t *testing.T, extractDir string), extractError error) {
+ var err error
+
+ // Directory structure.
+ // FIXME: We can't easily work on a MemFS since we would need to replace
+ // all the `os` calls in the extractor so using a temporary dir.
+ rootDir, err := os.MkdirTemp("", "tar-extraction-test")
+ assert.NoError(t, err)
+ extractDir := fp.Join(rootDir, tarOutRoot)
+ err = os.MkdirAll(extractDir, 0755)
+ assert.NoError(t, err)
+
+ // Generated TAR file.
+ tarFilename := fp.Join(rootDir, "generated.tar")
+ tarFile, err := os.Create(tarFilename)
+ assert.NoError(t, err)
+ defer tarFile.Close()
+ tw := tar.NewWriter(tarFile)
+ defer tw.Close()
+
+ if setup != nil {
+ setup(t, rootDir)
+ }
+
+ writeTarFile(t, tarFilename, tarEntries)
+
+ testExtract(t, tarFilename, extractDir, extractError)
+
+ if check != nil {
+ check(t, extractDir)
+ }
+}
+
+func testExtract(t *testing.T, tarFile string, extractDir string, expectedError error) {
+ var err error
+
+ tarReader, err := os.Open(tarFile)
+ assert.NoError(t, err)
+
+ extractor := &Extractor{Path: extractDir}
+ err = extractor.Extract(tarReader)
+
+ assert.ErrorIs(t, err, expectedError)
+}
+
+// Based on the `writeXXXHeader` family of functions in
+// github.com/ipfs/go-ipfs-files@v0.0.8/tarwriter.go.
+func writeTarFile(t *testing.T, path string, entries []tarEntry) {
+ tarFile, err := os.Create(path)
+ assert.NoError(t, err)
+ defer tarFile.Close()
+
+ tw := tar.NewWriter(tarFile)
+ defer tw.Close()
+
+ for _, e := range entries {
+ err = e.write(tw)
+ assert.NoError(t, err)
+ }
+}
+
+type tarEntry interface {
+ write(tw *tar.Writer) error
+}
+
+var _ tarEntry = (*fileTarEntry)(nil)
+var _ tarEntry = (*dirTarEntry)(nil)
+var _ tarEntry = (*symlinkTarEntry)(nil)
+
+type fileTarEntry struct {
+ path string
+ buf []byte
+}
+
+func (e *fileTarEntry) write(tw *tar.Writer) error {
+ if err := writeFileHeader(tw, e.path, uint64(len(e.buf))); err != nil {
+ return err
+ }
+
+ if _, err := io.Copy(tw, bytes.NewReader(e.buf)); err != nil {
+ return err
+ }
+
+ tw.Flush()
+ return nil
+}
+func writeFileHeader(w *tar.Writer, fpath string, size uint64) error {
+ return w.WriteHeader(&tar.Header{
+ Name: fpath,
+ Size: int64(size),
+ Typeflag: tar.TypeReg,
+ Mode: 0644,
+ ModTime: time.Now(),
+ // TODO: set mode, dates, etc. when added to unixFS
+ })
+}
+
+type dirTarEntry struct {
+ path string
+}
+
+func (e *dirTarEntry) write(tw *tar.Writer) error {
+ return tw.WriteHeader(&tar.Header{
+ Name: e.path,
+ Typeflag: tar.TypeDir,
+ Mode: 0777,
+ ModTime: time.Now(),
+ // TODO: set mode, dates, etc. when added to unixFS
+ })
+}
+
+type symlinkTarEntry struct {
+ target string
+ path string
+}
+
+func (e *symlinkTarEntry) write(w *tar.Writer) error {
+ return w.WriteHeader(&tar.Header{
+ Name: e.path,
+ Linkname: e.target,
+ Mode: 0777,
+ Typeflag: tar.TypeSymlink,
+ })
+}
diff --git a/tar/sanitize.go b/tar/sanitize.go
new file mode 100644
index 000000000..86fba1991
--- /dev/null
+++ b/tar/sanitize.go
@@ -0,0 +1,30 @@
+//go:build !windows
+
+package tar
+
+import (
+ "fmt"
+ "os"
+ "strings"
+)
+
+func isNullDevice(path string) bool {
+ return path == os.DevNull
+}
+
+func validatePlatformPath(platformPath string) error {
+ if strings.Contains(platformPath, "\x00") {
+ return fmt.Errorf("invalid platform path: path components cannot contain null: %q", platformPath)
+ }
+ return nil
+}
+
+func validatePathComponent(c string) error {
+ if c == ".." {
+ return fmt.Errorf("invalid platform path: path component cannot be '..'")
+ }
+ if strings.Contains(c, "\x00") {
+ return fmt.Errorf("invalid platform path: path components cannot contain null: %q", c)
+ }
+ return nil
+}
diff --git a/tar/sanitize_windows.go b/tar/sanitize_windows.go
new file mode 100644
index 000000000..4a788a484
--- /dev/null
+++ b/tar/sanitize_windows.go
@@ -0,0 +1,74 @@
+package tar
+
+import (
+ "fmt"
+ "path/filepath"
+ "strings"
+)
+
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
+var reservedNames = [...]string{"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"}
+
+const reservedCharsStr = `[<>:"\|?*]` + "\x00" //NOTE: `/` is not included as it is our standard path separator
+
+func isNullDevice(path string) bool {
+ // This is a case insensitive comparison to NUL
+ if len(path) != 3 {
+ return false
+ }
+ if path[0]|0x20 != 'n' {
+ return false
+ }
+ if path[1]|0x20 != 'u' {
+ return false
+ }
+ if path[2]|0x20 != 'l' {
+ return false
+ }
+ return true
+}
+
+// validatePathComponent returns an error if the given path component is not allowed on the platform
+func validatePathComponent(c string) error {
+ //MSDN: Do not end a file or directory name with a space or a period
+ if strings.HasSuffix(c, ".") {
+ return fmt.Errorf("invalid platform path: path components cannot end with '.' : %q", c)
+ }
+ if strings.HasSuffix(c, " ") {
+ return fmt.Errorf("invalid platform path: path components cannot end with ' ' : %q", c)
+ }
+
+ if c == ".." {
+ return fmt.Errorf("invalid platform path: path component cannot be '..'")
+ }
+ // error on reserved characters
+ if strings.ContainsAny(c, reservedCharsStr) {
+ return fmt.Errorf("invalid platform path: path components cannot contain any of %s : %q", reservedCharsStr, c)
+ }
+
+ // error on reserved names
+ for _, rn := range reservedNames {
+ if c == rn {
+ return fmt.Errorf("invalid platform path: path component is a reserved name: %s", c)
+ }
+ }
+
+ return nil
+}
+
+func validatePlatformPath(platformPath string) error {
+ // remove the volume name
+ p := platformPath[len(filepath.VolumeName(platformPath)):]
+
+ // convert to cleaned slash-path
+ p = filepath.ToSlash(p)
+ p = strings.Trim(p, "/")
+
+ // make sure all components of the path are valid
+ for _, e := range strings.Split(p, "/") {
+ if err := validatePathComponent(e); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/unixfs/README.md b/unixfs/README.md
new file mode 100644
index 000000000..6c80d07c8
--- /dev/null
+++ b/unixfs/README.md
@@ -0,0 +1,28 @@
+## Package Directory
+This package contains many subpackages, each of which can be very large on its own.
+
+### Top Level
+The top level unixfs package defines the unixfs format datastructures, and some helper methods around it.
+
+### importers
+The `importer` subpackage is what you'll use when you want to turn a normal file into a unixfs file.
+
+### io
+The `io` subpackage provides helpers for reading files and manipulating directories. The `DagReader` takes a
+reference to a unixfs file and returns a file handle that can be read from and seeked through. The `Directory`
+interface allows you to easily read items in a directory, add items to a directory, and do lookups.
+
+### mod
+The `mod` subpackage implements a `DagModifier` type that can be used to write to an existing unixfs file, or
+create a new one. The logic for this is significantly more complicated than for the dagreader, so its a separate
+type. (TODO: maybe it still belongs in the `io` subpackage though?)
+
+### hamt
+The `hamt` subpackage implements a CHAMP hamt that is used in unixfs directory sharding.
+
+### archive
+The `archive` subpackage implements a `tar` importer and exporter. The objects created here are not officially unixfs,
+but in the future, this may be integrated more directly.
+
+### test
+The `test` subpackage provides several utilities to make testing unixfs related things easier.
diff --git a/unixfs/file/unixfile.go b/unixfs/file/unixfile.go
new file mode 100644
index 000000000..3897f5e9a
--- /dev/null
+++ b/unixfs/file/unixfile.go
@@ -0,0 +1,183 @@
+package unixfile
+
+import (
+ "context"
+ "errors"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ uio "github.com/ipfs/boxo/unixfs/io"
+
+ "github.com/ipfs/boxo/files"
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// Number to file to prefetch in directories
+// TODO: should we allow setting this via context hint?
+const prefetchFiles = 4
+
+type ufsDirectory struct {
+ ctx context.Context
+ dserv ipld.DAGService
+ dir uio.Directory
+ size int64
+}
+
+type ufsIterator struct {
+ ctx context.Context
+ files chan *ipld.Link
+ dserv ipld.DAGService
+
+ curName string
+ curFile files.Node
+
+ err error
+ errCh chan error
+}
+
+func (it *ufsIterator) Name() string {
+ return it.curName
+}
+
+func (it *ufsIterator) Node() files.Node {
+ return it.curFile
+}
+
+func (it *ufsIterator) Next() bool {
+ if it.err != nil {
+ return false
+ }
+
+ var l *ipld.Link
+ var ok bool
+ for !ok {
+ if it.files == nil && it.errCh == nil {
+ return false
+ }
+ select {
+ case l, ok = <-it.files:
+ if !ok {
+ it.files = nil
+ }
+ case err := <-it.errCh:
+ it.errCh = nil
+ it.err = err
+
+ if err != nil {
+ return false
+ }
+ }
+ }
+
+ it.curFile = nil
+
+ nd, err := l.GetNode(it.ctx, it.dserv)
+ if err != nil {
+ it.err = err
+ return false
+ }
+
+ it.curName = l.Name
+ it.curFile, it.err = NewUnixfsFile(it.ctx, it.dserv, nd)
+ return it.err == nil
+}
+
+func (it *ufsIterator) Err() error {
+ return it.err
+}
+
+func (d *ufsDirectory) Close() error {
+ return nil
+}
+
+func (d *ufsDirectory) Entries() files.DirIterator {
+ fileCh := make(chan *ipld.Link, prefetchFiles)
+ errCh := make(chan error, 1)
+ go func() {
+ errCh <- d.dir.ForEachLink(d.ctx, func(link *ipld.Link) error {
+ if d.ctx.Err() != nil {
+ return d.ctx.Err()
+ }
+ select {
+ case fileCh <- link:
+ case <-d.ctx.Done():
+ return d.ctx.Err()
+ }
+ return nil
+ })
+
+ close(errCh)
+ close(fileCh)
+ }()
+
+ return &ufsIterator{
+ ctx: d.ctx,
+ files: fileCh,
+ errCh: errCh,
+ dserv: d.dserv,
+ }
+}
+
+func (d *ufsDirectory) Size() (int64, error) {
+ return d.size, nil
+}
+
+type ufsFile struct {
+ uio.DagReader
+}
+
+func (f *ufsFile) Size() (int64, error) {
+ return int64(f.DagReader.Size()), nil
+}
+
+func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) (files.Directory, error) {
+ dir, err := uio.NewDirectoryFromNode(dserv, nd)
+ if err != nil {
+ return nil, err
+ }
+
+ size, err := nd.Size()
+ if err != nil {
+ return nil, err
+ }
+
+ return &ufsDirectory{
+ ctx: ctx,
+ dserv: dserv,
+
+ dir: dir,
+ size: int64(size),
+ }, nil
+}
+
+func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) {
+ switch dn := nd.(type) {
+ case *dag.ProtoNode:
+ fsn, err := ft.FSNodeFromBytes(dn.Data())
+ if err != nil {
+ return nil, err
+ }
+ if fsn.IsDir() {
+ return newUnixfsDir(ctx, dserv, dn)
+ }
+ if fsn.Type() == ft.TSymlink {
+ return files.NewLinkFile(string(fsn.Data()), nil), nil
+ }
+
+ case *dag.RawNode:
+ default:
+ return nil, errors.New("unknown node type")
+ }
+
+ dr, err := uio.NewDagReader(ctx, nd, dserv)
+ if err != nil {
+ return nil, err
+ }
+
+ return &ufsFile{
+ DagReader: dr,
+ }, nil
+}
+
+var _ files.Directory = &ufsDirectory{}
+var _ files.File = &ufsFile{}
diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go
new file mode 100644
index 000000000..41a35fe15
--- /dev/null
+++ b/unixfs/hamt/hamt.go
@@ -0,0 +1,946 @@
+// Package hamt implements a Hash Array Mapped Trie over ipfs merkledag nodes.
+// It is implemented mostly as described in the wikipedia article on HAMTs,
+// however the table size is variable (usually 256 in our usages) as opposed to
+// 32 as suggested in the article. The hash function used is currently
+// Murmur3, but this value is configurable (the datastructure reports which
+// hash function its using).
+//
+// The one algorithmic change we implement that is not mentioned in the
+// wikipedia article is the collapsing of empty shards.
+// Given the following tree: ( '[' = shards, '{' = values )
+// [ 'A' ] -> [ 'B' ] -> { "ABC" }
+//
+// | L-> { "ABD" }
+// L-> { "ASDF" }
+//
+// If we simply removed "ABC", we would end up with a tree where shard 'B' only
+// has a single child. This causes two issues, the first, is that now we have
+// an extra lookup required to get to "ABD". The second issue is that now we
+// have a tree that contains only "ABD", but is not the same tree that we would
+// get by simply inserting "ABD" into a new tree. To address this, we always
+// check for empty shard nodes upon deletion and prune them to maintain a
+// consistent tree, independent of insertion order.
+package hamt
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "sync"
+
+ "golang.org/x/sync/errgroup"
+
+ format "github.com/ipfs/boxo/unixfs"
+ "github.com/ipfs/boxo/unixfs/internal"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ bitfield "github.com/ipfs/go-bitfield"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+const (
+ // HashMurmur3 is the multiformats identifier for Murmur3
+ HashMurmur3 uint64 = 0x22
+)
+
+func init() {
+ internal.HAMTHashFunction = murmur3Hash
+}
+
+func (ds *Shard) isValueNode() bool {
+ return ds.key != "" && ds.val != nil
+}
+
+// A Shard represents the HAMT. It should be initialized with NewShard().
+type Shard struct {
+ childer *childer
+
+ // Entries per node (number of possible childs indexed by the partial key).
+ tableSize int
+ // Bits needed to encode child indexes (log2 of number of entries). This is
+ // the number of bits taken from the hash key on each level of the tree.
+ tableSizeLg2 int
+
+ builder cid.Builder
+ hashFunc uint64
+
+ // String format with number of zeros that will be present in the hexadecimal
+ // encoding of the child index to always reach the fixed maxpadlen chars.
+ // Example: maxpadlen = 4 => prefixPadStr: "%04X" (print number in hexadecimal
+ // format padding with zeros to always reach 4 characters).
+ prefixPadStr string
+ // Length in chars of string that encodes child indexes. We encode indexes
+ // as hexadecimal strings to this is log4 of number of entries.
+ maxpadlen int
+
+ dserv ipld.DAGService
+
+ // FIXME: Remove. We don't actually store "value nodes". This confusing
+ // abstraction just removes the maxpadlen from the link names to extract
+ // the actual value link the trie is storing.
+ // leaf node
+ key string
+ val *ipld.Link
+}
+
+// NewShard creates a new, empty HAMT shard with the given size.
+func NewShard(dserv ipld.DAGService, size int) (*Shard, error) {
+ return NewShardValue(dserv, size, "", nil)
+}
+
+// NewShardValue creates a new, empty HAMT shard with the given key, value and size.
+func NewShardValue(dserv ipld.DAGService, size int, key string, value *ipld.Link) (*Shard, error) {
+ ds, err := makeShard(dserv, size, key, value)
+ if err != nil {
+ return nil, err
+ }
+
+ // FIXME: Make this at least a static configuration for testing.
+ ds.hashFunc = HashMurmur3
+ return ds, nil
+}
+
+func makeShard(ds ipld.DAGService, size int, key string, val *ipld.Link) (*Shard, error) {
+ lg2s, err := Logtwo(size)
+ if err != nil {
+ return nil, err
+ }
+ childer, err := newChilder(ds, size)
+ if err != nil {
+ return nil, err
+ }
+ maxpadding := fmt.Sprintf("%X", size-1)
+ s := &Shard{
+ tableSizeLg2: lg2s,
+ prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)),
+ maxpadlen: len(maxpadding),
+ childer: childer,
+ tableSize: size,
+ dserv: ds,
+
+ key: key,
+ val: val,
+ }
+
+ s.childer.sd = s
+
+ return s, nil
+}
+
+// NewHamtFromDag creates new a HAMT shard from the given DAG.
+func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
+ pbnd, ok := nd.(*dag.ProtoNode)
+ if !ok {
+ return nil, dag.ErrNotProtobuf
+ }
+
+ fsn, err := format.FSNodeFromBytes(pbnd.Data())
+ if err != nil {
+ return nil, err
+ }
+
+ if fsn.Type() != format.THAMTShard {
+ return nil, fmt.Errorf("node was not a dir shard")
+ }
+
+ if fsn.HashType() != HashMurmur3 {
+ return nil, fmt.Errorf("only murmur3 supported as hash function")
+ }
+
+ size := int(fsn.Fanout())
+
+ ds, err := makeShard(dserv, size, "", nil)
+ if err != nil {
+ return nil, err
+ }
+
+ ds.childer.makeChilder(fsn.Data(), pbnd.Links())
+
+ ds.hashFunc = fsn.HashType()
+ ds.builder = pbnd.CidBuilder()
+
+ return ds, nil
+}
+
+// SetCidBuilder sets the CID Builder
+func (ds *Shard) SetCidBuilder(builder cid.Builder) {
+ ds.builder = builder
+}
+
+// CidBuilder gets the CID Builder, may be nil if unset
+func (ds *Shard) CidBuilder() cid.Builder {
+ return ds.builder
+}
+
+// Node serializes the HAMT structure into a merkledag node with unixfs formatting
+func (ds *Shard) Node() (ipld.Node, error) {
+ out := new(dag.ProtoNode)
+ out.SetCidBuilder(ds.builder)
+
+ sliceIndex := 0
+ // TODO: optimized 'for each set bit'
+ for childIndex := 0; childIndex < ds.tableSize; childIndex++ {
+ if !ds.childer.has(childIndex) {
+ continue
+ }
+
+ ch := ds.childer.child(sliceIndex)
+ if ch != nil {
+ clnk, err := ch.Link()
+ if err != nil {
+ return nil, err
+ }
+
+ err = out.AddRawLink(ds.linkNamePrefix(childIndex)+ch.key, clnk)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ // child unloaded, just copy in link with updated name
+ lnk := ds.childer.link(sliceIndex)
+ label := lnk.Name[ds.maxpadlen:]
+
+ err := out.AddRawLink(ds.linkNamePrefix(childIndex)+label, lnk)
+ if err != nil {
+ return nil, err
+ }
+ }
+ sliceIndex++
+ }
+
+ data, err := format.HAMTShardData(ds.childer.bitfield.Bytes(), uint64(ds.tableSize), HashMurmur3)
+ if err != nil {
+ return nil, err
+ }
+
+ out.SetData(data)
+
+ err = ds.dserv.Add(context.TODO(), out)
+ if err != nil {
+ return nil, err
+ }
+
+ return out, nil
+}
+
+func (ds *Shard) makeShardValue(lnk *ipld.Link) (*Shard, error) {
+ lnk2 := *lnk
+ s, err := makeShard(ds.dserv, ds.tableSize, "", nil)
+ if err != nil {
+ return nil, err
+ }
+
+ s.key = lnk.Name[ds.maxpadlen:]
+ s.val = &lnk2
+
+ return s, nil
+}
+
+// Set sets 'name' = nd in the HAMT
+func (ds *Shard) Set(ctx context.Context, name string, nd ipld.Node) error {
+ _, err := ds.Swap(ctx, name, nd)
+ return err
+}
+
+// Set sets 'name' = nd in the HAMT, using directly the information in the
+// given link. This avoids writing the given node, then reading it to making a
+// link out of it.
+func (ds *Shard) SetLink(ctx context.Context, name string, lnk *ipld.Link) error {
+ hv := newHashBits(name)
+
+ newLink := ipld.Link{
+ Name: lnk.Name,
+ Size: lnk.Size,
+ Cid: lnk.Cid,
+ }
+
+ // FIXME: We don't need to set the name here, it will get overwritten.
+ // This is confusing, confirm and remove this line.
+ newLink.Name = ds.linkNamePrefix(0) + name
+
+ _, err := ds.swapValue(ctx, hv, name, &newLink)
+ return err
+}
+
+// Swap sets a link pointing to the passed node as the value under the
+// name key in this Shard or its children. It also returns the previous link
+// under that name key (if any).
+func (ds *Shard) Swap(ctx context.Context, name string, node ipld.Node) (*ipld.Link, error) {
+ hv := newHashBits(name)
+ err := ds.dserv.Add(ctx, node)
+ if err != nil {
+ return nil, err
+ }
+
+ lnk, err := ipld.MakeLink(node)
+ if err != nil {
+ return nil, err
+ }
+
+ // FIXME: We don't need to set the name here, it will get overwritten.
+ // This is confusing, confirm and remove this line.
+ lnk.Name = ds.linkNamePrefix(0) + name
+
+ return ds.swapValue(ctx, hv, name, lnk)
+}
+
+// Remove deletes the named entry if it exists. Otherwise, it returns
+// os.ErrNotExist.
+func (ds *Shard) Remove(ctx context.Context, name string) error {
+ _, err := ds.Take(ctx, name)
+ return err
+}
+
+// Take is similar to the public Remove but also returns the
+// old removed link (if it exists).
+func (ds *Shard) Take(ctx context.Context, name string) (*ipld.Link, error) {
+ hv := newHashBits(name)
+ return ds.swapValue(ctx, hv, name, nil)
+}
+
+// Find searches for a child node by 'name' within this hamt
+func (ds *Shard) Find(ctx context.Context, name string) (*ipld.Link, error) {
+ hv := newHashBits(name)
+
+ var out *ipld.Link
+ err := ds.getValue(ctx, hv, name, func(sv *Shard) error {
+ out = sv.val
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return out, nil
+}
+
+type linkType int
+
+const (
+ invalidLink linkType = iota
+ shardLink
+ shardValueLink
+)
+
+func (ds *Shard) childLinkType(lnk *ipld.Link) (linkType, error) {
+ if len(lnk.Name) < ds.maxpadlen {
+ return invalidLink, fmt.Errorf("invalid link name '%s'", lnk.Name)
+ }
+ if len(lnk.Name) == ds.maxpadlen {
+ return shardLink, nil
+ }
+ return shardValueLink, nil
+}
+
+// Link returns a merklelink to this shard node
+func (ds *Shard) Link() (*ipld.Link, error) {
+ if ds.isValueNode() {
+ return ds.val, nil
+ }
+
+ nd, err := ds.Node()
+ if err != nil {
+ return nil, err
+ }
+
+ err = ds.dserv.Add(context.TODO(), nd)
+ if err != nil {
+ return nil, err
+ }
+
+ return ipld.MakeLink(nd)
+}
+
+func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*Shard) error) error {
+ childIndex, err := hv.Next(ds.tableSizeLg2)
+ if err != nil {
+ return err
+ }
+
+ if ds.childer.has(childIndex) {
+ child, err := ds.childer.get(ctx, ds.childer.sliceIndex(childIndex))
+ if err != nil {
+ return err
+ }
+
+ if child.isValueNode() {
+ if child.key == key {
+ return cb(child)
+ }
+ } else {
+ return child.getValue(ctx, hv, key, cb)
+ }
+ }
+
+ return os.ErrNotExist
+}
+
+// EnumLinks collects all links in the Shard.
+func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
+ var links []*ipld.Link
+
+ linkResults := ds.EnumLinksAsync(ctx)
+
+ for linkResult := range linkResults {
+ if linkResult.Err != nil {
+ return links, linkResult.Err
+ }
+ links = append(links, linkResult.Link)
+ }
+ return links, nil
+}
+
+// ForEachLink walks the Shard and calls the given function.
+func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
+ return ds.walkTrie(ctx, func(sv *Shard) error {
+ lnk := sv.val
+ lnk.Name = sv.key
+
+ return f(lnk)
+ })
+}
+
+// EnumLinksAsync returns a channel which will receive Links in the directory
+// as they are enumerated, where order is not guaranteed
+func (ds *Shard) EnumLinksAsync(ctx context.Context) <-chan format.LinkResult {
+ linkResults := make(chan format.LinkResult)
+ ctx, cancel := context.WithCancel(ctx)
+ go func() {
+ defer close(linkResults)
+ defer cancel()
+
+ err := parallelShardWalk(ctx, ds, ds.dserv, func(formattedLink *ipld.Link) error {
+ emitResult(ctx, linkResults, format.LinkResult{Link: formattedLink, Err: nil})
+ return nil
+ })
+ if err != nil {
+ emitResult(ctx, linkResults, format.LinkResult{Link: nil, Err: err})
+ }
+ }()
+ return linkResults
+}
+
+type listCidsAndShards struct {
+ cids []cid.Cid
+ shards []*Shard
+}
+
+func (ds *Shard) walkChildren(processLinkValues func(formattedLink *ipld.Link) error) (*listCidsAndShards, error) {
+ res := &listCidsAndShards{}
+
+ for idx, lnk := range ds.childer.links {
+ if nextShard := ds.childer.children[idx]; nextShard == nil {
+ lnkLinkType, err := ds.childLinkType(lnk)
+ if err != nil {
+ return nil, err
+ }
+
+ switch lnkLinkType {
+ case shardValueLink:
+ sv, err := ds.makeShardValue(lnk)
+ if err != nil {
+ return nil, err
+ }
+ formattedLink := sv.val
+ formattedLink.Name = sv.key
+
+ if err := processLinkValues(formattedLink); err != nil {
+ return nil, err
+ }
+ case shardLink:
+ res.cids = append(res.cids, lnk.Cid)
+ default:
+ return nil, fmt.Errorf("unsupported shard link type")
+ }
+
+ } else {
+ if nextShard.val != nil {
+ formattedLink := &ipld.Link{
+ Name: nextShard.key,
+ Size: nextShard.val.Size,
+ Cid: nextShard.val.Cid,
+ }
+ if err := processLinkValues(formattedLink); err != nil {
+ return nil, err
+ }
+ } else {
+ res.shards = append(res.shards, nextShard)
+ }
+ }
+ }
+ return res, nil
+}
+
+// parallelShardWalk is quite similar to the DAG walking algorithm from https://github.com/ipfs/boxo/ipld/merkledag/blob/594e515f162e764183243b72c2ba84f743424c8c/merkledag.go#L464
+// However, there are a few notable differences:
+// 1. Some children are actualized Shard structs and some are in the blockstore, this will leverage walking over the in memory Shards as well as the stored blocks
+// 2. Instead of just passing each child into the worker pool by itself we group them so that we can leverage optimizations from GetMany.
+// This optimization also makes the walk a little more biased towards depth (as opposed to BFS) in the earlier part of the DAG.
+// This is particularly helpful for operations like estimating the directory size which should complete quickly when possible.
+// 3. None of the extra options from that package are needed
+func parallelShardWalk(ctx context.Context, root *Shard, dserv ipld.DAGService, processShardValues func(formattedLink *ipld.Link) error) error {
+ const concurrency = 32
+
+ var visitlk sync.Mutex
+ visitSet := cid.NewSet()
+ visit := visitSet.Visit
+
+ // Setup synchronization
+ grp, errGrpCtx := errgroup.WithContext(ctx)
+
+ // Input and output queues for workers.
+ feed := make(chan *listCidsAndShards)
+ out := make(chan *listCidsAndShards)
+ done := make(chan struct{})
+
+ for i := 0; i < concurrency; i++ {
+ grp.Go(func() error {
+ for feedChildren := range feed {
+ for _, nextShard := range feedChildren.shards {
+ nextChildren, err := nextShard.walkChildren(processShardValues)
+ if err != nil {
+ return err
+ }
+
+ select {
+ case out <- nextChildren:
+ case <-errGrpCtx.Done():
+ return nil
+ }
+ }
+
+ var linksToVisit []cid.Cid
+ for _, nextCid := range feedChildren.cids {
+ var shouldVisit bool
+
+ visitlk.Lock()
+ shouldVisit = visit(nextCid)
+ visitlk.Unlock()
+
+ if shouldVisit {
+ linksToVisit = append(linksToVisit, nextCid)
+ }
+ }
+
+ chNodes := dserv.GetMany(errGrpCtx, linksToVisit)
+ for optNode := range chNodes {
+ if optNode.Err != nil {
+ return optNode.Err
+ }
+
+ nextShard, err := NewHamtFromDag(dserv, optNode.Node)
+ if err != nil {
+ return err
+ }
+
+ nextChildren, err := nextShard.walkChildren(processShardValues)
+ if err != nil {
+ return err
+ }
+
+ select {
+ case out <- nextChildren:
+ case <-errGrpCtx.Done():
+ return nil
+ }
+ }
+
+ select {
+ case done <- struct{}{}:
+ case <-errGrpCtx.Done():
+ }
+ }
+ return nil
+ })
+ }
+
+ send := feed
+ var todoQueue []*listCidsAndShards
+ var inProgress int
+
+ next := &listCidsAndShards{
+ shards: []*Shard{root},
+ }
+
+dispatcherLoop:
+ for {
+ select {
+ case send <- next:
+ inProgress++
+ if len(todoQueue) > 0 {
+ next = todoQueue[0]
+ todoQueue = todoQueue[1:]
+ } else {
+ next = nil
+ send = nil
+ }
+ case <-done:
+ inProgress--
+ if inProgress == 0 && next == nil {
+ break dispatcherLoop
+ }
+ case nextNodes := <-out:
+ if next == nil {
+ next = nextNodes
+ send = feed
+ } else {
+ todoQueue = append(todoQueue, nextNodes)
+ }
+ case <-errGrpCtx.Done():
+ break dispatcherLoop
+ }
+ }
+ close(feed)
+ return grp.Wait()
+}
+
+func emitResult(ctx context.Context, linkResults chan<- format.LinkResult, r format.LinkResult) {
+ // make sure that context cancel is processed first
+ // the reason is due to the concurrency of EnumerateChildrenAsync
+ // it's possible for EnumLinksAsync to complete and close the linkResults
+ // channel before this code runs
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
+ select {
+ case linkResults <- r:
+ case <-ctx.Done():
+ }
+}
+
+func (ds *Shard) walkTrie(ctx context.Context, cb func(*Shard) error) error {
+ return ds.childer.each(ctx, func(s *Shard) error {
+ if s.isValueNode() {
+ if err := cb(s); err != nil {
+ return err
+ }
+ } else {
+ if err := s.walkTrie(ctx, cb); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
+
+// swapValue sets the link `value` in the given key, either creating the entry
+// if it didn't exist or overwriting the old one. It returns the old entry (if any).
+func (ds *Shard) swapValue(ctx context.Context, hv *hashBits, key string, value *ipld.Link) (*ipld.Link, error) {
+ idx, err := hv.Next(ds.tableSizeLg2)
+ if err != nil {
+ return nil, err
+ }
+
+ if !ds.childer.has(idx) {
+ // Entry does not exist, create a new one.
+ return nil, ds.childer.insert(key, value, idx)
+ }
+
+ i := ds.childer.sliceIndex(idx)
+ child, err := ds.childer.get(ctx, i)
+ if err != nil {
+ return nil, err
+ }
+
+ if child.isValueNode() {
+ // Leaf node. This is the base case of this recursive function.
+ if child.key == key {
+ // We are in the correct shard (tree level) so we modify this child
+ // and return.
+ oldValue := child.val
+
+ if value == nil { // Remove old entry.
+ return oldValue, ds.childer.rm(idx)
+ }
+
+ child.val = value // Overwrite entry.
+ return oldValue, nil
+ }
+
+ if value == nil {
+ return nil, os.ErrNotExist
+ }
+
+ // We are in the same slot with another entry with a different key
+ // so we need to fork this leaf node into a shard with two childs:
+ // the old entry and the new one being inserted here.
+ // We don't overwrite anything here so we keep:
+ // `oldValue = nil`
+
+ // The child of this shard will now be a new shard. The old child value
+ // will be a child of this new shard (along with the new value being
+ // inserted).
+ grandChild := child
+ child, err = NewShard(ds.dserv, ds.tableSize)
+ if err != nil {
+ return nil, err
+ }
+ child.builder = ds.builder
+ chhv := newConsumedHashBits(grandChild.key, hv.consumed)
+
+ // We explicitly ignore the oldValue returned by the next two insertions
+ // (which will be nil) to highlight there is no overwrite here: they are
+ // done with different keys to a new (empty) shard. (At best this shard
+ // will create new ones until we find different slots for both.)
+ _, err = child.swapValue(ctx, hv, key, value)
+ if err != nil {
+ return nil, err
+ }
+ _, err = child.swapValue(ctx, chhv, grandChild.key, grandChild.val)
+ if err != nil {
+ return nil, err
+ }
+
+ // Replace this leaf node with the new Shard node.
+ ds.childer.set(child, i)
+ return nil, nil
+ } else {
+ // We are in a Shard (internal node). We will recursively call this
+ // function until finding the leaf (the logic of the `if` case above).
+ oldValue, err := child.swapValue(ctx, hv, key, value)
+ if err != nil {
+ return nil, err
+ }
+
+ if value == nil {
+ // We have removed an entry, check if we should remove shards
+ // as well.
+ switch child.childer.length() {
+ case 0:
+ // empty sub-shard, prune it
+ // Note: this shouldnt normally ever happen
+ // in the event of another implementation creates flawed
+ // structures, this will help to normalize them.
+ return oldValue, ds.childer.rm(idx)
+ case 1:
+ // The single child _should_ be a value by
+ // induction. However, we allow for it to be a
+ // shard in case an implementation is broken.
+
+ // Have we loaded the child? Prefer that.
+ schild := child.childer.child(0)
+ if schild != nil {
+ if schild.isValueNode() {
+ ds.childer.set(schild, i)
+ }
+ return oldValue, nil
+ }
+
+ // Otherwise, work with the link.
+ slnk := child.childer.link(0)
+ var lnkType linkType
+ lnkType, err = child.childer.sd.childLinkType(slnk)
+ if err != nil {
+ return nil, err
+ }
+ if lnkType == shardValueLink {
+ // sub-shard with a single value element, collapse it
+ ds.childer.setLink(slnk, i)
+ }
+ return oldValue, nil
+ }
+ }
+
+ return oldValue, nil
+ }
+}
+
+// linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix
+func (ds *Shard) linkNamePrefix(idx int) string {
+ return fmt.Sprintf(ds.prefixPadStr, idx)
+}
+
+// childer wraps the links, children and bitfield
+// and provides basic operation (get, rm, insert and set) of manipulating children.
+// The slices `links` and `children` are always coordinated to have the entries
+// in the same index. A `childIndex` belonging to one of the original `Shard.size`
+// entries corresponds to a `sliceIndex` in `links` and `children` (the conversion
+// is done through `bitfield`).
+type childer struct {
+ sd *Shard
+ dserv ipld.DAGService
+ bitfield bitfield.Bitfield
+
+ // Only one of links/children will be non-nil for every child/link.
+ links []*ipld.Link
+ children []*Shard
+}
+
+const maximumHamtWidth = 1 << 10 // FIXME: Spec this and decide of a correct value
+
+func newChilder(ds ipld.DAGService, size int) (*childer, error) {
+ if size > maximumHamtWidth {
+ return nil, fmt.Errorf("hamt witdh (%d) exceed maximum allowed (%d)", size, maximumHamtWidth)
+ }
+ bf, err := bitfield.NewBitfield(size)
+ if err != nil {
+ return nil, err
+ }
+
+ return &childer{
+ dserv: ds,
+ bitfield: bf,
+ }, nil
+}
+
+func (s *childer) makeChilder(data []byte, links []*ipld.Link) *childer {
+ s.children = make([]*Shard, len(links))
+ s.bitfield.SetBytes(data)
+ if len(links) > 0 {
+ s.links = make([]*ipld.Link, len(links))
+ copy(s.links, links)
+ }
+
+ return s
+}
+
+// Return the `sliceIndex` associated with a child.
+func (s *childer) sliceIndex(childIndex int) (sliceIndex int) {
+ return s.bitfield.OnesBefore(childIndex)
+}
+
+func (s *childer) child(sliceIndex int) *Shard {
+ return s.children[sliceIndex]
+}
+
+func (s *childer) link(sliceIndex int) *ipld.Link {
+ return s.links[sliceIndex]
+}
+
+func (s *childer) insert(key string, lnk *ipld.Link, idx int) error {
+ if lnk == nil {
+ return os.ErrNotExist
+ }
+
+ lnk.Name = s.sd.linkNamePrefix(idx) + key
+ i := s.sliceIndex(idx)
+
+ sd, err := NewShardValue(s.dserv, 256, key, lnk)
+ if err != nil {
+ return err
+ }
+
+ s.children = append(s.children[:i], append([]*Shard{sd}, s.children[i:]...)...)
+ s.links = append(s.links[:i], append([]*ipld.Link{nil}, s.links[i:]...)...)
+ // Add a `nil` placeholder in `links` so the rest of the entries keep the same
+ // index as `children`.
+ s.bitfield.SetBit(idx)
+
+ return nil
+}
+
+func (s *childer) set(sd *Shard, i int) {
+ s.children[i] = sd
+ s.links[i] = nil
+}
+
+func (s *childer) setLink(lnk *ipld.Link, i int) {
+ s.children[i] = nil
+ s.links[i] = lnk
+}
+
+func (s *childer) rm(childIndex int) error {
+ i := s.sliceIndex(childIndex)
+
+ if err := s.check(i); err != nil {
+ return err
+ }
+
+ copy(s.children[i:], s.children[i+1:])
+ s.children = s.children[:len(s.children)-1]
+
+ copy(s.links[i:], s.links[i+1:])
+ s.links = s.links[:len(s.links)-1]
+
+ s.bitfield.UnsetBit(childIndex)
+
+ return nil
+}
+
+// get returns the i'th child of this shard. If it is cached in the
+// children array, it will return it from there. Otherwise, it loads the child
+// node from disk.
+func (s *childer) get(ctx context.Context, sliceIndex int) (*Shard, error) {
+ if err := s.check(sliceIndex); err != nil {
+ return nil, err
+ }
+
+ c := s.child(sliceIndex)
+ if c != nil {
+ return c, nil
+ }
+
+ return s.loadChild(ctx, sliceIndex)
+}
+
+// loadChild reads the i'th child node of this shard from disk and returns it
+// as a 'child' interface
+func (s *childer) loadChild(ctx context.Context, sliceIndex int) (*Shard, error) {
+ lnk := s.link(sliceIndex)
+ lnkLinkType, err := s.sd.childLinkType(lnk)
+ if err != nil {
+ return nil, err
+ }
+
+ var c *Shard
+ if lnkLinkType == shardLink {
+ nd, err := lnk.GetNode(ctx, s.dserv)
+ if err != nil {
+ return nil, err
+ }
+ cds, err := NewHamtFromDag(s.dserv, nd)
+ if err != nil {
+ return nil, err
+ }
+
+ c = cds
+ } else {
+ s, err := s.sd.makeShardValue(lnk)
+ if err != nil {
+ return nil, err
+ }
+ c = s
+ }
+
+ s.set(c, sliceIndex)
+
+ return c, nil
+}
+
+func (s *childer) has(childIndex int) bool {
+ return s.bitfield.Bit(childIndex)
+}
+
+func (s *childer) length() int {
+ return len(s.children)
+}
+
+func (s *childer) each(ctx context.Context, cb func(*Shard) error) error {
+ for i := range s.children {
+ c, err := s.get(ctx, i)
+ if err != nil {
+ return err
+ }
+
+ if err := cb(c); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *childer) check(sliceIndex int) error {
+ if sliceIndex >= len(s.children) || sliceIndex < 0 {
+ return fmt.Errorf("invalid index passed to operate children (likely corrupt bitfield)")
+ }
+
+ if len(s.children) != len(s.links) {
+ return fmt.Errorf("inconsistent lengths between children array and Links array")
+ }
+
+ return nil
+}
diff --git a/unixfs/hamt/hamt_stress_test.go b/unixfs/hamt/hamt_stress_test.go
new file mode 100644
index 000000000..e7983e3a9
--- /dev/null
+++ b/unixfs/hamt/hamt_stress_test.go
@@ -0,0 +1,291 @@
+package hamt
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "os"
+ "testing"
+ "time"
+
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+ ft "github.com/ipfs/boxo/unixfs"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+func getNames(prefix string, count int) []string {
+ out := make([]string, count)
+ for i := 0; i < count; i++ {
+ out[i] = fmt.Sprintf("%s%d", prefix, i)
+ }
+ return out
+}
+
+const (
+ opAdd = iota
+ opDel
+ opFind
+)
+
+type testOp struct {
+ Op int
+ Val string
+}
+
+func stringArrToSet(arr []string) map[string]bool {
+ out := make(map[string]bool)
+ for _, s := range arr {
+ out[s] = true
+ }
+ return out
+}
+
+// generate two different random sets of operations to result in the same
+// ending directory (same set of entries at the end) and execute each of them
+// in turn, then compare to ensure the output is the same on each.
+func TestOrderConsistency(t *testing.T) {
+ seed := time.Now().UnixNano()
+ t.Logf("using seed = %d", seed)
+ ds := mdtest.Mock()
+
+ shardWidth := 1024
+
+ keep := getNames("good", 4000)
+ temp := getNames("tempo", 6000)
+
+ ops := genOpSet(seed, keep, temp)
+ s, err := executeOpSet(t, ds, shardWidth, ops)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = validateOpSetCompletion(t, s, keep, temp)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ops2 := genOpSet(seed+1000, keep, temp)
+ s2, err := executeOpSet(t, ds, shardWidth, ops2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = validateOpSetCompletion(t, s2, keep, temp)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nd2, err := s2.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ k := nd.Cid()
+ k2 := nd2.Cid()
+
+ if !k.Equals(k2) {
+ t.Fatal("got different results: ", k, k2)
+ }
+}
+
+func validateOpSetCompletion(t *testing.T, s *Shard, keep, temp []string) error {
+ ctx := context.TODO()
+ for _, n := range keep {
+ _, err := s.Find(ctx, n)
+ if err != nil {
+ return fmt.Errorf("couldnt find %s: %s", n, err)
+ }
+ }
+
+ for _, n := range temp {
+ _, err := s.Find(ctx, n)
+ if err != os.ErrNotExist {
+ return fmt.Errorf("expected not to find: %s", err)
+ }
+ }
+
+ return nil
+}
+
+func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*Shard, error) {
+ ctx := context.TODO()
+ s, err := NewShard(ds, width)
+ if err != nil {
+ return nil, err
+ }
+
+ e := ft.EmptyDirNode()
+ ds.Add(ctx, e)
+
+ for _, o := range ops {
+ switch o.Op {
+ case opAdd:
+ err := s.Set(ctx, o.Val, e)
+ if err != nil {
+ return nil, fmt.Errorf("inserting %s: %s", o.Val, err)
+ }
+ case opDel:
+ err := s.Remove(ctx, o.Val)
+ if err != nil {
+ return nil, fmt.Errorf("deleting %s: %s", o.Val, err)
+ }
+ case opFind:
+ _, err := s.Find(ctx, o.Val)
+ if err != nil {
+ return nil, fmt.Errorf("finding %s: %s", o.Val, err)
+ }
+ }
+ }
+
+ return s, nil
+}
+
+func genOpSet(seed int64, keep, temp []string) []testOp {
+ tempset := stringArrToSet(temp)
+
+ allnames := append(keep, temp...)
+ shuffle(seed, allnames)
+
+ var todel []string
+
+ var ops []testOp
+
+ for {
+ n := len(allnames) + len(todel)
+ if n == 0 {
+ return ops
+ }
+
+ rn := rand.Intn(n)
+
+ if rn < len(allnames) {
+ next := allnames[0]
+ allnames = allnames[1:]
+ ops = append(ops, testOp{
+ Op: opAdd,
+ Val: next,
+ })
+
+ if tempset[next] {
+ todel = append(todel, next)
+ }
+ } else {
+ shuffle(seed+100, todel)
+ next := todel[0]
+ todel = todel[1:]
+
+ ops = append(ops, testOp{
+ Op: opDel,
+ Val: next,
+ })
+ }
+ }
+}
+
+// executes the given op set with a repl to allow easier debugging
+/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*Shard, error) {
+
+ s, err := NewShard(ds, width)
+ if err != nil {
+ return nil, err
+ }
+
+ e := ft.EmptyDirNode()
+ ds.Add(e)
+ ctx := context.TODO()
+
+ run := 0
+
+ opnames := map[int]string{
+ opAdd: "add",
+ opDel: "del",
+ }
+
+mainloop:
+ for i := 0; i < len(ops); i++ {
+ o := ops[i]
+
+ fmt.Printf("Op %d: %s %s\n", i, opnames[o.Op], o.Val)
+ for run == 0 {
+ cmd := readCommand()
+ parts := strings.Split(cmd, " ")
+ switch parts[0] {
+ case "":
+ run = 1
+ case "find":
+ _, err := s.Find(ctx, parts[1])
+ if err == nil {
+ fmt.Println("success")
+ } else {
+ fmt.Println(err)
+ }
+ case "run":
+ if len(parts) > 1 {
+ n, err := strconv.Atoi(parts[1])
+ if err != nil {
+ panic(err)
+ }
+
+ run = n
+ } else {
+ run = -1
+ }
+ case "lookop":
+ for k = 0; k < len(ops); k++ {
+ if ops[k].Val == parts[1] {
+ fmt.Printf(" Op %d: %s %s\n", k, opnames[ops[k].Op], parts[1])
+ }
+ }
+ case "restart":
+ var err error
+ s, err = NewShard(ds, width)
+ if err != nil {
+ panic(err)
+ }
+ i = -1
+ continue mainloop
+ case "print":
+ nd, err := s.Node()
+ if err != nil {
+ panic(err)
+ }
+ printDag(ds, nd.(*dag.ProtoNode), 0)
+ }
+ }
+ run--
+
+ switch o.Op {
+ case opAdd:
+ err := s.Set(ctx, o.Val, e)
+ if err != nil {
+ return nil, fmt.Errorf("inserting %s: %s", o.Val, err)
+ }
+ case opDel:
+ fmt.Println("deleting: ", o.Val)
+ err := s.Remove(ctx, o.Val)
+ if err != nil {
+ return nil, fmt.Errorf("deleting %s: %s", o.Val, err)
+ }
+ case opFind:
+ _, err := s.Find(ctx, o.Val)
+ if err != nil {
+ return nil, fmt.Errorf("finding %s: %s", o.Val, err)
+ }
+ }
+ }
+
+ return s, nil
+}
+
+func readCommand() string {
+ fmt.Print("> ")
+ scan := bufio.NewScanner(os.Stdin)
+ scan.Scan()
+ return scan.Text()
+}*/
diff --git a/unixfs/hamt/hamt_test.go b/unixfs/hamt/hamt_test.go
new file mode 100644
index 000000000..02c4ed2d3
--- /dev/null
+++ b/unixfs/hamt/hamt_test.go
@@ -0,0 +1,752 @@
+package hamt
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "os"
+ "sort"
+ "testing"
+ "time"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+ ft "github.com/ipfs/boxo/unixfs"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+func shuffle(seed int64, arr []string) {
+ r := rand.New(rand.NewSource(seed))
+ for i := 0; i < len(arr); i++ {
+ a := r.Intn(len(arr))
+ b := r.Intn(len(arr))
+ arr[a], arr[b] = arr[b], arr[a]
+ }
+}
+
+func makeDir(ds ipld.DAGService, size int) ([]string, *Shard, error) {
+ return makeDirWidth(ds, size, 256)
+}
+
+func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *Shard, error) {
+ ctx := context.Background()
+
+ s, err := NewShard(ds, width)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ var dirs []string
+ for i := 0; i < size; i++ {
+ dirs = append(dirs, fmt.Sprintf("DIRNAME%d", i))
+ }
+
+ shuffle(time.Now().UnixNano(), dirs)
+
+ for i := 0; i < len(dirs); i++ {
+ nd := ft.EmptyDirNode()
+ err := ds.Add(ctx, nd)
+ if err != nil {
+ return nil, nil, err
+ }
+ err = s.Set(ctx, dirs[i], nd)
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+
+ return dirs, s, nil
+}
+
+func assertLink(s *Shard, name string, found bool) error {
+ _, err := s.Find(context.Background(), name)
+ switch err {
+ case os.ErrNotExist:
+ if found {
+ return err
+ }
+
+ return nil
+ case nil:
+ if found {
+ return nil
+ }
+
+ return fmt.Errorf("expected not to find link named %s", name)
+ default:
+ return err
+ }
+}
+
+func assertLinksEqual(linksA []*ipld.Link, linksB []*ipld.Link) error {
+
+ if len(linksA) != len(linksB) {
+ return fmt.Errorf("links arrays are different sizes")
+ }
+
+ sort.Stable(dag.LinkSlice(linksA))
+ sort.Stable(dag.LinkSlice(linksB))
+ for i, a := range linksA {
+ b := linksB[i]
+ if a.Name != b.Name {
+ return fmt.Errorf("links names mismatch")
+ }
+
+ if a.Cid.String() != b.Cid.String() {
+ return fmt.Errorf("link hashes dont match")
+ }
+
+ if a.Size != b.Size {
+ return fmt.Errorf("link sizes not the same")
+ }
+ }
+
+ return nil
+}
+
+func assertSerializationWorks(ds ipld.DAGService, s *Shard) error {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ nd, err := s.Node()
+ if err != nil {
+ return err
+ }
+
+ nds, err := NewHamtFromDag(ds, nd)
+ if err != nil {
+ return err
+ }
+
+ linksA, err := s.EnumLinks(ctx)
+ if err != nil {
+ return err
+ }
+
+ linksB, err := nds.EnumLinks(ctx)
+ if err != nil {
+ return err
+ }
+
+ return assertLinksEqual(linksA, linksB)
+}
+
+func TestBasicSet(t *testing.T) {
+ ds := mdtest.Mock()
+ for _, w := range []int{128, 256, 512, 1024} {
+ t.Run(fmt.Sprintf("BasicSet%d", w), func(t *testing.T) {
+ names, s, err := makeDirWidth(ds, 1000, w)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ for _, d := range names {
+ _, err := s.Find(ctx, d)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+func TestDirBuilding(t *testing.T) {
+ ds := mdtest.Mock()
+ _, _ = NewShard(ds, 256)
+
+ _, s, err := makeDir(ds, 200)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ //printDag(ds, nd, 0)
+
+ k := nd.Cid()
+
+ if k.String() != "QmY89TkSEVHykWMHDmyejSWFj9CYNtvzw4UwnT9xbc4Zjc" {
+ t.Fatalf("output didnt match what we expected (got %s)", k.String())
+ }
+}
+
+func TestShardReload(t *testing.T) {
+ ds := mdtest.Mock()
+ _, _ = NewShard(ds, 256)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ _, s, err := makeDir(ds, 200)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nds, err := NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ lnks, err := nds.EnumLinks(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(lnks) != 200 {
+ t.Fatal("not enough links back")
+ }
+
+ _, err = nds.Find(ctx, "DIRNAME50")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Now test roundtrip marshal with no operations
+
+ nds, err = NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ond, err := nds.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ outk := ond.Cid()
+ ndk := nd.Cid()
+
+ if !outk.Equals(ndk) {
+ t.Fatal("roundtrip serialization failed")
+ }
+}
+
+func TestRemoveElems(t *testing.T) {
+ ds := mdtest.Mock()
+ dirs, s, err := makeDir(ds, 500)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ for i := 0; i < 100; i++ {
+ err := s.Remove(ctx, fmt.Sprintf("NOTEXIST%d", rand.Int()))
+ if err != os.ErrNotExist {
+ t.Fatal("shouldnt be able to remove things that don't exist")
+ }
+ }
+
+ for _, d := range dirs {
+ _, err := s.Find(ctx, d)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ shuffle(time.Now().UnixNano(), dirs)
+
+ for _, d := range dirs {
+ err := s.Remove(ctx, d)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(nd.Links()) > 0 {
+ t.Fatal("shouldnt have any links here")
+ }
+
+ err = s.Remove(ctx, "doesnt exist")
+ if err != os.ErrNotExist {
+ t.Fatal("expected error does not exist")
+ }
+}
+
+func TestRemoveAfterMarshal(t *testing.T) {
+ ds := mdtest.Mock()
+ dirs, s, err := makeDir(ds, 500)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ s, err = NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ctx := context.Background()
+
+ shuffle(time.Now().UnixNano(), dirs)
+
+ for i, d := range dirs {
+ err := s.Remove(ctx, d)
+ if err != nil {
+ t.Fatalf("%d/%d: %s", i, len(dirs), err)
+ }
+ }
+
+ nd, err = s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(nd.Links()) > 0 {
+ t.Fatal("shouldnt have any links here")
+ }
+
+ err = s.Remove(ctx, "doesnt exist")
+ if err != os.ErrNotExist {
+ t.Fatal("expected error does not exist")
+ }
+}
+
+func TestSetAfterMarshal(t *testing.T) {
+ ds := mdtest.Mock()
+ _, s, err := makeDir(ds, 300)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nds, err := NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i := 0; i < 100; i++ {
+ empty := ft.EmptyDirNode()
+ err := nds.Set(ctx, fmt.Sprintf("moredirs%d", i), empty)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ nd, err = nds.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+ nds, err = NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ links, err := nds.EnumLinks(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(links) != 400 {
+ t.Fatal("expected 400 links")
+ }
+
+ err = assertSerializationWorks(ds, nds)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestEnumLinksAsync(t *testing.T) {
+ ds := mdtest.Mock()
+ _, s, err := makeDir(ds, 300)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nds, err := NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ linksA, err := nds.EnumLinks(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ linkResults := nds.EnumLinksAsync(ctx)
+
+ var linksB []*ipld.Link
+
+ for linkResult := range linkResults {
+ if linkResult.Err != nil {
+ t.Fatal(linkResult.Err)
+ }
+ linksB = append(linksB, linkResult.Link)
+ }
+
+ err = assertLinksEqual(linksA, linksB)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestDuplicateAddShard(t *testing.T) {
+ ds := mdtest.Mock()
+ dir, _ := NewShard(ds, 256)
+ nd := new(dag.ProtoNode)
+ ctx := context.Background()
+
+ err := dir.Set(ctx, "test", nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = dir.Set(ctx, "test", nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ node, err := dir.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+ dir, err = NewHamtFromDag(ds, node)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ lnks, err := dir.EnumLinks(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(lnks) != 1 {
+ t.Fatal("expected only one link")
+ }
+}
+
+// fix https://github.com/ipfs/kubo/issues/9063
+func TestSetLink(t *testing.T) {
+ ds := mdtest.Mock()
+ dir, _ := NewShard(ds, 256)
+ _, s, err := makeDir(ds, 300)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ lnk, err := s.Link()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ctx := context.Background()
+
+ err = dir.SetLink(ctx, "test", lnk)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(dir.childer.children) != 1 {
+ t.Fatal("no child")
+ }
+
+ for _, sh := range dir.childer.children {
+ if sh.childer == nil {
+ t.Fatal("no childer on shard")
+ }
+ }
+}
+
+func TestLoadFailsFromNonShard(t *testing.T) {
+ ds := mdtest.Mock()
+ nd := ft.EmptyDirNode()
+
+ _, err := NewHamtFromDag(ds, nd)
+ if err == nil {
+ t.Fatal("expected dir shard creation to fail when given normal directory")
+ }
+
+ nd = new(dag.ProtoNode)
+
+ _, err = NewHamtFromDag(ds, nd)
+ if err == nil {
+ t.Fatal("expected dir shard creation to fail when given normal directory")
+ }
+}
+
+func TestFindNonExisting(t *testing.T) {
+ ds := mdtest.Mock()
+ _, s, err := makeDir(ds, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ for i := 0; i < 200; i++ {
+ _, err := s.Find(ctx, fmt.Sprintf("notfound%d", i))
+ if err != os.ErrNotExist {
+ t.Fatal("expected ErrNotExist")
+ }
+ }
+}
+
+func TestRemoveElemsAfterMarshal(t *testing.T) {
+ ds := mdtest.Mock()
+ dirs, s, err := makeDir(ds, 30)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ sort.Strings(dirs)
+
+ err = s.Remove(ctx, dirs[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := s.Find(ctx, dirs[0])
+ if err == nil {
+ t.Fatal("expected error, got: ", out)
+ }
+
+ nd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nds, err := NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = nds.Find(ctx, dirs[0])
+ if err == nil {
+ t.Fatal("expected not to find ", dirs[0])
+ }
+
+ for _, d := range dirs[1:] {
+ _, err := nds.Find(ctx, d)
+ if err != nil {
+ t.Fatal("could not find expected link after unmarshaling")
+ }
+ }
+
+ for _, d := range dirs[1:] {
+ err := nds.Remove(ctx, d)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ nd, err = nds.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+ nds, err = NewHamtFromDag(ds, nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ links, err := nds.EnumLinks(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(links) != 0 {
+ t.Fatal("expected all links to be removed")
+ }
+
+ err = assertSerializationWorks(ds, nds)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestBitfieldIndexing(t *testing.T) {
+ ds := mdtest.Mock()
+ s, _ := NewShard(ds, 256)
+
+ set := func(i int) {
+ s.childer.bitfield.SetBit(i)
+ }
+
+ assert := func(i int, val int) {
+ if s.childer.sliceIndex(i) != val {
+ t.Fatalf("expected index %d to be %d", i, val)
+ }
+ }
+
+ assert(50, 0)
+ set(4)
+ set(5)
+ set(60)
+
+ assert(10, 2)
+ set(3)
+ assert(10, 3)
+ assert(1, 0)
+
+ assert(100, 4)
+ set(50)
+ assert(45, 3)
+ set(100)
+ assert(100, 5)
+}
+
+// test adding a sharded directory node as the child of another directory node.
+// if improperly implemented, the parent hamt may assume the child is a part of
+// itself.
+func TestSetHamtChild(t *testing.T) {
+ ctx := context.Background()
+
+ ds := mdtest.Mock()
+ s, _ := NewShard(ds, 256)
+
+ e := ft.EmptyDirNode()
+ ds.Add(ctx, e)
+
+ err := s.Set(ctx, "bar", e)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ snd, err := s.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, ns, err := makeDir(ds, 50)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = ns.Set(ctx, "foo", snd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nsnd, err := ns.Node()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ hs, err := NewHamtFromDag(ds, nsnd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = assertLink(hs, "bar", false)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = assertLink(hs, "foo", true)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func BenchmarkHAMTWalk(b *testing.B) {
+ ctx := context.Background()
+
+ ds := mdtest.Mock()
+ sh, _ := NewShard(ds, 256)
+ nd, err := sh.Node()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ err = ds.Add(ctx, nd)
+ if err != nil {
+ b.Fatal(err)
+ }
+ ds.Add(ctx, ft.EmptyDirNode())
+
+ s, err := NewHamtFromDag(ds, nd)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ for j := 0; j < 1000; j++ {
+ err = s.Set(ctx, fmt.Sprintf("%d", j), ft.EmptyDirNode())
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+
+ for i := 0; i < b.N; i++ {
+ cnt := 0
+ err = s.ForEachLink(ctx, func(l *ipld.Link) error {
+ cnt++
+ return nil
+ })
+ if err != nil {
+ b.Fatal(err)
+ }
+ if cnt < 1000 {
+ b.Fatal("expected 100 children")
+ }
+ }
+}
+
+func BenchmarkHAMTSet(b *testing.B) {
+ ctx := context.Background()
+
+ ds := mdtest.Mock()
+ sh, _ := NewShard(ds, 256)
+ nd, err := sh.Node()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ err = ds.Add(ctx, nd)
+ if err != nil {
+ b.Fatal(err)
+ }
+ ds.Add(ctx, ft.EmptyDirNode())
+
+ for i := 0; i < b.N; i++ {
+ s, err := NewHamtFromDag(ds, nd)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ err = s.Set(context.TODO(), fmt.Sprint(i), ft.EmptyDirNode())
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ out, err := s.Node()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ nd = out
+ }
+}
+
+func TestHamtBadSize(t *testing.T) {
+ for _, size := range [...]int{-8, 7, 2, 1337, 1024 + 8, -3} {
+ _, err := NewShard(nil, size)
+ if err == nil {
+ t.Errorf("should have failed to construct hamt with bad size: %d", size)
+ }
+ }
+}
diff --git a/unixfs/hamt/util.go b/unixfs/hamt/util.go
new file mode 100644
index 000000000..2216a217f
--- /dev/null
+++ b/unixfs/hamt/util.go
@@ -0,0 +1,80 @@
+package hamt
+
+import (
+ "fmt"
+ "math/bits"
+
+ "github.com/ipfs/boxo/unixfs/internal"
+
+ "github.com/spaolacci/murmur3"
+)
+
+// hashBits is a helper that allows the reading of the 'next n bits' as an integer.
+type hashBits struct {
+ b []byte
+ consumed int
+}
+
+func newHashBits(val string) *hashBits {
+ return &hashBits{b: internal.HAMTHashFunction([]byte(val))}
+}
+
+func newConsumedHashBits(val string, consumed int) *hashBits {
+ hv := &hashBits{b: internal.HAMTHashFunction([]byte(val))}
+ hv.consumed = consumed
+ return hv
+}
+
+func mkmask(n int) byte {
+ return (1 << uint(n)) - 1
+}
+
+// Next returns the next 'i' bits of the hashBits value as an integer, or an
+// error if there aren't enough bits.
+func (hb *hashBits) Next(i int) (int, error) {
+ if hb.consumed+i > len(hb.b)*8 {
+ return 0, fmt.Errorf("sharded directory too deep")
+ }
+ return hb.next(i), nil
+}
+
+func (hb *hashBits) next(i int) int {
+ curbi := hb.consumed / 8
+ leftb := 8 - (hb.consumed % 8)
+
+ curb := hb.b[curbi]
+ if i == leftb {
+ out := int(mkmask(i) & curb)
+ hb.consumed += i
+ return out
+ } else if i < leftb {
+ a := curb & mkmask(leftb) // mask out the high bits we don't want
+ b := a & ^mkmask(leftb-i) // mask out the low bits we don't want
+ c := b >> uint(leftb-i) // shift whats left down
+ hb.consumed += i
+ return int(c)
+ } else {
+ out := int(mkmask(leftb) & curb)
+ out <<= uint(i - leftb)
+ hb.consumed += leftb
+ out += hb.next(i - leftb)
+ return out
+ }
+}
+
+func Logtwo(v int) (int, error) {
+ if v <= 0 {
+ return 0, fmt.Errorf("hamt size should be a power of two")
+ }
+ lg2 := bits.TrailingZeros(uint(v))
+ if 1<= 0; coff -= 4096 {
+ t.Log(coff)
+ n, err := rs.Seek(coff, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != coff {
+ t.Fatal("wasnt able to seek to the right position")
+ }
+ nread, err := rs.Read(out[coff : coff+4096])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if nread != 4096 {
+ t.Fatal("didnt read the correct number of bytes")
+ }
+ }
+
+ err = arrComp(out, should)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/unixfs/importer/balanced/builder.go b/unixfs/importer/balanced/builder.go
new file mode 100644
index 000000000..6b5a7e074
--- /dev/null
+++ b/unixfs/importer/balanced/builder.go
@@ -0,0 +1,267 @@
+// Package balanced provides methods to build balanced DAGs, which are generalistic
+// DAGs in which all leaves (nodes representing chunks of data) are at the same
+// distance from the root. Nodes can have only a maximum number of children; to be
+// able to store more leaf data nodes balanced DAGs are extended by increasing its
+// depth (and having more intermediary nodes).
+//
+// Internal nodes are always represented by UnixFS nodes (of type `File`) encoded
+// inside DAG nodes (see the `go-unixfs` package for details of UnixFS). In
+// contrast, leaf nodes with data have multiple possible representations: UnixFS
+// nodes as above, raw nodes with just the file data (no format) and Filestore
+// nodes (that directly link to the file on disk using a format stored on a raw
+// node, see the `go-ipfs/filestore` package for details of Filestore.)
+//
+// In the case the entire file fits into just one node it will be formatted as a
+// (single) leaf node (without parent) with the possible representations already
+// mentioned. This is the only scenario where the root can be of a type different
+// that the UnixFS node.
+//
+// Notes:
+//
+// 1. In the implementation. `FSNodeOverDag` structure is used for representing
+// the UnixFS node encoded inside the DAG node.
+// (see https://github.com/ipfs/go-ipfs/pull/5118.)
+//
+// 2. `TFile` is used for backwards-compatibility. It was a bug causing the leaf
+// nodes to be generated with this type instead of `TRaw`. The former one
+// should be used (like the trickle builder does).
+// (See https://github.com/ipfs/go-ipfs/pull/5120.)
+//
+// +-------------+
+// | Root 4 |
+// +-------------+
+// |
+// +--------------------------+----------------------------+
+// | |
+// +-------------+ +-------------+
+// | Node 2 | | Node 5 |
+// +-------------+ +-------------+
+// | |
+// +-------------+-------------+ +-------------+
+// | | |
+// +-------------+ +-------------+ +-------------+
+// | Node 1 | | Node 3 | | Node 6 |
+// +-------------+ +-------------+ +-------------+
+// | | |
+// +------+------+ +------+------+ +------+
+// | | | | |
+// +=========+ +=========+ +=========+ +=========+ +=========+
+// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 | | Chunk 5 |
+// +=========+ +=========+ +=========+ +=========+ +=========+
+package balanced
+
+import (
+ "errors"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ h "github.com/ipfs/boxo/unixfs/importer/helpers"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// Layout builds a balanced DAG layout. In a balanced DAG of depth 1, leaf nodes
+// with data are added to a single `root` until the maximum number of links is
+// reached. Then, to continue adding more data leaf nodes, a `newRoot` is created
+// pointing to the old `root` (which will now become and intermediary node),
+// increasing the depth of the DAG to 2. This will increase the maximum number of
+// data leaf nodes the DAG can have (`Maxlinks() ^ depth`). The `fillNodeRec`
+// function will add more intermediary child nodes to `newRoot` (which already has
+// `root` as child) that in turn will have leaf nodes with data added to them.
+// After that process is completed (the maximum number of links is reached),
+// `fillNodeRec` will return and the loop will be repeated: the `newRoot` created
+// will become the old `root` and a new root will be created again to increase the
+// depth of the DAG. The process is repeated until there is no more data to add
+// (i.e. the DagBuilderHelper’s Done() function returns true).
+//
+// The nodes are filled recursively, so the DAG is built from the bottom up. Leaf
+// nodes are created first using the chunked file data and its size. The size is
+// then bubbled up to the parent (internal) node, which aggregates all the sizes of
+// its children and bubbles that combined size up to its parent, and so on up to
+// the root. This way, a balanced DAG acts like a B-tree when seeking to a byte
+// offset in the file the graph represents: each internal node uses the file size
+// of its children as an index when seeking.
+//
+// `Layout` creates a root and hands it off to be filled:
+//
+// +-------------+
+// | Root 1 |
+// +-------------+
+// |
+// ( fillNodeRec fills in the )
+// ( chunks on the root. )
+// |
+// +------+------+
+// | |
+// + - - - - + + - - - - +
+// | Chunk 1 | | Chunk 2 |
+// + - - - - + + - - - - +
+//
+// ↓
+// When the root is full but there's more data...
+// ↓
+//
+// +-------------+
+// | Root 1 |
+// +-------------+
+// |
+// +------+------+
+// | |
+// +=========+ +=========+ + - - - - +
+// | Chunk 1 | | Chunk 2 | | Chunk 3 |
+// +=========+ +=========+ + - - - - +
+//
+// ↓
+// ...Layout's job is to create a new root.
+// ↓
+//
+// +-------------+
+// | Root 2 |
+// +-------------+
+// |
+// +-------------+ - - - - - - - - +
+// | |
+// +-------------+ ( fillNodeRec creates the )
+// | Node 1 | ( branch that connects )
+// +-------------+ ( "Root 2" to "Chunk 3." )
+// | |
+// +------+------+ + - - - - -+
+// | | |
+// +=========+ +=========+ + - - - - +
+// | Chunk 1 | | Chunk 2 | | Chunk 3 |
+// +=========+ +=========+ + - - - - +
+func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
+ if db.Done() {
+ // No data, return just an empty node.
+ root, err := db.NewLeafNode(nil, ft.TFile)
+ if err != nil {
+ return nil, err
+ }
+ // This works without Filestore support (`ProcessFileStore`).
+ // TODO: Why? Is there a test case missing?
+
+ return root, db.Add(root)
+ }
+
+ // The first `root` will be a single leaf node with data
+ // (corner case), after that subsequent `root` nodes will
+ // always be internal nodes (with a depth > 0) that can
+ // be handled by the loop.
+ root, fileSize, err := db.NewLeafDataNode(ft.TFile)
+ if err != nil {
+ return nil, err
+ }
+
+ // Each time a DAG of a certain `depth` is filled (because it
+ // has reached its maximum capacity of `db.Maxlinks()` per node)
+ // extend it by making it a sub-DAG of a bigger DAG with `depth+1`.
+ for depth := 1; !db.Done(); depth++ {
+
+ // Add the old `root` as a child of the `newRoot`.
+ newRoot := db.NewFSNodeOverDag(ft.TFile)
+ err = newRoot.AddChild(root, fileSize, db)
+ if err != nil {
+ return nil, err
+ }
+
+ // Fill the `newRoot` (that has the old `root` already as child)
+ // and make it the current `root` for the next iteration (when
+ // it will become "old").
+ root, fileSize, err = fillNodeRec(db, newRoot, depth)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return root, db.Add(root)
+}
+
+// fillNodeRec will "fill" the given internal (non-leaf) `node` with data by
+// adding child nodes to it, either leaf data nodes (if `depth` is 1) or more
+// internal nodes with higher depth (and calling itself recursively on them
+// until *they* are filled with data). The data to fill the node with is
+// provided by DagBuilderHelper.
+//
+// `node` represents a (sub-)DAG root that is being filled. If called recursively,
+// it is `nil`, a new node is created. If it has been called from `Layout` (see
+// diagram below) it points to the new root (that increases the depth of the DAG),
+// it already has a child (the old root). New children will be added to this new
+// root, and those children will in turn be filled (calling `fillNodeRec`
+// recursively).
+//
+// +-------------+
+// | `node` |
+// | (new root) |
+// +-------------+
+// |
+// +-------------+ - - - - - - + - - - - - - - - - - - +
+// | | |
+// +--------------+ + - - - - - + + - - - - - +
+// | (old root) | | new child | | |
+// +--------------+ + - - - - - + + - - - - - +
+// | | |
+// +------+------+ + - - + - - - +
+// | | | |
+// +=========+ +=========+ + - - - - + + - - - - +
+// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 |
+// +=========+ +=========+ + - - - - + + - - - - +
+//
+// The `node` to be filled uses the `FSNodeOverDag` abstraction that allows adding
+// child nodes without packing/unpacking the UnixFS layer node (having an internal
+// `ft.FSNode` cache).
+//
+// It returns the `ipld.Node` representation of the passed `node` filled with
+// children and the `nodeFileSize` with the total size of the file chunk (leaf)
+// nodes stored under this node (parent nodes store this to enable efficient
+// seeking through the DAG when reading data later).
+//
+// warning: **children** pinned indirectly, but input node IS NOT pinned.
+func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (filledNode ipld.Node, nodeFileSize uint64, err error) {
+ if depth < 1 {
+ return nil, 0, errors.New("attempt to fillNode at depth < 1")
+ }
+
+ if node == nil {
+ node = db.NewFSNodeOverDag(ft.TFile)
+ }
+
+ // Child node created on every iteration to add to parent `node`.
+ // It can be a leaf node or another internal node.
+ var childNode ipld.Node
+ // File size from the child node needed to update the `FSNode`
+ // in `node` when adding the child.
+ var childFileSize uint64
+
+ // While we have room and there is data available to be added.
+ for node.NumChildren() < db.Maxlinks() && !db.Done() {
+
+ if depth == 1 {
+ // Base case: add leaf node with data.
+ childNode, childFileSize, err = db.NewLeafDataNode(ft.TFile)
+ if err != nil {
+ return nil, 0, err
+ }
+ } else {
+ // Recursion case: create an internal node to in turn keep
+ // descending in the DAG and adding child nodes to it.
+ childNode, childFileSize, err = fillNodeRec(db, nil, depth-1)
+ if err != nil {
+ return nil, 0, err
+ }
+ }
+
+ err = node.AddChild(childNode, childFileSize, db)
+ if err != nil {
+ return nil, 0, err
+ }
+ }
+
+ nodeFileSize = node.FileSize()
+
+ // Get the final `dag.ProtoNode` with the `FSNode` data encoded inside.
+ filledNode, err = node.Commit()
+ if err != nil {
+ return nil, 0, err
+ }
+
+ return filledNode, nodeFileSize, nil
+}
diff --git a/unixfs/importer/helpers/dagbuilder.go b/unixfs/importer/helpers/dagbuilder.go
new file mode 100644
index 000000000..920c053da
--- /dev/null
+++ b/unixfs/importer/helpers/dagbuilder.go
@@ -0,0 +1,399 @@
+package helpers
+
+import (
+ "context"
+ "errors"
+ "io"
+ "os"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ pb "github.com/ipfs/boxo/unixfs/pb"
+
+ chunker "github.com/ipfs/boxo/chunker"
+ "github.com/ipfs/boxo/files"
+ pi "github.com/ipfs/boxo/filestore/posinfo"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+var ErrMissingFsRef = errors.New("missing file path or URL, can't create filestore reference")
+
+// DagBuilderHelper wraps together a bunch of objects needed to
+// efficiently create unixfs dag trees
+type DagBuilderHelper struct {
+ dserv ipld.DAGService
+ spl chunker.Splitter
+ recvdErr error
+ rawLeaves bool
+ nextData []byte // the next item to return.
+ maxlinks int
+ cidBuilder cid.Builder
+
+ // Filestore support variables.
+ // ----------------------------
+ // TODO: Encapsulate in `FilestoreNode` (which is basically what they are).
+ //
+ // Besides having the path this variable (if set) is used as a flag
+ // to indicate that Filestore should be used.
+ fullPath string
+ stat os.FileInfo
+ // Keeps track of the current file size added to the DAG (used in
+ // the balanced builder). It is assumed that the `DagBuilderHelper`
+ // is not reused to construct another DAG, but a new one (with a
+ // zero `offset`) is created.
+ offset uint64
+}
+
+// DagBuilderParams wraps configuration options to create a DagBuilderHelper
+// from a chunker.Splitter.
+type DagBuilderParams struct {
+ // Maximum number of links per intermediate node
+ Maxlinks int
+
+ // RawLeaves signifies that the importer should use raw ipld nodes as leaves
+ // instead of using the unixfs TRaw type
+ RawLeaves bool
+
+ // CID Builder to use if set
+ CidBuilder cid.Builder
+
+ // DAGService to write blocks to (required)
+ Dagserv ipld.DAGService
+
+ // NoCopy signals to the chunker that it should track fileinfo for
+ // filestore adds
+ NoCopy bool
+}
+
+// New generates a new DagBuilderHelper from the given params and a given
+// chunker.Splitter as data source.
+func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error) {
+ db := &DagBuilderHelper{
+ dserv: dbp.Dagserv,
+ spl: spl,
+ rawLeaves: dbp.RawLeaves,
+ cidBuilder: dbp.CidBuilder,
+ maxlinks: dbp.Maxlinks,
+ }
+ if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok {
+ db.fullPath = fi.AbsPath()
+ db.stat = fi.Stat()
+ }
+
+ if dbp.NoCopy && db.fullPath == "" { // Enforce NoCopy
+ return nil, ErrMissingFsRef
+ }
+
+ return db, nil
+}
+
+// prepareNext consumes the next item from the splitter and puts it
+// in the nextData field. it is idempotent-- if nextData is full
+// it will do nothing.
+func (db *DagBuilderHelper) prepareNext() {
+ // if we already have data waiting to be consumed, we're ready
+ if db.nextData != nil || db.recvdErr != nil {
+ return
+ }
+
+ db.nextData, db.recvdErr = db.spl.NextBytes()
+ if db.recvdErr == io.EOF {
+ db.recvdErr = nil
+ }
+}
+
+// Done returns whether or not we're done consuming the incoming data.
+func (db *DagBuilderHelper) Done() bool {
+ // ensure we have an accurate perspective on data
+ // as `done` this may be called before `next`.
+ db.prepareNext() // idempotent
+ if db.recvdErr != nil {
+ return false
+ }
+ return db.nextData == nil
+}
+
+// Next returns the next chunk of data to be inserted into the dag
+// if it returns nil, that signifies that the stream is at an end, and
+// that the current building operation should finish.
+func (db *DagBuilderHelper) Next() ([]byte, error) {
+ db.prepareNext() // idempotent
+ d := db.nextData
+ db.nextData = nil // signal we've consumed it
+ if db.recvdErr != nil {
+ return nil, db.recvdErr
+ }
+ return d, nil
+}
+
+// GetDagServ returns the dagservice object this Helper is using
+func (db *DagBuilderHelper) GetDagServ() ipld.DAGService {
+ return db.dserv
+}
+
+// GetCidBuilder returns the internal `cid.CidBuilder` set in the builder.
+func (db *DagBuilderHelper) GetCidBuilder() cid.Builder {
+ return db.cidBuilder
+}
+
+// NewLeafNode creates a leaf node filled with data. If rawLeaves is
+// defined then a raw leaf will be returned. Otherwise, it will create
+// and return `FSNodeOverDag` with `fsNodeType`.
+func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) {
+ if len(data) > BlockSizeLimit {
+ return nil, ErrSizeLimitExceeded
+ }
+
+ if db.rawLeaves {
+ // Encapsulate the data in a raw node.
+ if db.cidBuilder == nil {
+ return dag.NewRawNode(data), nil
+ }
+ rawnode, err := dag.NewRawNodeWPrefix(data, db.cidBuilder)
+ if err != nil {
+ return nil, err
+ }
+ return rawnode, nil
+ }
+
+ // Encapsulate the data in UnixFS node (instead of a raw node).
+ fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType)
+ fsNodeOverDag.SetFileData(data)
+ node, err := fsNodeOverDag.Commit()
+ if err != nil {
+ return nil, err
+ }
+ // TODO: Encapsulate this sequence of calls into a function that
+ // just returns the final `ipld.Node` avoiding going through
+ // `FSNodeOverDag`.
+
+ return node, nil
+}
+
+// FillNodeLayer will add datanodes as children to the give node until
+// it is full in this layer or no more data.
+// NOTE: This function creates raw data nodes so it only works
+// for the `trickle.Layout`.
+func (db *DagBuilderHelper) FillNodeLayer(node *FSNodeOverDag) error {
+
+ // while we have room AND we're not done
+ for node.NumChildren() < db.maxlinks && !db.Done() {
+ child, childFileSize, err := db.NewLeafDataNode(ft.TRaw)
+ if err != nil {
+ return err
+ }
+
+ if err := node.AddChild(child, childFileSize, db); err != nil {
+ return err
+ }
+ }
+ // TODO: Do we need to commit here? The caller who created the
+ // `FSNodeOverDag` should be in charge of that.
+ _, err := node.Commit()
+ return err
+}
+
+// NewLeafDataNode builds the `node` with the data obtained from the
+// Splitter with the given constraints (BlockSizeLimit, RawLeaves)
+// specified when creating the DagBuilderHelper. It returns
+// `ipld.Node` with the `dataSize` (that will be used to keep track of
+// the DAG file size). The size of the data is computed here because
+// after that it will be hidden by `NewLeafNode` inside a generic
+// `ipld.Node` representation.
+func (db *DagBuilderHelper) NewLeafDataNode(fsNodeType pb.Data_DataType) (node ipld.Node, dataSize uint64, err error) {
+ fileData, err := db.Next()
+ if err != nil {
+ return nil, 0, err
+ }
+ dataSize = uint64(len(fileData))
+
+ // Create a new leaf node containing the file chunk data.
+ node, err = db.NewLeafNode(fileData, fsNodeType)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ // Convert this leaf to a `FilestoreNode` if needed.
+ node = db.ProcessFileStore(node, dataSize)
+
+ return node, dataSize, nil
+}
+
+// ProcessFileStore generates, if Filestore is being used, the
+// `FilestoreNode` representation of the `ipld.Node` that
+// contains the file data. If Filestore is not being used just
+// return the same node to continue with its addition to the DAG.
+//
+// The `db.offset` is updated at this point (instead of when
+// `NewLeafDataNode` is called, both work in tandem but the
+// offset is more related to this function).
+func (db *DagBuilderHelper) ProcessFileStore(node ipld.Node, dataSize uint64) ipld.Node {
+ // Check if Filestore is being used.
+ if db.fullPath != "" {
+ // Check if the node is actually a raw node (needed for
+ // Filestore support).
+ if _, ok := node.(*dag.RawNode); ok {
+ fn := &pi.FilestoreNode{
+ Node: node,
+ PosInfo: &pi.PosInfo{
+ Offset: db.offset,
+ FullPath: db.fullPath,
+ Stat: db.stat,
+ },
+ }
+
+ // Update `offset` with the size of the data generated by `db.Next`.
+ db.offset += dataSize
+
+ return fn
+ }
+ }
+
+ // Filestore is not used, return the same `node` argument.
+ return node
+}
+
+// Add inserts the given node in the DAGService.
+func (db *DagBuilderHelper) Add(node ipld.Node) error {
+ return db.dserv.Add(context.TODO(), node)
+}
+
+// Maxlinks returns the configured maximum number for links
+// for nodes built with this helper.
+func (db *DagBuilderHelper) Maxlinks() int {
+ return db.maxlinks
+}
+
+// FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a
+// `dag.ProtoNode`. Instead of just having a single `ipld.Node` that
+// would need to be constantly (un)packed to access and modify its
+// internal `FSNode` in the process of creating a UnixFS DAG, this
+// structure stores an `FSNode` cache to manipulate it (add child nodes)
+// directly , and only when the node has reached its final (immutable) state
+// (signaled by calling `Commit()`) is it committed to a single (indivisible)
+// `ipld.Node`.
+//
+// It is used mainly for internal (non-leaf) nodes, and for some
+// representations of data leaf nodes (that don't use raw nodes or
+// Filestore).
+//
+// It aims to replace the `UnixfsNode` structure which encapsulated too
+// many possible node state combinations.
+//
+// TODO: Revisit the name.
+type FSNodeOverDag struct {
+ dag *dag.ProtoNode
+ file *ft.FSNode
+}
+
+// NewFSNodeOverDag creates a new `dag.ProtoNode` and `ft.FSNode`
+// decoupled from one onther (and will continue in that way until
+// `Commit` is called), with `fsNodeType` specifying the type of
+// the UnixFS layer node (either `File` or `Raw`).
+func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag {
+ node := new(FSNodeOverDag)
+ node.dag = new(dag.ProtoNode)
+ node.dag.SetCidBuilder(db.GetCidBuilder())
+
+ node.file = ft.NewFSNode(fsNodeType)
+
+ return node
+}
+
+// NewFSNFromDag reconstructs a FSNodeOverDag node from a given dag node
+func (db *DagBuilderHelper) NewFSNFromDag(nd *dag.ProtoNode) (*FSNodeOverDag, error) {
+ return NewFSNFromDag(nd)
+}
+
+// NewFSNFromDag reconstructs a FSNodeOverDag node from a given dag node
+func NewFSNFromDag(nd *dag.ProtoNode) (*FSNodeOverDag, error) {
+ mb, err := ft.FSNodeFromBytes(nd.Data())
+ if err != nil {
+ return nil, err
+ }
+
+ return &FSNodeOverDag{
+ dag: nd,
+ file: mb,
+ }, nil
+}
+
+// AddChild adds a `child` `ipld.Node` to both node layers. The
+// `dag.ProtoNode` creates a link to the child node while the
+// `ft.FSNode` stores its file size (that is, not the size of the
+// node but the size of the file data that it is storing at the
+// UnixFS layer). The child is also stored in the `DAGService`.
+func (n *FSNodeOverDag) AddChild(child ipld.Node, fileSize uint64, db *DagBuilderHelper) error {
+ err := n.dag.AddNodeLink("", child)
+ if err != nil {
+ return err
+ }
+
+ n.file.AddBlockSize(fileSize)
+
+ return db.Add(child)
+}
+
+// RemoveChild deletes the child node at the given index.
+func (n *FSNodeOverDag) RemoveChild(index int, dbh *DagBuilderHelper) {
+ n.file.RemoveBlockSize(index)
+ n.dag.SetLinks(append(n.dag.Links()[:index], n.dag.Links()[index+1:]...))
+}
+
+// Commit unifies (resolves) the cache nodes into a single `ipld.Node`
+// that represents them: the `ft.FSNode` is encoded inside the
+// `dag.ProtoNode`.
+//
+// TODO: Make it read-only after committing, allow to commit only once.
+func (n *FSNodeOverDag) Commit() (ipld.Node, error) {
+ fileData, err := n.file.GetBytes()
+ if err != nil {
+ return nil, err
+ }
+ n.dag.SetData(fileData)
+
+ return n.dag, nil
+}
+
+// NumChildren returns the number of children of the `ft.FSNode`.
+func (n *FSNodeOverDag) NumChildren() int {
+ return n.file.NumChildren()
+}
+
+// FileSize returns the `Filesize` attribute from the underlying
+// representation of the `ft.FSNode`.
+func (n *FSNodeOverDag) FileSize() uint64 {
+ return n.file.FileSize()
+}
+
+// SetFileData stores the `fileData` in the `ft.FSNode`. It
+// should be used only when `FSNodeOverDag` represents a leaf
+// node (internal nodes don't carry data, just file sizes).
+func (n *FSNodeOverDag) SetFileData(fileData []byte) {
+ n.file.SetData(fileData)
+}
+
+// GetDagNode fills out the proper formatting for the FSNodeOverDag node
+// inside of a DAG node and returns the dag node.
+// TODO: Check if we have committed (passed the UnixFS information
+// to the DAG layer) before returning this.
+func (n *FSNodeOverDag) GetDagNode() (ipld.Node, error) {
+ return n.dag, nil
+}
+
+// GetChild gets the ith child of this node from the given DAGService.
+func (n *FSNodeOverDag) GetChild(ctx context.Context, i int, ds ipld.DAGService) (*FSNodeOverDag, error) {
+ nd, err := n.dag.Links()[i].GetNode(ctx, ds)
+ if err != nil {
+ return nil, err
+ }
+
+ pbn, ok := nd.(*dag.ProtoNode)
+ if !ok {
+ return nil, dag.ErrNotProtobuf
+ }
+
+ return NewFSNFromDag(pbn)
+}
diff --git a/unixfs/importer/helpers/helpers.go b/unixfs/importer/helpers/helpers.go
new file mode 100644
index 000000000..20cb598e6
--- /dev/null
+++ b/unixfs/importer/helpers/helpers.go
@@ -0,0 +1,31 @@
+package helpers
+
+import (
+ "fmt"
+)
+
+// BlockSizeLimit specifies the maximum size an imported block can have.
+var BlockSizeLimit = 1048576 // 1 MB
+
+// rough estimates on expected sizes
+var roughLinkBlockSize = 1 << 13 // 8KB
+var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf framing
+
+// DefaultLinksPerBlock governs how the importer decides how many links there
+// will be per block. This calculation is based on expected distributions of:
+// - the expected distribution of block sizes
+// - the expected distribution of link sizes
+// - desired access speed
+//
+// For now, we use:
+//
+// var roughLinkBlockSize = 1 << 13 // 8KB
+// var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name
+// // + protobuf framing
+// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
+// = ( 8192 / 47 )
+// = (approximately) 174
+var DefaultLinksPerBlock = roughLinkBlockSize / roughLinkSize
+
+// ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
+var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
diff --git a/unixfs/importer/importer.go b/unixfs/importer/importer.go
new file mode 100644
index 000000000..64fb96a63
--- /dev/null
+++ b/unixfs/importer/importer.go
@@ -0,0 +1,41 @@
+// Package importer implements utilities used to create IPFS DAGs from files
+// and readers.
+package importer
+
+import (
+ bal "github.com/ipfs/boxo/unixfs/importer/balanced"
+ h "github.com/ipfs/boxo/unixfs/importer/helpers"
+ trickle "github.com/ipfs/boxo/unixfs/importer/trickle"
+
+ chunker "github.com/ipfs/boxo/chunker"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// BuildDagFromReader creates a DAG given a DAGService and a Splitter
+// implementation (Splitters are io.Readers), using a Balanced layout.
+func BuildDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.Node, error) {
+ dbp := h.DagBuilderParams{
+ Dagserv: ds,
+ Maxlinks: h.DefaultLinksPerBlock,
+ }
+ db, err := dbp.New(spl)
+ if err != nil {
+ return nil, err
+ }
+ return bal.Layout(db)
+}
+
+// BuildTrickleDagFromReader creates a DAG given a DAGService and a Splitter
+// implementation (Splitters are io.Readers), using a Trickle Layout.
+func BuildTrickleDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.Node, error) {
+ dbp := h.DagBuilderParams{
+ Dagserv: ds,
+ Maxlinks: h.DefaultLinksPerBlock,
+ }
+
+ db, err := dbp.New(spl)
+ if err != nil {
+ return nil, err
+ }
+ return trickle.Layout(db)
+}
diff --git a/unixfs/importer/importer_test.go b/unixfs/importer/importer_test.go
new file mode 100644
index 000000000..24517ebe8
--- /dev/null
+++ b/unixfs/importer/importer_test.go
@@ -0,0 +1,152 @@
+package importer
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "testing"
+
+ uio "github.com/ipfs/boxo/unixfs/io"
+
+ chunker "github.com/ipfs/boxo/chunker"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+ u "github.com/ipfs/boxo/util"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+func getBalancedDag(t testing.TB, size int64, blksize int64) (ipld.Node, ipld.DAGService) {
+ ds := mdtest.Mock()
+ r := io.LimitReader(u.NewTimeSeededRand(), size)
+ nd, err := BuildDagFromReader(ds, chunker.NewSizeSplitter(r, blksize))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return nd, ds
+}
+
+func getTrickleDag(t testing.TB, size int64, blksize int64) (ipld.Node, ipld.DAGService) {
+ ds := mdtest.Mock()
+ r := io.LimitReader(u.NewTimeSeededRand(), size)
+ nd, err := BuildTrickleDagFromReader(ds, chunker.NewSizeSplitter(r, blksize))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return nd, ds
+}
+
+func TestStableCid(t *testing.T) {
+ ds := mdtest.Mock()
+ buf := make([]byte, 10*1024*1024)
+ u.NewSeededRand(0xdeadbeef).Read(buf)
+ r := bytes.NewReader(buf)
+
+ nd, err := BuildDagFromReader(ds, chunker.DefaultSplitter(r))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ expected, err := cid.Decode("QmZN1qquw84zhV4j6vT56tCcmFxaDaySL1ezTXFvMdNmrK")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !expected.Equals(nd.Cid()) {
+ t.Fatalf("expected CID %s, got CID %s", expected, nd)
+ }
+
+ dr, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(dr)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(out, buf) {
+ t.Fatal("bad read")
+ }
+}
+
+func TestBalancedDag(t *testing.T) {
+ ds := mdtest.Mock()
+ buf := make([]byte, 10000)
+ u.NewTimeSeededRand().Read(buf)
+ r := bytes.NewReader(buf)
+
+ nd, err := BuildDagFromReader(ds, chunker.DefaultSplitter(r))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ dr, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(dr)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(out, buf) {
+ t.Fatal("bad read")
+ }
+}
+
+func BenchmarkBalancedReadSmallBlock(b *testing.B) {
+ b.StopTimer()
+ nbytes := int64(10000000)
+ nd, ds := getBalancedDag(b, nbytes, 4096)
+
+ b.SetBytes(nbytes)
+ b.StartTimer()
+ runReadBench(b, nd, ds)
+}
+
+func BenchmarkTrickleReadSmallBlock(b *testing.B) {
+ b.StopTimer()
+ nbytes := int64(10000000)
+ nd, ds := getTrickleDag(b, nbytes, 4096)
+
+ b.SetBytes(nbytes)
+ b.StartTimer()
+ runReadBench(b, nd, ds)
+}
+
+func BenchmarkBalancedReadFull(b *testing.B) {
+ b.StopTimer()
+ nbytes := int64(10000000)
+ nd, ds := getBalancedDag(b, nbytes, chunker.DefaultBlockSize)
+
+ b.SetBytes(nbytes)
+ b.StartTimer()
+ runReadBench(b, nd, ds)
+}
+
+func BenchmarkTrickleReadFull(b *testing.B) {
+ b.StopTimer()
+ nbytes := int64(10000000)
+ nd, ds := getTrickleDag(b, nbytes, chunker.DefaultBlockSize)
+
+ b.SetBytes(nbytes)
+ b.StartTimer()
+ runReadBench(b, nd, ds)
+}
+
+func runReadBench(b *testing.B, nd ipld.Node, ds ipld.DAGService) {
+ for i := 0; i < b.N; i++ {
+ ctx, cancel := context.WithCancel(context.Background())
+ read, err := uio.NewDagReader(ctx, nd, ds)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ _, err = read.WriteTo(io.Discard)
+ if err != nil && err != io.EOF {
+ b.Fatal(err)
+ }
+ cancel()
+ }
+}
diff --git a/unixfs/importer/trickle/trickle_test.go b/unixfs/importer/trickle/trickle_test.go
new file mode 100644
index 000000000..a78e2d252
--- /dev/null
+++ b/unixfs/importer/trickle/trickle_test.go
@@ -0,0 +1,666 @@
+package trickle
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ mrand "math/rand"
+ "testing"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ h "github.com/ipfs/boxo/unixfs/importer/helpers"
+ uio "github.com/ipfs/boxo/unixfs/io"
+
+ chunker "github.com/ipfs/boxo/chunker"
+ merkledag "github.com/ipfs/boxo/ipld/merkledag"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+ u "github.com/ipfs/boxo/util"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+type UseRawLeaves bool
+
+const (
+ ProtoBufLeaves UseRawLeaves = false
+ RawLeaves UseRawLeaves = true
+)
+
+func runBothSubtests(t *testing.T, tfunc func(*testing.T, UseRawLeaves)) {
+ t.Run("leaves=ProtoBuf", func(t *testing.T) { tfunc(t, ProtoBufLeaves) })
+ t.Run("leaves=Raw", func(t *testing.T) { tfunc(t, RawLeaves) })
+}
+
+func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeaves) (*merkledag.ProtoNode, error) {
+ dbp := h.DagBuilderParams{
+ Dagserv: ds,
+ Maxlinks: h.DefaultLinksPerBlock,
+ RawLeaves: bool(rawLeaves),
+ }
+
+ db, err := dbp.New(spl)
+ if err != nil {
+ return nil, err
+ }
+
+ nd, err := Layout(db)
+ if err != nil {
+ return nil, err
+ }
+
+ pbnd, ok := nd.(*merkledag.ProtoNode)
+ if !ok {
+ return nil, merkledag.ErrNotProtobuf
+ }
+
+ return pbnd, VerifyTrickleDagStructure(pbnd, VerifyParams{
+ Getter: ds,
+ Direct: dbp.Maxlinks,
+ LayerRepeat: depthRepeat,
+ RawLeaves: bool(rawLeaves),
+ })
+}
+
+// Test where calls to read are smaller than the chunk size
+func TestSizeBasedSplit(t *testing.T) {
+ runBothSubtests(t, testSizeBasedSplit)
+}
+
+func testSizeBasedSplit(t *testing.T, rawLeaves UseRawLeaves) {
+ if testing.Short() {
+ t.SkipNow()
+ }
+ bs := chunker.SizeSplitterGen(512)
+ testFileConsistency(t, bs, 32*512, rawLeaves)
+
+ bs = chunker.SizeSplitterGen(4096)
+ testFileConsistency(t, bs, 32*4096, rawLeaves)
+
+ // Uneven offset
+ testFileConsistency(t, bs, 31*4095, rawLeaves)
+}
+
+func dup(b []byte) []byte {
+ o := make([]byte, len(b))
+ copy(o, b)
+ return o
+}
+
+func testFileConsistency(t *testing.T, bs chunker.SplitterGen, nbytes int, rawLeaves UseRawLeaves) {
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, bs(read), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ r, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = arrComp(out, should)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestBuilderConsistency(t *testing.T) {
+ runBothSubtests(t, testBuilderConsistency)
+}
+
+func testBuilderConsistency(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := 100000
+ buf := new(bytes.Buffer)
+ io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes))
+ should := dup(buf.Bytes())
+ dagserv := mdtest.Mock()
+ nd, err := buildTestDag(dagserv, chunker.DefaultSplitter(buf), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+ r, err := uio.NewDagReader(context.Background(), nd, dagserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = arrComp(out, should)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func arrComp(a, b []byte) error {
+ if len(a) != len(b) {
+ return fmt.Errorf("arrays differ in length. %d != %d", len(a), len(b))
+ }
+ for i, v := range a {
+ if v != b[i] {
+ return fmt.Errorf("arrays differ at index: %d", i)
+ }
+ }
+ return nil
+}
+
+func TestIndirectBlocks(t *testing.T) {
+ runBothSubtests(t, testIndirectBlocks)
+}
+
+func testIndirectBlocks(t *testing.T, rawLeaves UseRawLeaves) {
+ splitter := chunker.SizeSplitterGen(512)
+ nbytes := 1024 * 1024
+ buf := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(buf)
+
+ read := bytes.NewReader(buf)
+
+ ds := mdtest.Mock()
+ dag, err := buildTestDag(ds, splitter(read), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ reader, err := uio.NewDagReader(context.Background(), dag, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bytes.Equal(out, buf) {
+ t.Fatal("Not equal!")
+ }
+}
+
+func TestSeekingBasic(t *testing.T) {
+ runBothSubtests(t, testSeekingBasic)
+}
+
+func testSeekingBasic(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(10 * 1024)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 512), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rs, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ start := int64(4000)
+ n, err := rs.Seek(start, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != start {
+ t.Fatal("Failed to seek to correct offset")
+ }
+
+ out, err := io.ReadAll(rs)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = arrComp(out, should[start:])
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSeekToBegin(t *testing.T) {
+ runBothSubtests(t, testSeekToBegin)
+}
+
+func testSeekToBegin(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(10 * 1024)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rs, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ n, err := io.CopyN(io.Discard, rs, 1024*4)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != 4096 {
+ t.Fatal("Copy didnt copy enough bytes")
+ }
+
+ seeked, err := rs.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if seeked != 0 {
+ t.Fatal("Failed to seek to beginning")
+ }
+
+ out, err := io.ReadAll(rs)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = arrComp(out, should)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSeekToAlmostBegin(t *testing.T) {
+ runBothSubtests(t, testSeekToAlmostBegin)
+}
+
+func testSeekToAlmostBegin(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(10 * 1024)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rs, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ n, err := io.CopyN(io.Discard, rs, 1024*4)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != 4096 {
+ t.Fatal("Copy didnt copy enough bytes")
+ }
+
+ seeked, err := rs.Seek(1, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if seeked != 1 {
+ t.Fatal("Failed to seek to almost beginning")
+ }
+
+ out, err := io.ReadAll(rs)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = arrComp(out, should[1:])
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSeekEnd(t *testing.T) {
+ runBothSubtests(t, testSeekEnd)
+}
+
+func testSeekEnd(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(50 * 1024)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rs, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ seeked, err := rs.Seek(0, io.SeekEnd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if seeked != nbytes {
+ t.Fatal("Failed to seek to end")
+ }
+}
+
+func TestSeekEndSingleBlockFile(t *testing.T) {
+ runBothSubtests(t, testSeekEndSingleBlockFile)
+}
+
+func testSeekEndSingleBlockFile(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(100)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 5000), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rs, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ seeked, err := rs.Seek(0, io.SeekEnd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if seeked != nbytes {
+ t.Fatal("Failed to seek to end")
+ }
+}
+
+func TestSeekingStress(t *testing.T) {
+ runBothSubtests(t, testSeekingStress)
+}
+
+func testSeekingStress(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(1024 * 1024)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 1000), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rs, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ testbuf := make([]byte, nbytes)
+ for i := 0; i < 50; i++ {
+ offset := mrand.Intn(int(nbytes))
+ l := int(nbytes) - offset
+ n, err := rs.Seek(int64(offset), io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != int64(offset) {
+ t.Fatal("Seek failed to move to correct position")
+ }
+
+ nread, err := rs.Read(testbuf[:l])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if nread != l {
+ t.Fatal("Failed to read enough bytes")
+ }
+
+ err = arrComp(testbuf[:l], should[offset:offset+l])
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+}
+
+func TestSeekingConsistency(t *testing.T) {
+ runBothSubtests(t, testSeekingConsistency)
+}
+
+func testSeekingConsistency(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(128 * 1024)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(should)
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rs, err := uio.NewDagReader(context.Background(), nd, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out := make([]byte, nbytes)
+
+ for coff := nbytes - 4096; coff >= 0; coff -= 4096 {
+ t.Log(coff)
+ n, err := rs.Seek(coff, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != coff {
+ t.Fatal("wasnt able to seek to the right position")
+ }
+ nread, err := rs.Read(out[coff : coff+4096])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if nread != 4096 {
+ t.Fatal("didnt read the correct number of bytes")
+ }
+ }
+
+ err = arrComp(out, should)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestAppend(t *testing.T) {
+ runBothSubtests(t, testAppend)
+}
+
+func testAppend(t *testing.T, rawLeaves UseRawLeaves) {
+ nbytes := int64(128 * 1024)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ // Reader for half the bytes
+ read := bytes.NewReader(should[:nbytes/2])
+ ds := mdtest.Mock()
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ dbp := &h.DagBuilderParams{
+ Dagserv: ds,
+ Maxlinks: h.DefaultLinksPerBlock,
+ RawLeaves: bool(rawLeaves),
+ }
+
+ r := bytes.NewReader(should[nbytes/2:])
+
+ ctx := context.Background()
+
+ db, err := dbp.New(chunker.NewSizeSplitter(r, 500))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nnode, err := Append(ctx, nd, db)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = VerifyTrickleDagStructure(nnode, VerifyParams{
+ Getter: ds,
+ Direct: dbp.Maxlinks,
+ LayerRepeat: depthRepeat,
+ RawLeaves: bool(rawLeaves),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fread, err := uio.NewDagReader(ctx, nnode, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(fread)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = arrComp(out, should)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+// This test appends one byte at a time to an empty file
+func TestMultipleAppends(t *testing.T) {
+ runBothSubtests(t, testMultipleAppends)
+}
+
+func testMultipleAppends(t *testing.T, rawLeaves UseRawLeaves) {
+ ds := mdtest.Mock()
+
+ // TODO: fix small size appends and make this number bigger
+ nbytes := int64(1000)
+ should := make([]byte, nbytes)
+ u.NewTimeSeededRand().Read(should)
+
+ read := bytes.NewReader(nil)
+ nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ dbp := &h.DagBuilderParams{
+ Dagserv: ds,
+ Maxlinks: 4,
+ RawLeaves: bool(rawLeaves),
+ }
+
+ spl := chunker.SizeSplitterGen(500)
+
+ ctx := context.Background()
+ for i := 0; i < len(should); i++ {
+
+ db, err := dbp.New(spl(bytes.NewReader(should[i : i+1])))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nnode, err := Append(ctx, nd, db)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = VerifyTrickleDagStructure(nnode, VerifyParams{
+ Getter: ds,
+ Direct: dbp.Maxlinks,
+ LayerRepeat: depthRepeat,
+ RawLeaves: bool(rawLeaves),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fread, err := uio.NewDagReader(ctx, nnode, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(fread)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = arrComp(out, should[:i+1])
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func TestAppendSingleBytesToEmpty(t *testing.T) {
+ ds := mdtest.Mock()
+
+ data := []byte("AB")
+
+ nd := new(merkledag.ProtoNode)
+ nd.SetData(ft.FilePBData(nil, 0))
+
+ dbp := &h.DagBuilderParams{
+ Dagserv: ds,
+ Maxlinks: 4,
+ }
+
+ spl := chunker.SizeSplitterGen(500)
+
+ ctx := context.Background()
+
+ db, err := dbp.New(spl(bytes.NewReader(data[:1])))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nnode, err := Append(ctx, nd, db)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ db, err = dbp.New(spl(bytes.NewReader(data[1:])))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ nnode, err = Append(ctx, nnode, db)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fread, err := uio.NewDagReader(ctx, nnode, ds)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(fread)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fmt.Println(out, data)
+ err = arrComp(out, data)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/unixfs/importer/trickle/trickledag.go b/unixfs/importer/trickle/trickledag.go
new file mode 100644
index 000000000..bc878b1a8
--- /dev/null
+++ b/unixfs/importer/trickle/trickledag.go
@@ -0,0 +1,389 @@
+// Package trickle allows to build trickle DAGs.
+// In this type of DAG, non-leave nodes are first filled
+// with data leaves, and then incorporate "layers" of subtrees
+// as additional links.
+//
+// Each layer is a trickle sub-tree and is limited by an increasing
+// maximum depth. Thus, the nodes first layer
+// can only hold leaves (depth 1) but subsequent layers can grow deeper.
+// By default, this module places 4 nodes per layer (that is, 4 subtrees
+// of the same maximum depth before increasing it).
+//
+// Trickle DAGs are very good for sequentially reading data, as the
+// first data leaves are directly reachable from the root and those
+// coming next are always nearby. They are
+// suited for things like streaming applications.
+package trickle
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ h "github.com/ipfs/boxo/unixfs/importer/helpers"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// depthRepeat specifies how many times to append a child tree of a
+// given depth. Higher values increase the width of a given node, which
+// improves seek speeds.
+const depthRepeat = 4
+
+// Layout builds a new DAG with the trickle format using the provided
+// DagBuilderHelper. See the module's description for a more detailed
+// explanation.
+func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
+ newRoot := db.NewFSNodeOverDag(ft.TFile)
+ root, _, err := fillTrickleRec(db, newRoot, -1)
+ if err != nil {
+ return nil, err
+ }
+
+ return root, db.Add(root)
+}
+
+// fillTrickleRec creates a trickle (sub-)tree with an optional maximum specified depth
+// in the case maxDepth is greater than zero, or with unlimited depth otherwise
+// (where the DAG builder will signal the end of data to end the function).
+func fillTrickleRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, maxDepth int) (filledNode ipld.Node, nodeFileSize uint64, err error) {
+ // Always do this, even in the base case
+ if err := db.FillNodeLayer(node); err != nil {
+ return nil, 0, err
+ }
+
+ // For each depth in [1, `maxDepth`) (or without limit if `maxDepth` is -1,
+ // initial call from `Layout`) add `depthRepeat` sub-graphs of that depth.
+ for depth := 1; maxDepth == -1 || depth < maxDepth; depth++ {
+ if db.Done() {
+ break
+ // No more data, stop here, posterior append calls will figure out
+ // where we left off.
+ }
+
+ for repeatIndex := 0; repeatIndex < depthRepeat && !db.Done(); repeatIndex++ {
+
+ childNode, childFileSize, err := fillTrickleRec(db, db.NewFSNodeOverDag(ft.TFile), depth)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ if err := node.AddChild(childNode, childFileSize, db); err != nil {
+ return nil, 0, err
+ }
+ }
+ }
+
+ // Get the final `dag.ProtoNode` with the `FSNode` data encoded inside.
+ filledNode, err = node.Commit()
+ if err != nil {
+ return nil, 0, err
+ }
+
+ return filledNode, node.FileSize(), nil
+}
+
+// Append appends the data in `db` to the dag, using the Trickledag format
+func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out ipld.Node, errOut error) {
+ base, ok := basen.(*dag.ProtoNode)
+ if !ok {
+ return nil, dag.ErrNotProtobuf
+ }
+
+ // Convert to unixfs node for working with easily
+
+ fsn, err := h.NewFSNFromDag(base)
+ if err != nil {
+ return nil, err
+ }
+
+ // Get depth of this 'tree'
+ depth, repeatNumber := trickleDepthInfo(fsn, db.Maxlinks())
+ if depth == 0 {
+ // If direct blocks not filled...
+ if err := db.FillNodeLayer(fsn); err != nil {
+ return nil, err
+ }
+
+ if db.Done() {
+ // TODO: If `FillNodeLayer` stop `Commit`ing this should be
+ // the place (besides the function end) to call it.
+ return fsn.GetDagNode()
+ }
+
+ // If continuing, our depth has increased by one
+ depth++
+ }
+
+ // Last child in this node may not be a full tree, lets fill it up.
+ if err := appendFillLastChild(ctx, fsn, depth-1, repeatNumber, db); err != nil {
+ return nil, err
+ }
+
+ // after appendFillLastChild, our depth is now increased by one
+ if !db.Done() {
+ depth++
+ }
+
+ // Now, continue filling out tree like normal
+ for i := depth; !db.Done(); i++ {
+ for j := 0; j < depthRepeat && !db.Done(); j++ {
+ nextChild := db.NewFSNodeOverDag(ft.TFile)
+ childNode, childFileSize, err := fillTrickleRec(db, nextChild, i)
+ if err != nil {
+ return nil, err
+ }
+ err = fsn.AddChild(childNode, childFileSize, db)
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+ _, err = fsn.Commit()
+ if err != nil {
+ return nil, err
+ }
+ return fsn.GetDagNode()
+}
+
+func appendFillLastChild(ctx context.Context, fsn *h.FSNodeOverDag, depth int, repeatNumber int, db *h.DagBuilderHelper) error {
+ if fsn.NumChildren() <= db.Maxlinks() {
+ return nil
+ }
+ // TODO: Why do we need this check, didn't the caller already take
+ // care of this?
+
+ // Recursive step, grab last child
+ last := fsn.NumChildren() - 1
+ lastChild, err := fsn.GetChild(ctx, last, db.GetDagServ())
+ if err != nil {
+ return err
+ }
+
+ // Fill out last child (may not be full tree)
+ newChild, nchildSize, err := appendRec(ctx, lastChild, db, depth-1)
+ if err != nil {
+ return err
+ }
+
+ // Update changed child in parent node
+ fsn.RemoveChild(last, db)
+ filledNode, err := newChild.Commit()
+ if err != nil {
+ return err
+ }
+ err = fsn.AddChild(filledNode, nchildSize, db)
+ if err != nil {
+ return err
+ }
+
+ // Partially filled depth layer
+ if repeatNumber != 0 {
+ for ; repeatNumber < depthRepeat && !db.Done(); repeatNumber++ {
+ nextChild := db.NewFSNodeOverDag(ft.TFile)
+ childNode, childFileSize, err := fillTrickleRec(db, nextChild, depth)
+ if err != nil {
+ return err
+ }
+
+ if err := fsn.AddChild(childNode, childFileSize, db); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+// recursive call for Append
+func appendRec(ctx context.Context, fsn *h.FSNodeOverDag, db *h.DagBuilderHelper, maxDepth int) (*h.FSNodeOverDag, uint64, error) {
+ if maxDepth == 0 || db.Done() {
+ return fsn, fsn.FileSize(), nil
+ }
+
+ // Get depth of this 'tree'
+ depth, repeatNumber := trickleDepthInfo(fsn, db.Maxlinks())
+ if depth == 0 {
+ // If direct blocks not filled...
+ if err := db.FillNodeLayer(fsn); err != nil {
+ return nil, 0, err
+ }
+ depth++
+ }
+ // TODO: Same as `appendFillLastChild`, when is this case possible?
+
+ // If at correct depth, no need to continue
+ if depth == maxDepth {
+ return fsn, fsn.FileSize(), nil
+ }
+
+ if err := appendFillLastChild(ctx, fsn, depth, repeatNumber, db); err != nil {
+ return nil, 0, err
+ }
+
+ // after appendFillLastChild, our depth is now increased by one
+ if !db.Done() {
+ depth++
+ }
+
+ // Now, continue filling out tree like normal
+ for i := depth; i < maxDepth && !db.Done(); i++ {
+ for j := 0; j < depthRepeat && !db.Done(); j++ {
+ nextChild := db.NewFSNodeOverDag(ft.TFile)
+ childNode, childFileSize, err := fillTrickleRec(db, nextChild, i)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ if err := fsn.AddChild(childNode, childFileSize, db); err != nil {
+ return nil, 0, err
+ }
+ }
+ }
+
+ return fsn, fsn.FileSize(), nil
+}
+
+// Deduce where we left off in `fillTrickleRec`, returns the `depth`
+// with which new sub-graphs were being added and, within that depth,
+// in which `repeatNumber` of the total `depthRepeat` we should add.
+func trickleDepthInfo(node *h.FSNodeOverDag, maxlinks int) (depth int, repeatNumber int) {
+ n := node.NumChildren()
+
+ if n < maxlinks {
+ // We didn't even added the initial `maxlinks` leaf nodes (`FillNodeLayer`).
+ return 0, 0
+ }
+
+ nonLeafChildren := n - maxlinks
+ // The number of non-leaf child nodes added in `fillTrickleRec` (after
+ // the `FillNodeLayer` call).
+
+ depth = nonLeafChildren/depthRepeat + 1
+ // "Deduplicate" the added `depthRepeat` sub-graphs at each depth
+ // (rounding it up since we may be on an unfinished depth with less
+ // than `depthRepeat` sub-graphs).
+
+ repeatNumber = nonLeafChildren % depthRepeat
+ // What's left after taking full depths of `depthRepeat` sub-graphs
+ // is the current `repeatNumber` we're at (this fractional part is
+ // what we rounded up before).
+
+ return
+}
+
+// VerifyParams is used by VerifyTrickleDagStructure
+type VerifyParams struct {
+ Getter ipld.NodeGetter
+ Direct int
+ LayerRepeat int
+ Prefix *cid.Prefix
+ RawLeaves bool
+}
+
+// VerifyTrickleDagStructure checks that the given dag matches exactly the trickle dag datastructure
+// layout
+func VerifyTrickleDagStructure(nd ipld.Node, p VerifyParams) error {
+ return verifyTDagRec(nd, -1, p)
+}
+
+// Recursive call for verifying the structure of a trickledag
+func verifyTDagRec(n ipld.Node, depth int, p VerifyParams) error {
+ codec := cid.DagProtobuf
+ if depth == 0 {
+ if len(n.Links()) > 0 {
+ return errors.New("expected direct block")
+ }
+ // zero depth dag is raw data block
+ switch nd := n.(type) {
+ case *dag.ProtoNode:
+ fsn, err := ft.FSNodeFromBytes(nd.Data())
+ if err != nil {
+ return err
+ }
+
+ if fsn.Type() != ft.TRaw {
+ return errors.New("expected raw block")
+ }
+
+ if p.RawLeaves {
+ return errors.New("expected raw leaf, got a protobuf node")
+ }
+ case *dag.RawNode:
+ if !p.RawLeaves {
+ return errors.New("expected protobuf node as leaf")
+ }
+ codec = cid.Raw
+ default:
+ return errors.New("expected ProtoNode or RawNode")
+ }
+ }
+
+ // verify prefix
+ if p.Prefix != nil {
+ prefix := n.Cid().Prefix()
+ expect := *p.Prefix // make a copy
+ expect.Codec = uint64(codec)
+ if codec == cid.Raw && expect.Version == 0 {
+ expect.Version = 1
+ }
+ if expect.MhLength == -1 {
+ expect.MhLength = prefix.MhLength
+ }
+ if prefix != expect {
+ return fmt.Errorf("unexpected cid prefix: expected: %v; got %v", expect, prefix)
+ }
+ }
+
+ if depth == 0 {
+ return nil
+ }
+
+ nd, ok := n.(*dag.ProtoNode)
+ if !ok {
+ return errors.New("expected ProtoNode")
+ }
+
+ // Verify this is a branch node
+ fsn, err := ft.FSNodeFromBytes(nd.Data())
+ if err != nil {
+ return err
+ }
+
+ if fsn.Type() != ft.TFile {
+ return fmt.Errorf("expected file as branch node, got: %s", fsn.Type())
+ }
+
+ if len(fsn.Data()) > 0 {
+ return errors.New("branch node should not have data")
+ }
+
+ for i := 0; i < len(nd.Links()); i++ {
+ child, err := nd.Links()[i].GetNode(context.TODO(), p.Getter)
+ if err != nil {
+ return err
+ }
+
+ if i < p.Direct {
+ // Direct blocks
+ err := verifyTDagRec(child, 0, p)
+ if err != nil {
+ return err
+ }
+ } else {
+ // Recursive trickle dags
+ rdepth := ((i - p.Direct) / p.LayerRepeat) + 1
+ if rdepth >= depth && depth > 0 {
+ return errors.New("child dag was too deep")
+ }
+ err := verifyTDagRec(child, rdepth, p)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
diff --git a/unixfs/internal/config.go b/unixfs/internal/config.go
new file mode 100644
index 000000000..9250ae2ae
--- /dev/null
+++ b/unixfs/internal/config.go
@@ -0,0 +1,3 @@
+package internal
+
+var HAMTHashFunction func(val []byte) []byte
diff --git a/unixfs/io/completehamt_test.go b/unixfs/io/completehamt_test.go
new file mode 100644
index 000000000..231765de1
--- /dev/null
+++ b/unixfs/io/completehamt_test.go
@@ -0,0 +1,101 @@
+package io
+
+import (
+ "context"
+ "encoding/binary"
+ "fmt"
+ "math"
+ "testing"
+
+ "github.com/ipfs/boxo/unixfs/internal"
+
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/ipfs/boxo/unixfs"
+ "github.com/ipfs/boxo/unixfs/hamt"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// CreateCompleteHAMT creates a HAMT the following properties:
+// - its height (distance/edges from root to deepest node) is specified by treeHeight.
+// - all leaf Shard nodes have the same depth (and have only 'value' links).
+// - all internal Shard nodes point only to other Shards (and hence have zero 'value' links).
+// - the total number of 'value' links (directory entries) is:
+// childsPerNode ^ (treeHeight).
+// treeHeight: The number of layers of non-value HAMT nodes (e.g. height = 1 is a single shard pointing to some values)
+//
+// FIXME: HAMTHashFunction needs to be set to idHash by the caller. We depend on
+//
+// this simplification for the current logic to work.
+func CreateCompleteHAMT(ds ipld.DAGService, treeHeight int, childsPerNode int) (ipld.Node, error) {
+ if treeHeight < 1 {
+ panic("treeHeight < 1")
+ }
+ if treeHeight > 8 {
+ panic("treeHeight > 8: we don't allow a key larger than what can be encoded in a 64-bit word")
+ }
+
+ rootShard, err := hamt.NewShard(ds, childsPerNode)
+ if err != nil {
+ return nil, err
+ }
+
+ // Assuming we are using the ID hash function we can just insert all
+ // the combinations of a byte slice that will reach the desired height.
+ totalChildren := int(math.Pow(float64(childsPerNode), float64(treeHeight)))
+ log2ofChilds, err := hamt.Logtwo(childsPerNode)
+ if err != nil {
+ return nil, err
+ }
+ if log2ofChilds*treeHeight%8 != 0 {
+ return nil, fmt.Errorf("childsPerNode * treeHeight should be multiple of 8")
+ }
+ bytesInKey := log2ofChilds * treeHeight / 8
+ for i := 0; i < totalChildren; i++ {
+ var hashbuf [8]byte
+ binary.LittleEndian.PutUint64(hashbuf[:], uint64(i))
+ var oldLink *ipld.Link
+ oldLink, err = rootShard.Swap(context.Background(), string(hashbuf[:bytesInKey]), unixfs.EmptyFileNode())
+ if err != nil {
+ return nil, err
+ }
+ if oldLink != nil {
+ // We shouldn't be overwriting any value, otherwise the tree
+ // won't be complete.
+ return nil, fmt.Errorf("we have overwritten entry %s",
+ oldLink.Cid)
+ }
+ }
+
+ return rootShard.Node()
+}
+
+// Return the same value as the hash.
+func idHash(val []byte) []byte {
+ return val
+}
+
+// FIXME: This is not checking the exact height of the tree but just making
+//
+// sure there are as many children as we would have with a complete HAMT.
+func TestCreateCompleteShard(t *testing.T) {
+ oldHashFunc := internal.HAMTHashFunction
+ defer func() { internal.HAMTHashFunction = oldHashFunc }()
+ internal.HAMTHashFunction = idHash
+
+ ds := mdtest.Mock()
+ childsPerNode := 16
+ treeHeight := 2
+ node, err := CreateCompleteHAMT(ds, treeHeight, childsPerNode)
+ assert.NoError(t, err)
+
+ shard, err := hamt.NewHamtFromDag(ds, node)
+ assert.NoError(t, err)
+ links, err := shard.EnumLinks(context.Background())
+ assert.NoError(t, err)
+
+ childNodes := int(math.Pow(float64(childsPerNode), float64(treeHeight)))
+ assert.Equal(t, childNodes, len(links))
+}
diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go
new file mode 100644
index 000000000..2767fab6a
--- /dev/null
+++ b/unixfs/io/dagreader.go
@@ -0,0 +1,488 @@
+package io
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ unixfs "github.com/ipfs/boxo/unixfs"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// Common errors
+var (
+ ErrIsDir = errors.New("this dag node is a directory")
+ ErrCantReadSymlinks = errors.New("cannot currently read symlinks")
+ ErrUnkownNodeType = errors.New("unknown node type")
+ ErrSeekNotSupported = errors.New("file does not support seeking")
+)
+
+// TODO: Rename the `DagReader` interface, this doesn't read *any* DAG, just
+// DAGs with UnixFS node (and it *belongs* to the `unixfs` package). Some
+// alternatives: `FileReader`, `UnixFSFileReader`, `UnixFSReader`.
+
+// A DagReader provides read-only read and seek acess to a unixfs file.
+// Different implementations of readers are used for the different
+// types of unixfs/protobuf-encoded nodes.
+type DagReader interface {
+ ReadSeekCloser
+ Size() uint64
+ CtxReadFull(context.Context, []byte) (int, error)
+}
+
+// A ReadSeekCloser implements interfaces to read, copy, seek and close.
+type ReadSeekCloser interface {
+ io.Reader
+ io.Seeker
+ io.Closer
+ io.WriterTo
+}
+
+// NewDagReader creates a new reader object that reads the data represented by
+// the given node, using the passed in DAGService for data retrieval.
+func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) {
+ var size uint64
+
+ switch n := n.(type) {
+ case *mdag.RawNode:
+ size = uint64(len(n.RawData()))
+
+ case *mdag.ProtoNode:
+ fsNode, err := unixfs.FSNodeFromBytes(n.Data())
+ if err != nil {
+ return nil, err
+ }
+
+ switch fsNode.Type() {
+ case unixfs.TFile, unixfs.TRaw:
+ size = fsNode.FileSize()
+
+ case unixfs.TDirectory, unixfs.THAMTShard:
+ // Dont allow reading directories
+ return nil, ErrIsDir
+
+ case unixfs.TMetadata:
+ if len(n.Links()) == 0 {
+ return nil, errors.New("incorrectly formatted metadata object")
+ }
+ child, err := n.Links()[0].GetNode(ctx, serv)
+ if err != nil {
+ return nil, err
+ }
+
+ childpb, ok := child.(*mdag.ProtoNode)
+ if !ok {
+ return nil, mdag.ErrNotProtobuf
+ }
+ return NewDagReader(ctx, childpb, serv)
+ case unixfs.TSymlink:
+ return nil, ErrCantReadSymlinks
+ default:
+ return nil, unixfs.ErrUnrecognizedType
+ }
+ default:
+ return nil, ErrUnkownNodeType
+ }
+
+ ctxWithCancel, cancel := context.WithCancel(ctx)
+
+ return &dagReader{
+ ctx: ctxWithCancel,
+ cancel: cancel,
+ serv: serv,
+ size: size,
+ rootNode: n,
+ dagWalker: ipld.NewWalker(ctxWithCancel, ipld.NewNavigableIPLDNode(n, serv)),
+ }, nil
+}
+
+// dagReader provides a way to easily read the data contained in a dag.
+type dagReader struct {
+
+ // Structure to perform the DAG iteration and search, the reader
+ // just needs to add logic to the `Visitor` callback passed to
+ // `Iterate` and `Seek`.
+ dagWalker *ipld.Walker
+
+ // Buffer with the data extracted from the current node being visited.
+ // To avoid revisiting a node to complete a (potential) partial read
+ // (or read after seek) the node's data is fully extracted in a single
+ // `readNodeDataBuffer` operation.
+ currentNodeData *bytes.Reader
+
+ // Implements the `Size()` API.
+ size uint64
+
+ // Current offset for the read head within the DAG file.
+ offset int64
+
+ // Root node of the DAG, stored to re-create the `dagWalker` (effectively
+ // re-setting the position of the reader, used during `Seek`).
+ rootNode ipld.Node
+
+ // Context passed to the `dagWalker`, the `cancel` function is used to
+ // cancel read operations (cancelling requested child node promises,
+ // see `ipld.NavigableIPLDNode.FetchChild` for details).
+ ctx context.Context
+ cancel func()
+
+ // Passed to the `dagWalker` that will use it to request nodes.
+ // TODO: Revisit name.
+ serv ipld.NodeGetter
+}
+
+// Size returns the total size of the data from the DAG structured file.
+func (dr *dagReader) Size() uint64 {
+ return dr.size
+}
+
+// Read implements the `io.Reader` interface through the `CtxReadFull`
+// method using the DAG reader's internal context.
+func (dr *dagReader) Read(b []byte) (int, error) {
+ return dr.CtxReadFull(dr.ctx, b)
+}
+
+// CtxReadFull reads data from the DAG structured file. It always
+// attempts a full read of the DAG until the `out` buffer is full.
+// It uses the `Walker` structure to iterate the file DAG and read
+// every node's data into the `out` buffer.
+func (dr *dagReader) CtxReadFull(ctx context.Context, out []byte) (n int, err error) {
+ // Set the `dagWalker`'s context to the `ctx` argument, it will be used
+ // to fetch the child node promises (see
+ // `ipld.NavigableIPLDNode.FetchChild` for details).
+ dr.dagWalker.SetContext(ctx)
+
+ // If there was a partially read buffer from the last visited
+ // node read it before visiting a new one.
+ if dr.currentNodeData != nil {
+ // TODO: Move this check inside `readNodeDataBuffer`?
+ n = dr.readNodeDataBuffer(out)
+
+ if n == len(out) {
+ return n, nil
+ // Output buffer full, no need to traverse the DAG.
+ }
+ }
+
+ // Iterate the DAG calling the passed `Visitor` function on every node
+ // to read its data into the `out` buffer, stop if there is an error or
+ // if the entire DAG is traversed (`EndOfDag`).
+ err = dr.dagWalker.Iterate(func(visitedNode ipld.NavigableNode) error {
+ node := ipld.ExtractIPLDNode(visitedNode)
+
+ // Skip internal nodes, they shouldn't have any file data
+ // (see the `balanced` package for more details).
+ if len(node.Links()) > 0 {
+ return nil
+ }
+
+ err = dr.saveNodeData(node)
+ if err != nil {
+ return err
+ }
+ // Save the leaf node file data in a buffer in case it is only
+ // partially read now and future `CtxReadFull` calls reclaim the
+ // rest (as each node is visited only once during `Iterate`).
+ //
+ // TODO: We could check if the entire node's data can fit in the
+ // remaining `out` buffer free space to skip this intermediary step.
+
+ n += dr.readNodeDataBuffer(out[n:])
+
+ if n == len(out) {
+ // Output buffer full, no need to keep traversing the DAG,
+ // signal the `Walker` to pause the iteration.
+ dr.dagWalker.Pause()
+ }
+
+ return nil
+ })
+
+ if err == ipld.EndOfDag {
+ return n, io.EOF
+ // Reached the end of the (DAG) file, no more data to read.
+ } else if err != nil {
+ return n, err
+ // Pass along any other errors from the `Visitor`.
+ }
+
+ return n, nil
+}
+
+// Save the UnixFS `node`'s data into the internal `currentNodeData` buffer to
+// later move it to the output buffer (`Read`) or seek into it (`Seek`).
+func (dr *dagReader) saveNodeData(node ipld.Node) error {
+ extractedNodeData, err := unixfs.ReadUnixFSNodeData(node)
+ if err != nil {
+ return err
+ }
+
+ dr.currentNodeData = bytes.NewReader(extractedNodeData)
+ return nil
+}
+
+// Read the `currentNodeData` buffer into `out`. This function can't have
+// any errors as it's always reading from a `bytes.Reader` and asking only
+// the available data in it.
+func (dr *dagReader) readNodeDataBuffer(out []byte) int {
+
+ n, _ := dr.currentNodeData.Read(out)
+ // Ignore the error as the EOF may not be returned in the first
+ // `Read` call, explicitly ask for an empty buffer below to check
+ // if we've reached the end.
+
+ if dr.currentNodeData.Len() == 0 {
+ dr.currentNodeData = nil
+ // Signal that the buffer was consumed (for later `Read` calls).
+ // This shouldn't return an EOF error as it's just the end of a
+ // single node's data, not the entire DAG.
+ }
+
+ dr.offset += int64(n)
+ // TODO: Should `offset` be incremented here or in the calling function?
+ // (Doing it here saves LoC but may be confusing as it's more hidden).
+
+ return n
+}
+
+// Similar to `readNodeDataBuffer` but it writes the contents to
+// an `io.Writer` argument.
+//
+// TODO: Check what part of the logic between the two functions
+// can be extracted away.
+func (dr *dagReader) writeNodeDataBuffer(w io.Writer) (int64, error) {
+
+ n, err := dr.currentNodeData.WriteTo(w)
+ if err != nil {
+ return n, err
+ }
+
+ if dr.currentNodeData.Len() == 0 {
+ dr.currentNodeData = nil
+ // Signal that the buffer was consumed (for later `Read` calls).
+ // This shouldn't return an EOF error as it's just the end of a
+ // single node's data, not the entire DAG.
+ }
+
+ dr.offset += int64(n)
+ return n, nil
+}
+
+// WriteTo writes to the given writer.
+// This follows the `bytes.Reader.WriteTo` implementation
+// where it starts from the internal index that may have
+// been modified by other `Read` calls.
+//
+// TODO: This implementation is very similar to `CtxReadFull`,
+// the common parts should be abstracted away.
+func (dr *dagReader) WriteTo(w io.Writer) (n int64, err error) {
+ // Use the internal reader's context to fetch the child node promises
+ // (see `ipld.NavigableIPLDNode.FetchChild` for details).
+ dr.dagWalker.SetContext(dr.ctx)
+
+ // If there was a partially read buffer from the last visited
+ // node read it before visiting a new one.
+ if dr.currentNodeData != nil {
+ n, err = dr.writeNodeDataBuffer(w)
+ if err != nil {
+ return n, err
+ }
+ }
+
+ // Iterate the DAG calling the passed `Visitor` function on every node
+ // to read its data into the `out` buffer, stop if there is an error or
+ // if the entire DAG is traversed (`EndOfDag`).
+ err = dr.dagWalker.Iterate(func(visitedNode ipld.NavigableNode) error {
+ node := ipld.ExtractIPLDNode(visitedNode)
+
+ // Skip internal nodes, they shouldn't have any file data
+ // (see the `balanced` package for more details).
+ if len(node.Links()) > 0 {
+ return nil
+ }
+
+ err = dr.saveNodeData(node)
+ if err != nil {
+ return err
+ }
+ // Save the leaf node file data in a buffer in case it is only
+ // partially read now and future `CtxReadFull` calls reclaim the
+ // rest (as each node is visited only once during `Iterate`).
+
+ written, err := dr.writeNodeDataBuffer(w)
+ n += written
+ if err != nil {
+ return err
+ }
+
+ return nil
+ })
+
+ if err == ipld.EndOfDag {
+ return n, nil
+ }
+
+ return n, err
+}
+
+// Close the reader (cancelling fetch node operations requested with
+// the internal context, that is, `Read` calls but not `CtxReadFull`
+// with user-supplied contexts).
+func (dr *dagReader) Close() error {
+ dr.cancel()
+ return nil
+}
+
+// Seek implements `io.Seeker` seeking to a given offset in the DAG file,
+// it matches the standard unix `seek`. It moves the position of the internal
+// `dagWalker` and may also leave a `currentNodeData` buffer loaded in case
+// the seek is performed to the middle of the data in a node.
+//
+// TODO: Support seeking from the current position (relative seek)
+// through the `dagWalker` in `io.SeekCurrent`.
+func (dr *dagReader) Seek(offset int64, whence int) (int64, error) {
+ switch whence {
+ case io.SeekStart:
+ if offset < 0 {
+ return dr.offset, errors.New("invalid offset")
+ }
+
+ if offset == dr.offset {
+ return offset, nil
+ // Already at the requested `offset`, nothing to do.
+ }
+
+ left := offset
+ // Amount left to seek.
+
+ // Seek from the beginning of the DAG.
+ dr.resetPosition()
+
+ // Shortcut seeking to the beginning, we're already there.
+ if offset == 0 {
+ return 0, nil
+ }
+
+ // Use the internal reader's context to fetch the child node promises
+ // (see `ipld.NavigableIPLDNode.FetchChild` for details).
+ dr.dagWalker.SetContext(dr.ctx)
+ // TODO: Performance: we could adjust here `preloadSize` of
+ // `ipld.NavigableIPLDNode` also, when seeking we only want
+ // to fetch one child at a time.
+
+ // Seek the DAG by calling the provided `Visitor` function on every
+ // node the `dagWalker` descends to while searching which can be
+ // either an internal or leaf node. In the internal node case, check
+ // the child node sizes and set the corresponding child index to go
+ // down to next. In the leaf case (last visit of the search), if there
+ // is still an amount `left` to seek do it inside the node's data
+ // saved in the `currentNodeData` buffer, leaving it ready for a `Read`
+ // call.
+ err := dr.dagWalker.Seek(func(visitedNode ipld.NavigableNode) error {
+ node := ipld.ExtractIPLDNode(visitedNode)
+
+ if len(node.Links()) > 0 {
+ // Internal node, should be a `mdag.ProtoNode` containing a
+ // `unixfs.FSNode` (see the `balanced` package for more details).
+ fsNode, err := unixfs.ExtractFSNode(node)
+ if err != nil {
+ return err
+ }
+
+ // If there aren't enough size hints don't seek
+ // (see the `io.EOF` handling error comment below).
+ if fsNode.NumChildren() != len(node.Links()) {
+ return ErrSeekNotSupported
+ }
+
+ // Internal nodes have no data, so just iterate through the
+ // sizes of its children (advancing the child index of the
+ // `dagWalker`) to find where we need to go down to next in
+ // the search.
+ for {
+ childSize := fsNode.BlockSize(int(dr.dagWalker.ActiveChildIndex()))
+
+ if childSize > uint64(left) {
+ // This child's data contains the position requested
+ // in `offset`, go down this child.
+ return nil
+ }
+
+ // Else, skip this child.
+ left -= int64(childSize)
+ err := dr.dagWalker.NextChild()
+ if err == ipld.ErrNextNoChild {
+ // No more child nodes available, nothing to do,
+ // the `Seek` will stop on its own.
+ return nil
+ } else if err != nil {
+ return err
+ // Pass along any other errors (that may in future
+ // implementations be returned by `Next`) to stop
+ // the search.
+ }
+ }
+
+ } else {
+ // Leaf node, seek inside its data.
+ err := dr.saveNodeData(node)
+ if err != nil {
+ return err
+ }
+
+ _, err = dr.currentNodeData.Seek(left, io.SeekStart)
+ if err != nil {
+ return err
+ }
+ // The corner case of a DAG consisting only of a single (leaf)
+ // node should make no difference here. In that case, where the
+ // node doesn't have a parent UnixFS node with size hints, this
+ // implementation would allow this `Seek` to be called with an
+ // argument larger than the buffer size which normally wouldn't
+ // happen (because we would skip the node based on the size
+ // hint) but that would just mean that a future `CtxReadFull`
+ // call would read no data from the `currentNodeData` buffer.
+ // TODO: Re-check this reasoning.
+
+ return nil
+ // In the leaf node case the search will stop here.
+ }
+ })
+
+ if err != nil {
+ return 0, err
+ }
+
+ dr.offset = offset
+ return dr.offset, nil
+
+ case io.SeekCurrent:
+ if offset == 0 {
+ return dr.offset, nil
+ }
+
+ return dr.Seek(dr.offset+offset, io.SeekStart)
+ // TODO: Performance. This can be improved supporting relative
+ // searches in the `Walker` (see `Walker.Seek`).
+
+ case io.SeekEnd:
+ return dr.Seek(int64(dr.Size())+offset, io.SeekStart)
+
+ default:
+ return 0, errors.New("invalid whence")
+ }
+}
+
+// Reset the reader position by resetting the `dagWalker` and discarding
+// any partially used node's data in the `currentNodeData` buffer, used
+// in the `SeekStart` case.
+func (dr *dagReader) resetPosition() {
+ dr.currentNodeData = nil
+ dr.offset = 0
+
+ dr.dagWalker = ipld.NewWalker(dr.ctx, ipld.NewNavigableIPLDNode(dr.rootNode, dr.serv))
+ // TODO: This could be avoided (along with storing the `dr.rootNode` and
+ // `dr.serv` just for this call) if `Reset` is supported in the `Walker`.
+}
diff --git a/unixfs/io/dagreader_test.go b/unixfs/io/dagreader_test.go
new file mode 100644
index 000000000..f9c547e44
--- /dev/null
+++ b/unixfs/io/dagreader_test.go
@@ -0,0 +1,326 @@
+package io
+
+import (
+ "bytes"
+ "io"
+ "strings"
+ "testing"
+
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ "github.com/ipfs/boxo/unixfs"
+
+ context "context"
+
+ testu "github.com/ipfs/boxo/unixfs/test"
+)
+
+func TestBasicRead(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ reader, err := NewDagReader(ctx, node, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ outbuf, err := io.ReadAll(reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = testu.ArrComp(inbuf, outbuf)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSeekAndRead(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ inbuf := make([]byte, 256)
+ for i := 0; i <= 255; i++ {
+ inbuf[i] = byte(i)
+ }
+
+ node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ reader, err := NewDagReader(ctx, node, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i := 255; i >= 0; i-- {
+ reader.Seek(int64(i), io.SeekStart)
+
+ if getOffset(reader) != int64(i) {
+ t.Fatal("expected offset to be increased by one after read")
+ }
+
+ out := readByte(t, reader)
+
+ if int(out) != i {
+ t.Fatalf("read %d at index %d, expected %d", out, i, i)
+ }
+
+ if getOffset(reader) != int64(i+1) {
+ t.Fatal("expected offset to be increased by one after read")
+ }
+ }
+}
+
+func TestSeekWithoutBlocksizes(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ inbuf := make([]byte, 1024)
+
+ for i := 0; i < 256; i++ {
+ inbuf[i*4] = byte(i)
+ }
+
+ inbuf[1023] = 1 // force the reader to be 1024 bytes
+ node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
+
+ // remove the blocksizes
+ pbnode := node.Copy().(*mdag.ProtoNode)
+ fsnode, err := unixfs.FSNodeFromBytes(pbnode.Data())
+ if err != nil {
+ t.Fatal(err)
+ }
+ fsnode.RemoveAllBlockSizes()
+ newData, err := fsnode.GetBytes()
+ if err != nil {
+ t.Fatal(err)
+ }
+ pbnode.SetData(newData)
+ err = dserv.Add(ctx, pbnode)
+ if err != nil {
+ t.Fatal(err)
+ }
+ node = pbnode
+
+ reader, err := NewDagReader(ctx, node, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = reader.Seek(-4, io.SeekEnd)
+ if err == nil {
+ t.Fatal("seeking shouldn't work without blocksizes")
+ }
+
+ _, err = reader.Seek(4, io.SeekStart)
+ if err == nil {
+ t.Fatal("seeking shouldn't work without blocksizes")
+ }
+
+ _, err = reader.Seek(4, io.SeekCurrent)
+ if err == nil {
+ t.Fatal("seeking shouldn't work without blocksizes")
+ }
+
+ // Seeking to the current position or the end should still work.
+
+ _, err = reader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = reader.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestRelativeSeek(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ inbuf := make([]byte, 1024)
+
+ for i := 0; i < 256; i++ {
+ inbuf[i*4] = byte(i)
+ }
+
+ inbuf[1023] = 1 // force the reader to be 1024 bytes
+ node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
+
+ reader, err := NewDagReader(ctx, node, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i := 0; i < 256; i++ {
+ if getOffset(reader) != int64(i*4) {
+ t.Fatalf("offset should be %d, was %d", i*4, getOffset(reader))
+ }
+ out := readByte(t, reader)
+ if int(out) != i {
+ t.Fatalf("expected to read: %d at %d, read %d", i, getOffset(reader)-1, out)
+ }
+ if i != 255 {
+ _, err := reader.Seek(3, io.SeekCurrent)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+
+ _, err = reader.Seek(-4, io.SeekEnd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i := 0; i < 256; i++ {
+ if getOffset(reader) != int64(1020-i*4) {
+ t.Fatalf("offset should be %d, was %d", 1020-i*4, getOffset(reader))
+ }
+ out := readByte(t, reader)
+ if int(out) != 255-i {
+ t.Fatalf("expected to read: %d at %d, read %d", 255-i, getOffset(reader)-1, out)
+ }
+ reader.Seek(-5, io.SeekCurrent) // seek 4 bytes but we read one byte every time so 5 bytes
+ }
+
+}
+
+func TestTypeFailures(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ node := unixfs.EmptyDirNode()
+ if _, err := NewDagReader(ctx, node, dserv); err != ErrIsDir {
+ t.Fatalf("excepted to get %v, got %v", ErrIsDir, err)
+ }
+
+ data, err := unixfs.SymlinkData("/somelink")
+ if err != nil {
+ t.Fatal(err)
+ }
+ node = mdag.NodeWithData(data)
+
+ if _, err := NewDagReader(ctx, node, dserv); err != ErrCantReadSymlinks {
+ t.Fatalf("excepted to get %v, got %v", ErrCantReadSymlinks, err)
+ }
+}
+
+func TestBadPBData(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ node := mdag.NodeWithData([]byte{42})
+ _, err := NewDagReader(ctx, node, dserv)
+ if err == nil {
+ t.Fatal("excepted error, got nil")
+ }
+}
+
+func TestMetadataNode(t *testing.T) {
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ dserv := testu.GetDAGServ()
+ rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.UseProtoBufLeaves)
+ err := dserv.Add(ctx, rnode)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ data, err := unixfs.BytesForMetadata(&unixfs.Metadata{
+ MimeType: "text",
+ Size: 125,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ node := mdag.NodeWithData(data)
+
+ _, err = NewDagReader(ctx, node, dserv)
+ if err == nil {
+ t.Fatal("expected an error")
+ }
+ if !strings.Contains(err.Error(), "incorrectly formatted") {
+ t.Fatal("expected different error")
+ }
+
+ node.AddNodeLink("", rnode)
+
+ reader, err := NewDagReader(ctx, node, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ readdata, err := io.ReadAll(reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := testu.ArrComp(rdata, readdata); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestWriteTo(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ reader, err := NewDagReader(ctx, node, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ outbuf := new(bytes.Buffer)
+ reader.WriteTo(outbuf)
+
+ err = testu.ArrComp(inbuf, outbuf.Bytes())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+}
+
+func TestReaderSzie(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ size := int64(1024)
+ _, node := testu.GetRandomNode(t, dserv, size, testu.UseProtoBufLeaves)
+ ctx, closer := context.WithCancel(context.Background())
+ defer closer()
+
+ reader, err := NewDagReader(ctx, node, dserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if reader.Size() != uint64(size) {
+ t.Fatal("wrong reader size")
+ }
+}
+
+func readByte(t testing.TB, reader DagReader) byte {
+ out := make([]byte, 1)
+ c, err := reader.Read(out)
+
+ if c != 1 {
+ t.Fatal("reader should have read just one byte")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return out[0]
+}
+
+func getOffset(reader DagReader) int64 {
+ offset, err := reader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ panic("failed to retrieve offset: " + err.Error())
+ }
+ return offset
+}
diff --git a/unixfs/io/directory.go b/unixfs/io/directory.go
new file mode 100644
index 000000000..76dd8defe
--- /dev/null
+++ b/unixfs/io/directory.go
@@ -0,0 +1,610 @@
+package io
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/ipfs/boxo/unixfs/hamt"
+ "github.com/ipfs/boxo/unixfs/private/linksize"
+
+ "github.com/alecthomas/units"
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ format "github.com/ipfs/boxo/unixfs"
+ "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+ logging "github.com/ipfs/go-log"
+)
+
+var log = logging.Logger("unixfs")
+
+// HAMTShardingSize is a global option that allows switching to a HAMTDirectory
+// when the BasicDirectory grows above the size (in bytes) signalled by this
+// flag. The default size of 0 disables the option.
+// The size is not the *exact* block size of the encoded BasicDirectory but just
+// the estimated size based byte length of links name and CID (BasicDirectory's
+// ProtoNode doesn't use the Data field so this estimate is pretty accurate).
+var HAMTShardingSize = int(256 * units.KiB)
+
+// DefaultShardWidth is the default value used for hamt sharding width.
+// Needs to be a power of two (shard entry size) and multiple of 8 (bitfield size).
+var DefaultShardWidth = 256
+
+// Directory defines a UnixFS directory. It is used for creating, reading and
+// editing directories. It allows to work with different directory schemes,
+// like the basic or the HAMT implementation.
+//
+// It just allows to perform explicit edits on a single directory, working with
+// directory trees is out of its scope, they are managed by the MFS layer
+// (which is the main consumer of this interface).
+type Directory interface {
+
+ // SetCidBuilder sets the CID Builder of the root node.
+ SetCidBuilder(cid.Builder)
+
+ // AddChild adds a (name, key) pair to the root node.
+ AddChild(context.Context, string, ipld.Node) error
+
+ // ForEachLink applies the given function to Links in the directory.
+ ForEachLink(context.Context, func(*ipld.Link) error) error
+
+ // EnumLinksAsync returns a channel which will receive Links in the directory
+ // as they are enumerated, where order is not gauranteed
+ EnumLinksAsync(context.Context) <-chan format.LinkResult
+
+ // Links returns the all the links in the directory node.
+ Links(context.Context) ([]*ipld.Link, error)
+
+ // Find returns the root node of the file named 'name' within this directory.
+ // In the case of HAMT-directories, it will traverse the tree.
+ //
+ // Returns os.ErrNotExist if the child does not exist.
+ Find(context.Context, string) (ipld.Node, error)
+
+ // RemoveChild removes the child with the given name.
+ //
+ // Returns os.ErrNotExist if the child doesn't exist.
+ RemoveChild(context.Context, string) error
+
+ // GetNode returns the root of this directory.
+ GetNode() (ipld.Node, error)
+
+ // GetCidBuilder returns the CID Builder used.
+ GetCidBuilder() cid.Builder
+}
+
+// TODO: Evaluate removing `dserv` from this layer and providing it in MFS.
+// (The functions should in that case add a `DAGService` argument.)
+
+// Link size estimation function. For production it's usually the one here
+// but during test we may mock it to get fixed sizes.
+func productionLinkSize(linkName string, linkCid cid.Cid) int {
+ return len(linkName) + linkCid.ByteLen()
+}
+
+func init() {
+ linksize.LinkSizeFunction = productionLinkSize
+}
+
+// BasicDirectory is the basic implementation of `Directory`. All the entries
+// are stored in a single node.
+type BasicDirectory struct {
+ node *mdag.ProtoNode
+ dserv ipld.DAGService
+
+ // Internal variable used to cache the estimated size of the basic directory:
+ // for each link, aggregate link name + link CID. DO NOT CHANGE THIS
+ // as it will affect the HAMT transition behavior in HAMTShardingSize.
+ // (We maintain this value up to date even if the HAMTShardingSize is off
+ // since potentially the option could be activated on the fly.)
+ estimatedSize int
+}
+
+// HAMTDirectory is the HAMT implementation of `Directory`.
+// (See package `hamt` for more information.)
+type HAMTDirectory struct {
+ shard *hamt.Shard
+ dserv ipld.DAGService
+
+ // Track the changes in size by the AddChild and RemoveChild calls
+ // for the HAMTShardingSize option.
+ sizeChange int
+}
+
+func newEmptyBasicDirectory(dserv ipld.DAGService) *BasicDirectory {
+ return newBasicDirectoryFromNode(dserv, format.EmptyDirNode())
+}
+
+func newBasicDirectoryFromNode(dserv ipld.DAGService, node *mdag.ProtoNode) *BasicDirectory {
+ basicDir := new(BasicDirectory)
+ basicDir.node = node
+ basicDir.dserv = dserv
+
+ // Scan node links (if any) to restore estimated size.
+ basicDir.computeEstimatedSize()
+
+ return basicDir
+}
+
+// NewDirectory returns a Directory implemented by DynamicDirectory
+// containing a BasicDirectory that can be converted to a HAMTDirectory.
+func NewDirectory(dserv ipld.DAGService) Directory {
+ return &DynamicDirectory{newEmptyBasicDirectory(dserv)}
+}
+
+// ErrNotADir implies that the given node was not a unixfs directory
+var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard")
+
+// NewDirectoryFromNode loads a unixfs directory from the given IPLD node and
+// DAGService.
+func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, error) {
+ protoBufNode, ok := node.(*mdag.ProtoNode)
+ if !ok {
+ return nil, ErrNotADir
+ }
+
+ fsNode, err := format.FSNodeFromBytes(protoBufNode.Data())
+ if err != nil {
+ return nil, err
+ }
+
+ switch fsNode.Type() {
+ case format.TDirectory:
+ return &DynamicDirectory{newBasicDirectoryFromNode(dserv, protoBufNode.Copy().(*mdag.ProtoNode))}, nil
+ case format.THAMTShard:
+ shard, err := hamt.NewHamtFromDag(dserv, node)
+ if err != nil {
+ return nil, err
+ }
+ return &DynamicDirectory{&HAMTDirectory{shard, dserv, 0}}, nil
+ }
+
+ return nil, ErrNotADir
+}
+
+func (d *BasicDirectory) computeEstimatedSize() {
+ d.estimatedSize = 0
+ // err is just breaking the iteration and we always return nil
+ _ = d.ForEachLink(context.TODO(), func(l *ipld.Link) error {
+ d.addToEstimatedSize(l.Name, l.Cid)
+ return nil
+ })
+ // ForEachLink will never fail traversing the BasicDirectory
+ // and neither the inner callback `addToEstimatedSize`.
+}
+
+func (d *BasicDirectory) addToEstimatedSize(name string, linkCid cid.Cid) {
+ d.estimatedSize += linksize.LinkSizeFunction(name, linkCid)
+}
+
+func (d *BasicDirectory) removeFromEstimatedSize(name string, linkCid cid.Cid) {
+ d.estimatedSize -= linksize.LinkSizeFunction(name, linkCid)
+ if d.estimatedSize < 0 {
+ // Something has gone very wrong. Log an error and recompute the
+ // size from scratch.
+ log.Error("BasicDirectory's estimatedSize went below 0")
+ d.computeEstimatedSize()
+ }
+}
+
+// SetCidBuilder implements the `Directory` interface.
+func (d *BasicDirectory) SetCidBuilder(builder cid.Builder) {
+ d.node.SetCidBuilder(builder)
+}
+
+// AddChild implements the `Directory` interface. It adds (or replaces)
+// a link to the given `node` under `name`.
+func (d *BasicDirectory) AddChild(ctx context.Context, name string, node ipld.Node) error {
+ link, err := ipld.MakeLink(node)
+ if err != nil {
+ return err
+ }
+
+ return d.addLinkChild(ctx, name, link)
+}
+
+func (d *BasicDirectory) needsToSwitchToHAMTDir(name string, nodeToAdd ipld.Node) (bool, error) {
+ if HAMTShardingSize == 0 { // Option disabled.
+ return false, nil
+ }
+
+ operationSizeChange := 0
+ // Find if there is an old entry under that name that will be overwritten.
+ entryToRemove, err := d.node.GetNodeLink(name)
+ if err != mdag.ErrLinkNotFound {
+ if err != nil {
+ return false, err
+ }
+ operationSizeChange -= linksize.LinkSizeFunction(name, entryToRemove.Cid)
+ }
+ if nodeToAdd != nil {
+ operationSizeChange += linksize.LinkSizeFunction(name, nodeToAdd.Cid())
+ }
+
+ return d.estimatedSize+operationSizeChange >= HAMTShardingSize, nil
+}
+
+// addLinkChild adds the link as an entry to this directory under the given
+// name. Plumbing function for the AddChild API.
+func (d *BasicDirectory) addLinkChild(ctx context.Context, name string, link *ipld.Link) error {
+ // Remove old link and account for size change (if it existed; ignore
+ // `ErrNotExist` otherwise).
+ err := d.RemoveChild(ctx, name)
+ if err != nil && err != os.ErrNotExist {
+ return err
+ }
+
+ err = d.node.AddRawLink(name, link)
+ if err != nil {
+ return err
+ }
+ d.addToEstimatedSize(name, link.Cid)
+ return nil
+}
+
+// EnumLinksAsync returns a channel which will receive Links in the directory
+// as they are enumerated, where order is not gauranteed
+func (d *BasicDirectory) EnumLinksAsync(ctx context.Context) <-chan format.LinkResult {
+ linkResults := make(chan format.LinkResult)
+ go func() {
+ defer close(linkResults)
+ for _, l := range d.node.Links() {
+ select {
+ case linkResults <- format.LinkResult{
+ Link: l,
+ Err: nil,
+ }:
+ case <-ctx.Done():
+ return
+ }
+ }
+ }()
+ return linkResults
+}
+
+// ForEachLink implements the `Directory` interface.
+func (d *BasicDirectory) ForEachLink(_ context.Context, f func(*ipld.Link) error) error {
+ for _, l := range d.node.Links() {
+ if err := f(l); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Links implements the `Directory` interface.
+func (d *BasicDirectory) Links(ctx context.Context) ([]*ipld.Link, error) {
+ return d.node.Links(), nil
+}
+
+// Find implements the `Directory` interface.
+func (d *BasicDirectory) Find(ctx context.Context, name string) (ipld.Node, error) {
+ lnk, err := d.node.GetNodeLink(name)
+ if err == mdag.ErrLinkNotFound {
+ err = os.ErrNotExist
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ return d.dserv.Get(ctx, lnk.Cid)
+}
+
+// RemoveChild implements the `Directory` interface.
+func (d *BasicDirectory) RemoveChild(ctx context.Context, name string) error {
+ // We need to *retrieve* the link before removing it to update the estimated
+ // size. This means we may iterate the links slice twice: if traversing this
+ // becomes a problem, a factor of 2 isn't going to make much of a difference.
+ // We'd likely need to cache a link resolution map in that case.
+ link, err := d.node.GetNodeLink(name)
+ if err == mdag.ErrLinkNotFound {
+ return os.ErrNotExist
+ }
+ if err != nil {
+ return err // at the moment there is no other error besides ErrLinkNotFound
+ }
+
+ // The name actually existed so we should update the estimated size.
+ d.removeFromEstimatedSize(link.Name, link.Cid)
+
+ return d.node.RemoveNodeLink(name)
+ // GetNodeLink didn't return ErrLinkNotFound so this won't fail with that
+ // and we don't need to convert the error again.
+}
+
+// GetNode implements the `Directory` interface.
+func (d *BasicDirectory) GetNode() (ipld.Node, error) {
+ return d.node, nil
+}
+
+// GetCidBuilder implements the `Directory` interface.
+func (d *BasicDirectory) GetCidBuilder() cid.Builder {
+ return d.node.CidBuilder()
+}
+
+// switchToSharding returns a HAMT implementation of this directory.
+func (d *BasicDirectory) switchToSharding(ctx context.Context) (*HAMTDirectory, error) {
+ hamtDir := new(HAMTDirectory)
+ hamtDir.dserv = d.dserv
+
+ shard, err := hamt.NewShard(d.dserv, DefaultShardWidth)
+ if err != nil {
+ return nil, err
+ }
+ shard.SetCidBuilder(d.node.CidBuilder())
+ hamtDir.shard = shard
+
+ for _, lnk := range d.node.Links() {
+ err = hamtDir.shard.SetLink(ctx, lnk.Name, lnk)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return hamtDir, nil
+}
+
+// SetCidBuilder implements the `Directory` interface.
+func (d *HAMTDirectory) SetCidBuilder(builder cid.Builder) {
+ d.shard.SetCidBuilder(builder)
+}
+
+// AddChild implements the `Directory` interface.
+func (d *HAMTDirectory) AddChild(ctx context.Context, name string, nd ipld.Node) error {
+ oldChild, err := d.shard.Swap(ctx, name, nd)
+ if err != nil {
+ return err
+ }
+
+ if oldChild != nil {
+ d.removeFromSizeChange(oldChild.Name, oldChild.Cid)
+ }
+ d.addToSizeChange(name, nd.Cid())
+ return nil
+}
+
+// ForEachLink implements the `Directory` interface.
+func (d *HAMTDirectory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
+ return d.shard.ForEachLink(ctx, f)
+}
+
+// EnumLinksAsync returns a channel which will receive Links in the directory
+// as they are enumerated, where order is not gauranteed
+func (d *HAMTDirectory) EnumLinksAsync(ctx context.Context) <-chan format.LinkResult {
+ return d.shard.EnumLinksAsync(ctx)
+}
+
+// Links implements the `Directory` interface.
+func (d *HAMTDirectory) Links(ctx context.Context) ([]*ipld.Link, error) {
+ return d.shard.EnumLinks(ctx)
+}
+
+// Find implements the `Directory` interface. It will traverse the tree.
+func (d *HAMTDirectory) Find(ctx context.Context, name string) (ipld.Node, error) {
+ lnk, err := d.shard.Find(ctx, name)
+ if err != nil {
+ return nil, err
+ }
+
+ return lnk.GetNode(ctx, d.dserv)
+}
+
+// RemoveChild implements the `Directory` interface.
+func (d *HAMTDirectory) RemoveChild(ctx context.Context, name string) error {
+ oldChild, err := d.shard.Take(ctx, name)
+ if err != nil {
+ return err
+ }
+
+ if oldChild != nil {
+ d.removeFromSizeChange(oldChild.Name, oldChild.Cid)
+ }
+
+ return nil
+}
+
+// GetNode implements the `Directory` interface.
+func (d *HAMTDirectory) GetNode() (ipld.Node, error) {
+ return d.shard.Node()
+}
+
+// GetCidBuilder implements the `Directory` interface.
+func (d *HAMTDirectory) GetCidBuilder() cid.Builder {
+ return d.shard.CidBuilder()
+}
+
+// switchToBasic returns a BasicDirectory implementation of this directory.
+func (d *HAMTDirectory) switchToBasic(ctx context.Context) (*BasicDirectory, error) {
+ basicDir := newEmptyBasicDirectory(d.dserv)
+ basicDir.SetCidBuilder(d.GetCidBuilder())
+
+ err := d.ForEachLink(ctx, func(lnk *ipld.Link) error {
+ err := basicDir.addLinkChild(ctx, lnk.Name, lnk)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ // This function enumerates all the links in the Directory requiring all
+ // shards to be accessible but it is only called *after* sizeBelowThreshold
+ // returns true, which means we have already enumerated and fetched *all*
+ // shards in the first place (that's the only way we can be really sure
+ // we are actually below the threshold).
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return basicDir, nil
+}
+
+func (d *HAMTDirectory) addToSizeChange(name string, linkCid cid.Cid) {
+ d.sizeChange += linksize.LinkSizeFunction(name, linkCid)
+}
+
+func (d *HAMTDirectory) removeFromSizeChange(name string, linkCid cid.Cid) {
+ d.sizeChange -= linksize.LinkSizeFunction(name, linkCid)
+}
+
+// Evaluate a switch from HAMTDirectory to BasicDirectory in case the size will
+// go above the threshold when we are adding or removing an entry.
+// In both the add/remove operations any old name will be removed, and for the
+// add operation in particular a new entry will be added under that name (otherwise
+// nodeToAdd is nil). We compute both (potential) future subtraction and
+// addition to the size change.
+func (d *HAMTDirectory) needsToSwitchToBasicDir(ctx context.Context, name string, nodeToAdd ipld.Node) (switchToBasic bool, err error) {
+ if HAMTShardingSize == 0 { // Option disabled.
+ return false, nil
+ }
+
+ operationSizeChange := 0
+
+ // Find if there is an old entry under that name that will be overwritten
+ // (AddEntry) or flat out removed (RemoveEntry).
+ entryToRemove, err := d.shard.Find(ctx, name)
+ if err != os.ErrNotExist {
+ if err != nil {
+ return false, err
+ }
+ operationSizeChange -= linksize.LinkSizeFunction(name, entryToRemove.Cid)
+ }
+
+ // For the AddEntry case compute the size addition of the new entry.
+ if nodeToAdd != nil {
+ operationSizeChange += linksize.LinkSizeFunction(name, nodeToAdd.Cid())
+ }
+
+ if d.sizeChange+operationSizeChange >= 0 {
+ // We won't have reduced the HAMT net size.
+ return false, nil
+ }
+
+ // We have reduced the directory size, check if went below the
+ // HAMTShardingSize threshold to trigger a switch.
+ return d.sizeBelowThreshold(ctx, operationSizeChange)
+}
+
+// Evaluate directory size and a future sizeChange and check if it will be below
+// HAMTShardingSize threshold (to trigger a transition to a BasicDirectory).
+// Instead of enumerating the entire tree we eagerly call EnumLinksAsync
+// until we either reach a value above the threshold (in that case no need
+// to keep counting) or an error occurs (like the context being canceled
+// if we take too much time fetching the necessary shards).
+func (d *HAMTDirectory) sizeBelowThreshold(ctx context.Context, sizeChange int) (below bool, err error) {
+ if HAMTShardingSize == 0 {
+ panic("asked to compute HAMT size with HAMTShardingSize option off (0)")
+ }
+
+ // We don't necessarily compute the full size of *all* shards as we might
+ // end early if we already know we're above the threshold or run out of time.
+ partialSize := 0
+
+ // We stop the enumeration once we have enough information and exit this function.
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ for linkResult := range d.EnumLinksAsync(ctx) {
+ if linkResult.Err != nil {
+ return false, linkResult.Err
+ }
+
+ partialSize += linksize.LinkSizeFunction(linkResult.Link.Name, linkResult.Link.Cid)
+ if partialSize+sizeChange >= HAMTShardingSize {
+ // We have already fetched enough shards to assert we are
+ // above the threshold, so no need to keep fetching.
+ return false, nil
+ }
+ }
+
+ // We enumerated *all* links in all shards and didn't reach the threshold.
+ return true, nil
+}
+
+// DynamicDirectory wraps a Directory interface and provides extra logic
+// to switch from BasicDirectory to HAMTDirectory and backwards based on
+// size.
+type DynamicDirectory struct {
+ Directory
+}
+
+var _ Directory = (*DynamicDirectory)(nil)
+
+// AddChild implements the `Directory` interface. We check when adding new entries
+// if we should switch to HAMTDirectory according to global option(s).
+func (d *DynamicDirectory) AddChild(ctx context.Context, name string, nd ipld.Node) error {
+ hamtDir, ok := d.Directory.(*HAMTDirectory)
+ if ok {
+ // We evaluate a switch in the HAMTDirectory case even for an AddChild
+ // as it may overwrite an existing entry and end up actually reducing
+ // the directory size.
+ switchToBasic, err := hamtDir.needsToSwitchToBasicDir(ctx, name, nd)
+ if err != nil {
+ return err
+ }
+
+ if switchToBasic {
+ basicDir, err := hamtDir.switchToBasic(ctx)
+ if err != nil {
+ return err
+ }
+ err = basicDir.AddChild(ctx, name, nd)
+ if err != nil {
+ return err
+ }
+ d.Directory = basicDir
+ return nil
+ }
+
+ return d.Directory.AddChild(ctx, name, nd)
+ }
+
+ // BasicDirectory
+ basicDir := d.Directory.(*BasicDirectory)
+ switchToHAMT, err := basicDir.needsToSwitchToHAMTDir(name, nd)
+ if err != nil {
+ return err
+ }
+ if !switchToHAMT {
+ return basicDir.AddChild(ctx, name, nd)
+ }
+ hamtDir, err = basicDir.switchToSharding(ctx)
+ if err != nil {
+ return err
+ }
+ err = hamtDir.AddChild(ctx, name, nd)
+ if err != nil {
+ return err
+ }
+ d.Directory = hamtDir
+ return nil
+}
+
+// RemoveChild implements the `Directory` interface. Used in the case where we wrap
+// a HAMTDirectory that might need to be downgraded to a BasicDirectory. The
+// upgrade path is in AddChild.
+func (d *DynamicDirectory) RemoveChild(ctx context.Context, name string) error {
+ hamtDir, ok := d.Directory.(*HAMTDirectory)
+ if !ok {
+ return d.Directory.RemoveChild(ctx, name)
+ }
+
+ switchToBasic, err := hamtDir.needsToSwitchToBasicDir(ctx, name, nil)
+ if err != nil {
+ return err
+ }
+
+ if !switchToBasic {
+ return hamtDir.RemoveChild(ctx, name)
+ }
+
+ basicDir, err := hamtDir.switchToBasic(ctx)
+ if err != nil {
+ return err
+ }
+ err = basicDir.RemoveChild(ctx, name)
+ if err != nil {
+ return err
+ }
+ d.Directory = basicDir
+ return nil
+}
diff --git a/unixfs/io/directory_test.go b/unixfs/io/directory_test.go
new file mode 100644
index 000000000..bcde53bf7
--- /dev/null
+++ b/unixfs/io/directory_test.go
@@ -0,0 +1,638 @@
+package io
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ blocks "github.com/ipfs/boxo/blocks"
+ bsrv "github.com/ipfs/boxo/blockservice"
+ blockstore "github.com/ipfs/boxo/blockstore"
+ offline "github.com/ipfs/boxo/exchange/offline"
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ mdtest "github.com/ipfs/boxo/ipld/merkledag/test"
+ cid "github.com/ipfs/go-cid"
+ ds "github.com/ipfs/go-datastore"
+ dssync "github.com/ipfs/go-datastore/sync"
+ ipld "github.com/ipfs/go-ipld-format"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ "github.com/ipfs/boxo/unixfs/hamt"
+ "github.com/ipfs/boxo/unixfs/internal"
+ "github.com/ipfs/boxo/unixfs/private/linksize"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestEmptyNode(t *testing.T) {
+ n := ft.EmptyDirNode()
+ if len(n.Links()) != 0 {
+ t.Fatal("empty node should have 0 links")
+ }
+}
+
+func TestDirectoryGrowth(t *testing.T) {
+ ds := mdtest.Mock()
+ dir := NewDirectory(ds)
+ ctx := context.Background()
+
+ d := ft.EmptyDirNode()
+ ds.Add(ctx, d)
+
+ nelems := 10000
+
+ for i := 0; i < nelems; i++ {
+ err := dir.AddChild(ctx, fmt.Sprintf("dir%d", i), d)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ _, err := dir.GetNode()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ links, err := dir.Links(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(links) != nelems {
+ t.Fatal("didnt get right number of elements")
+ }
+
+ dirc := d.Cid()
+
+ names := make(map[string]bool)
+ for _, l := range links {
+ names[l.Name] = true
+ if !l.Cid.Equals(dirc) {
+ t.Fatal("link wasnt correct")
+ }
+ }
+
+ for i := 0; i < nelems; i++ {
+ dn := fmt.Sprintf("dir%d", i)
+ if !names[dn] {
+ t.Fatal("didnt find directory: ", dn)
+ }
+
+ _, err := dir.Find(context.Background(), dn)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func TestDuplicateAddDir(t *testing.T) {
+ ds := mdtest.Mock()
+ dir := NewDirectory(ds)
+ ctx := context.Background()
+ nd := ft.EmptyDirNode()
+
+ err := dir.AddChild(ctx, "test", nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = dir.AddChild(ctx, "test", nd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ lnks, err := dir.Links(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(lnks) != 1 {
+ t.Fatal("expected only one link")
+ }
+}
+
+func TestBasicDirectory_estimatedSize(t *testing.T) {
+ ds := mdtest.Mock()
+ basicDir := newEmptyBasicDirectory(ds)
+
+ testDirectorySizeEstimation(t, basicDir, ds, func(dir Directory) int {
+ return dir.(*BasicDirectory).estimatedSize
+ })
+}
+
+func TestHAMTDirectory_sizeChange(t *testing.T) {
+ ds := mdtest.Mock()
+ hamtDir, err := newEmptyHAMTDirectory(ds, DefaultShardWidth)
+ assert.NoError(t, err)
+
+ testDirectorySizeEstimation(t, hamtDir, ds, func(dir Directory) int {
+ // Since we created a HAMTDirectory from scratch with size 0 its
+ // internal sizeChange delta will in fact track the directory size
+ // throughout this run.
+ return dir.(*HAMTDirectory).sizeChange
+ })
+}
+
+func fullSizeEnumeration(dir Directory) int {
+ size := 0
+ dir.ForEachLink(context.Background(), func(l *ipld.Link) error {
+ size += linksize.LinkSizeFunction(l.Name, l.Cid)
+ return nil
+ })
+ return size
+}
+
+func testDirectorySizeEstimation(t *testing.T, dir Directory, ds ipld.DAGService, size func(Directory) int) {
+ linksize.LinkSizeFunction = mockLinkSizeFunc(1)
+ defer func() { linksize.LinkSizeFunction = productionLinkSize }()
+
+ ctx := context.Background()
+ child := ft.EmptyFileNode()
+ assert.NoError(t, ds.Add(ctx, child))
+
+ // Several overwrites should not corrupt the size estimation.
+ assert.NoError(t, dir.AddChild(ctx, "child", child))
+ assert.NoError(t, dir.AddChild(ctx, "child", child))
+ assert.NoError(t, dir.AddChild(ctx, "child", child))
+ assert.NoError(t, dir.RemoveChild(ctx, "child"))
+ assert.NoError(t, dir.AddChild(ctx, "child", child))
+ assert.NoError(t, dir.RemoveChild(ctx, "child"))
+ assert.Equal(t, 0, size(dir), "estimated size is not zero after removing all entries")
+
+ dirEntries := 100
+ for i := 0; i < dirEntries; i++ {
+ assert.NoError(t, dir.AddChild(ctx, fmt.Sprintf("child-%03d", i), child))
+ }
+ assert.Equal(t, dirEntries, size(dir), "estimated size inaccurate after adding many entries")
+
+ assert.NoError(t, dir.RemoveChild(ctx, "child-045")) // just random values
+ assert.NoError(t, dir.RemoveChild(ctx, "child-063"))
+ assert.NoError(t, dir.RemoveChild(ctx, "child-011"))
+ assert.NoError(t, dir.RemoveChild(ctx, "child-000"))
+ assert.NoError(t, dir.RemoveChild(ctx, "child-099"))
+ dirEntries -= 5
+ assert.Equal(t, dirEntries, size(dir), "estimated size inaccurate after removing some entries")
+
+ // All of the following remove operations will fail (won't impact dirEntries):
+ assert.Error(t, dir.RemoveChild(ctx, "nonexistent-name"))
+ assert.Error(t, dir.RemoveChild(ctx, "child-045")) // already removed
+ assert.Error(t, dir.RemoveChild(ctx, "child-100"))
+ assert.Equal(t, dirEntries, size(dir), "estimated size inaccurate after failed remove attempts")
+
+ // Restore a directory from original's node and check estimated size consistency.
+ dirNode, err := dir.GetNode()
+ assert.NoError(t, err)
+ restoredDir, err := NewDirectoryFromNode(ds, dirNode.(*mdag.ProtoNode))
+ assert.NoError(t, err)
+ assert.Equal(t, size(dir), fullSizeEnumeration(restoredDir), "restored directory's size doesn't match original's")
+ // We don't use the estimation size function for the restored directory
+ // because in the HAMT case this function depends on the sizeChange variable
+ // that will be cleared when loading the directory from the node.
+ // This also covers the case of comparing the size estimation `size()` with
+ // the full enumeration function `fullSizeEnumeration()` to make sure it's
+ // correct.
+}
+
+// Any entry link size will have the fixedSize passed.
+func mockLinkSizeFunc(fixedSize int) func(linkName string, linkCid cid.Cid) int {
+ return func(_ string, _ cid.Cid) int {
+ return fixedSize
+ }
+}
+
+func checkBasicDirectory(t *testing.T, dir Directory, errorMessage string) {
+ if _, ok := dir.(*DynamicDirectory).Directory.(*BasicDirectory); !ok {
+ t.Fatal(errorMessage)
+ }
+}
+
+func checkHAMTDirectory(t *testing.T, dir Directory, errorMessage string) {
+ if _, ok := dir.(*DynamicDirectory).Directory.(*HAMTDirectory); !ok {
+ t.Fatal(errorMessage)
+ }
+}
+
+func TestProductionLinkSize(t *testing.T) {
+ link, err := ipld.MakeLink(ft.EmptyDirNode())
+ assert.NoError(t, err)
+ link.Name = "directory_link_name"
+ assert.Equal(t, 53, productionLinkSize(link.Name, link.Cid))
+
+ link, err = ipld.MakeLink(ft.EmptyFileNode())
+ assert.NoError(t, err)
+ link.Name = "file_link_name"
+ assert.Equal(t, 48, productionLinkSize(link.Name, link.Cid))
+
+ ds := mdtest.Mock()
+ basicDir := newEmptyBasicDirectory(ds)
+ assert.NoError(t, err)
+ for i := 0; i < 10; i++ {
+ basicDir.AddChild(context.Background(), strconv.FormatUint(uint64(i), 10), ft.EmptyFileNode())
+ }
+ basicDirNode, err := basicDir.GetNode()
+ assert.NoError(t, err)
+ link, err = ipld.MakeLink(basicDirNode)
+ assert.NoError(t, err)
+ link.Name = "basic_dir"
+ assert.Equal(t, 43, productionLinkSize(link.Name, link.Cid))
+}
+
+// Test HAMTDirectory <-> BasicDirectory switch based on directory size. The
+// switch is managed by the DynamicDirectory abstraction.
+func TestDynamicDirectorySwitch(t *testing.T) {
+ oldHamtOption := HAMTShardingSize
+ defer func() { HAMTShardingSize = oldHamtOption }()
+ HAMTShardingSize = 0 // Disable automatic switch at the start.
+ linksize.LinkSizeFunction = mockLinkSizeFunc(1)
+ defer func() { linksize.LinkSizeFunction = productionLinkSize }()
+
+ ds := mdtest.Mock()
+ dir := NewDirectory(ds)
+ checkBasicDirectory(t, dir, "new dir is not BasicDirectory")
+
+ ctx := context.Background()
+ child := ft.EmptyDirNode()
+ err := ds.Add(ctx, child)
+ assert.NoError(t, err)
+
+ err = dir.AddChild(ctx, "1", child)
+ assert.NoError(t, err)
+ checkBasicDirectory(t, dir, "added child, option still disabled")
+
+ // Set a threshold so big a new entry won't trigger the change.
+ HAMTShardingSize = math.MaxInt32
+
+ err = dir.AddChild(ctx, "2", child)
+ assert.NoError(t, err)
+ checkBasicDirectory(t, dir, "added child, option now enabled but at max")
+
+ // Now set it so low to make sure any new entry will trigger the upgrade.
+ HAMTShardingSize = 1
+
+ // We are already above the threshold, we trigger the switch with an overwrite
+ // (any AddChild() should reevaluate the size).
+ err = dir.AddChild(ctx, "2", child)
+ assert.NoError(t, err)
+ checkHAMTDirectory(t, dir, "added child, option at min, should switch up")
+
+ // Set threshold at the number of current entries and delete the last one
+ // to trigger a switch and evaluate if the rest of the entries are conserved.
+ HAMTShardingSize = 2
+ err = dir.RemoveChild(ctx, "2")
+ assert.NoError(t, err)
+ checkBasicDirectory(t, dir, "removed threshold entry, option at min, should switch down")
+}
+
+func TestIntegrityOfDirectorySwitch(t *testing.T) {
+ ds := mdtest.Mock()
+ dir := NewDirectory(ds)
+ checkBasicDirectory(t, dir, "new dir is not BasicDirectory")
+
+ ctx := context.Background()
+ child := ft.EmptyDirNode()
+ err := ds.Add(ctx, child)
+ assert.NoError(t, err)
+
+ basicDir := newEmptyBasicDirectory(ds)
+ hamtDir, err := newEmptyHAMTDirectory(ds, DefaultShardWidth)
+ assert.NoError(t, err)
+ for i := 0; i < 1000; i++ {
+ basicDir.AddChild(ctx, strconv.FormatUint(uint64(i), 10), child)
+ hamtDir.AddChild(ctx, strconv.FormatUint(uint64(i), 10), child)
+ }
+ compareDirectoryEntries(t, basicDir, hamtDir)
+
+ hamtDirFromSwitch, err := basicDir.switchToSharding(ctx)
+ assert.NoError(t, err)
+ basicDirFromSwitch, err := hamtDir.switchToBasic(ctx)
+ assert.NoError(t, err)
+ compareDirectoryEntries(t, basicDir, basicDirFromSwitch)
+ compareDirectoryEntries(t, hamtDir, hamtDirFromSwitch)
+}
+
+// This is the value of concurrent fetches during dag.Walk. Used in
+// test to better predict how many nodes will be fetched.
+var defaultConcurrentFetch = 32
+
+// FIXME: Taken from private github.com/ipfs/boxo/ipld/merkledag@v0.2.3/merkledag.go.
+// (We can also pass an explicit concurrency value in `(*Shard).EnumLinksAsync()`
+// and take ownership of this configuration, but departing from the more
+// standard and reliable one in `go-merkledag`.
+
+// Test that we fetch as little nodes as needed to reach the HAMTShardingSize
+// during the sizeBelowThreshold computation.
+func TestHAMTEnumerationWhenComputingSize(t *testing.T) {
+ // Adjust HAMT global/static options for the test to simplify its logic.
+ // FIXME: These variables weren't designed to be modified and we should
+ // review in depth side effects.
+
+ // Set all link sizes to a uniform 1 so the estimated directory size
+ // is just the count of its entry links (in HAMT/Shard terminology these
+ // are the "value" links pointing to anything that is *not* another Shard).
+ linksize.LinkSizeFunction = mockLinkSizeFunc(1)
+ defer func() { linksize.LinkSizeFunction = productionLinkSize }()
+
+ // Use an identity hash function to ease the construction of "complete" HAMTs
+ // (see CreateCompleteHAMT below for more details). (Ideally this should be
+ // a parameter we pass and not a global option we modify in the caller.)
+ oldHashFunc := internal.HAMTHashFunction
+ defer func() { internal.HAMTHashFunction = oldHashFunc }()
+ internal.HAMTHashFunction = idHash
+
+ oldHamtOption := HAMTShardingSize
+ defer func() { HAMTShardingSize = oldHamtOption }()
+
+ // --- End of test static configuration adjustments. ---
+
+ // Some arbitrary values below that make this test not that expensive.
+ treeHeight := 4
+ // How many leaf shards nodes (with value links,
+ // i.e., directory entries) do we need to reach the threshold.
+ thresholdToWidthRatio := 4
+ // Departing from DefaultShardWidth of 256 to reduce HAMT size in
+ // CreateCompleteHAMT.
+ shardWidth := 16
+ HAMTShardingSize = shardWidth * thresholdToWidthRatio
+
+ // We create a "complete" HAMT (see CreateCompleteHAMT for more details)
+ // with a regular structure to be able to predict how many Shard nodes we
+ // will need to fetch in order to reach the HAMTShardingSize threshold in
+ // sizeBelowThreshold (assuming a sequential DAG walk function).
+
+ bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
+ countGetsDS := newCountGetsDS(bstore)
+ dsrv := mdag.NewDAGService(bsrv.New(countGetsDS, offline.Exchange(countGetsDS)))
+ completeHAMTRoot, err := CreateCompleteHAMT(dsrv, treeHeight, shardWidth)
+ assert.NoError(t, err)
+
+ // Calculate the optimal number of nodes to traverse
+ optimalNodesToFetch := 0
+ nodesToProcess := HAMTShardingSize
+ for i := 0; i < treeHeight-1; i++ {
+ // divide by the shard width to get the parents and continue up the tree
+ parentNodes := int(math.Ceil(float64(nodesToProcess) / float64(shardWidth)))
+ optimalNodesToFetch += parentNodes
+ nodesToProcess = parentNodes
+ }
+
+ // With this structure and a BFS traversal (from `parallelWalkDepth`) then
+ // we would roughly fetch the following nodes:
+ nodesToFetch := 0
+ // * all layers up to (but not including) the last one with leaf nodes
+ // (because it's a BFS)
+ for i := 0; i < treeHeight-1; i++ {
+ nodesToFetch += int(math.Pow(float64(shardWidth), float64(i)))
+ }
+ // * `thresholdToWidthRatio` leaf Shards with enough value links to reach
+ // the HAMTShardingSize threshold.
+ nodesToFetch += thresholdToWidthRatio
+
+ hamtDir, err := newHAMTDirectoryFromNode(dsrv, completeHAMTRoot)
+ assert.NoError(t, err)
+
+ countGetsDS.resetCounter()
+ countGetsDS.setRequestDelay(10 * time.Millisecond)
+ // (Without the `setRequestDelay` above the number of nodes fetched
+ // drops dramatically and unpredictably as the BFS starts to behave
+ // more like a DFS because some search paths are fetched faster than
+ // others.)
+ below, err := hamtDir.sizeBelowThreshold(context.TODO(), 0)
+ assert.NoError(t, err)
+ assert.False(t, below)
+ t.Logf("fetched %d nodes (predicted range: %d-%d)",
+ countGetsDS.uniqueCidsFetched(), optimalNodesToFetch, nodesToFetch+defaultConcurrentFetch)
+ // Check that the actual number of nodes fetched is within the margin of the
+ // estimated `nodesToFetch` plus an extra of `defaultConcurrentFetch` since
+ // we are fetching in parallel.
+ assert.True(t, countGetsDS.uniqueCidsFetched() <= nodesToFetch+defaultConcurrentFetch)
+ assert.True(t, countGetsDS.uniqueCidsFetched() >= optimalNodesToFetch)
+}
+
+// Compare entries in the leftDir against the rightDir and possibly
+// missingEntries in the second.
+func compareDirectoryEntries(t *testing.T, leftDir Directory, rightDir Directory) {
+ leftLinks, err := getAllLinksSortedByName(leftDir)
+ assert.NoError(t, err)
+ rightLinks, err := getAllLinksSortedByName(rightDir)
+ assert.NoError(t, err)
+
+ assert.Equal(t, len(leftLinks), len(rightLinks))
+
+ for i, leftLink := range leftLinks {
+ assert.Equal(t, leftLink, rightLinks[i]) // FIXME: Can we just compare the entire struct?
+ }
+}
+
+func getAllLinksSortedByName(d Directory) ([]*ipld.Link, error) {
+ entries, err := d.Links(context.Background())
+ if err != nil {
+ return nil, err
+ }
+ sortLinksByName(entries)
+ return entries, nil
+}
+
+func sortLinksByName(l []*ipld.Link) {
+ sort.SliceStable(l, func(i, j int) bool {
+ return strings.Compare(l[i].Name, l[j].Name) == -1 // FIXME: Is this correct?
+ })
+}
+
+func TestDirBuilder(t *testing.T) {
+ ds := mdtest.Mock()
+ dir := NewDirectory(ds)
+ ctx := context.Background()
+
+ child := ft.EmptyDirNode()
+ err := ds.Add(ctx, child)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ count := 5000
+
+ for i := 0; i < count; i++ {
+ err := dir.AddChild(ctx, fmt.Sprintf("entry %d", i), child)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ dirnd, err := dir.GetNode()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ links, err := dir.Links(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(links) != count {
+ t.Fatal("not enough links dawg", len(links), count)
+ }
+
+ adir, err := NewDirectoryFromNode(ds, dirnd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ links, err = adir.Links(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ names := make(map[string]bool)
+ for _, lnk := range links {
+ names[lnk.Name] = true
+ }
+
+ for i := 0; i < count; i++ {
+ n := fmt.Sprintf("entry %d", i)
+ if !names[n] {
+ t.Fatal("COULDNT FIND: ", n)
+ }
+ }
+
+ if len(links) != count {
+ t.Fatal("wrong number of links", len(links), count)
+ }
+
+ linkResults := dir.EnumLinksAsync(ctx)
+
+ asyncNames := make(map[string]bool)
+ var asyncLinks []*ipld.Link
+
+ for linkResult := range linkResults {
+ if linkResult.Err != nil {
+ t.Fatal(linkResult.Err)
+ }
+ asyncNames[linkResult.Link.Name] = true
+ asyncLinks = append(asyncLinks, linkResult.Link)
+ }
+
+ for i := 0; i < count; i++ {
+ n := fmt.Sprintf("entry %d", i)
+ if !asyncNames[n] {
+ t.Fatal("COULDNT FIND: ", n)
+ }
+ }
+
+ if len(asyncLinks) != count {
+ t.Fatal("wrong number of links", len(asyncLinks), count)
+ }
+}
+
+func newHAMTDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (*HAMTDirectory, error) {
+ shard, err := hamt.NewHamtFromDag(dserv, node)
+ if err != nil {
+ return nil, err
+ }
+ return &HAMTDirectory{
+ dserv: dserv,
+ shard: shard,
+ }, nil
+}
+
+func newEmptyHAMTDirectory(dserv ipld.DAGService, shardWidth int) (*HAMTDirectory, error) {
+ shard, err := hamt.NewShard(dserv, shardWidth)
+ if err != nil {
+ return nil, err
+ }
+
+ return &HAMTDirectory{
+ dserv: dserv,
+ shard: shard,
+ }, nil
+}
+
+// countGetsDS is a DAG service that keeps track of the number of
+// unique CIDs fetched.
+type countGetsDS struct {
+ blockstore.Blockstore
+
+ cidsFetched map[cid.Cid]struct{}
+ mapLock sync.Mutex
+ started bool
+
+ getRequestDelay time.Duration
+}
+
+var _ blockstore.Blockstore = (*countGetsDS)(nil)
+
+func newCountGetsDS(bs blockstore.Blockstore) *countGetsDS {
+ return &countGetsDS{
+ bs,
+ make(map[cid.Cid]struct{}),
+ sync.Mutex{},
+ false,
+ 0,
+ }
+}
+
+func (d *countGetsDS) resetCounter() {
+ d.mapLock.Lock()
+ defer d.mapLock.Unlock()
+ d.cidsFetched = make(map[cid.Cid]struct{})
+ d.started = true
+}
+
+func (d *countGetsDS) uniqueCidsFetched() int {
+ d.mapLock.Lock()
+ defer d.mapLock.Unlock()
+ return len(d.cidsFetched)
+}
+
+func (d *countGetsDS) setRequestDelay(timeout time.Duration) {
+ d.getRequestDelay = timeout
+}
+
+func (d *countGetsDS) maybeSleep(c cid.Cid) {
+ d.mapLock.Lock()
+ _, cidRequestedBefore := d.cidsFetched[c]
+ d.cidsFetched[c] = struct{}{}
+ d.mapLock.Unlock()
+
+ if d.getRequestDelay != 0 && !cidRequestedBefore {
+ // First request gets a timeout to simulate a network fetch.
+ // Subsequent requests get no timeout simulating an in-disk cache.
+ time.Sleep(d.getRequestDelay)
+ }
+}
+
+func (d *countGetsDS) Has(ctx context.Context, c cid.Cid) (bool, error) {
+ if d.started {
+ panic("implement me")
+ }
+ return d.Blockstore.Has(ctx, c)
+}
+
+func (d *countGetsDS) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+ blk, err := d.Blockstore.Get(ctx, c)
+ if err != nil {
+ return nil, err
+ }
+
+ d.maybeSleep(c)
+ return blk, nil
+}
+
+func (d *countGetsDS) GetSize(ctx context.Context, c cid.Cid) (int, error) {
+ if d.started {
+ panic("implement me")
+ }
+ return d.Blockstore.GetSize(ctx, c)
+}
+
+func (d *countGetsDS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
+ if d.started {
+ panic("implement me")
+ }
+ return d.Blockstore.AllKeysChan(ctx)
+}
diff --git a/unixfs/io/doc.go b/unixfs/io/doc.go
new file mode 100644
index 000000000..cf844bd23
--- /dev/null
+++ b/unixfs/io/doc.go
@@ -0,0 +1,3 @@
+// Package io implements convenience objects for working with the ipfs
+// unixfs data format.
+package io
diff --git a/unixfs/io/resolve.go b/unixfs/io/resolve.go
new file mode 100644
index 000000000..e290db9d2
--- /dev/null
+++ b/unixfs/io/resolve.go
@@ -0,0 +1,41 @@
+package io
+
+import (
+ "context"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ ft "github.com/ipfs/boxo/unixfs"
+ hamt "github.com/ipfs/boxo/unixfs/hamt"
+
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// ResolveUnixfsOnce resolves a single hop of a path through a graph in a
+// unixfs context. This includes handling traversing sharded directories.
+func ResolveUnixfsOnce(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) {
+ pn, ok := nd.(*dag.ProtoNode)
+ if ok {
+ fsn, err := ft.FSNodeFromBytes(pn.Data())
+ if err != nil {
+ // Not a unixfs node, use standard object traversal code
+ return nd.ResolveLink(names)
+ }
+
+ if fsn.Type() == ft.THAMTShard {
+ rods := dag.NewReadOnlyDagService(ds)
+ s, err := hamt.NewHamtFromDag(rods, nd)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ out, err := s.Find(ctx, names[0])
+ if err != nil {
+ return nil, nil, err
+ }
+
+ return out, names[1:], nil
+ }
+ }
+
+ return nd.ResolveLink(names)
+}
diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go
new file mode 100644
index 000000000..70bf80233
--- /dev/null
+++ b/unixfs/mod/dagmodifier.go
@@ -0,0 +1,605 @@
+// Package mod provides DAG modification utilities to, for example,
+// insert additional nodes in a unixfs DAG or truncate them.
+package mod
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ help "github.com/ipfs/boxo/unixfs/importer/helpers"
+ trickle "github.com/ipfs/boxo/unixfs/importer/trickle"
+ uio "github.com/ipfs/boxo/unixfs/io"
+
+ chunker "github.com/ipfs/boxo/chunker"
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// Common errors
+var (
+ ErrSeekFail = errors.New("failed to seek properly")
+ ErrUnrecognizedWhence = errors.New("unrecognized whence")
+ ErrNotUnixfs = errors.New("dagmodifier only supports unixfs nodes (proto or raw)")
+)
+
+// 2MB
+var writebufferSize = 1 << 21
+
+// DagModifier is the only struct licensed and able to correctly
+// perform surgery on a DAG 'file'
+// Dear god, please rename this to something more pleasant
+type DagModifier struct {
+ dagserv ipld.DAGService
+ curNode ipld.Node
+
+ splitter chunker.SplitterGen
+ ctx context.Context
+ readCancel func()
+
+ writeStart uint64
+ curWrOff uint64
+ wrBuf *bytes.Buffer
+
+ Prefix cid.Prefix
+ RawLeaves bool
+
+ read uio.DagReader
+}
+
+// NewDagModifier returns a new DagModifier, the Cid prefix for newly
+// created nodes will be inhered from the passed in node. If the Cid
+// version if not 0 raw leaves will also be enabled. The Prefix and
+// RawLeaves options can be overridden by changing them after the call.
+func NewDagModifier(ctx context.Context, from ipld.Node, serv ipld.DAGService, spl chunker.SplitterGen) (*DagModifier, error) {
+ switch from.(type) {
+ case *mdag.ProtoNode, *mdag.RawNode:
+ // ok
+ default:
+ return nil, ErrNotUnixfs
+ }
+
+ prefix := from.Cid().Prefix()
+ prefix.Codec = cid.DagProtobuf
+ rawLeaves := false
+ if prefix.Version > 0 {
+ rawLeaves = true
+ }
+
+ return &DagModifier{
+ curNode: from.Copy(),
+ dagserv: serv,
+ splitter: spl,
+ ctx: ctx,
+ Prefix: prefix,
+ RawLeaves: rawLeaves,
+ }, nil
+}
+
+// WriteAt will modify a dag file in place
+func (dm *DagModifier) WriteAt(b []byte, offset int64) (int, error) {
+ // TODO: this is currently VERY inefficient
+ // each write that happens at an offset other than the current one causes a
+ // flush to disk, and dag rewrite
+ if offset == int64(dm.writeStart) && dm.wrBuf != nil {
+ // If we would overwrite the previous write
+ if len(b) >= dm.wrBuf.Len() {
+ dm.wrBuf.Reset()
+ }
+ } else if uint64(offset) != dm.curWrOff {
+ size, err := dm.Size()
+ if err != nil {
+ return 0, err
+ }
+ if offset > size {
+ err := dm.expandSparse(offset - size)
+ if err != nil {
+ return 0, err
+ }
+ }
+
+ err = dm.Sync()
+ if err != nil {
+ return 0, err
+ }
+ dm.writeStart = uint64(offset)
+ }
+
+ return dm.Write(b)
+}
+
+// A reader that just returns zeros
+type zeroReader struct{}
+
+func (zr zeroReader) Read(b []byte) (int, error) {
+ for i := range b {
+ b[i] = 0
+ }
+ return len(b), nil
+}
+
+// expandSparse grows the file with zero blocks of 4096
+// A small blocksize is chosen to aid in deduplication
+func (dm *DagModifier) expandSparse(size int64) error {
+ r := io.LimitReader(zeroReader{}, size)
+ spl := chunker.NewSizeSplitter(r, 4096)
+ nnode, err := dm.appendData(dm.curNode, spl)
+ if err != nil {
+ return err
+ }
+ err = dm.dagserv.Add(dm.ctx, nnode)
+ return err
+}
+
+// Write continues writing to the dag at the current offset
+func (dm *DagModifier) Write(b []byte) (int, error) {
+ if dm.read != nil {
+ dm.read = nil
+ }
+ if dm.wrBuf == nil {
+ dm.wrBuf = new(bytes.Buffer)
+ }
+
+ n, err := dm.wrBuf.Write(b)
+ if err != nil {
+ return n, err
+ }
+ dm.curWrOff += uint64(n)
+ if dm.wrBuf.Len() > writebufferSize {
+ err := dm.Sync()
+ if err != nil {
+ return n, err
+ }
+ }
+ return n, nil
+}
+
+// Size returns the Filesize of the node
+func (dm *DagModifier) Size() (int64, error) {
+ fileSize, err := fileSize(dm.curNode)
+ if err != nil {
+ return 0, err
+ }
+ if dm.wrBuf != nil && int64(dm.wrBuf.Len())+int64(dm.writeStart) > int64(fileSize) {
+ return int64(dm.wrBuf.Len()) + int64(dm.writeStart), nil
+ }
+ return int64(fileSize), nil
+}
+
+func fileSize(n ipld.Node) (uint64, error) {
+ switch nd := n.(type) {
+ case *mdag.ProtoNode:
+ fsn, err := ft.FSNodeFromBytes(nd.Data())
+ if err != nil {
+ return 0, err
+ }
+ return fsn.FileSize(), nil
+ case *mdag.RawNode:
+ return uint64(len(nd.RawData())), nil
+ default:
+ return 0, ErrNotUnixfs
+ }
+}
+
+// Sync writes changes to this dag to disk
+func (dm *DagModifier) Sync() error {
+ // No buffer? Nothing to do
+ if dm.wrBuf == nil {
+ return nil
+ }
+
+ // If we have an active reader, kill it
+ if dm.read != nil {
+ dm.read = nil
+ dm.readCancel()
+ }
+
+ // Number of bytes we're going to write
+ buflen := dm.wrBuf.Len()
+
+ fs, err := fileSize(dm.curNode)
+ if err != nil {
+ return err
+ }
+ if fs < dm.writeStart {
+ if err := dm.expandSparse(int64(dm.writeStart - fs)); err != nil {
+ return err
+ }
+ }
+
+ // overwrite existing dag nodes
+ thisc, err := dm.modifyDag(dm.curNode, dm.writeStart)
+ if err != nil {
+ return err
+ }
+
+ dm.curNode, err = dm.dagserv.Get(dm.ctx, thisc)
+ if err != nil {
+ return err
+ }
+
+ // need to write past end of current dag
+ if dm.wrBuf.Len() > 0 {
+ dm.curNode, err = dm.appendData(dm.curNode, dm.splitter(dm.wrBuf))
+ if err != nil {
+ return err
+ }
+
+ err = dm.dagserv.Add(dm.ctx, dm.curNode)
+ if err != nil {
+ return err
+ }
+ }
+
+ dm.writeStart += uint64(buflen)
+ dm.wrBuf = nil
+
+ return nil
+}
+
+// modifyDag writes the data in 'dm.wrBuf' over the data in 'node' starting at 'offset'
+// returns the new key of the passed in node.
+func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (cid.Cid, error) {
+ // If we've reached a leaf node.
+ if len(n.Links()) == 0 {
+ switch nd0 := n.(type) {
+ case *mdag.ProtoNode:
+ fsn, err := ft.FSNodeFromBytes(nd0.Data())
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ _, err = dm.wrBuf.Read(fsn.Data()[offset:])
+ if err != nil && err != io.EOF {
+ return cid.Cid{}, err
+ }
+
+ // Update newly written node..
+ b, err := fsn.GetBytes()
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ nd := new(mdag.ProtoNode)
+ nd.SetData(b)
+ nd.SetCidBuilder(nd0.CidBuilder())
+ err = dm.dagserv.Add(dm.ctx, nd)
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ return nd.Cid(), nil
+ case *mdag.RawNode:
+ origData := nd0.RawData()
+ bytes := make([]byte, len(origData))
+
+ // copy orig data up to offset
+ copy(bytes, origData[:offset])
+
+ // copy in new data
+ n, err := dm.wrBuf.Read(bytes[offset:])
+ if err != nil && err != io.EOF {
+ return cid.Cid{}, err
+ }
+
+ // copy remaining data
+ offsetPlusN := int(offset) + n
+ if offsetPlusN < len(origData) {
+ copy(bytes[offsetPlusN:], origData[offsetPlusN:])
+ }
+
+ nd, err := mdag.NewRawNodeWPrefix(bytes, nd0.Cid().Prefix())
+ if err != nil {
+ return cid.Cid{}, err
+ }
+ err = dm.dagserv.Add(dm.ctx, nd)
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ return nd.Cid(), nil
+ }
+ }
+
+ node, ok := n.(*mdag.ProtoNode)
+ if !ok {
+ return cid.Cid{}, ErrNotUnixfs
+ }
+
+ fsn, err := ft.FSNodeFromBytes(node.Data())
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ var cur uint64
+ for i, bs := range fsn.BlockSizes() {
+ // We found the correct child to write into
+ if cur+bs > offset {
+ child, err := node.Links()[i].GetNode(dm.ctx, dm.dagserv)
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ k, err := dm.modifyDag(child, offset-cur)
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ node.Links()[i].Cid = k
+
+ // Recache serialized node
+ _, err = node.EncodeProtobuf(true)
+ if err != nil {
+ return cid.Cid{}, err
+ }
+
+ if dm.wrBuf.Len() == 0 {
+ // No more bytes to write!
+ break
+ }
+ offset = cur + bs
+ }
+ cur += bs
+ }
+
+ err = dm.dagserv.Add(dm.ctx, node)
+ return node.Cid(), err
+}
+
+// appendData appends the blocks from the given chan to the end of this dag
+func (dm *DagModifier) appendData(nd ipld.Node, spl chunker.Splitter) (ipld.Node, error) {
+ switch nd := nd.(type) {
+ case *mdag.ProtoNode, *mdag.RawNode:
+ dbp := &help.DagBuilderParams{
+ Dagserv: dm.dagserv,
+ Maxlinks: help.DefaultLinksPerBlock,
+ CidBuilder: dm.Prefix,
+ RawLeaves: dm.RawLeaves,
+ }
+ db, err := dbp.New(spl)
+ if err != nil {
+ return nil, err
+ }
+ return trickle.Append(dm.ctx, nd, db)
+ default:
+ return nil, ErrNotUnixfs
+ }
+}
+
+// Read data from this dag starting at the current offset
+func (dm *DagModifier) Read(b []byte) (int, error) {
+ err := dm.readPrep()
+ if err != nil {
+ return 0, err
+ }
+
+ n, err := dm.read.Read(b)
+ dm.curWrOff += uint64(n)
+ return n, err
+}
+
+func (dm *DagModifier) readPrep() error {
+ err := dm.Sync()
+ if err != nil {
+ return err
+ }
+
+ if dm.read == nil {
+ ctx, cancel := context.WithCancel(dm.ctx)
+ dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv)
+ if err != nil {
+ cancel()
+ return err
+ }
+
+ i, err := dr.Seek(int64(dm.curWrOff), io.SeekStart)
+ if err != nil {
+ cancel()
+ return err
+ }
+
+ if i != int64(dm.curWrOff) {
+ cancel()
+ return ErrSeekFail
+ }
+
+ dm.readCancel = cancel
+ dm.read = dr
+ }
+
+ return nil
+}
+
+// CtxReadFull reads data from this dag starting at the current offset
+func (dm *DagModifier) CtxReadFull(ctx context.Context, b []byte) (int, error) {
+ err := dm.readPrep()
+ if err != nil {
+ return 0, err
+ }
+
+ n, err := dm.read.CtxReadFull(ctx, b)
+ dm.curWrOff += uint64(n)
+ return n, err
+}
+
+// GetNode gets the modified DAG Node
+func (dm *DagModifier) GetNode() (ipld.Node, error) {
+ err := dm.Sync()
+ if err != nil {
+ return nil, err
+ }
+ return dm.curNode.Copy(), nil
+}
+
+// HasChanges returned whether or not there are unflushed changes to this dag
+func (dm *DagModifier) HasChanges() bool {
+ return dm.wrBuf != nil
+}
+
+// Seek modifies the offset according to whence. See unixfs/io for valid whence
+// values.
+func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) {
+ err := dm.Sync()
+ if err != nil {
+ return 0, err
+ }
+
+ fisize, err := dm.Size()
+ if err != nil {
+ return 0, err
+ }
+
+ var newoffset uint64
+ switch whence {
+ case io.SeekCurrent:
+ newoffset = dm.curWrOff + uint64(offset)
+ case io.SeekStart:
+ newoffset = uint64(offset)
+ case io.SeekEnd:
+ newoffset = uint64(fisize) - uint64(offset)
+ default:
+ return 0, ErrUnrecognizedWhence
+ }
+
+ if int64(newoffset) > fisize {
+ if err := dm.expandSparse(int64(newoffset) - fisize); err != nil {
+ return 0, err
+ }
+ }
+ dm.curWrOff = newoffset
+ dm.writeStart = newoffset
+
+ if dm.read != nil {
+ _, err = dm.read.Seek(offset, whence)
+ if err != nil {
+ return 0, err
+ }
+ }
+
+ return int64(dm.curWrOff), nil
+}
+
+// Truncate truncates the current Node to 'size' and replaces it with the
+// new one.
+func (dm *DagModifier) Truncate(size int64) error {
+ err := dm.Sync()
+ if err != nil {
+ return err
+ }
+
+ realSize, err := dm.Size()
+ if err != nil {
+ return err
+ }
+ if size == int64(realSize) {
+ return nil
+ }
+
+ // Truncate can also be used to expand the file
+ if size > int64(realSize) {
+ return dm.expandSparse(int64(size) - realSize)
+ }
+
+ nnode, err := dm.dagTruncate(dm.ctx, dm.curNode, uint64(size))
+ if err != nil {
+ return err
+ }
+
+ err = dm.dagserv.Add(dm.ctx, nnode)
+ if err != nil {
+ return err
+ }
+
+ dm.curNode = nnode
+ return nil
+}
+
+// dagTruncate truncates the given node to 'size' and returns the modified Node
+func (dm *DagModifier) dagTruncate(ctx context.Context, n ipld.Node, size uint64) (ipld.Node, error) {
+ if len(n.Links()) == 0 {
+ switch nd := n.(type) {
+ case *mdag.ProtoNode:
+ // TODO: this can likely be done without marshaling and remarshaling
+ fsn, err := ft.FSNodeFromBytes(nd.Data())
+ if err != nil {
+ return nil, err
+ }
+ nd.SetData(ft.WrapData(fsn.Data()[:size]))
+ return nd, nil
+ case *mdag.RawNode:
+ return mdag.NewRawNodeWPrefix(nd.RawData()[:size], nd.Cid().Prefix())
+ }
+ }
+
+ nd, ok := n.(*mdag.ProtoNode)
+ if !ok {
+ return nil, ErrNotUnixfs
+ }
+
+ var cur uint64
+ end := 0
+ var modified ipld.Node
+ ndata, err := ft.FSNodeFromBytes(nd.Data())
+ if err != nil {
+ return nil, err
+ }
+ // Reset the block sizes of the node to adjust them
+ // with the new values of the truncated children.
+ ndata.RemoveAllBlockSizes()
+ for i, lnk := range nd.Links() {
+ child, err := lnk.GetNode(ctx, dm.dagserv)
+ if err != nil {
+ return nil, err
+ }
+
+ childsize, err := fileSize(child)
+ if err != nil {
+ return nil, err
+ }
+
+ // found the child we want to cut
+ if size < cur+childsize {
+ nchild, err := dm.dagTruncate(ctx, child, size-cur)
+ if err != nil {
+ return nil, err
+ }
+
+ ndata.AddBlockSize(size - cur)
+
+ modified = nchild
+ end = i
+ break
+ }
+ cur += childsize
+ ndata.AddBlockSize(childsize)
+ }
+
+ err = dm.dagserv.Add(ctx, modified)
+ if err != nil {
+ return nil, err
+ }
+
+ nd.SetLinks(nd.Links()[:end])
+ err = nd.AddNodeLink("", modified)
+ if err != nil {
+ return nil, err
+ }
+
+ d, err := ndata.GetBytes()
+ if err != nil {
+ return nil, err
+ }
+ // Save the new block sizes to the original node.
+ nd.SetData(d)
+
+ // invalidate cache and recompute serialized data
+ _, err = nd.EncodeProtobuf(true)
+ if err != nil {
+ return nil, err
+ }
+
+ return nd, nil
+}
diff --git a/unixfs/mod/dagmodifier_test.go b/unixfs/mod/dagmodifier_test.go
new file mode 100644
index 000000000..d41bb35fb
--- /dev/null
+++ b/unixfs/mod/dagmodifier_test.go
@@ -0,0 +1,859 @@
+package mod
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "testing"
+
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+ h "github.com/ipfs/boxo/unixfs/importer/helpers"
+ trickle "github.com/ipfs/boxo/unixfs/importer/trickle"
+ uio "github.com/ipfs/boxo/unixfs/io"
+ testu "github.com/ipfs/boxo/unixfs/test"
+
+ "github.com/ipfs/boxo/unixfs"
+ u "github.com/ipfs/boxo/util"
+)
+
+func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, opts testu.NodeOpts) []byte {
+ newdata := make([]byte, size)
+ r := u.NewTimeSeededRand()
+ r.Read(newdata)
+
+ if size+beg > uint64(len(orig)) {
+ orig = append(orig, make([]byte, (size+beg)-uint64(len(orig)))...)
+ }
+ copy(orig[beg:], newdata)
+
+ nmod, err := dm.WriteAt(newdata, int64(beg))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if nmod != int(size) {
+ t.Fatalf("Mod length not correct! %d != %d", nmod, size)
+ }
+
+ verifyNode(t, orig, dm, opts)
+
+ return orig
+}
+
+func verifyNode(t *testing.T, orig []byte, dm *DagModifier, opts testu.NodeOpts) {
+ nd, err := dm.GetNode()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = trickle.VerifyTrickleDagStructure(nd, trickle.VerifyParams{
+ Getter: dm.dagserv,
+ Direct: h.DefaultLinksPerBlock,
+ LayerRepeat: 4,
+ Prefix: &opts.Prefix,
+ RawLeaves: opts.RawLeavesUsed,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ after, err := io.ReadAll(rd)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = testu.ArrComp(after, orig)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func runAllSubtests(t *testing.T, tfunc func(*testing.T, testu.NodeOpts)) {
+ t.Run("opts=ProtoBufLeaves", func(t *testing.T) { tfunc(t, testu.UseProtoBufLeaves) })
+ t.Run("opts=RawLeaves", func(t *testing.T) { tfunc(t, testu.UseRawLeaves) })
+ t.Run("opts=CidV1", func(t *testing.T) { tfunc(t, testu.UseCidV1) })
+ t.Run("opts=Blake2b256", func(t *testing.T) { tfunc(t, testu.UseBlake2b256) })
+}
+
+func TestDagModifierBasic(t *testing.T) {
+ runAllSubtests(t, testDagModifierBasic)
+}
+func testDagModifierBasic(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ b, n := testu.GetRandomNode(t, dserv, 50000, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ // Within zero block
+ beg := uint64(15)
+ length := uint64(60)
+
+ t.Log("Testing mod within zero block")
+ b = testModWrite(t, beg, length, b, dagmod, opts)
+
+ // Within bounds of existing file
+ beg = 1000
+ length = 4000
+ t.Log("Testing mod within bounds of existing multiblock file.")
+ b = testModWrite(t, beg, length, b, dagmod, opts)
+
+ // Extend bounds
+ beg = 49500
+ length = 4000
+
+ t.Log("Testing mod that extends file.")
+ b = testModWrite(t, beg, length, b, dagmod, opts)
+
+ // "Append"
+ beg = uint64(len(b))
+ length = 3000
+ t.Log("Testing pure append")
+ _ = testModWrite(t, beg, length, b, dagmod, opts)
+
+ // Verify reported length
+ node, err := dagmod.GetNode()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ size, err := fileSize(node)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ expected := uint64(50000 + 3500 + 3000)
+ if size != expected {
+ t.Fatalf("Final reported size is incorrect [%d != %d]", size, expected)
+ }
+}
+
+func TestMultiWrite(t *testing.T) {
+ runAllSubtests(t, testMultiWrite)
+}
+func testMultiWrite(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ data := make([]byte, 4000)
+ u.NewTimeSeededRand().Read(data)
+
+ for i := 0; i < len(data); i++ {
+ n, err := dagmod.WriteAt(data[i:i+1], int64(i))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != 1 {
+ t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
+ }
+
+ size, err := dagmod.Size()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if size != int64(i+1) {
+ t.Fatal("Size was reported incorrectly")
+ }
+ }
+
+ verifyNode(t, data, dagmod, opts)
+}
+
+func TestMultiWriteAndFlush(t *testing.T) {
+ runAllSubtests(t, testMultiWriteAndFlush)
+}
+func testMultiWriteAndFlush(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ data := make([]byte, 20)
+ u.NewTimeSeededRand().Read(data)
+
+ for i := 0; i < len(data); i++ {
+ n, err := dagmod.WriteAt(data[i:i+1], int64(i))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != 1 {
+ t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
+ }
+ err = dagmod.Sync()
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ verifyNode(t, data, dagmod, opts)
+}
+
+func TestWriteNewFile(t *testing.T) {
+ runAllSubtests(t, testWriteNewFile)
+}
+func testWriteNewFile(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ towrite := make([]byte, 2000)
+ u.NewTimeSeededRand().Read(towrite)
+
+ nw, err := dagmod.Write(towrite)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if nw != len(towrite) {
+ t.Fatal("Wrote wrong amount")
+ }
+
+ verifyNode(t, towrite, dagmod, opts)
+}
+
+func TestMultiWriteCoal(t *testing.T) {
+ runAllSubtests(t, testMultiWriteCoal)
+}
+func testMultiWriteCoal(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ data := make([]byte, 1000)
+ u.NewTimeSeededRand().Read(data)
+
+ for i := 0; i < len(data); i++ {
+ n, err := dagmod.WriteAt(data[:i+1], 0)
+ if err != nil {
+ fmt.Println("FAIL AT ", i)
+ t.Fatal(err)
+ }
+ if n != i+1 {
+ t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
+ }
+
+ }
+
+ verifyNode(t, data, dagmod, opts)
+}
+
+func TestLargeWriteChunks(t *testing.T) {
+ runAllSubtests(t, testLargeWriteChunks)
+}
+func testLargeWriteChunks(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ wrsize := 1000
+ datasize := 10000000
+ data := make([]byte, datasize)
+
+ u.NewTimeSeededRand().Read(data)
+
+ for i := 0; i < datasize/wrsize; i++ {
+ n, err := dagmod.WriteAt(data[i*wrsize:(i+1)*wrsize], int64(i*wrsize))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if n != wrsize {
+ t.Fatal("failed to write buffer")
+ }
+ }
+
+ _, err = dagmod.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(dagmod)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err = testu.ArrComp(out, data); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestDagTruncate(t *testing.T) {
+ runAllSubtests(t, testDagTruncate)
+}
+func testDagTruncate(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ b, n := testu.GetRandomNode(t, dserv, 50000, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ err = dagmod.Truncate(12345)
+ if err != nil {
+ t.Fatal(err)
+ }
+ size, err := dagmod.Size()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if size != 12345 {
+ t.Fatal("size was incorrect!")
+ }
+
+ _, err = dagmod.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(dagmod)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err = testu.ArrComp(out, b[:12345]); err != nil {
+ t.Fatal(err)
+ }
+
+ err = dagmod.Truncate(10)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ size, err = dagmod.Size()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if size != 10 {
+ t.Fatal("size was incorrect!")
+ }
+
+ err = dagmod.Truncate(0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ size, err = dagmod.Size()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if size != 0 {
+ t.Fatal("size was incorrect!")
+ }
+}
+
+// TestDagSync tests that a DAG will expand sparse during sync
+// if offset > curNode's size.
+func TestDagSync(t *testing.T) {
+ dserv := testu.GetDAGServ()
+ nd := dag.NodeWithData(unixfs.FilePBData(nil, 0))
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, nd, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = dagmod.Write([]byte("test1"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = dagmod.Sync()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Truncate leave the offset at 5 and filesize at 0
+ err = dagmod.Truncate(0)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = dagmod.Write([]byte("test2"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // When Offset > filesize , Sync will call enpandSparse
+ err = dagmod.Sync()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = dagmod.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(dagmod)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err = testu.ArrComp(out[5:], []byte("test2")); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestDagTruncateSameSize tests that a DAG truncated
+// to the same size (i.e., doing nothing) doesn't modify
+// the DAG (its hash).
+func TestDagTruncateSameSize(t *testing.T) {
+ runAllSubtests(t, testDagTruncateSameSize)
+}
+func testDagTruncateSameSize(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ _, n := testu.GetRandomNode(t, dserv, 50000, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Copied from `TestDagTruncate`.
+
+ size, err := dagmod.Size()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = dagmod.Truncate(size)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ modifiedNode, err := dagmod.GetNode()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if modifiedNode.Cid().Equals(n.Cid()) == false {
+ t.Fatal("the node has been modified!")
+ }
+}
+
+func TestSparseWrite(t *testing.T) {
+ runAllSubtests(t, testSparseWrite)
+}
+func testSparseWrite(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ buf := make([]byte, 5000)
+ u.NewTimeSeededRand().Read(buf[2500:])
+
+ wrote, err := dagmod.WriteAt(buf[2500:], 2500)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if wrote != 2500 {
+ t.Fatal("incorrect write amount")
+ }
+
+ _, err = dagmod.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(dagmod)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err = testu.ArrComp(out, buf); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSeekPastEndWrite(t *testing.T) {
+ runAllSubtests(t, testSeekPastEndWrite)
+}
+func testSeekPastEndWrite(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ buf := make([]byte, 5000)
+ u.NewTimeSeededRand().Read(buf[2500:])
+
+ nseek, err := dagmod.Seek(2500, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if nseek != 2500 {
+ t.Fatal("failed to seek")
+ }
+
+ wrote, err := dagmod.Write(buf[2500:])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if wrote != 2500 {
+ t.Fatal("incorrect write amount")
+ }
+
+ _, err = dagmod.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ out, err := io.ReadAll(dagmod)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err = testu.ArrComp(out, buf); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestRelativeSeek(t *testing.T) {
+ runAllSubtests(t, testRelativeSeek)
+}
+func testRelativeSeek(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ for i := 0; i < 64; i++ {
+ dagmod.Write([]byte{byte(i)})
+ if _, err := dagmod.Seek(1, io.SeekCurrent); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ out, err := io.ReadAll(dagmod)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i, v := range out {
+ if v != 0 && i/2 != int(v) {
+ t.Errorf("expected %d, at index %d, got %d", i/2, i, v)
+ }
+ }
+}
+
+func TestInvalidSeek(t *testing.T) {
+ runAllSubtests(t, testInvalidSeek)
+}
+func testInvalidSeek(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(t, dserv, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ _, err = dagmod.Seek(10, -10)
+
+ if err != ErrUnrecognizedWhence {
+ t.Fatal(err)
+ }
+}
+
+func TestEndSeek(t *testing.T) {
+ runAllSubtests(t, testEndSeek)
+}
+func testEndSeek(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+
+ n := testu.GetEmptyNode(t, dserv, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ _, err = dagmod.Write(make([]byte, 100))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ offset, err := dagmod.Seek(0, io.SeekCurrent)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if offset != 100 {
+ t.Fatal("expected the relative seek 0 to return current location")
+ }
+
+ offset, err = dagmod.Seek(0, io.SeekStart)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if offset != 0 {
+ t.Fatal("expected the absolute seek to set offset at 0")
+ }
+
+ offset, err = dagmod.Seek(0, io.SeekEnd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if offset != 100 {
+ t.Fatal("expected the end seek to set offset at end")
+ }
+}
+
+func TestReadAndSeek(t *testing.T) {
+ runAllSubtests(t, testReadAndSeek)
+}
+func testReadAndSeek(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+
+ n := testu.GetEmptyNode(t, dserv, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ writeBuf := []byte{0, 1, 2, 3, 4, 5, 6, 7}
+ dagmod.Write(writeBuf)
+
+ if !dagmod.HasChanges() {
+ t.Fatal("there are changes, this should be true")
+ }
+
+ readBuf := make([]byte, 4)
+ offset, err := dagmod.Seek(0, io.SeekStart)
+ if offset != 0 {
+ t.Fatal("expected offset to be 0")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // read 0,1,2,3
+ c, err := dagmod.Read(readBuf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if c != 4 {
+ t.Fatalf("expected length of 4 got %d", c)
+ }
+
+ for i := byte(0); i < 4; i++ {
+ if readBuf[i] != i {
+ t.Fatalf("wrong value %d [at index %d]", readBuf[i], i)
+ }
+ }
+
+ // skip 4
+ _, err = dagmod.Seek(1, io.SeekCurrent)
+ if err != nil {
+ t.Fatalf("error: %s, offset %d, reader offset %d", err, dagmod.curWrOff, getOffset(dagmod.read))
+ }
+
+ //read 5,6,7
+ readBuf = make([]byte, 3)
+ c, err = dagmod.Read(readBuf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if c != 3 {
+ t.Fatalf("expected length of 3 got %d", c)
+ }
+
+ for i := byte(0); i < 3; i++ {
+ if readBuf[i] != i+5 {
+ t.Fatalf("wrong value %d [at index %d]", readBuf[i], i)
+ }
+
+ }
+
+}
+
+func TestCtxRead(t *testing.T) {
+ runAllSubtests(t, testCtxRead)
+}
+func testCtxRead(t *testing.T, opts testu.NodeOpts) {
+ dserv := testu.GetDAGServ()
+
+ n := testu.GetEmptyNode(t, dserv, opts)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if opts.ForceRawLeaves {
+ dagmod.RawLeaves = true
+ }
+
+ _, err = dagmod.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7})
+ if err != nil {
+ t.Fatal(err)
+ }
+ dagmod.Seek(0, io.SeekStart)
+
+ readBuf := make([]byte, 4)
+ _, err = dagmod.CtxReadFull(ctx, readBuf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = testu.ArrComp(readBuf, []byte{0, 1, 2, 3})
+ if err != nil {
+ t.Fatal(err)
+ }
+ // TODO(Kubuxu): context cancel case, I will do it after I figure out dagreader tests,
+ // because this is exacelly the same.
+}
+
+func BenchmarkDagmodWrite(b *testing.B) {
+ b.StopTimer()
+ dserv := testu.GetDAGServ()
+ n := testu.GetEmptyNode(b, dserv, testu.UseProtoBufLeaves)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ wrsize := 4096
+
+ dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ buf := make([]byte, b.N*wrsize)
+ u.NewTimeSeededRand().Read(buf)
+ b.StartTimer()
+ b.SetBytes(int64(wrsize))
+ for i := 0; i < b.N; i++ {
+ n, err := dagmod.Write(buf[i*wrsize : (i+1)*wrsize])
+ if err != nil {
+ b.Fatal(err)
+ }
+ if n != wrsize {
+ b.Fatal("Wrote bad size")
+ }
+ }
+}
+
+func getOffset(reader uio.DagReader) int64 {
+ offset, err := reader.Seek(0, io.SeekCurrent)
+ if err != nil {
+ panic("failed to retrieve offset: " + err.Error())
+ }
+ return offset
+}
diff --git a/unixfs/pb/Makefile b/unixfs/pb/Makefile
new file mode 100644
index 000000000..51552a096
--- /dev/null
+++ b/unixfs/pb/Makefile
@@ -0,0 +1,11 @@
+PB = $(wildcard *.proto)
+GO = $(PB:.proto=.pb.go)
+
+all: $(GO)
+
+%.pb.go: %.proto
+ protoc --proto_path=$(GOPATH)/src:. --gogo_out=. $<
+
+clean:
+ rm -f *.pb.go
+ rm -f *.go
diff --git a/unixfs/pb/unixfs.pb.go b/unixfs/pb/unixfs.pb.go
new file mode 100644
index 000000000..e52314007
--- /dev/null
+++ b/unixfs/pb/unixfs.pb.go
@@ -0,0 +1,218 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: unixfs.proto
+
+package unixfs_pb
+
+import (
+ fmt "fmt"
+ math "math"
+
+ proto "github.com/gogo/protobuf/proto"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+
+type Data_DataType int32
+
+const (
+ Data_Raw Data_DataType = 0
+ Data_Directory Data_DataType = 1
+ Data_File Data_DataType = 2
+ Data_Metadata Data_DataType = 3
+ Data_Symlink Data_DataType = 4
+ Data_HAMTShard Data_DataType = 5
+)
+
+var Data_DataType_name = map[int32]string{
+ 0: "Raw",
+ 1: "Directory",
+ 2: "File",
+ 3: "Metadata",
+ 4: "Symlink",
+ 5: "HAMTShard",
+}
+
+var Data_DataType_value = map[string]int32{
+ "Raw": 0,
+ "Directory": 1,
+ "File": 2,
+ "Metadata": 3,
+ "Symlink": 4,
+ "HAMTShard": 5,
+}
+
+func (x Data_DataType) Enum() *Data_DataType {
+ p := new(Data_DataType)
+ *p = x
+ return p
+}
+
+func (x Data_DataType) String() string {
+ return proto.EnumName(Data_DataType_name, int32(x))
+}
+
+func (x *Data_DataType) UnmarshalJSON(data []byte) error {
+ value, err := proto.UnmarshalJSONEnum(Data_DataType_value, data, "Data_DataType")
+ if err != nil {
+ return err
+ }
+ *x = Data_DataType(value)
+ return nil
+}
+
+func (Data_DataType) EnumDescriptor() ([]byte, []int) {
+ return fileDescriptor_e2fd76cc44dfc7c3, []int{0, 0}
+}
+
+type Data struct {
+ Type *Data_DataType `protobuf:"varint,1,req,name=Type,enum=unixfs.pb.Data_DataType" json:"Type,omitempty"`
+ Data []byte `protobuf:"bytes,2,opt,name=Data" json:"Data,omitempty"`
+ Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"`
+ Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"`
+ HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"`
+ Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Data) Reset() { *m = Data{} }
+func (m *Data) String() string { return proto.CompactTextString(m) }
+func (*Data) ProtoMessage() {}
+func (*Data) Descriptor() ([]byte, []int) {
+ return fileDescriptor_e2fd76cc44dfc7c3, []int{0}
+}
+func (m *Data) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Data.Unmarshal(m, b)
+}
+func (m *Data) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Data.Marshal(b, m, deterministic)
+}
+func (m *Data) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Data.Merge(m, src)
+}
+func (m *Data) XXX_Size() int {
+ return xxx_messageInfo_Data.Size(m)
+}
+func (m *Data) XXX_DiscardUnknown() {
+ xxx_messageInfo_Data.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Data proto.InternalMessageInfo
+
+func (m *Data) GetType() Data_DataType {
+ if m != nil && m.Type != nil {
+ return *m.Type
+ }
+ return Data_Raw
+}
+
+func (m *Data) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
+func (m *Data) GetFilesize() uint64 {
+ if m != nil && m.Filesize != nil {
+ return *m.Filesize
+ }
+ return 0
+}
+
+func (m *Data) GetBlocksizes() []uint64 {
+ if m != nil {
+ return m.Blocksizes
+ }
+ return nil
+}
+
+func (m *Data) GetHashType() uint64 {
+ if m != nil && m.HashType != nil {
+ return *m.HashType
+ }
+ return 0
+}
+
+func (m *Data) GetFanout() uint64 {
+ if m != nil && m.Fanout != nil {
+ return *m.Fanout
+ }
+ return 0
+}
+
+type Metadata struct {
+ MimeType *string `protobuf:"bytes,1,opt,name=MimeType" json:"MimeType,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *Metadata) Reset() { *m = Metadata{} }
+func (m *Metadata) String() string { return proto.CompactTextString(m) }
+func (*Metadata) ProtoMessage() {}
+func (*Metadata) Descriptor() ([]byte, []int) {
+ return fileDescriptor_e2fd76cc44dfc7c3, []int{1}
+}
+func (m *Metadata) XXX_Unmarshal(b []byte) error {
+ return xxx_messageInfo_Metadata.Unmarshal(m, b)
+}
+func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+ return xxx_messageInfo_Metadata.Marshal(b, m, deterministic)
+}
+func (m *Metadata) XXX_Merge(src proto.Message) {
+ xxx_messageInfo_Metadata.Merge(m, src)
+}
+func (m *Metadata) XXX_Size() int {
+ return xxx_messageInfo_Metadata.Size(m)
+}
+func (m *Metadata) XXX_DiscardUnknown() {
+ xxx_messageInfo_Metadata.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Metadata proto.InternalMessageInfo
+
+func (m *Metadata) GetMimeType() string {
+ if m != nil && m.MimeType != nil {
+ return *m.MimeType
+ }
+ return ""
+}
+
+func init() {
+ proto.RegisterEnum("unixfs.pb.Data_DataType", Data_DataType_name, Data_DataType_value)
+ proto.RegisterType((*Data)(nil), "unixfs.pb.Data")
+ proto.RegisterType((*Metadata)(nil), "unixfs.pb.Metadata")
+}
+
+func init() { proto.RegisterFile("unixfs.proto", fileDescriptor_e2fd76cc44dfc7c3) }
+
+var fileDescriptor_e2fd76cc44dfc7c3 = []byte{
+ // 254 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x90, 0xb1, 0x6a, 0xeb, 0x30,
+ 0x18, 0x85, 0xaf, 0x6c, 0x25, 0xb1, 0xff, 0xeb, 0x16, 0xf1, 0x0f, 0x45, 0x74, 0x28, 0xc6, 0x43,
+ 0xd1, 0x50, 0x3c, 0xf4, 0x0d, 0x0a, 0xa1, 0x74, 0xf1, 0xa2, 0x84, 0xee, 0x4a, 0x22, 0x63, 0x11,
+ 0xc7, 0x0a, 0xb6, 0x42, 0xeb, 0x3e, 0x45, 0x1f, 0xb9, 0xc8, 0x8e, 0xdd, 0x2e, 0x82, 0x4f, 0xe7,
+ 0x7c, 0xe2, 0x20, 0x48, 0x2e, 0x8d, 0xf9, 0x2c, 0xbb, 0xfc, 0xdc, 0x5a, 0x67, 0x31, 0x9e, 0x68,
+ 0x97, 0x7d, 0x07, 0x40, 0xd7, 0xca, 0x29, 0x7c, 0x02, 0xba, 0xed, 0xcf, 0x9a, 0x93, 0x34, 0x10,
+ 0xb7, 0xcf, 0x3c, 0x9f, 0x2b, 0xb9, 0x8f, 0x87, 0xc3, 0xe7, 0x72, 0x68, 0x21, 0x8e, 0x16, 0x0f,
+ 0x52, 0x22, 0x12, 0x39, 0xbe, 0x70, 0x0f, 0x51, 0x69, 0x6a, 0xdd, 0x99, 0x2f, 0xcd, 0xc3, 0x94,
+ 0x08, 0x2a, 0x67, 0xc6, 0x07, 0x80, 0x5d, 0x6d, 0xf7, 0x47, 0x0f, 0x1d, 0xa7, 0x69, 0x28, 0xa8,
+ 0xfc, 0x73, 0xe3, 0xdd, 0x4a, 0x75, 0xd5, 0xb0, 0x60, 0x31, 0xba, 0x13, 0xe3, 0x1d, 0x2c, 0x4b,
+ 0xd5, 0xd8, 0x8b, 0xe3, 0xcb, 0x21, 0xb9, 0x52, 0xf6, 0x0e, 0xd1, 0xb4, 0x0a, 0x57, 0x10, 0x4a,
+ 0xf5, 0xc1, 0xfe, 0xe1, 0x0d, 0xc4, 0x6b, 0xd3, 0xea, 0xbd, 0xb3, 0x6d, 0xcf, 0x08, 0x46, 0x40,
+ 0x5f, 0x4d, 0xad, 0x59, 0x80, 0x09, 0x44, 0x85, 0x76, 0xea, 0xa0, 0x9c, 0x62, 0x21, 0xfe, 0x87,
+ 0xd5, 0xa6, 0x3f, 0xd5, 0xa6, 0x39, 0x32, 0xea, 0x9d, 0xb7, 0x97, 0x62, 0xbb, 0xa9, 0x54, 0x7b,
+ 0x60, 0x8b, 0xec, 0xf1, 0xb7, 0xe9, 0x77, 0x15, 0xe6, 0xa4, 0xaf, 0x3f, 0x43, 0x44, 0x2c, 0x67,
+ 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, 0xe9, 0xa0, 0x51, 0x10, 0x54, 0x01, 0x00, 0x00,
+}
diff --git a/unixfs/pb/unixfs.proto b/unixfs/pb/unixfs.proto
new file mode 100644
index 000000000..ffc059e8b
--- /dev/null
+++ b/unixfs/pb/unixfs.proto
@@ -0,0 +1,26 @@
+syntax = "proto2";
+
+package unixfs.pb;
+
+message Data {
+ enum DataType {
+ Raw = 0;
+ Directory = 1;
+ File = 2;
+ Metadata = 3;
+ Symlink = 4;
+ HAMTShard = 5;
+ }
+
+ required DataType Type = 1;
+ optional bytes Data = 2;
+ optional uint64 filesize = 3;
+ repeated uint64 blocksizes = 4;
+
+ optional uint64 hashType = 5;
+ optional uint64 fanout = 6;
+}
+
+message Metadata {
+ optional string MimeType = 1;
+}
diff --git a/unixfs/private/linksize/linksize.go b/unixfs/private/linksize/linksize.go
new file mode 100644
index 000000000..e7ae098b6
--- /dev/null
+++ b/unixfs/private/linksize/linksize.go
@@ -0,0 +1,5 @@
+package linksize
+
+import "github.com/ipfs/go-cid"
+
+var LinkSizeFunction func(linkName string, linkCid cid.Cid) int
diff --git a/unixfs/test/utils.go b/unixfs/test/utils.go
new file mode 100644
index 000000000..146426559
--- /dev/null
+++ b/unixfs/test/utils.go
@@ -0,0 +1,138 @@
+package testu
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "testing"
+
+ ft "github.com/ipfs/boxo/unixfs"
+ h "github.com/ipfs/boxo/unixfs/importer/helpers"
+ trickle "github.com/ipfs/boxo/unixfs/importer/trickle"
+
+ chunker "github.com/ipfs/boxo/chunker"
+ mdag "github.com/ipfs/boxo/ipld/merkledag"
+ mdagmock "github.com/ipfs/boxo/ipld/merkledag/test"
+ u "github.com/ipfs/boxo/util"
+ cid "github.com/ipfs/go-cid"
+ ipld "github.com/ipfs/go-ipld-format"
+ mh "github.com/multiformats/go-multihash"
+)
+
+// SizeSplitterGen creates a generator.
+func SizeSplitterGen(size int64) chunker.SplitterGen {
+ return func(r io.Reader) chunker.Splitter {
+ return chunker.NewSizeSplitter(r, size)
+ }
+}
+
+// GetDAGServ returns a mock DAGService.
+func GetDAGServ() ipld.DAGService {
+ return mdagmock.Mock()
+}
+
+// NodeOpts is used by GetNode, GetEmptyNode and GetRandomNode
+type NodeOpts struct {
+ Prefix cid.Prefix
+ // ForceRawLeaves if true will force the use of raw leaves
+ ForceRawLeaves bool
+ // RawLeavesUsed is true if raw leaves or either implicitly or explicitly enabled
+ RawLeavesUsed bool
+}
+
+// Some shorthands for NodeOpts.
+var (
+ UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()}
+ UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true}
+ UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true}
+ UseBlake2b256 NodeOpts
+)
+
+func init() {
+ UseBlake2b256 = UseCidV1
+ UseBlake2b256.Prefix.MhType = mh.Names["blake2b-256"]
+ UseBlake2b256.Prefix.MhLength = -1
+}
+
+// GetNode returns a unixfs file node with the specified data.
+func GetNode(t testing.TB, dserv ipld.DAGService, data []byte, opts NodeOpts) ipld.Node {
+ in := bytes.NewReader(data)
+
+ dbp := h.DagBuilderParams{
+ Dagserv: dserv,
+ Maxlinks: h.DefaultLinksPerBlock,
+ CidBuilder: opts.Prefix,
+ RawLeaves: opts.RawLeavesUsed,
+ }
+
+ db, err := dbp.New(SizeSplitterGen(500)(in))
+ if err != nil {
+ t.Fatal(err)
+ }
+ node, err := trickle.Layout(db)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return node
+}
+
+// GetEmptyNode returns an empty unixfs file node.
+func GetEmptyNode(t testing.TB, dserv ipld.DAGService, opts NodeOpts) ipld.Node {
+ return GetNode(t, dserv, []byte{}, opts)
+}
+
+// GetRandomNode returns a random unixfs file node.
+func GetRandomNode(t testing.TB, dserv ipld.DAGService, size int64, opts NodeOpts) ([]byte, ipld.Node) {
+ in := io.LimitReader(u.NewTimeSeededRand(), size)
+ buf, err := io.ReadAll(in)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ node := GetNode(t, dserv, buf, opts)
+ return buf, node
+}
+
+// ArrComp checks if two byte slices are the same.
+func ArrComp(a, b []byte) error {
+ if len(a) != len(b) {
+ return fmt.Errorf("arrays differ in length. %d != %d", len(a), len(b))
+ }
+ for i, v := range a {
+ if v != b[i] {
+ return fmt.Errorf("arrays differ at index: %d", i)
+ }
+ }
+ return nil
+}
+
+// PrintDag pretty-prints the given dag to stdout.
+func PrintDag(nd *mdag.ProtoNode, ds ipld.DAGService, indent int) {
+ fsn, err := ft.FSNodeFromBytes(nd.Data())
+ if err != nil {
+ panic(err)
+ }
+
+ for i := 0; i < indent; i++ {
+ fmt.Print(" ")
+ }
+ fmt.Printf("{size = %d, type = %s, children = %d", fsn.FileSize(), fsn.Type().String(), fsn.NumChildren())
+ if len(nd.Links()) > 0 {
+ fmt.Println()
+ }
+ for _, lnk := range nd.Links() {
+ child, err := lnk.GetNode(context.Background(), ds)
+ if err != nil {
+ panic(err)
+ }
+ PrintDag(child.(*mdag.ProtoNode), ds, indent+1)
+ }
+ if len(nd.Links()) > 0 {
+ for i := 0; i < indent; i++ {
+ fmt.Print(" ")
+ }
+ }
+ fmt.Println("}")
+}
diff --git a/unixfs/unixfs.go b/unixfs/unixfs.go
new file mode 100644
index 000000000..6f68bdfb6
--- /dev/null
+++ b/unixfs/unixfs.go
@@ -0,0 +1,418 @@
+// Package unixfs implements a data format for files in the IPFS filesystem It
+// is not the only format in ipfs, but it is the one that the filesystem
+// assumes
+package unixfs
+
+import (
+ "errors"
+ "fmt"
+
+ proto "github.com/gogo/protobuf/proto"
+ dag "github.com/ipfs/boxo/ipld/merkledag"
+
+ pb "github.com/ipfs/boxo/unixfs/pb"
+ ipld "github.com/ipfs/go-ipld-format"
+)
+
+// A LinkResult for any parallel enumeration of links
+// TODO: Should this live in go-ipld-format?
+type LinkResult struct {
+ Link *ipld.Link
+ Err error
+}
+
+// Shorthands for protobuffer types
+const (
+ TRaw = pb.Data_Raw
+ TFile = pb.Data_File
+ TDirectory = pb.Data_Directory
+ TMetadata = pb.Data_Metadata
+ TSymlink = pb.Data_Symlink
+ THAMTShard = pb.Data_HAMTShard
+)
+
+// Common errors
+var (
+ ErrMalformedFileFormat = errors.New("malformed data in file format")
+ ErrUnrecognizedType = errors.New("unrecognized node type")
+)
+
+// FromBytes unmarshals a byte slice as protobuf Data.
+// Deprecated: Use `FSNodeFromBytes` instead to avoid direct manipulation of `pb.Data`.
+func FromBytes(data []byte) (*pb.Data, error) {
+ pbdata := new(pb.Data)
+ err := proto.Unmarshal(data, pbdata)
+ if err != nil {
+ return nil, err
+ }
+ return pbdata, nil
+}
+
+// FilePBData creates a protobuf File with the given
+// byte slice and returns the marshaled protobuf bytes representing it.
+func FilePBData(data []byte, totalsize uint64) []byte {
+ pbfile := new(pb.Data)
+ typ := pb.Data_File
+ pbfile.Type = &typ
+ pbfile.Data = data
+ pbfile.Filesize = proto.Uint64(totalsize)
+
+ data, err := proto.Marshal(pbfile)
+ if err != nil {
+ // This really shouldnt happen, i promise
+ // The only failure case for marshal is if required fields
+ // are not filled out, and they all are. If the proto object
+ // gets changed and nobody updates this function, the code
+ // should panic due to programmer error
+ panic(err)
+ }
+ return data
+}
+
+// FolderPBData returns Bytes that represent a Directory.
+func FolderPBData() []byte {
+ pbfile := new(pb.Data)
+ typ := pb.Data_Directory
+ pbfile.Type = &typ
+
+ data, err := proto.Marshal(pbfile)
+ if err != nil {
+ //this really shouldnt happen, i promise
+ panic(err)
+ }
+ return data
+}
+
+// WrapData marshals raw bytes into a `Data_Raw` type protobuf message.
+func WrapData(b []byte) []byte {
+ pbdata := new(pb.Data)
+ typ := pb.Data_Raw
+ pbdata.Data = b
+ pbdata.Type = &typ
+ pbdata.Filesize = proto.Uint64(uint64(len(b)))
+
+ out, err := proto.Marshal(pbdata)
+ if err != nil {
+ // This shouldnt happen. seriously.
+ panic(err)
+ }
+
+ return out
+}
+
+// SymlinkData returns a `Data_Symlink` protobuf message for the path you specify.
+func SymlinkData(path string) ([]byte, error) {
+ pbdata := new(pb.Data)
+ typ := pb.Data_Symlink
+ pbdata.Data = []byte(path)
+ pbdata.Type = &typ
+
+ out, err := proto.Marshal(pbdata)
+ if err != nil {
+ return nil, err
+ }
+
+ return out, nil
+}
+
+// HAMTShardData return a `Data_HAMTShard` protobuf message
+func HAMTShardData(data []byte, fanout uint64, hashType uint64) ([]byte, error) {
+ pbdata := new(pb.Data)
+ typ := pb.Data_HAMTShard
+ pbdata.Type = &typ
+ pbdata.HashType = proto.Uint64(hashType)
+ pbdata.Data = data
+ pbdata.Fanout = proto.Uint64(fanout)
+
+ out, err := proto.Marshal(pbdata)
+ if err != nil {
+ return nil, err
+ }
+
+ return out, nil
+}
+
+// UnwrapData unmarshals a protobuf messages and returns the contents.
+func UnwrapData(data []byte) ([]byte, error) {
+ pbdata := new(pb.Data)
+ err := proto.Unmarshal(data, pbdata)
+ if err != nil {
+ return nil, err
+ }
+ return pbdata.GetData(), nil
+}
+
+// DataSize returns the size of the contents in protobuf wrapped slice.
+// For raw data it simply provides the length of it. For Data_Files, it
+// will return the associated filesize. Note that Data_Directories will
+// return an error.
+func DataSize(data []byte) (uint64, error) {
+ pbdata := new(pb.Data)
+ err := proto.Unmarshal(data, pbdata)
+ if err != nil {
+ return 0, err
+ }
+ return size(pbdata)
+}
+
+func size(pbdata *pb.Data) (uint64, error) {
+ switch pbdata.GetType() {
+ case pb.Data_Directory, pb.Data_HAMTShard:
+ return 0, errors.New("can't get data size of directory")
+ case pb.Data_File, pb.Data_Raw:
+ return pbdata.GetFilesize(), nil
+ case pb.Data_Symlink:
+ return uint64(len(pbdata.GetData())), nil
+ default:
+ return 0, errors.New("unrecognized node data type")
+ }
+}
+
+// An FSNode represents a filesystem object using the UnixFS specification.
+//
+// The `NewFSNode` constructor should be used instead of just calling `new(FSNode)`
+// to guarantee that the required (`Type` and `Filesize`) fields in the `format`
+// structure are initialized before marshaling (in `GetBytes()`).
+type FSNode struct {
+
+ // UnixFS format defined as a protocol buffers message.
+ format pb.Data
+}
+
+// FSNodeFromBytes unmarshal a protobuf message onto an FSNode.
+func FSNodeFromBytes(b []byte) (*FSNode, error) {
+ n := new(FSNode)
+ err := proto.Unmarshal(b, &n.format)
+ if err != nil {
+ return nil, err
+ }
+
+ return n, nil
+}
+
+// NewFSNode creates a new FSNode structure with the given `dataType`.
+//
+// It initializes the (required) `Type` field (that doesn't have a `Set()`
+// accessor so it must be specified at creation), otherwise the `Marshal()`
+// method in `GetBytes()` would fail (`required field "Type" not set`).
+//
+// It also initializes the `Filesize` pointer field to ensure its value
+// is never nil before marshaling, this is not a required field but it is
+// done to be backwards compatible with previous `go-ipfs` versions hash.
+// (If it wasn't initialized there could be cases where `Filesize` could
+// have been left at nil, when the `FSNode` was created but no data or
+// child nodes were set to adjust it, as is the case in `NewLeaf()`.)
+func NewFSNode(dataType pb.Data_DataType) *FSNode {
+ n := new(FSNode)
+ n.format.Type = &dataType
+
+ // Initialize by `Filesize` by updating it with a dummy (zero) value.
+ n.UpdateFilesize(0)
+
+ return n
+}
+
+// HashType gets hash type of format
+func (n *FSNode) HashType() uint64 {
+ return n.format.GetHashType()
+}
+
+// Fanout gets fanout of format
+func (n *FSNode) Fanout() uint64 {
+ return n.format.GetFanout()
+}
+
+// AddBlockSize adds the size of the next child block of this node
+func (n *FSNode) AddBlockSize(s uint64) {
+ n.UpdateFilesize(int64(s))
+ n.format.Blocksizes = append(n.format.Blocksizes, s)
+}
+
+// RemoveBlockSize removes the given child block's size.
+func (n *FSNode) RemoveBlockSize(i int) {
+ n.UpdateFilesize(-int64(n.format.Blocksizes[i]))
+ n.format.Blocksizes = append(n.format.Blocksizes[:i], n.format.Blocksizes[i+1:]...)
+}
+
+// BlockSize returns the block size indexed by `i`.
+// TODO: Evaluate if this function should be bounds checking.
+func (n *FSNode) BlockSize(i int) uint64 {
+ return n.format.Blocksizes[i]
+}
+
+// BlockSizes gets blocksizes of format
+func (n *FSNode) BlockSizes() []uint64 {
+ return n.format.GetBlocksizes()
+}
+
+// RemoveAllBlockSizes removes all the child block sizes of this node.
+func (n *FSNode) RemoveAllBlockSizes() {
+ n.format.Blocksizes = []uint64{}
+ n.format.Filesize = proto.Uint64(uint64(len(n.Data())))
+}
+
+// GetBytes marshals this node as a protobuf message.
+func (n *FSNode) GetBytes() ([]byte, error) {
+ return proto.Marshal(&n.format)
+}
+
+// FileSize returns the size of the file.
+func (n *FSNode) FileSize() uint64 {
+ // XXX: This needs to be able to return an error when we don't know the
+ // size.
+ size, _ := size(&n.format)
+ return size
+}
+
+// NumChildren returns the number of child blocks of this node
+func (n *FSNode) NumChildren() int {
+ return len(n.format.Blocksizes)
+}
+
+// Data retrieves the `Data` field from the internal `format`.
+func (n *FSNode) Data() []byte {
+ return n.format.GetData()
+}
+
+// SetData sets the `Data` field from the internal `format`
+// updating its `Filesize`.
+func (n *FSNode) SetData(newData []byte) {
+ n.UpdateFilesize(int64(len(newData) - len(n.Data())))
+ n.format.Data = newData
+}
+
+// UpdateFilesize updates the `Filesize` field from the internal `format`
+// by a signed difference (`filesizeDiff`).
+// TODO: Add assert to check for `Filesize` > 0?
+func (n *FSNode) UpdateFilesize(filesizeDiff int64) {
+ n.format.Filesize = proto.Uint64(uint64(
+ int64(n.format.GetFilesize()) + filesizeDiff))
+}
+
+// Type retrieves the `Type` field from the internal `format`.
+func (n *FSNode) Type() pb.Data_DataType {
+ return n.format.GetType()
+}
+
+// IsDir checks whether the node represents a directory
+func (n *FSNode) IsDir() bool {
+ switch n.Type() {
+ case pb.Data_Directory, pb.Data_HAMTShard:
+ return true
+ default:
+ return false
+ }
+}
+
+// Metadata is used to store additional FSNode information.
+type Metadata struct {
+ MimeType string
+ Size uint64
+}
+
+// MetadataFromBytes Unmarshals a protobuf Data message into Metadata.
+// The provided slice should have been encoded with BytesForMetadata().
+func MetadataFromBytes(b []byte) (*Metadata, error) {
+ pbd := new(pb.Data)
+ err := proto.Unmarshal(b, pbd)
+ if err != nil {
+ return nil, err
+ }
+ if pbd.GetType() != pb.Data_Metadata {
+ return nil, errors.New("incorrect node type")
+ }
+
+ pbm := new(pb.Metadata)
+ err = proto.Unmarshal(pbd.Data, pbm)
+ if err != nil {
+ return nil, err
+ }
+ md := new(Metadata)
+ md.MimeType = pbm.GetMimeType()
+ return md, nil
+}
+
+// Bytes marshals Metadata as a protobuf message of Metadata type.
+func (m *Metadata) Bytes() ([]byte, error) {
+ pbm := new(pb.Metadata)
+ pbm.MimeType = &m.MimeType
+ return proto.Marshal(pbm)
+}
+
+// BytesForMetadata wraps the given Metadata as a profobuf message of Data type,
+// setting the DataType to Metadata. The wrapped bytes are itself the
+// result of calling m.Bytes().
+func BytesForMetadata(m *Metadata) ([]byte, error) {
+ pbd := new(pb.Data)
+ pbd.Filesize = proto.Uint64(m.Size)
+ typ := pb.Data_Metadata
+ pbd.Type = &typ
+ mdd, err := m.Bytes()
+ if err != nil {
+ return nil, err
+ }
+
+ pbd.Data = mdd
+ return proto.Marshal(pbd)
+}
+
+// EmptyDirNode creates an empty folder Protonode.
+func EmptyDirNode() *dag.ProtoNode {
+ return dag.NodeWithData(FolderPBData())
+}
+
+// EmptyFileNode creates an empty file Protonode.
+func EmptyFileNode() *dag.ProtoNode {
+ return dag.NodeWithData(FilePBData(nil, 0))
+}
+
+// ReadUnixFSNodeData extracts the UnixFS data from an IPLD node.
+// Raw nodes are (also) processed because they are used as leaf
+// nodes containing (only) UnixFS data.
+func ReadUnixFSNodeData(node ipld.Node) (data []byte, err error) {
+ switch node := node.(type) {
+
+ case *dag.ProtoNode:
+ fsNode, err := FSNodeFromBytes(node.Data())
+ if err != nil {
+ return nil, fmt.Errorf("incorrectly formatted protobuf: %s", err)
+ }
+
+ switch fsNode.Type() {
+ case pb.Data_File, pb.Data_Raw:
+ return fsNode.Data(), nil
+ // Only leaf nodes (of type `Data_Raw`) contain data but due to a
+ // bug the `Data_File` type (normally used for internal nodes) is
+ // also used for leaf nodes, so both types are accepted here
+ // (see the `balanced` package for more details).
+ default:
+ return nil, fmt.Errorf("found %s node in unexpected place",
+ fsNode.Type().String())
+ }
+
+ case *dag.RawNode:
+ return node.RawData(), nil
+
+ default:
+ return nil, ErrUnrecognizedType
+ // TODO: To avoid rewriting the error message, but a different error from
+ // `unixfs.ErrUnrecognizedType` should be used (defining it in the
+ // `merkledag` or `go-ipld-format` packages).
+ }
+}
+
+// Extract the `unixfs.FSNode` from the `ipld.Node` (assuming this
+// was implemented by a `mdag.ProtoNode`).
+func ExtractFSNode(node ipld.Node) (*FSNode, error) {
+ protoNode, ok := node.(*dag.ProtoNode)
+ if !ok {
+ return nil, errors.New("expected a ProtoNode as internal node")
+ }
+
+ fsNode, err := FSNodeFromBytes(protoNode.Data())
+ if err != nil {
+ return nil, err
+ }
+
+ return fsNode, nil
+}
diff --git a/unixfs/unixfs_test.go b/unixfs/unixfs_test.go
new file mode 100644
index 000000000..bbd2bd3b3
--- /dev/null
+++ b/unixfs/unixfs_test.go
@@ -0,0 +1,186 @@
+package unixfs
+
+import (
+ "bytes"
+ "testing"
+
+ proto "github.com/gogo/protobuf/proto"
+
+ pb "github.com/ipfs/boxo/unixfs/pb"
+)
+
+func TestFSNode(t *testing.T) {
+ fsn := NewFSNode(TFile)
+ for i := 0; i < 16; i++ {
+ fsn.AddBlockSize(100)
+ }
+ fsn.RemoveBlockSize(15)
+
+ fsn.SetData(make([]byte, 128))
+
+ b, err := fsn.GetBytes()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ pbn := new(pb.Data)
+ err = proto.Unmarshal(b, pbn)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ds, err := DataSize(b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nKids := fsn.NumChildren()
+ if nKids != 15 {
+ t.Fatal("Wrong number of child nodes")
+ }
+
+ if ds != (100*15)+128 {
+ t.Fatal("Datasize calculations incorrect!")
+ }
+
+ nfsn, err := FSNodeFromBytes(b)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if nfsn.FileSize() != (100*15)+128 {
+ t.Fatal("fsNode FileSize calculations incorrect")
+ }
+}
+
+func TestPBdataTools(t *testing.T) {
+ raw := []byte{0x00, 0x01, 0x02, 0x17, 0xA1}
+ rawPB := WrapData(raw)
+
+ pbDataSize, err := DataSize(rawPB)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ same := len(raw) == int(pbDataSize)
+ if !same {
+ t.Fatal("WrapData changes the size of data.")
+ }
+
+ rawPBBytes, err := UnwrapData(rawPB)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ same = bytes.Equal(raw, rawPBBytes)
+ if !same {
+ t.Fatal("Unwrap failed to produce the correct wrapped data.")
+ }
+
+ rawPBdata, err := FSNodeFromBytes(rawPB)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ isRaw := rawPBdata.Type() == TRaw
+ if !isRaw {
+ t.Fatal("WrapData does not create pb.Data_Raw!")
+ }
+
+ catFile := []byte("Mr_Meowgie.gif")
+ catPBfile := FilePBData(catFile, 17)
+ catSize, err := DataSize(catPBfile)
+ if catSize != 17 {
+ t.Fatal("FilePBData is the wrong size.")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ dirPB := FolderPBData()
+ dir, err := FSNodeFromBytes(dirPB)
+ isDir := dir.Type() == TDirectory
+ if !isDir {
+ t.Fatal("FolderPBData does not create a directory!")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, dirErr := DataSize(dirPB)
+ if dirErr == nil {
+ t.Fatal("DataSize didn't throw an error when taking the size of a directory.")
+ }
+
+ catSym, err := SymlinkData("/ipfs/adad123123/meowgie.gif")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ catSymPB, err := FSNodeFromBytes(catSym)
+ isSym := catSymPB.Type() == TSymlink
+ if !isSym {
+ t.Fatal("Failed to make a Symlink.")
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSymlinkFilesize(t *testing.T) {
+ path := "/ipfs/adad123123/meowgie.gif"
+ sym, err := SymlinkData(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ size, err := DataSize(sym)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if int(size) != len(path) {
+ t.Fatalf("size mismatch: %d != %d", size, len(path))
+ }
+}
+
+func TestMetadata(t *testing.T) {
+ meta := &Metadata{
+ MimeType: "audio/aiff",
+ Size: 12345,
+ }
+
+ _, err := meta.Bytes()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ metaPB, err := BytesForMetadata(meta)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ meta, err = MetadataFromBytes(metaPB)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ mimeAiff := meta.MimeType == "audio/aiff"
+ if !mimeAiff {
+ t.Fatal("Metadata does not Marshal and Unmarshal properly!")
+ }
+
+}
+
+func TestIsDir(t *testing.T) {
+ prepares := map[pb.Data_DataType]bool{
+ TDirectory: true,
+ THAMTShard: true,
+ TFile: false,
+ TMetadata: false,
+ TRaw: false,
+ TSymlink: false,
+ }
+ for typ, v := range prepares {
+ fsn := NewFSNode(typ)
+ if fsn.IsDir() != v {
+ t.Fatalf("type %v, IsDir() should be %v, but %v", typ, v, fsn.IsDir())
+ }
+ }
+}
diff --git a/util/.gitignore b/util/.gitignore
new file mode 100644
index 000000000..1377554eb
--- /dev/null
+++ b/util/.gitignore
@@ -0,0 +1 @@
+*.swp
diff --git a/util/file.go b/util/file.go
new file mode 100644
index 000000000..e6e30df4d
--- /dev/null
+++ b/util/file.go
@@ -0,0 +1,12 @@
+package util
+
+import "os"
+
+// FileExists check if the file with the given path exits.
+func FileExists(filename string) bool {
+ fi, err := os.Lstat(filename)
+ if fi != nil || (err != nil && !os.IsNotExist(err)) {
+ return true
+ }
+ return false
+}
diff --git a/util/file_test.go b/util/file_test.go
new file mode 100644
index 000000000..040b22927
--- /dev/null
+++ b/util/file_test.go
@@ -0,0 +1,10 @@
+package util
+
+import "testing"
+
+func TestFileDoesNotExist(t *testing.T) {
+ t.Parallel()
+ if FileExists("i would be surprised to discover that this file exists") {
+ t.Fail()
+ }
+}
diff --git a/util/time.go b/util/time.go
new file mode 100644
index 000000000..37d720fb1
--- /dev/null
+++ b/util/time.go
@@ -0,0 +1,22 @@
+package util
+
+import "time"
+
+// TimeFormatIpfs is the format ipfs uses to represent time in string form.
+var TimeFormatIpfs = time.RFC3339Nano
+
+// ParseRFC3339 parses an RFC3339Nano-formatted time stamp and
+// returns the UTC time.
+func ParseRFC3339(s string) (time.Time, error) {
+ t, err := time.Parse(TimeFormatIpfs, s)
+ if err != nil {
+ return time.Time{}, err
+ }
+ return t.UTC(), nil
+}
+
+// FormatRFC3339 returns the string representation of the
+// UTC value of the given time in RFC3339Nano format.
+func FormatRFC3339(t time.Time) string {
+ return t.UTC().Format(TimeFormatIpfs)
+}
diff --git a/util/time_test.go b/util/time_test.go
new file mode 100644
index 000000000..b5a98caa6
--- /dev/null
+++ b/util/time_test.go
@@ -0,0 +1,16 @@
+package util
+
+import (
+ "testing"
+ "time"
+)
+
+func TestTimeFormatParseInversion(t *testing.T) {
+ v, err := ParseRFC3339(FormatRFC3339(time.Now()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v.Location() != time.UTC {
+ t.Fatal("Time should be UTC")
+ }
+}
diff --git a/util/util.go b/util/util.go
new file mode 100644
index 000000000..ffcab2f33
--- /dev/null
+++ b/util/util.go
@@ -0,0 +1,158 @@
+// Package util implements various utility functions used within ipfs
+// that do not currently have a better place to live.
+package util
+
+import (
+ "errors"
+ "io"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "runtime/debug"
+ "strings"
+ "time"
+
+ b58 "github.com/mr-tron/base58/base58"
+ mh "github.com/multiformats/go-multihash"
+)
+
+// DefaultIpfsHash is the current default hash function used by IPFS.
+const DefaultIpfsHash = mh.SHA2_256
+
+// Debug is a global flag for debugging.
+var Debug bool
+
+// ErrNotImplemented signifies a function has not been implemented yet.
+var ErrNotImplemented = errors.New("error: not implemented yet")
+
+// ErrTimeout implies that a timeout has been triggered
+var ErrTimeout = errors.New("error: call timed out")
+
+// ErrSearchIncomplete implies that a search type operation didn't
+// find the expected node, but did find 'a' node.
+var ErrSearchIncomplete = errors.New("error: search incomplete")
+
+// ErrCast is returned when a cast fails AND the program should not panic.
+func ErrCast() error {
+ debug.PrintStack()
+ return errCast
+}
+
+var errCast = errors.New("cast error")
+
+// ExpandPathnames takes a set of paths and turns them into absolute paths
+func ExpandPathnames(paths []string) ([]string, error) {
+ var out []string
+ for _, p := range paths {
+ abspath, err := filepath.Abs(p)
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, abspath)
+ }
+ return out, nil
+}
+
+type randGen struct {
+ rand.Rand
+}
+
+// NewTimeSeededRand returns a random bytes reader
+// which has been initialized with the current time.
+func NewTimeSeededRand() io.Reader {
+ src := rand.NewSource(time.Now().UnixNano())
+ return &randGen{
+ Rand: *rand.New(src),
+ }
+}
+
+// NewSeededRand returns a random bytes reader
+// initialized with the given seed.
+func NewSeededRand(seed int64) io.Reader {
+ src := rand.NewSource(seed)
+ return &randGen{
+ Rand: *rand.New(src),
+ }
+}
+
+func (r *randGen) Read(p []byte) (n int, err error) {
+ for i := 0; i < len(p); i++ {
+ p[i] = byte(r.Rand.Intn(255))
+ }
+ return len(p), nil
+}
+
+// GetenvBool is the way to check an env var as a boolean
+func GetenvBool(name string) bool {
+ v := strings.ToLower(os.Getenv(name))
+ return v == "true" || v == "t" || v == "1"
+}
+
+// MultiErr is a util to return multiple errors
+type MultiErr []error
+
+func (m MultiErr) Error() string {
+ if len(m) == 0 {
+ return "no errors"
+ }
+
+ s := "Multiple errors: "
+ for i, e := range m {
+ if i != 0 {
+ s += ", "
+ }
+ s += e.Error()
+ }
+ return s
+}
+
+// Partition splits a subject 3 parts: prefix, separator, suffix.
+// The first occurrence of the separator will be matched.
+// ie. Partition("Ready, steady, go!", ", ") -> ["Ready", ", ", "steady, go!"]
+func Partition(subject string, sep string) (string, string, string) {
+ if i := strings.Index(subject, sep); i != -1 {
+ return subject[:i], subject[i : i+len(sep)], subject[i+len(sep):]
+ }
+ return subject, "", ""
+}
+
+// RPartition splits a subject 3 parts: prefix, separator, suffix.
+// The last occurrence of the separator will be matched.
+// ie. RPartition("Ready, steady, go!", ", ") -> ["Ready, steady", ", ", "go!"]
+func RPartition(subject string, sep string) (string, string, string) {
+ if i := strings.LastIndex(subject, sep); i != -1 {
+ return subject[:i], subject[i : i+len(sep)], subject[i+len(sep):]
+ }
+ return subject, "", ""
+}
+
+// Hash is the global IPFS hash function. uses multihash SHA2_256, 256 bits
+func Hash(data []byte) mh.Multihash {
+ h, err := mh.Sum(data, DefaultIpfsHash, -1)
+ if err != nil {
+ // this error can be safely ignored (panic) because multihash only fails
+ // from the selection of hash function. If the fn + length are valid, it
+ // won't error.
+ panic("multihash failed to hash using SHA2_256.")
+ }
+ return h
+}
+
+// IsValidHash checks whether a given hash is valid (b58 decodable, len > 0)
+func IsValidHash(s string) bool {
+ out, err := b58.Decode(s)
+ if err != nil {
+ return false
+ }
+ _, err = mh.Cast(out)
+ return err == nil
+}
+
+// XOR takes two byte slices, XORs them together, returns the resulting slice.
+func XOR(a, b []byte) []byte {
+ c := make([]byte, len(a))
+ for i := 0; i < len(a); i++ {
+ c[i] = a[i] ^ b[i]
+ }
+ return c
+}
diff --git a/util/util_test.go b/util/util_test.go
new file mode 100644
index 000000000..70747ad90
--- /dev/null
+++ b/util/util_test.go
@@ -0,0 +1,63 @@
+package util
+
+import (
+ "bytes"
+ "testing"
+)
+
+func TestXOR(t *testing.T) {
+ cases := [][3][]byte{
+ {
+ {0xFF, 0xFF, 0xFF},
+ {0xFF, 0xFF, 0xFF},
+ {0x00, 0x00, 0x00},
+ },
+ {
+ {0x00, 0xFF, 0x00},
+ {0xFF, 0xFF, 0xFF},
+ {0xFF, 0x00, 0xFF},
+ },
+ {
+ {0x55, 0x55, 0x55},
+ {0x55, 0xFF, 0xAA},
+ {0x00, 0xAA, 0xFF},
+ },
+ }
+
+ for _, c := range cases {
+ r := XOR(c[0], c[1])
+ if !bytes.Equal(r, c[2]) {
+ t.Error("XOR failed")
+ }
+ }
+}
+
+func BenchmarkHash256K(b *testing.B) {
+ buf := make([]byte, 256*1024)
+ NewTimeSeededRand().Read(buf)
+ b.SetBytes(int64(256 * 1024))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ Hash(buf)
+ }
+}
+
+func BenchmarkHash512K(b *testing.B) {
+ buf := make([]byte, 512*1024)
+ NewTimeSeededRand().Read(buf)
+ b.SetBytes(int64(512 * 1024))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ Hash(buf)
+ }
+}
+
+func BenchmarkHash1M(b *testing.B) {
+ buf := make([]byte, 1024*1024)
+ NewTimeSeededRand().Read(buf)
+ b.SetBytes(int64(1024 * 1024))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ Hash(buf)
+ }
+}
diff --git a/verifcid/validate.go b/verifcid/validate.go
new file mode 100644
index 000000000..7b27debc9
--- /dev/null
+++ b/verifcid/validate.go
@@ -0,0 +1,69 @@
+package verifcid
+
+import (
+ "fmt"
+
+ cid "github.com/ipfs/go-cid"
+ mh "github.com/multiformats/go-multihash"
+)
+
+var ErrPossiblyInsecureHashFunction = fmt.Errorf("potentially insecure hash functions not allowed")
+var ErrBelowMinimumHashLength = fmt.Errorf("hashes must be at least %d bytes long", minimumHashLength)
+var ErrAboveMaximumHashLength = fmt.Errorf("hashes must be at most %d bytes long", maximumHashLength)
+
+const minimumHashLength = 20
+const maximumHashLength = 128
+
+var goodset = map[uint64]bool{
+ mh.SHA2_256: true,
+ mh.SHA2_512: true,
+ mh.SHA3_224: true,
+ mh.SHA3_256: true,
+ mh.SHA3_384: true,
+ mh.SHA3_512: true,
+ mh.SHAKE_256: true,
+ mh.DBL_SHA2_256: true,
+ mh.KECCAK_224: true,
+ mh.KECCAK_256: true,
+ mh.KECCAK_384: true,
+ mh.KECCAK_512: true,
+ mh.BLAKE3: true,
+ mh.IDENTITY: true,
+
+ mh.SHA1: true, // not really secure but still useful
+}
+
+func IsGoodHash(code uint64) bool {
+ good, found := goodset[code]
+ if good {
+ return true
+ }
+
+ if !found {
+ if code >= mh.BLAKE2B_MIN+19 && code <= mh.BLAKE2B_MAX {
+ return true
+ }
+ if code >= mh.BLAKE2S_MIN+19 && code <= mh.BLAKE2S_MAX {
+ return true
+ }
+ }
+
+ return false
+}
+
+func ValidateCid(c cid.Cid) error {
+ pref := c.Prefix()
+ if !IsGoodHash(pref.MhType) {
+ return ErrPossiblyInsecureHashFunction
+ }
+
+ if pref.MhType != mh.IDENTITY && pref.MhLength < minimumHashLength {
+ return ErrBelowMinimumHashLength
+ }
+
+ if pref.MhType != mh.IDENTITY && pref.MhLength > maximumHashLength {
+ return ErrAboveMaximumHashLength
+ }
+
+ return nil
+}
diff --git a/verifcid/validate_test.go b/verifcid/validate_test.go
new file mode 100644
index 000000000..5129b861a
--- /dev/null
+++ b/verifcid/validate_test.go
@@ -0,0 +1,70 @@
+package verifcid
+
+import (
+ "testing"
+
+ mh "github.com/multiformats/go-multihash"
+
+ cid "github.com/ipfs/go-cid"
+)
+
+func TestValidateCids(t *testing.T) {
+ assertTrue := func(v bool) {
+ t.Helper()
+ if !v {
+ t.Fatal("expected success")
+ }
+ }
+ assertFalse := func(v bool) {
+ t.Helper()
+ if v {
+ t.Fatal("expected failure")
+ }
+ }
+
+ assertTrue(IsGoodHash(mh.SHA2_256))
+ assertTrue(IsGoodHash(mh.BLAKE2B_MIN + 32))
+ assertTrue(IsGoodHash(mh.DBL_SHA2_256))
+ assertTrue(IsGoodHash(mh.KECCAK_256))
+ assertTrue(IsGoodHash(mh.SHA3))
+
+ assertTrue(IsGoodHash(mh.SHA1))
+
+ assertFalse(IsGoodHash(mh.BLAKE2B_MIN + 5))
+
+ mhcid := func(code uint64, length int) cid.Cid {
+ mhash, err := mh.Sum([]byte{}, code, length)
+ if err != nil {
+ t.Fatalf("%v: code: %x length: %d", err, code, length)
+ }
+ return cid.NewCidV1(cid.DagCBOR, mhash)
+ }
+
+ cases := []struct {
+ cid cid.Cid
+ err error
+ }{
+ {mhcid(mh.SHA2_256, 32), nil},
+ {mhcid(mh.SHA2_256, 16), ErrBelowMinimumHashLength},
+ {mhcid(mh.MURMUR3X64_64, 4), ErrPossiblyInsecureHashFunction},
+ {mhcid(mh.BLAKE3, 32), nil},
+ {mhcid(mh.BLAKE3, 69), nil},
+ {mhcid(mh.BLAKE3, 128), nil},
+ }
+
+ for i, cas := range cases {
+ if ValidateCid(cas.cid) != cas.err {
+ t.Errorf("wrong result in case of %s (index %d). Expected: %s, got %s",
+ cas.cid, i, cas.err, ValidateCid(cas.cid))
+ }
+ }
+
+ longBlake3Hex := "1e810104e0bb39f30b1a3feb89f536c93be15055482df748674b00d26e5a75777702e9791074b7511b59d31c71c62f5a745689fa6c9497f68bdf1061fe07f518d410c0b0c27f41b3cf083f8a7fdc67a877e21790515762a754a45dcb8a356722698a7af5ed2bb608983d5aa75d4d61691ef132efe8631ce0afc15553a08fffc60ee9369b"
+ longBlake3Mh, err := mh.FromHexString(longBlake3Hex)
+ if err != nil {
+ t.Fatalf("failed to produce a multihash from the long blake3 hash: %v", err)
+ }
+ if ValidateCid(cid.NewCidV1(cid.DagCBOR, longBlake3Mh)) != ErrAboveMaximumHashLength {
+ t.Errorf("a CID that was longer than the maximum hash length did not error with ErrAboveMaximumHashLength")
+ }
+}
diff --git a/version.json b/version.json
new file mode 100644
index 000000000..9f6d6fca3
--- /dev/null
+++ b/version.json
@@ -0,0 +1,3 @@
+{
+ "version": "v0.7.0"
+}
+