[Experiment] WASM IPLD Codecs and ADLs

cmd/ipfs/daemon.go

@@ -786,7 +786,7 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 	var opts = []corehttp.ServeOption{
 		corehttp.MetricsCollectionOption("gateway"),
 		corehttp.HostnameOption(),
-		corehttp.GatewayOption(writable, "/ipfs", "/ipns"),
+		corehttp.GatewayOption(writable, "/ipfs", "/ipns", "/ipld"),
 		corehttp.VersionOption(),
 		corehttp.CheckVersionOption(),
 		corehttp.CommandsROOption(cmdctx),

core/coreapi/coreapi.go

@@ -14,9 +14,15 @@ Interfaces here aren't yet completely stable.
 package coreapi
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	ipld2 "github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/datamodel"
+	"github.com/ipld/go-ipld-prime/linking"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"io"
 
 	bserv "github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-fetcher"
@@ -91,6 +97,25 @@ func NewCoreAPI(n *core.IpfsNode, opts ...options.ApiOption) (coreiface.CoreAPI,
 	return (&CoreAPI{nd: n, parentOpts: *parentOpts}).WithOptions(opts...)
 }
 
+var KnownReifiers map[string]ipld2.NodeReifier = make(map[string]ipld2.NodeReifier)
+
+func (api *CoreAPI) LinkSystem() ipld2.LinkSystem {
+	lsys := cidlink.DefaultLinkSystem()
+	lsys.KnownReifiers = KnownReifiers
+	lsys.StorageReadOpener = func(linkContext linking.LinkContext, link datamodel.Link) (io.Reader, error) {
+		if cl, ok := link.(cidlink.Link); !ok {
+			return nil, fmt.Errorf("cannot process link: %v", link)
+		} else {
+			block, err := api.blocks.GetBlock(linkContext.Ctx, cl.Cid)
+			if err != nil {
+				return nil, err
+			}
+			return bytes.NewReader(block.RawData()), nil
+		}
+	}
+	return lsys
+}
+
 // Unixfs returns the UnixfsAPI interface implementation backed by the go-ipfs node
 func (api *CoreAPI) Unixfs() coreiface.UnixfsAPI {
 	return (*UnixfsAPI)(api)

core/corehttp/gateway_handler.go

@@ -3,6 +3,12 @@ package corehttp
 import (
 	"context"
 	"fmt"
+	"github.com/ipld/go-ipld-prime"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
+	"github.com/multiformats/go-multicodec"
+	"github.com/multiformats/go-multihash"
 	"html/template"
 	"io"
 	"mime"
@@ -364,22 +370,35 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	// Resolve path to the final DAG node for the ETag
-	resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
-	switch err {
-	case nil:
-	case coreiface.ErrOffline:
-		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
-		return
-	default:
-		// if Accept is text/html, see if ipfs-404.html is present
-		if i.servePretty404IfPresent(w, r, contentPath) {
-			logger.Debugw("serve pretty 404 if present")
+	ns := contentPath.Namespace()
+
+	var resolvedPath ipath.Resolved
+	var err error
+	if ns != "ipld" {
+		// Resolve path to the final DAG node for the ETag
+		resolvedPath, err = i.api.ResolvePath(r.Context(), contentPath)
+		switch err {
+		case nil:
+		case coreiface.ErrOffline:
+			webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
 			return
-		}
+		default:
+			// if Accept is text/html, see if ipfs-404.html is present
+			if i.servePretty404IfPresent(w, r, contentPath) {
+				logger.Debugw("serve pretty 404 if present")
+				return
+			}
 
-		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound)
-		return
+			webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound)
+			return
+		}
+	} else {
+		cstr := strings.Split(contentPath.String(), "/")[2]
+		c, err := cid.Decode(cstr)
+		if err != nil {
+			webError(w, "resolve /ipld error "+debugStr(contentPath.String()), err, http.StatusNotFound)
+		}
+		resolvedPath = ipath.IpfsPath(c)
 	}
 
 	// Detect when explicit Accept header or ?format parameter are present
@@ -415,11 +434,95 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
+	var querySelector ipld.Node
+	var selectorCID cid.Cid
+	if selectorParam := r.URL.Query().Get("selector"); selectorParam != "" {
+		selectorCID, err = cid.Decode(selectorParam)
+		if err != nil {
+			webError(w, "selector is not a valid CID", err, http.StatusInternalServerError)
+			return
+		}
+		selectorNode, err := i.api.Dag().Get(r.Context(), selectorCID)
+		if err != nil {
+			webError(w, "could not get selector", err, http.StatusInternalServerError)
+			return
+		}
+		querySelector = selectorNode.(ipld.Node)
+	}
+
+	if ipldPathParam := r.URL.Query().Get("ipld-path"); ipldPathParam != "" || ns == "ipld" {
+		var pcs []string
+		if ipldPathParam != "" {
+			pcs = strings.Split(ipldPathParam, "|")
+		} else {
+			pcs = strings.Split(contentPath.String(), "/")[3:]
+		}
+		buildSel := builder.NewSelectorSpecBuilder(basicnode.Prototype__Any{})
+		var fnBuildSel func(comps []string) (builder.SelectorSpec, error)
+		fnBuildSel = func(comps []string) (builder.SelectorSpec, error) {
+			if len(comps) == 0 {
+				return buildSel.Matcher(), nil
+			}
+			comp := comps[0]
+			if comp[0] != '[' {
+				next, err := fnBuildSel(comps[1:])
+				if err != nil {
+					return nil, err
+				}
+				return buildSel.ExploreFields(func(specBuilder builder.ExploreFieldsSpecBuilder) {
+					specBuilder.Insert(comp, next)
+				}), nil
+			} else {
+				matched, err := regexp.MatchString(`[ADL=(.*)]`, comp)
+				if err != nil {
+					return nil, err
+				}
+				if !matched {
+					return nil, fmt.Errorf("invalid path component %s", comp)
+				}
+				adlName := comp[5 : len(comp)-1]
+				next, err := fnBuildSel(comps[1:])
+				if err != nil {
+					return nil, err
+				}
+				return buildSel.ExploreInterpretAs(adlName, next), nil
+			}
+		}
+
+		sel, err := fnBuildSel(pcs)
+		if err != nil {
+			webError(w, "problem with ipld pathing", err, http.StatusBadRequest)
+			return
+		}
+
+		querySelector = sel.Node()
+		lsys := cidlink.DefaultLinkSystem()
+		lnk, err := lsys.ComputeLink(cidlink.LinkPrototype{Prefix: cid.Prefix{
+			Version:  1,
+			Codec:    uint64(multicodec.DagJson),
+			MhType:   multihash.IDENTITY,
+			MhLength: -1,
+		}}, querySelector)
+
+		selectorCIDLnk, ok := lnk.(cidlink.Link)
+		if !ok {
+			webError(w, "could not compute selector CID", err, http.StatusInternalServerError)
+			return
+		}
+		selectorCID = selectorCIDLnk.Cid
+	}
+
 	// Support custom response formats passed via ?format or Accept HTTP header
 	switch responseFormat {
 	case "": // The implicit response format is UnixFS
-		logger.Debugw("serving unixfs", "path", contentPath)
-		i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
+		// If there is a selector we're in IPLD land
+		if querySelector != nil {
+			logger.Debugw("serving ipld selector request", "path", contentPath, "selector", selectorCID)
+			i.serveIPLD(w, r, resolvedPath, contentPath, querySelector, selectorCID, begin, logger)
+		} else {
+			logger.Debugw("serving unixfs", "path", contentPath)
+			i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
+		}
 		return
 	case "application/vnd.ipld.raw":
 		logger.Debugw("serving raw block", "path", contentPath)
@@ -1071,11 +1174,12 @@ func (i *gatewayHandler) setCommonHeaders(w http.ResponseWriter, r *http.Request
 	i.addUserHeaders(w) // ok, _now_ write user's headers.
 	w.Header().Set("X-Ipfs-Path", contentPath.String())
 
-	if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil {
-		w.Header().Set("X-Ipfs-Roots", rootCids)
-	} else { // this should never happen, as we resolved the contentPath already
-		return newRequestError("error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError)
+	if contentPath.Namespace() != "ipld" {
+		if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil {
+			w.Header().Set("X-Ipfs-Roots", rootCids)
+		} else { // this should never happen, as we resolved the contentPath already
+			return newRequestError("error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError)
+		}
 	}
-
 	return nil
 }

core/corehttp/gateway_handler_selector.go

@@ -0,0 +1,107 @@
+package corehttp
+
+import (
+	"fmt"
+	"github.com/ipfs/go-cid"
+	"net/http"
+	"time"
+
+	"go.uber.org/zap"
+
+	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+
+	dagpb "github.com/ipld/go-codec-dagpb"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/datamodel"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+	"github.com/ipld/go-ipld-prime/traversal"
+	"github.com/ipld/go-ipld-prime/traversal/selector"
+)
+
+func (i *gatewayHandler) serveIPLD(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, selectorNode ipld.Node, selectorCID cid.Cid, begin time.Time, logger *zap.SugaredLogger) {
+	if resolvedPath.Remainder() != "" {
+		http.Error(w, "serving ipld cannot handle path remainders", http.StatusInternalServerError)
+		return
+	}
+
+	nsc := func(lnk ipld.Link, lctx ipld.LinkContext) (ipld.NodePrototype, error) {
+		// We can decode all nodes into basicnode's Any, except for
+		// dagpb nodes, which must explicitly use the PBNode prototype.
+		if lnk, ok := lnk.(cidlink.Link); ok && lnk.Cid.Prefix().Codec == 0x70 {
+			return dagpb.Type.PBNode, nil
+		}
+		return basicnode.Prototype.Any, nil
+	}
+
+	compiledSelector, err := selector.CompileSelector(selectorNode)
+	if err != nil {
+		webError(w, "could not compile selector", err, http.StatusInternalServerError)
+		return
+	}
+
+	lnk := cidlink.Link{Cid: resolvedPath.Cid()}
+	ns, _ := nsc(lnk, ipld.LinkContext{}) // nsc won't error
+
+	type HasLinksystem interface {
+		LinkSystem() ipld.LinkSystem
+	}
+
+	lsHaver, ok := i.api.(HasLinksystem)
+	if !ok {
+		webError(w, "could not find linksystem", err, http.StatusInternalServerError)
+		return
+	}
+	lsys := lsHaver.LinkSystem()
+
+	nd, err := lsys.Load(ipld.LinkContext{Ctx: r.Context()}, lnk, ns)
+	if err != nil {
+		webError(w, "could not load root", err, http.StatusInternalServerError)
+		return
+	}
+
+	prog := traversal.Progress{
+		Cfg: &traversal.Config{
+			Ctx:                            r.Context(),
+			LinkSystem:                     lsys,
+			LinkTargetNodePrototypeChooser: nsc,
+			LinkVisitOnlyOnce:              true,
+		},
+	}
+
+	var latestMatchedNode ipld.Node
+
+	err = prog.WalkAdv(nd, compiledSelector, func(progress traversal.Progress, node datamodel.Node, reason traversal.VisitReason) error {
+		if reason == traversal.VisitReason_SelectionMatch {
+			if latestMatchedNode == nil {
+				latestMatchedNode = node
+			} else {
+				return fmt.Errorf("can only use selectors that match a single node")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		webError(w, "could not execute selector", err, http.StatusInternalServerError)
+		return
+	}
+
+	if latestMatchedNode == nil {
+		webError(w, "selector did not match anything", err, http.StatusInternalServerError)
+		return
+	}
+
+	lbnNode, ok := latestMatchedNode.(datamodel.LargeBytesNode)
+	if !ok {
+		webError(w, "matched node was not bytes", err, http.StatusInternalServerError)
+		return
+	}
+	if data, err := lbnNode.AsLargeBytes(); err != nil {
+		webError(w, "matched node was not bytes", err, http.StatusInternalServerError)
+		return
+	} else {
+		modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid())
+		name := resolvedPath.Cid().String() + "-" + selectorCID.String()
+		http.ServeContent(w, r, name, modtime, data)
+	}
+}

core/corehttp/hostname.go

@@ -24,7 +24,7 @@ import (
 	nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys"
 )
 
-var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}
+var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/", "/ipld/"}
 
 var subdomainGatewaySpec = &config.GatewaySpec{
 	Paths:         defaultPaths,