feat: add HTTP Gateway support for /ipns/ paths (#2020)

> Part of an effort to run embedded js-ipfs in Brave 🦁 ipfs/ipfs-companion#716
> Fixes #1918 

This PR will add support for `/ipns/` paths at HTTP Gateway.
Smoke test: [/ipns/tr.wikipedia-on-ipfs.org](http://127.0.0.1:9090/ipns/tr.wikipedia-on-ipfs.org/wiki/Anasayfa.html) (IPNS+DNSLink+HAMT-sharded website)
 
This PR depends on the following merged PRs:

  - Gateway Improvements from #1989
    (after merging #1989 I will rebase this PR, which will remove first two commits)
  - PeerID 
    eg. `/ipns/<PeerId-as-multihash-b58>`
    - requires #2002 to land first
  - `/ipns/<libp2p-key-in-cidv1>` 
    - requires multiformats/js-multicodec#45
  - DNSLink 
    eg. `/ipns/<fqdn>/path/file` like `/ipns/docs.ipfs.io/assets/logo.svg`
    - requires #2002 to land first
  - HAMT shard support 
    eg. `/ipns/tr.wikipedia-on-ipfs.org/wiki/Anasayfa.html` (`wiki` is a sharded directory)
    - requires ipfs/js-ipfs-http-response#22 and ipfs-inactive/js-ipfs-mfs#48  to land first
  - Tests for `/ipns/` 

License: MIT
Signed-off-by: Marcin Rataj <lidel@lidel.org>

package.json

@@ -95,8 +95,8 @@
     "ipfs-block": "~0.8.1",
     "ipfs-block-service": "~0.15.1",
     "ipfs-http-client": "^32.0.1",
-    "ipfs-http-response": "~0.3.0",
-    "ipfs-mfs": "~0.11.4",
+    "ipfs-http-response": "~0.3.1",
+    "ipfs-mfs": "~0.11.5",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.26.6",
     "ipfs-unixfs": "~0.1.16",

src/http/api/routes/webui.js

@@ -1,17 +1,26 @@
 'use strict'
 
+const Joi = require('@hapi/joi')
 const resources = require('../../gateway/resources')
 
 module.exports = [
   {
     method: '*',
-    path: '/ipfs/{cid*}',
+    path: '/ipfs/{path*}',
     options: {
-      pre: [
-        { method: resources.gateway.checkCID, assign: 'args' }
-      ]
-    },
-    handler: resources.gateway.handler
+      handler: resources.gateway.handler,
+      validate: {
+        params: {
+          path: Joi.string().required()
+        }
+      },
+      response: {
+        ranges: false // disable built-in support, handler does it manually
+      },
+      ext: {
+        onPostHandler: { method: resources.gateway.afterHandler }
+      }
+    }
   },
   {
     method: '*',

src/http/gateway/resources/gateway.js

@@ -11,21 +11,23 @@ const Boom = require('boom')
 const Ammo = require('@hapi/ammo') // HTTP Range processing utilities
 const peek = require('buffer-peek-stream')
 
+const multibase = require('multibase')
 const { resolver } = require('ipfs-http-response')
 const PathUtils = require('../utils/path')
 const { cidToString } = require('../../../utils/cid')
+const isIPFS = require('is-ipfs')
 
-function detectContentType (ref, chunk) {
+function detectContentType (path, chunk) {
   let fileSignature
 
   // try to guess the filetype based on the first bytes
   // note that `file-type` doesn't support svgs, therefore we assume it's a svg if ref looks like it
-  if (!ref.endsWith('.svg')) {
+  if (!path.endsWith('.svg')) {
     fileSignature = fileType(chunk)
   }
 
-  // if we were unable to, fallback to the `ref` which might contain the extension
-  const mimeType = mime.lookup(fileSignature ? fileSignature.ext : ref)
+  // if we were unable to, fallback to the path which might contain the extension
+  const mimeType = mime.lookup(fileSignature ? fileSignature.ext : path)
 
   return mime.contentType(mimeType)
 }
@@ -45,44 +47,45 @@ class ResponseStream extends PassThrough {
 }
 
 module.exports = {
-  checkCID (request, h) {
-    if (!request.params.cid) {
-      throw Boom.badRequest('Path Resolve error: path must contain at least one component')
-    }
-
-    return { ref: `/ipfs/${request.params.cid}` }
-  },
 
   async handler (request, h) {
-    const { ref } = request.pre.args
     const { ipfs } = request.server.app
+    const path = request.path
+
+    // The resolver from ipfs-http-response supports only immutable /ipfs/ for now,
+    // so we convert /ipns/ to /ipfs/ before passing it to the resolver ¯\_(ツ)_/¯
+    // This could be removed if a solution proposed in
+    //  https://github.com/ipfs/js-ipfs-http-response/issues/22 lands upstream
+    const ipfsPath = decodeURI(path.startsWith('/ipns/')
+      ? await ipfs.name.resolve(path, { recursive: true })
+      : path)
 
     let data
     try {
-      data = await resolver.cid(ipfs, ref)
+      data = await resolver.cid(ipfs, ipfsPath)
     } catch (err) {
       const errorToString = err.toString()
       log.error('err: ', errorToString, ' fileName: ', err.fileName)
 
       // switch case with true feels so wrong.
       switch (true) {
         case (errorToString === 'Error: This dag node is a directory'):
-          data = await resolver.directory(ipfs, ref, err.cid)
+          data = await resolver.directory(ipfs, ipfsPath, err.cid)
 
           if (typeof data === 'string') {
             // no index file found
-            if (!ref.endsWith('/')) {
+            if (!path.endsWith('/')) {
               // for a directory, if URL doesn't end with a /
               // append / and redirect permanent to that URL
-              return h.redirect(`${ref}/`).permanent(true)
+              return h.redirect(`${path}/`).permanent(true)
             }
             // send directory listing
             return h.response(data)
           }
 
           // found index file
           // redirect to URL/<found-index-file>
-          return h.redirect(PathUtils.joinURLParts(ref, data[0].Name))
+          return h.redirect(PathUtils.joinURLParts(path, data[0].Name))
         case (errorToString.startsWith('Error: no link named')):
           throw Boom.boomify(err, { statusCode: 404 })
         case (errorToString.startsWith('Error: multihash length inconsistent')):
@@ -94,9 +97,9 @@ module.exports = {
       }
     }
 
-    if (ref.endsWith('/')) {
+    if (path.endsWith('/')) {
       // remove trailing slash for files
-      return h.redirect(PathUtils.removeTrailingSlash(ref)).permanent(true)
+      return h.redirect(PathUtils.removeTrailingSlash(path)).permanent(true)
     }
 
     // Support If-None-Match & Etag (Conditional Requests from RFC7232)
@@ -108,7 +111,7 @@ module.exports = {
     }
 
     // Immutable content produces 304 Not Modified for all values of If-Modified-Since
-    if (ref.startsWith('/ipfs/') && request.headers['if-modified-since']) {
+    if (path.startsWith('/ipfs/') && request.headers['if-modified-since']) {
       return h.response().code(304) // Not Modified
     }
 
@@ -150,7 +153,7 @@ module.exports = {
           log.error(err)
           return reject(err)
         }
-        resolve({ peekedStream, contentType: detectContentType(ref, streamHead) })
+        resolve({ peekedStream, contentType: detectContentType(path, streamHead) })
       })
     })
 
@@ -163,11 +166,11 @@ module.exports = {
     res.header('etag', etag)
 
     // Set headers specific to the immutable namespace
-    if (ref.startsWith('/ipfs/')) {
+    if (path.startsWith('/ipfs/')) {
       res.header('Cache-Control', 'public, max-age=29030400, immutable')
     }
 
-    log('ref ', ref)
+    log('path ', path)
     log('content-type ', contentType)
 
     if (contentType) {
@@ -200,18 +203,25 @@ module.exports = {
     const { response } = request
     // Add headers to successfult responses (regular or range)
     if (response.statusCode === 200 || response.statusCode === 206) {
-      const { ref } = request.pre.args
-      response.header('X-Ipfs-Path', ref)
-      if (ref.startsWith('/ipfs/')) {
+      const path = request.path
+      response.header('X-Ipfs-Path', path)
+      if (path.startsWith('/ipfs/')) {
         // "set modtime to a really long time ago, since files are immutable and should stay cached"
         // Source: https://github.com/ipfs/go-ipfs/blob/v0.4.20/core/corehttp/gateway_handler.go#L228-L229
         response.header('Last-Modified', 'Thu, 01 Jan 1970 00:00:01 GMT')
-        // Suborigins: https://github.com/ipfs/in-web-browsers/issues/66
-        const rootCid = ref.split('/')[2]
+        // Suborigin for /ipfs/: https://github.com/ipfs/in-web-browsers/issues/66
+        const rootCid = path.split('/')[2]
         const ipfsOrigin = cidToString(rootCid, { base: 'base32' })
-        response.header('Suborigin', 'ipfs000' + ipfsOrigin)
+        response.header('Suborigin', `ipfs000${ipfsOrigin}`)
+      } else if (path.startsWith('/ipns/')) {
+        // Suborigin for /ipns/: https://github.com/ipfs/in-web-browsers/issues/66
+        const root = path.split('/')[2]
+        // encode CID/FQDN in base32 (Suborigin allows only a-z)
+        const ipnsOrigin = isIPFS.cid(root)
+          ? cidToString(root, { base: 'base32' })
+          : multibase.encode('base32', Buffer.from(root)).toString()
+        response.header('Suborigin', `ipns000${ipnsOrigin}`)
       }
-      // TODO: we don't have case-insensitive solution for /ipns/ yet (https://github.com/ipfs/go-ipfs/issues/5287)
     }
     return h.continue
   }

src/http/gateway/routes/gateway.js

@@ -1,20 +1,43 @@
 'use strict'
 
+const Joi = require('@hapi/joi')
 const resources = require('../resources')
 
-module.exports = {
-  method: '*',
-  path: '/ipfs/{cid*}',
-  options: {
-    handler: resources.gateway.handler,
-    pre: [
-      { method: resources.gateway.checkCID, assign: 'args' }
-    ],
-    response: {
-      ranges: false // disable built-in support, we do it manually
-    },
-    ext: {
-      onPostHandler: { method: resources.gateway.afterHandler }
+module.exports = [
+  {
+    method: '*',
+    path: '/ipfs/{path*}',
+    options: {
+      handler: resources.gateway.handler,
+      validate: {
+        params: {
+          path: Joi.string().required()
+        }
+      },
+      response: {
+        ranges: false // disable built-in support, handler does it manually
+      },
+      ext: {
+        onPostHandler: { method: resources.gateway.afterHandler }
+      }
+    }
+  },
+  {
+    method: '*',
+    path: '/ipns/{path*}',
+    options: {
+      handler: resources.gateway.handler,
+      validate: {
+        params: {
+          path: Joi.string().required()
+        }
+      },
+      response: {
+        ranges: false // disable built-in support, handler does it manually
+      },
+      ext: {
+        onPostHandler: { method: resources.gateway.afterHandler }
+      }
     }
   }
-}
+]

src/http/gateway/routes/index.js

@@ -1,3 +1,3 @@
 'use strict'
 
-module.exports = [require('./gateway')]
+module.exports = require('./gateway')

test/gateway/index.js

@@ -12,6 +12,7 @@ const os = require('os')
 const path = require('path')
 const hat = require('hat')
 const fileType = require('file-type')
+const CID = require('cids')
 
 const bigFile = loadFixture('test/fixtures/15mb.random', 'interface-ipfs-core')
 const directoryContent = {
@@ -20,6 +21,7 @@ const directoryContent = {
   'nested-folder/ipfs.txt': loadFixture('test/gateway/test-folder/nested-folder/ipfs.txt'),
   'nested-folder/nested.html': loadFixture('test/gateway/test-folder/nested-folder/nested.html'),
   'cat-folder/cat.jpg': loadFixture('test/gateway/test-folder/cat-folder/cat.jpg'),
+  'utf8/cat-with-óąśśł-and-أعظم._.jpg': loadFixture('test/gateway/test-folder/cat-folder/cat.jpg'),
   'unsniffable-folder/hexagons-xml.svg': loadFixture('test/gateway/test-folder/unsniffable-folder/hexagons-xml.svg'),
   'unsniffable-folder/hexagons.svg': loadFixture('test/gateway/test-folder/unsniffable-folder/hexagons.svg')
 }
@@ -84,6 +86,10 @@ describe('HTTP Gateway', function () {
       content('unsniffable-folder/hexagons-xml.svg'),
       content('unsniffable-folder/hexagons.svg')
     ])
+    // QmaRdtkDark8TgXPdDczwBneadyF44JvFGbrKLTkmTUhHk
+    await http.api._ipfs.add([content('utf8/cat-with-óąśśł-and-أعظم._.jpg')])
+    // Publish QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ to IPNS using self key
+    await http.api._ipfs.name.publish('QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ', { resolve: false })
   })
 
   after(() => http.api.stop())
@@ -526,4 +532,73 @@ describe('HTTP Gateway', function () {
     expect(res.headers.location).to.equal('/ipfs/QmbQD7EMEL1zeebwBsWEfA3ndgSS6F7S6iTuwuqasPgVRi/index.html')
     expect(res.headers['x-ipfs-path']).to.equal(undefined)
   })
+
+  it('test(gateway): load from URI-encoded path', async () => {
+    // non-ascii characters will be URI-encoded by the browser
+    const utf8path = '/ipfs/QmaRdtkDark8TgXPdDczwBneadyF44JvFGbrKLTkmTUhHk/cat-with-óąśśł-and-أعظم._.jpg'
+    const escapedPath = encodeURI(utf8path) // this is what will be actually requested
+    const res = await gateway.inject({
+      method: 'GET',
+      url: escapedPath
+    })
+
+    expect(res.statusCode).to.equal(200)
+    expect(res.headers['content-type']).to.equal('image/jpeg')
+    expect(res.headers['x-ipfs-path']).to.equal(escapedPath)
+    expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable')
+    expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT')
+    expect(res.headers['content-length']).to.equal(res.rawPayload.length)
+    expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"')
+    expect(res.headers.suborigin).to.equal('ipfs000bafybeiftsm4u7cn24bn2suwg3x7sldx2uplvfylsk3e4bgylyxwjdevhqm')
+  })
+
+  it('load a file from IPNS', async () => {
+    const { id } = await http.api._ipfs.id()
+    const ipnsPath = `/ipns/${id}/cat.jpg`
+
+    const res = await gateway.inject({
+      method: 'GET',
+      url: ipnsPath
+    })
+
+    const kittyDirectCid = 'Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u'
+
+    expect(res.statusCode).to.equal(200)
+    expect(res.headers['content-type']).to.equal('image/jpeg')
+    expect(res.headers['content-length']).to.equal(res.rawPayload.length).to.equal(443230)
+    expect(res.headers['x-ipfs-path']).to.equal(ipnsPath)
+    expect(res.headers['etag']).to.equal(`"${kittyDirectCid}"`)
+    expect(res.headers['cache-control']).to.equal('no-cache') // TODO: should be record TTL
+    expect(res.headers['last-modified']).to.equal(undefined)
+    expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"')
+    expect(res.headers.suborigin).to.equal(`ipns000${new CID(id).toV1().toBaseEncodedString('base32')}`)
+
+    let fileSignature = fileType(res.rawPayload)
+    expect(fileSignature.mime).to.equal('image/jpeg')
+    expect(fileSignature.ext).to.equal('jpg')
+  })
+
+  it('load a directory from IPNS', async () => {
+    const { id } = await http.api._ipfs.id()
+    const ipnsPath = `/ipns/${id}/`
+
+    const res = await gateway.inject({
+      method: 'GET',
+      url: ipnsPath
+    })
+
+    expect(res.statusCode).to.equal(200)
+    expect(res.headers['content-type']).to.equal('text/html; charset=utf-8')
+    expect(res.headers['x-ipfs-path']).to.equal(ipnsPath)
+    expect(res.headers['cache-control']).to.equal('no-cache')
+    expect(res.headers['last-modified']).to.equal(undefined)
+    expect(res.headers['content-length']).to.equal(res.rawPayload.length)
+    expect(res.headers.etag).to.equal(undefined)
+    expect(res.headers.suborigin).to.equal(`ipns000${new CID(id).toV1().toBaseEncodedString('base32')}`)
+
+    // check if the cat picture is in the payload as a way to check
+    // if this is an index of this directory
+    let listedFile = res.payload.match(/\/cat\.jpg/g)
+    expect(listedFile).to.have.lengthOf(1)
+  })
 })