From 43ac3051a3066786d5ce0d6a305ddf46156d0c75 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 4 Jul 2019 18:00:24 +0200 Subject: [PATCH] feat: add HTTP Gateway support for /ipns/ paths (#2020) > Part of an effort to run embedded js-ipfs in Brave :lion: https://github.com/ipfs-shipyard/ipfs-companion/issues/716 > Fixes https://github.com/ipfs/js-ipfs/issues/1918 This PR will add support for `/ipns/` paths at HTTP Gateway. Smoke test: [/ipns/tr.wikipedia-on-ipfs.org](http://127.0.0.1:9090/ipns/tr.wikipedia-on-ipfs.org/wiki/Anasayfa.html) (IPNS+DNSLink+HAMT-sharded website) This PR depends on the following merged PRs: - Gateway Improvements from https://github.com/ipfs/js-ipfs/pull/1989 (after merging #1989 I will rebase this PR, which will remove first two commits) - PeerID eg. `/ipns/` - requires https://github.com/ipfs/js-ipfs/pull/2002 to land first - `/ipns/` - requires https://github.com/multiformats/js-multicodec/pull/45 - DNSLink eg. `/ipns//path/file` like `/ipns/docs.ipfs.io/assets/logo.svg` - requires https://github.com/ipfs/js-ipfs/pull/2002 to land first - HAMT shard support eg. `/ipns/tr.wikipedia-on-ipfs.org/wiki/Anasayfa.html` (`wiki` is a sharded directory) - requires https://github.com/ipfs/js-ipfs-http-response/issues/22 and https://github.com/ipfs/js-ipfs-mfs/pull/48 to land first - Tests for `/ipns/` License: MIT Signed-off-by: Marcin Rataj --- package.json | 4 +- src/http/api/routes/webui.js | 21 +++++--- src/http/gateway/resources/gateway.js | 70 ++++++++++++++----------- src/http/gateway/routes/gateway.js | 51 +++++++++++++----- src/http/gateway/routes/index.js | 2 +- test/gateway/index.js | 75 +++++++++++++++++++++++++++ 6 files changed, 170 insertions(+), 53 deletions(-) diff --git a/package.json b/package.json index 942488f893..ce09553dc9 100644 --- a/package.json +++ b/package.json @@ -95,8 +95,8 @@ "ipfs-block": "~0.8.1", "ipfs-block-service": "~0.15.1", "ipfs-http-client": "^32.0.1", - "ipfs-http-response": "~0.3.0", - "ipfs-mfs": "~0.11.4", + "ipfs-http-response": "~0.3.1", + "ipfs-mfs": "~0.11.5", "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.26.6", "ipfs-unixfs": "~0.1.16", diff --git a/src/http/api/routes/webui.js b/src/http/api/routes/webui.js index 10e7533956..9d6926f6aa 100644 --- a/src/http/api/routes/webui.js +++ b/src/http/api/routes/webui.js @@ -1,17 +1,26 @@ 'use strict' +const Joi = require('@hapi/joi') const resources = require('../../gateway/resources') module.exports = [ { method: '*', - path: '/ipfs/{cid*}', + path: '/ipfs/{path*}', options: { - pre: [ - { method: resources.gateway.checkCID, assign: 'args' } - ] - }, - handler: resources.gateway.handler + handler: resources.gateway.handler, + validate: { + params: { + path: Joi.string().required() + } + }, + response: { + ranges: false // disable built-in support, handler does it manually + }, + ext: { + onPostHandler: { method: resources.gateway.afterHandler } + } + } }, { method: '*', diff --git a/src/http/gateway/resources/gateway.js b/src/http/gateway/resources/gateway.js index 8f538aa22c..6f2a42b6c2 100644 --- a/src/http/gateway/resources/gateway.js +++ b/src/http/gateway/resources/gateway.js @@ -11,21 +11,23 @@ const Boom = require('boom') const Ammo = require('@hapi/ammo') // HTTP Range processing utilities const peek = require('buffer-peek-stream') +const multibase = require('multibase') const { resolver } = require('ipfs-http-response') const PathUtils = require('../utils/path') const { cidToString } = require('../../../utils/cid') +const isIPFS = require('is-ipfs') -function detectContentType (ref, chunk) { +function detectContentType (path, chunk) { let fileSignature // try to guess the filetype based on the first bytes // note that `file-type` doesn't support svgs, therefore we assume it's a svg if ref looks like it - if (!ref.endsWith('.svg')) { + if (!path.endsWith('.svg')) { fileSignature = fileType(chunk) } - // if we were unable to, fallback to the `ref` which might contain the extension - const mimeType = mime.lookup(fileSignature ? fileSignature.ext : ref) + // if we were unable to, fallback to the path which might contain the extension + const mimeType = mime.lookup(fileSignature ? fileSignature.ext : path) return mime.contentType(mimeType) } @@ -45,21 +47,22 @@ class ResponseStream extends PassThrough { } module.exports = { - checkCID (request, h) { - if (!request.params.cid) { - throw Boom.badRequest('Path Resolve error: path must contain at least one component') - } - - return { ref: `/ipfs/${request.params.cid}` } - }, async handler (request, h) { - const { ref } = request.pre.args const { ipfs } = request.server.app + const path = request.path + + // The resolver from ipfs-http-response supports only immutable /ipfs/ for now, + // so we convert /ipns/ to /ipfs/ before passing it to the resolver ¯\_(ツ)_/¯ + // This could be removed if a solution proposed in + // https://github.com/ipfs/js-ipfs-http-response/issues/22 lands upstream + const ipfsPath = decodeURI(path.startsWith('/ipns/') + ? await ipfs.name.resolve(path, { recursive: true }) + : path) let data try { - data = await resolver.cid(ipfs, ref) + data = await resolver.cid(ipfs, ipfsPath) } catch (err) { const errorToString = err.toString() log.error('err: ', errorToString, ' fileName: ', err.fileName) @@ -67,14 +70,14 @@ module.exports = { // switch case with true feels so wrong. switch (true) { case (errorToString === 'Error: This dag node is a directory'): - data = await resolver.directory(ipfs, ref, err.cid) + data = await resolver.directory(ipfs, ipfsPath, err.cid) if (typeof data === 'string') { // no index file found - if (!ref.endsWith('/')) { + if (!path.endsWith('/')) { // for a directory, if URL doesn't end with a / // append / and redirect permanent to that URL - return h.redirect(`${ref}/`).permanent(true) + return h.redirect(`${path}/`).permanent(true) } // send directory listing return h.response(data) @@ -82,7 +85,7 @@ module.exports = { // found index file // redirect to URL/ - return h.redirect(PathUtils.joinURLParts(ref, data[0].Name)) + return h.redirect(PathUtils.joinURLParts(path, data[0].Name)) case (errorToString.startsWith('Error: no link named')): throw Boom.boomify(err, { statusCode: 404 }) case (errorToString.startsWith('Error: multihash length inconsistent')): @@ -94,9 +97,9 @@ module.exports = { } } - if (ref.endsWith('/')) { + if (path.endsWith('/')) { // remove trailing slash for files - return h.redirect(PathUtils.removeTrailingSlash(ref)).permanent(true) + return h.redirect(PathUtils.removeTrailingSlash(path)).permanent(true) } // Support If-None-Match & Etag (Conditional Requests from RFC7232) @@ -108,7 +111,7 @@ module.exports = { } // Immutable content produces 304 Not Modified for all values of If-Modified-Since - if (ref.startsWith('/ipfs/') && request.headers['if-modified-since']) { + if (path.startsWith('/ipfs/') && request.headers['if-modified-since']) { return h.response().code(304) // Not Modified } @@ -150,7 +153,7 @@ module.exports = { log.error(err) return reject(err) } - resolve({ peekedStream, contentType: detectContentType(ref, streamHead) }) + resolve({ peekedStream, contentType: detectContentType(path, streamHead) }) }) }) @@ -163,11 +166,11 @@ module.exports = { res.header('etag', etag) // Set headers specific to the immutable namespace - if (ref.startsWith('/ipfs/')) { + if (path.startsWith('/ipfs/')) { res.header('Cache-Control', 'public, max-age=29030400, immutable') } - log('ref ', ref) + log('path ', path) log('content-type ', contentType) if (contentType) { @@ -200,18 +203,25 @@ module.exports = { const { response } = request // Add headers to successfult responses (regular or range) if (response.statusCode === 200 || response.statusCode === 206) { - const { ref } = request.pre.args - response.header('X-Ipfs-Path', ref) - if (ref.startsWith('/ipfs/')) { + const path = request.path + response.header('X-Ipfs-Path', path) + if (path.startsWith('/ipfs/')) { // "set modtime to a really long time ago, since files are immutable and should stay cached" // Source: https://github.com/ipfs/go-ipfs/blob/v0.4.20/core/corehttp/gateway_handler.go#L228-L229 response.header('Last-Modified', 'Thu, 01 Jan 1970 00:00:01 GMT') - // Suborigins: https://github.com/ipfs/in-web-browsers/issues/66 - const rootCid = ref.split('/')[2] + // Suborigin for /ipfs/: https://github.com/ipfs/in-web-browsers/issues/66 + const rootCid = path.split('/')[2] const ipfsOrigin = cidToString(rootCid, { base: 'base32' }) - response.header('Suborigin', 'ipfs000' + ipfsOrigin) + response.header('Suborigin', `ipfs000${ipfsOrigin}`) + } else if (path.startsWith('/ipns/')) { + // Suborigin for /ipns/: https://github.com/ipfs/in-web-browsers/issues/66 + const root = path.split('/')[2] + // encode CID/FQDN in base32 (Suborigin allows only a-z) + const ipnsOrigin = isIPFS.cid(root) + ? cidToString(root, { base: 'base32' }) + : multibase.encode('base32', Buffer.from(root)).toString() + response.header('Suborigin', `ipns000${ipnsOrigin}`) } - // TODO: we don't have case-insensitive solution for /ipns/ yet (https://github.com/ipfs/go-ipfs/issues/5287) } return h.continue } diff --git a/src/http/gateway/routes/gateway.js b/src/http/gateway/routes/gateway.js index 4fe5d640b0..57d563aa6b 100644 --- a/src/http/gateway/routes/gateway.js +++ b/src/http/gateway/routes/gateway.js @@ -1,20 +1,43 @@ 'use strict' +const Joi = require('@hapi/joi') const resources = require('../resources') -module.exports = { - method: '*', - path: '/ipfs/{cid*}', - options: { - handler: resources.gateway.handler, - pre: [ - { method: resources.gateway.checkCID, assign: 'args' } - ], - response: { - ranges: false // disable built-in support, we do it manually - }, - ext: { - onPostHandler: { method: resources.gateway.afterHandler } +module.exports = [ + { + method: '*', + path: '/ipfs/{path*}', + options: { + handler: resources.gateway.handler, + validate: { + params: { + path: Joi.string().required() + } + }, + response: { + ranges: false // disable built-in support, handler does it manually + }, + ext: { + onPostHandler: { method: resources.gateway.afterHandler } + } + } + }, + { + method: '*', + path: '/ipns/{path*}', + options: { + handler: resources.gateway.handler, + validate: { + params: { + path: Joi.string().required() + } + }, + response: { + ranges: false // disable built-in support, handler does it manually + }, + ext: { + onPostHandler: { method: resources.gateway.afterHandler } + } } } -} +] diff --git a/src/http/gateway/routes/index.js b/src/http/gateway/routes/index.js index 2cbf163b04..24f57fbbd7 100644 --- a/src/http/gateway/routes/index.js +++ b/src/http/gateway/routes/index.js @@ -1,3 +1,3 @@ 'use strict' -module.exports = [require('./gateway')] +module.exports = require('./gateway') diff --git a/test/gateway/index.js b/test/gateway/index.js index 22fc6ea7b6..9cab5b6d7d 100644 --- a/test/gateway/index.js +++ b/test/gateway/index.js @@ -12,6 +12,7 @@ const os = require('os') const path = require('path') const hat = require('hat') const fileType = require('file-type') +const CID = require('cids') const bigFile = loadFixture('test/fixtures/15mb.random', 'interface-ipfs-core') const directoryContent = { @@ -20,6 +21,7 @@ const directoryContent = { 'nested-folder/ipfs.txt': loadFixture('test/gateway/test-folder/nested-folder/ipfs.txt'), 'nested-folder/nested.html': loadFixture('test/gateway/test-folder/nested-folder/nested.html'), 'cat-folder/cat.jpg': loadFixture('test/gateway/test-folder/cat-folder/cat.jpg'), + 'utf8/cat-with-óąśśł-and-أعظم._.jpg': loadFixture('test/gateway/test-folder/cat-folder/cat.jpg'), 'unsniffable-folder/hexagons-xml.svg': loadFixture('test/gateway/test-folder/unsniffable-folder/hexagons-xml.svg'), 'unsniffable-folder/hexagons.svg': loadFixture('test/gateway/test-folder/unsniffable-folder/hexagons.svg') } @@ -84,6 +86,10 @@ describe('HTTP Gateway', function () { content('unsniffable-folder/hexagons-xml.svg'), content('unsniffable-folder/hexagons.svg') ]) + // QmaRdtkDark8TgXPdDczwBneadyF44JvFGbrKLTkmTUhHk + await http.api._ipfs.add([content('utf8/cat-with-óąśśł-and-أعظم._.jpg')]) + // Publish QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ to IPNS using self key + await http.api._ipfs.name.publish('QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ', { resolve: false }) }) after(() => http.api.stop()) @@ -526,4 +532,73 @@ describe('HTTP Gateway', function () { expect(res.headers.location).to.equal('/ipfs/QmbQD7EMEL1zeebwBsWEfA3ndgSS6F7S6iTuwuqasPgVRi/index.html') expect(res.headers['x-ipfs-path']).to.equal(undefined) }) + + it('test(gateway): load from URI-encoded path', async () => { + // non-ascii characters will be URI-encoded by the browser + const utf8path = '/ipfs/QmaRdtkDark8TgXPdDczwBneadyF44JvFGbrKLTkmTUhHk/cat-with-óąśśł-and-أعظم._.jpg' + const escapedPath = encodeURI(utf8path) // this is what will be actually requested + const res = await gateway.inject({ + method: 'GET', + url: escapedPath + }) + + expect(res.statusCode).to.equal(200) + expect(res.headers['content-type']).to.equal('image/jpeg') + expect(res.headers['x-ipfs-path']).to.equal(escapedPath) + expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') + expect(res.headers['content-length']).to.equal(res.rawPayload.length) + expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"') + expect(res.headers.suborigin).to.equal('ipfs000bafybeiftsm4u7cn24bn2suwg3x7sldx2uplvfylsk3e4bgylyxwjdevhqm') + }) + + it('load a file from IPNS', async () => { + const { id } = await http.api._ipfs.id() + const ipnsPath = `/ipns/${id}/cat.jpg` + + const res = await gateway.inject({ + method: 'GET', + url: ipnsPath + }) + + const kittyDirectCid = 'Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u' + + expect(res.statusCode).to.equal(200) + expect(res.headers['content-type']).to.equal('image/jpeg') + expect(res.headers['content-length']).to.equal(res.rawPayload.length).to.equal(443230) + expect(res.headers['x-ipfs-path']).to.equal(ipnsPath) + expect(res.headers['etag']).to.equal(`"${kittyDirectCid}"`) + expect(res.headers['cache-control']).to.equal('no-cache') // TODO: should be record TTL + expect(res.headers['last-modified']).to.equal(undefined) + expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"') + expect(res.headers.suborigin).to.equal(`ipns000${new CID(id).toV1().toBaseEncodedString('base32')}`) + + let fileSignature = fileType(res.rawPayload) + expect(fileSignature.mime).to.equal('image/jpeg') + expect(fileSignature.ext).to.equal('jpg') + }) + + it('load a directory from IPNS', async () => { + const { id } = await http.api._ipfs.id() + const ipnsPath = `/ipns/${id}/` + + const res = await gateway.inject({ + method: 'GET', + url: ipnsPath + }) + + expect(res.statusCode).to.equal(200) + expect(res.headers['content-type']).to.equal('text/html; charset=utf-8') + expect(res.headers['x-ipfs-path']).to.equal(ipnsPath) + expect(res.headers['cache-control']).to.equal('no-cache') + expect(res.headers['last-modified']).to.equal(undefined) + expect(res.headers['content-length']).to.equal(res.rawPayload.length) + expect(res.headers.etag).to.equal(undefined) + expect(res.headers.suborigin).to.equal(`ipns000${new CID(id).toV1().toBaseEncodedString('base32')}`) + + // check if the cat picture is in the payload as a way to check + // if this is an index of this directory + let listedFile = res.payload.match(/\/cat\.jpg/g) + expect(listedFile).to.have.lengthOf(1) + }) })