diff --git a/examples/custom-ipld-formats/README.md b/examples/custom-ipld-formats/README.md new file mode 100644 index 0000000000..e74bca8f98 --- /dev/null +++ b/examples/custom-ipld-formats/README.md @@ -0,0 +1,31 @@ +# Custom IPLD formats + +This example shows you how to configure an IPFS daemon with the ability to load extra IPLD formats so you can use them in your applications. + +## Before you start + +First clone this repo, install dependencies in the project root and build the project. + +```console +$ git clone https://github.com/ipfs/js-ipfs.git +$ cd js-ipfs +$ npm install +$ npm run build +``` + +## Running the example + +Running this example should result in metrics being logged out to the console every few seconds. + +``` +> npm start +``` + +## Play with the configuration! + +By default, IPFS is only configured to support a few common IPLD formats. Your application may require extra or more esoteric formats, in which case you can configure your node to support them using `options.ipld.formats` passed to the client or an in-process node or even a daemon if you start it with a wrapper. + +See the following files for different configuration: + +* [./in-process-node.js](./in-process-node.js) for running an in-process node as part of your confiugration +* [./daemon-node.js](./daemon-node.js) for running a node as a separate daemon process diff --git a/examples/custom-ipld-formats/daemon-node.js b/examples/custom-ipld-formats/daemon-node.js new file mode 100644 index 0000000000..939e6cd410 --- /dev/null +++ b/examples/custom-ipld-formats/daemon-node.js @@ -0,0 +1,96 @@ +// ordinarily we'd open a PR against the multicodec module to get our +// codec number added but since we're just testing we shim our new +// codec into the base-table.json file - this has to be done +// before requiring other modules as the int table will become read-only +const codecName = 'dag-test' +const codecNumber = 392091 + +const baseTable = require('multicodec/src/base-table.json') +baseTable[codecName] = codecNumber + +// now require modules as usual +const IPFSDaemon = require('ipfs-cli/src/daemon') +const multihashing = require('multihashing-async') +const multihash = multihashing.multihash +const multicodec = require('multicodec') +const CID = require('cids') +const ipfsHttpClient = require('ipfs-http-client') +const uint8ArrayToString = require('uint8arrays/to-string') + +async function main () { + // see https://github.com/ipld/interface-ipld-format for the interface definition + const format = { + codec: codecNumber, + defaultHashAlg: multicodec.SHA2_256, + util: { + serialize (data) { + return Buffer.from(JSON.stringify(data)) + }, + deserialize (buf) { + return JSON.parse(uint8ArrayToString(buf)) + }, + async cid (buf) { + const multihash = await multihashing(buf, format.defaultHashAlg) + + return new CID(1, format.codec, multihash) + } + }, + resolver: { + resolve: (buf, path) => { + return { + value: format.util.deserialize(buf), + remainderPath: path + } + } + } + } + + // start an IPFS Daemon + const daemon = new IPFSDaemon({ + ipld: { + formats: [ + format + ] + } + }) + await daemon.start() + + // in another process: + const client = ipfsHttpClient({ + url: `http://localhost:${daemon._httpApi._apiServers[0].info.port}`, + ipld: { + formats: [ + format + ] + } + }) + + const data = { + hello: 'world' + } + + const cid = await client.dag.put(data, { + format: codecName, + hashAlg: multihash.codes[format.defaultHashAlg] + }) + + console.info(`Put ${JSON.stringify(data)} = CID(${cid})`) + + const { + value + } = await client.dag.get(cid) + + console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`) + + await daemon.stop() +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) + .then(() => { + // https://github.com/libp2p/js-libp2p/issues/779 + process.exit(0) + }) diff --git a/examples/custom-ipld-formats/in-process-node.js b/examples/custom-ipld-formats/in-process-node.js new file mode 100644 index 0000000000..9fa19214ff --- /dev/null +++ b/examples/custom-ipld-formats/in-process-node.js @@ -0,0 +1,73 @@ +// ordinarily we'd open a PR against the multicodec module to get our +// codec number added but since we're just testing we shim our new +// codec into the base-table.json file - this has to be done +// before requiring other modules as the int table will become read-only +const codecName = 'dag-test' +const codecNumber = 392091 + +const baseTable = require('multicodec/src/base-table.json') +baseTable[codecName] = codecNumber + +// now require modules as usual +const IPFS = require('ipfs-core') +const multihashing = require('multihashing-async') +const multicodec = require('multicodec') +const CID = require('cids') + +async function main () { + // see https://github.com/ipld/interface-ipld-format for the interface definition + const format = { + codec: codecNumber, + defaultHashAlg: multicodec.SHA2_256, + util: { + serialize (data) { + return Buffer.from(JSON.stringify(data)) + }, + deserialize (buf) { + return JSON.parse(buf.toString('utf8')) + }, + async cid (buf) { + const multihash = await multihashing(buf, format.defaultHashAlg) + + return new CID(1, format.codec, multihash) + } + } + } + + const node = await IPFS.create({ + ipld: { + formats: [ + format + ] + } + }) + + const data = { + hello: 'world' + } + + const cid = await node.dag.put(data, { + format: codecName, + hashAlg: format.defaultHashAlg + }) + + console.info(`Put ${JSON.stringify(data)} = CID(${cid})`) + + const { + value + } = await node.dag.get(cid) + + console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`) + + await node.stop() +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) + .then(() => { + // https://github.com/libp2p/js-libp2p/issues/779 + process.exit(0) + }) diff --git a/examples/custom-ipld-formats/package.json b/examples/custom-ipld-formats/package.json new file mode 100644 index 0000000000..8d07a86782 --- /dev/null +++ b/examples/custom-ipld-formats/package.json @@ -0,0 +1,22 @@ +{ + "name": "example-custom-ipld-formats", + "version": "1.0.0", + "private": true, + "scripts": { + "test": "test-ipfs-example" + }, + "license": "MIT", + "devDependencies": { + "execa": "^4.0.3", + "test-ipfs-example": "^2.0.3" + }, + "dependencies": { + "cids": "^1.0.0", + "ipfs-cli": "0.0.1", + "ipfs-core": "0.0.1", + "ipfs-http-client": "^47.0.0", + "multicodec": "^2.0.1", + "multihashing-async": "^2.0.1", + "uint8arrays": "^1.1.0" + } +} diff --git a/examples/custom-ipld-formats/test.js b/examples/custom-ipld-formats/test.js new file mode 100644 index 0000000000..0ae52257f0 --- /dev/null +++ b/examples/custom-ipld-formats/test.js @@ -0,0 +1,28 @@ +'use strict' + +const path = require('path') +const { + waitForOutput +} = require('test-ipfs-example/utils') + +const testInProcessNode = async () => { + await waitForOutput( + 'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' + + 'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'in-process-node.js')) +} + +const testDaemonNode = async () => { + await waitForOutput( + 'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' + + 'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'daemon-node.js')) +} + +async function test () { + console.info('Testing in-process node') + await testInProcessNode() + + console.info('Testing daemon node') + await testDaemonNode() +} + +module.exports = test diff --git a/packages/ipfs-cli/src/daemon.js b/packages/ipfs-cli/src/daemon.js index 30492473fe..9aef331312 100644 --- a/packages/ipfs-cli/src/daemon.js +++ b/packages/ipfs-cli/src/daemon.js @@ -28,6 +28,11 @@ class Daemon { } } + /** + * Starts the IPFS HTTP server + * + * @returns {Promise} + */ async start () { log('starting') diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index b461c8e81f..8cd8384960 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -77,7 +77,7 @@ "ipfs-unixfs-exporter": "^3.0.4", "ipfs-unixfs-importer": "^3.0.4", "ipfs-utils": "^4.0.0", - "ipld": "^0.27.1", + "ipld": "^0.27.2", "ipld-bitcoin": "^0.4.0", "ipld-block": "^0.10.1", "ipld-dag-cbor": "^0.17.0", diff --git a/packages/ipfs-core/src/components/start.js b/packages/ipfs-core/src/components/start.js index ec72580c83..1f1f286eb2 100644 --- a/packages/ipfs-core/src/components/start.js +++ b/packages/ipfs-core/src/components/start.js @@ -294,6 +294,7 @@ function createApi ({ id: Components.id({ peerId, libp2p }), init: async () => { throw new AlreadyInitializedError() }, // eslint-disable-line require-await isOnline, + ipld, key: { export: Components.key.export({ keychain }), gen: Components.key.gen({ keychain }), diff --git a/packages/ipfs-http-client/src/dag/get.js b/packages/ipfs-http-client/src/dag/get.js index b3f1b3a402..e63f058e6b 100644 --- a/packages/ipfs-http-client/src/dag/get.js +++ b/packages/ipfs-http-client/src/dag/get.js @@ -1,19 +1,13 @@ 'use strict' -const dagPB = require('ipld-dag-pb') -const dagCBOR = require('ipld-dag-cbor') -const raw = require('ipld-raw') const configure = require('../lib/configure') +const multicodec = require('multicodec') +const loadFormat = require('../lib/ipld-formats') -const resolvers = { - 'dag-cbor': dagCBOR.resolver, - 'dag-pb': dagPB.resolver, - raw: raw.resolver -} - -module.exports = configure((api, options) => { - const getBlock = require('../block/get')(options) - const dagResolve = require('./resolve')(options) +module.exports = configure((api, opts) => { + const getBlock = require('../block/get')(opts) + const dagResolve = require('./resolve')(opts) + const load = loadFormat(opts.ipld) /** * @type {import('..').Implements} @@ -21,20 +15,15 @@ module.exports = configure((api, options) => { const get = async (cid, options = {}) => { const resolved = await dagResolve(cid, options) const block = await getBlock(resolved.cid, options) - const dagResolver = resolvers[resolved.cid.codec] - if (!dagResolver) { - throw Object.assign( - new Error(`Missing IPLD format "${resolved.cid.codec}"`), - { missingMulticodec: resolved.cid.codec } - ) - } + const codecName = multicodec.getName(resolved.cid.code) + const format = await load(codecName) - if (resolved.cid.codec === 'raw' && !resolved.remainderPath) { + if (resolved.cid.code === multicodec.RAW && !resolved.remainderPath) { resolved.remainderPath = '/' } - return dagResolver.resolve(block.data, resolved.remainderPath) + return format.resolver.resolve(block.data, resolved.remainderPath) } return get diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index dad5ccc231..2a22176591 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -1,8 +1,5 @@ 'use strict' -const dagCBOR = require('ipld-dag-cbor') -const dagPB = require('ipld-dag-pb') -const ipldRaw = require('ipld-raw') const CID = require('cids') const multihash = require('multihashes') const configure = require('../lib/configure') @@ -11,19 +8,10 @@ const toUrlSearchParams = require('../lib/to-url-search-params') const { anySignal } = require('any-signal') const AbortController = require('native-abort-controller') const multicodec = require('multicodec') +const loadFormat = require('../lib/ipld-formats') module.exports = configure((api, opts) => { - const formats = { - [multicodec.DAG_PB]: dagPB, - [multicodec.DAG_CBOR]: dagCBOR, - [multicodec.RAW]: ipldRaw - } - - const ipldOptions = (opts && opts.ipld) || {} - const configuredFormats = (ipldOptions && ipldOptions.formats) || [] - configuredFormats.forEach(format => { - formats[format.codec] = format - }) + const load = loadFormat(opts.ipld) /** * @type {import('..').Implements} @@ -39,7 +27,7 @@ module.exports = configure((api, opts) => { const cid = new CID(options.cid) options = { ...options, - format: cid.codec, + format: multicodec.getName(cid.code), hashAlg: multihash.decode(cid.multihash).name } delete options.cid @@ -52,24 +40,7 @@ module.exports = configure((api, opts) => { ...options } - const number = multicodec.getNumber(settings.format) - let format = formats[number] - - if (!format) { - if (opts && opts.ipld && opts.ipld.loadFormat) { - // @ts-ignore - loadFormat expect string but it could be a number - format = await opts.ipld.loadFormat(settings.format) - } - - if (!format) { - throw new Error('Format unsupported - please add support using the options.ipld.formats or options.ipld.loadFormat options') - } - } - - if (!format.util || !format.util.serialize) { - throw new Error('Format does not support utils.serialize function') - } - + const format = await load(settings.format) const serialized = format.util.serialize(dagNode) // allow aborting requests on body errors diff --git a/packages/ipfs-http-client/src/lib/ipld-formats.js b/packages/ipfs-http-client/src/lib/ipld-formats.js new file mode 100644 index 0000000000..1ef4c4e8c6 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/ipld-formats.js @@ -0,0 +1,57 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const dagCBOR = require('ipld-dag-cbor') +const raw = require('ipld-raw') +const multicodec = require('multicodec') + +const noop = () => {} + +/** + * @typedef {import('cids')} CID + */ + +/** + * Return an object containing supported IPLD Formats + * + * @param {object} [options] - IPLD options passed to the http client constructor + * @param {Array} [options.formats] - A list of IPLD Formats to use + * @param {Function} [options.loadFormat] - An async function that can load a format when passed a codec number + * @returns {Function} + */ +module.exports = ({ formats = [], loadFormat = noop } = {}) => { + formats = formats || [] + loadFormat = loadFormat || noop + + const configuredFormats = { + [multicodec.DAG_PB]: dagPB, + [multicodec.DAG_CBOR]: dagCBOR, + [multicodec.RAW]: raw + } + + formats.forEach(format => { + configuredFormats[format.codec] = format + }) + + /** + * Attempts to load an IPLD format for the passed CID + * + * @param {string} codec - The code to load the format for + * @returns {Promise} - An IPLD format + */ + const loadResolver = async (codec) => { + const number = multicodec.getNumber(codec) + const format = configuredFormats[number] || await loadFormat(codec) + + if (!format) { + throw Object.assign( + new Error(`Missing IPLD format "${codec}"`), + { missingMulticodec: codec } + ) + } + + return format + } + + return loadResolver +} diff --git a/packages/ipfs-http-client/test/dag.spec.js b/packages/ipfs-http-client/test/dag.spec.js index a5f861aeb9..fb3bdc438b 100644 --- a/packages/ipfs-http-client/test/dag.spec.js +++ b/packages/ipfs-http-client/test/dag.spec.js @@ -73,7 +73,7 @@ describe('.dag', function () { it('should error when putting node with esoteric format', () => { const node = uint8ArrayFromString('some data') - return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Format unsupported/) + return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Missing IPLD format/) }) it('should attempt to load an unsupported format', async () => { diff --git a/packages/ipfs-http-server/package.json b/packages/ipfs-http-server/package.json index 9fdd0a6897..1548d70788 100644 --- a/packages/ipfs-http-server/package.json +++ b/packages/ipfs-http-server/package.json @@ -40,13 +40,7 @@ "ipfs-core-utils": "^0.4.0", "ipfs-http-gateway": "0.0.1", "ipfs-unixfs": "^2.0.3", - "ipld-bitcoin": "^0.4.0", - "ipld-dag-cbor": "^0.17.0", "ipld-dag-pb": "^0.20.0", - "ipld-ethereum": "^5.0.1", - "ipld-git": "^0.6.1", - "ipld-raw": "^6.0.0", - "ipld-zcash": "^0.5.0", "it-all": "^1.0.4", "it-drain": "^1.0.3", "it-first": "^1.0.4", diff --git a/packages/ipfs-http-server/src/api/resources/dag.js b/packages/ipfs-http-server/src/api/resources/dag.js index 7f3c4a7bf3..b76e19a33c 100644 --- a/packages/ipfs-http-server/src/api/resources/dag.js +++ b/packages/ipfs-http-server/src/api/resources/dag.js @@ -3,7 +3,6 @@ const multipart = require('../../utils/multipart-request-parser') const mh = require('multihashing-async').multihash const Joi = require('../../utils/joi') -const multicodec = require('multicodec') const Boom = require('@hapi/boom') const { cidToString @@ -11,48 +10,6 @@ const { const all = require('it-all') const uint8ArrayToString = require('uint8arrays/to-string') -const IpldFormats = { - get [multicodec.RAW] () { - return require('ipld-raw') - }, - get [multicodec.DAG_PB] () { - return require('ipld-dag-pb') - }, - get [multicodec.DAG_CBOR] () { - return require('ipld-dag-cbor') - }, - get [multicodec.BITCOIN_BLOCK] () { - return require('ipld-bitcoin') - }, - get [multicodec.ETH_ACCOUNT_SNAPSHOT] () { - return require('ipld-ethereum').ethAccountSnapshot - }, - get [multicodec.ETH_BLOCK] () { - return require('ipld-ethereum').ethBlock - }, - get [multicodec.ETH_BLOCK_LIST] () { - return require('ipld-ethereum').ethBlockList - }, - get [multicodec.ETH_STATE_TRIE] () { - return require('ipld-ethereum').ethStateTrie - }, - get [multicodec.ETH_STORAGE_TRIE] () { - return require('ipld-ethereum').ethStorageTrie - }, - get [multicodec.ETH_TX] () { - return require('ipld-ethereum').ethTx - }, - get [multicodec.ETH_TX_TRIE] () { - return require('ipld-ethereum').ethTxTrie - }, - get [multicodec.GIT_RAW] () { - return require('ipld-git') - }, - get [multicodec.ZCASH_BLOCK] () { - return require('ipld-zcash') - } -} - const encodeBufferKeys = (obj, encoding) => { if (!obj) { return obj @@ -198,13 +155,15 @@ exports.put = { throw Boom.badRequest('Failed to parse the JSON: ' + err) } } else { - const codec = multicodec[format.toUpperCase().replace(/-/g, '_')] + // the node is an uncommon format which the client should have + // serialized so deserialize it before continuing + const ipldFormat = await request.server.app.ipfs.ipld.getFormat(format) - if (!IpldFormats[codec]) { - throw new Error(`Missing IPLD format "${codec}"`) + if (!ipldFormat) { + throw new Error(`Missing IPLD format "${format}"`) } - node = await IpldFormats[codec].util.deserialize(data) + node = await ipldFormat.util.deserialize(data) } return { diff --git a/packages/ipfs-http-server/src/index.js b/packages/ipfs-http-server/src/index.js index 4b6fb5cf06..5439be23d2 100644 --- a/packages/ipfs-http-server/src/index.js +++ b/packages/ipfs-http-server/src/index.js @@ -48,6 +48,11 @@ class HttpApi { }) } + /** + * Starts the IPFS HTTP server + * + * @returns {Promise} + */ async start () { this._log('starting') diff --git a/packages/ipfs-http-server/test/inject/dag.js b/packages/ipfs-http-server/test/inject/dag.js index f62870069e..ab1a273d48 100644 --- a/packages/ipfs-http-server/test/inject/dag.js +++ b/packages/ipfs-http-server/test/inject/dag.js @@ -37,6 +37,9 @@ describe('/dag', () => { get: sinon.stub(), put: sinon.stub(), resolve: sinon.stub() + }, + ipld: { + getFormat: sinon.stub() } } }) @@ -292,6 +295,32 @@ describe('/dag', () => { expect(res).to.have.deep.nested.property('result.Cid', { '/': cid.toString() }) }) + it('should attempt to load an unsupported format', async () => { + const data = Buffer.from('some data') + const codec = 'git-raw' + const format = { + util: { + deserialize: (buf) => buf + } + } + ipfs.ipld.getFormat.withArgs(codec).returns(format) + + ipfs.dag.put.withArgs(data, { + ...defaultOptions, + format: codec + }).returns(cid) + + const res = await http({ + method: 'POST', + url: '/api/v0/dag/put?format=git-raw&input-enc=raw', + ...await toHeadersAndPayload(data) + }, { ipfs }) + + expect(ipfs.ipld.getFormat.calledWith(codec)).to.be.true() + expect(res).to.have.property('statusCode', 200) + expect(res).to.have.deep.nested.property('result.Cid', { '/': cid.toString() }) + }) + it('accepts a timeout', async () => { const node = { foo: 'bar'