From 6c7f4ec26b1c76b4ee1dd991fbbc704ee9b08250 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 19 May 2019 23:00:38 +0100 Subject: [PATCH] chore: update ipld formats, async/await mfs and unixfs and base32 cids --- .../get-path-accross-formats.js | 19 +- examples/traverse-ipld-graphs/tree.js | 19 +- package.json | 47 ++--- src/cli/bin.js | 14 +- src/cli/commands/dag/get.js | 2 +- src/cli/commands/object/get.js | 14 +- src/cli/commands/object/links.js | 4 +- src/cli/commands/object/patch/add-link.js | 12 +- src/core/components/bitswap.js | 2 +- src/core/components/dag.js | 18 +- src/core/components/files-mfs.js | 67 ++++--- .../files-regular/add-pull-stream.js | 10 +- src/core/components/init.js | 8 +- src/core/components/object.js | 183 +++++++++++++----- src/core/components/pin-set.js | 43 ++-- src/core/components/pin.js | 48 ++++- src/core/components/resolve.js | 9 +- src/core/runtime/ipld-nodejs.js | 34 ++-- src/http/api/resources/dag.js | 17 +- src/http/api/resources/object.js | 36 ++-- src/utils/cid.js | 2 +- test/cli/dag.js | 3 +- test/cli/dht.js | 4 +- test/cli/name-pubsub.js | 8 +- test/cli/name.js | 10 +- test/cli/object.js | 24 ++- test/cli/pin.js | 4 +- test/cli/ping.js | 6 +- test/cli/pubsub.js | 4 +- test/cli/resolve.js | 6 +- test/cli/swarm.js | 6 +- test/core/bitswap.spec.js | 4 +- test/core/files.spec.js | 4 +- test/core/kad-dht.node.js | 4 +- test/core/pin-set.js | 27 +-- test/core/ping.spec.js | 4 +- test/core/preload.spec.js | 2 +- test/http-api/block.js | 4 +- test/http-api/bootstrap.js | 4 +- test/http-api/config.js | 2 +- test/http-api/dns.js | 4 +- test/http-api/files.js | 4 +- test/http-api/id.js | 2 +- test/http-api/inject/dag.js | 14 +- test/http-api/interface.js | 3 +- test/http-api/object.js | 4 +- test/http-api/version.js | 4 +- test/utils/ipfs-exec.js | 7 +- test/utils/on-and-off.js | 3 +- 49 files changed, 476 insertions(+), 307 deletions(-) diff --git a/examples/traverse-ipld-graphs/get-path-accross-formats.js b/examples/traverse-ipld-graphs/get-path-accross-formats.js index 984e9f234d..a5fa1f101c 100644 --- a/examples/traverse-ipld-graphs/get-path-accross-formats.js +++ b/examples/traverse-ipld-graphs/get-path-accross-formats.js @@ -17,19 +17,20 @@ createNode((err, ipfs) => { series([ (cb) => { const someData = Buffer.from('capoeira') + let node - dagPB.DAGNode.create(someData, (err, node) => { + try { + node = dagPB.DAGNode.create(someData) + } catch (err) { + return cb(err) + } + + ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => { if (err) { cb(err) } - - ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => { - if (err) { - cb(err) - } - cidPBNode = cid - cb() - }) + cidPBNode = cid + cb() }) }, (cb) => { diff --git a/examples/traverse-ipld-graphs/tree.js b/examples/traverse-ipld-graphs/tree.js index 920f7a2bb3..e4c2da6b34 100644 --- a/examples/traverse-ipld-graphs/tree.js +++ b/examples/traverse-ipld-graphs/tree.js @@ -17,19 +17,20 @@ createNode((err, ipfs) => { series([ (cb) => { const someData = Buffer.from('capoeira') + let node - dagPB.DAGNode.create(someData, (err, node) => { + try { + dagPB.DAGNode.create(someData) + } catch (err) { + return cb(err) + } + + ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => { if (err) { cb(err) } - - ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => { - if (err) { - cb(err) - } - cidPBNode = cid - cb() - }) + cidPBNode = cid + cb() }) }, (cb) => { diff --git a/package.json b/package.json index 49a1dba616..d8cc89e724 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "execa": "^1.0.0", "form-data": "^2.3.3", "hat": "0.0.3", - "interface-ipfs-core": "~0.102.0", + "interface-ipfs-core": "ipfs/interface-js-ipfs-core#update-cids-and-oh-wait", "ipfsd-ctl": "~0.42.0", "libp2p-websocket-star": "~0.10.2", "ncp": "^2.0.0", @@ -97,8 +97,9 @@ "bs58": "^4.0.1", "buffer-peek-stream": "^1.0.1", "byteman": "^1.3.5", - "cid-tool": "~0.2.0", - "cids": "~0.5.8", + "callbackify": "^1.1.0", + "cid-tool": "~0.3.0", + "cids": "~0.7.1", "class-is": "^1.1.0", "datastore-core": "~0.6.0", "datastore-pubsub": "~0.1.1", @@ -112,28 +113,28 @@ "glob": "^7.1.3", "hapi-pino": "^6.0.0", "human-to-milliseconds": "^1.0.0", - "interface-datastore": "~0.7.0", - "ipfs-bitswap": "~0.23.0", - "ipfs-block": "~0.8.0", + "interface-datastore": "~0.6.0", + "ipfs-bitswap": "~0.24.0", + "ipfs-block": "~0.8.1", "ipfs-block-service": "~0.15.1", - "ipfs-http-client": "^31.1.0", - "ipfs-http-response": "~0.2.1", - "ipfs-mfs": "~0.10.2", + "ipfs-http-client": "ipfs/js-ipfs-http-client#update-ipld-formats", + "ipfs-http-response": "ipfs/js-ipfs-http-response#update-ipld-formats", + "ipfs-mfs": "~0.11.0", "ipfs-multipart": "~0.1.0", - "ipfs-repo": "~0.26.5", + "ipfs-repo": "~0.26.6", "ipfs-unixfs": "~0.1.16", - "ipfs-unixfs-exporter": "~0.36.1", - "ipfs-unixfs-importer": "~0.38.5", - "ipld": "~0.22.0", - "ipld-bitcoin": "~0.1.8", - "ipld-dag-cbor": "~0.13.1", - "ipld-dag-pb": "~0.15.3", - "ipld-ethereum": "^2.0.1", - "ipld-git": "~0.3.0", - "ipld-raw": "^2.0.1", - "ipld-zcash": "~0.1.6", + "ipfs-unixfs-exporter": "~0.37.0", + "ipfs-unixfs-importer": "~0.39.4", + "ipld": "~0.24.1", + "ipld-bitcoin": "~0.3.0", + "ipld-dag-cbor": "~0.15.0", + "ipld-dag-pb": "~0.17.0", + "ipld-ethereum": "^4.0.0", + "ipld-git": "~0.5.0", + "ipld-raw": "^4.0.0", + "ipld-zcash": "~0.3.0", "ipns": "~0.5.0", - "is-ipfs": "~0.6.0", + "is-ipfs": "~0.6.1", "is-pull-stream": "~0.0.0", "is-stream": "^2.0.0", "iso-url": "~0.4.6", @@ -144,7 +145,7 @@ "libp2p": "~0.25.0", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.16.0", - "libp2p-kad-dht": "~0.14.12", + "libp2p-kad-dht": "~0.15.0", "libp2p-keychain": "~0.4.1", "libp2p-mdns": "~0.12.0", "libp2p-record": "~0.6.1", @@ -161,7 +162,7 @@ "multiaddr": "^6.0.5", "multiaddr-to-uri": "^4.0.1", "multibase": "~0.6.0", - "multicodec": "~0.5.0", + "multicodec": "~0.5.1", "multihashes": "~0.4.14", "multihashing-async": "~0.6.0", "node-fetch": "^2.3.0", diff --git a/src/cli/bin.js b/src/cli/bin.js index 7c283bff9b..8473c7a9c1 100755 --- a/src/cli/bin.js +++ b/src/cli/bin.js @@ -2,6 +2,18 @@ 'use strict' +process.on('uncaughtException', (err) => { + console.info(err) + + throw err +}) + +process.on('unhandledRejection', (err) => { + console.info(err) + + throw err +}) + const YargsPromise = require('yargs-promise') const updateNotifier = require('update-notifier') const utils = require('./utils') @@ -35,7 +47,7 @@ async function main (args) { } }) .catch(({ error, argv }) => { - getIpfs = argv.getIpfs + getIpfs = argv && argv.getIpfs debug(error) // the argument can have a different shape depending on where the error came from if (error.message || (error.error && error.error.message)) { diff --git a/src/cli/commands/dag/get.js b/src/cli/commands/dag/get.js index 0fb720ca5c..bf9fe04663 100644 --- a/src/cli/commands/dag/get.js +++ b/src/cli/commands/dag/get.js @@ -32,7 +32,7 @@ module.exports = { try { result = await ipfs.dag.get(cid, path, options) } catch (err) { - return print(`dag get failed: ${err.message}`) + return print(`dag get failed: ${err}`) } if (options.localResolve) { diff --git a/src/cli/commands/object/get.js b/src/cli/commands/object/get.js index 6bbaa887e0..ac569ab62f 100644 --- a/src/cli/commands/object/get.js +++ b/src/cli/commands/object/get.js @@ -25,21 +25,21 @@ module.exports = { resolve((async () => { const ipfs = await getIpfs() const node = await ipfs.object.get(key, { enc: 'base58' }) - let data = node.data + let data = node.Data || '' if (Buffer.isBuffer(data)) { - data = node.data.toString(dataEncoding || undefined) + data = node.Data.toString(dataEncoding || undefined) } const answer = { Data: data, Hash: cidToString(key, { base: cidBase, upgrade: false }), - Size: node.size, - Links: node.links.map((l) => { + Size: node.Size, + Links: node.Links.map((l) => { return { - Name: l.name, - Size: l.size, - Hash: cidToString(l.cid, { base: cidBase, upgrade: false }) + Name: l.Name, + Size: l.Tsize, + Hash: cidToString(l.Hash, { base: cidBase, upgrade: false }) } }) } diff --git a/src/cli/commands/object/links.js b/src/cli/commands/object/links.js index feb93596c7..21bb9ca6b8 100644 --- a/src/cli/commands/object/links.js +++ b/src/cli/commands/object/links.js @@ -23,8 +23,8 @@ module.exports = { const links = await ipfs.object.links(key, { enc: 'base58' }) links.forEach((link) => { - const cidStr = cidToString(link.cid, { base: cidBase, upgrade: false }) - print(`${cidStr} ${link.size} ${link.name}`) + const cidStr = cidToString(link.Hash, { base: cidBase, upgrade: false }) + print(`${cidStr} ${link.Tsize} ${link.Name}`) }) })()) } diff --git a/src/cli/commands/object/patch/add-link.js b/src/cli/commands/object/patch/add-link.js index 069060052e..e9ae2d082c 100644 --- a/src/cli/commands/object/patch/add-link.js +++ b/src/cli/commands/object/patch/add-link.js @@ -3,7 +3,6 @@ const dagPB = require('ipld-dag-pb') const DAGLink = dagPB.DAGLink const multibase = require('multibase') -const promisify = require('promisify-es6') const { print } = require('../../../utils') const { cidToString } = require('../../../../utils/cid') @@ -17,14 +16,21 @@ module.exports = { describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.', type: 'string', choices: multibase.names + }, + 'cid-version': { + describe: 'The CID version of the DAGNode to link to', + type: 'number', + default: 0 } }, - handler ({ getIpfs, root, name, ref, cidBase, resolve }) { + handler ({ getIpfs, root, name, ref, cidBase, cidVersion, resolve }) { resolve((async () => { const ipfs = await getIpfs() const nodeA = await ipfs.object.get(ref, { enc: 'base58' }) - const result = await promisify(dagPB.util.cid)(nodeA) + const result = await dagPB.util.cid(dagPB.util.serialize(nodeA), { + cidVersion + }) const link = new DAGLink(name, nodeA.size, result) const cid = await ipfs.object.patch.addLink(root, link, { enc: 'base58' }) print(cidToString(cid, { base: cidBase, upgrade: false })) diff --git a/src/core/components/bitswap.js b/src/core/components/bitswap.js index 13787534aa..c25bebacc3 100644 --- a/src/core/components/bitswap.js +++ b/src/core/components/bitswap.js @@ -9,7 +9,7 @@ const PeerId = require('peer-id') const errCode = require('err-code') function formatWantlist (list, cidBase) { - return Array.from(list).map((e) => ({ '/': e[1].cid.toBaseEncodedString() })) + return Array.from(list).map((e) => ({ '/': e[1].cid.toBaseEncodedString(cidBase) })) } module.exports = function bitswap (self) { diff --git a/src/core/components/dag.js b/src/core/components/dag.js index 2b754a6b27..5e750c35b0 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -45,10 +45,12 @@ module.exports = function dag (self) { // js-ipld defaults to verion 1 CIDs. Hence set version 0 explicitly for // dag-pb nodes - if (options.format === multicodec.DAG_PB && - options.hashAlg === multicodec.SHA2_256 && - options.version === undefined) { - options.version = 0 + if (options.version === undefined) { + if (options.format === multicodec.DAG_PB && options.hashAlg === multicodec.SHA2_256) { + options.version = 0 + } else { + options.version = 1 + } } self._ipld.put(dagNode, options.format, { @@ -125,9 +127,7 @@ module.exports = function dag (self) { ) } else { const result = self._ipld.resolve(cid, path) - const promisedValue = options.localResolve ? - result.first() : - result.last() + const promisedValue = options.localResolve ? result.first() : result.last() promisedValue.then( (value) => callback(null, value), (error) => callback(error) @@ -204,8 +204,8 @@ module.exports = function dag (self) { self.dag.get(cid, '', options, (err, res) => { if (err) { return callback(err) } - mapAsync(res.value.links, (link, cb) => { - self.dag._getRecursive(link.cid, options, cb) + mapAsync(res.value.Links, (link, cb) => { + self.dag._getRecursive(link.Hash, options, cb) }, (err, nodes) => { // console.log('nodes:', nodes) if (err) return callback(err) diff --git a/src/core/components/files-mfs.js b/src/core/components/files-mfs.js index 43e86e6418..3807e8bf16 100644 --- a/src/core/components/files-mfs.js +++ b/src/core/components/files-mfs.js @@ -1,10 +1,12 @@ 'use strict' const mfs = require('ipfs-mfs/core') +const isPullStream = require('is-pull-stream') const toPullStream = require('async-iterator-to-pull-stream') const toReadableStream = require('async-iterator-to-stream') +const pullStreamToAsyncIterator = require('pull-stream-to-async-iterator') const all = require('async-iterator-all') -const callbackify = require('util').callbackify +const callbackify = require('callbackify') const PassThrough = require('stream').PassThrough const pull = require('pull-stream/pull') const map = require('pull-stream/throughs/map') @@ -13,8 +15,6 @@ const mapLsFile = (options = {}) => { const long = options.long || options.l return (file) => { - console.info(file) - return { hash: long ? file.cid.toBaseEncodedString(options.cidBase) : '', name: file.name, @@ -33,43 +33,66 @@ module.exports = self => { }) return { - cp: callbackify(methods.cp), - flush: callbackify(methods.flush), - ls: callbackify(async (path, options) => { + cp: callbackify.variadic(methods.cp), + flush: callbackify.variadic(methods.flush), + ls: callbackify.variadic(async (path, options = {}) => { const files = await all(methods.ls(path, options)) return files.map(mapLsFile(options)) }), - lsReadableStream: (path, options) => { + lsReadableStream: (path, options = {}) => { const stream = toReadableStream.obj(methods.ls(path, options)) const through = new PassThrough({ objectMode: true }) - stream.on('data', (file) => through.emit('data', mapLsFile(options)(file))) - stream.on('error', through.emit.bind(through, 'error')) - stream.on('end', through.emit.bind(through, 'end')) + stream.on('data', (file) => { + through.write(mapLsFile(options)(file)) + }) + stream.on('error', (err) => { + through.destroy(err) + }) + stream.on('end', (file, enc, cb) => { + if (file) { + file = mapLsFile(options)(file) + } + + through.end(file, enc, cb) + }) return through }, - lsPullStream: (path, options) => { + lsPullStream: (path, options = {}) => { return pull( toPullStream.source(methods.ls(path, options)), map(mapLsFile(options)) ) }, - mkdir: callbackify(methods.mkdir), - mv: callbackify(methods.mv), - read: callbackify(async (path, options) => { + mkdir: callbackify.variadic(methods.mkdir), + mv: callbackify.variadic(methods.mv), + read: callbackify(async (path, options = {}) => { return Buffer.concat(await all(methods.read(path, options))) }), - readPullStream: (path, options) => { - return toReadableStream(methods.read(path, options)) - }, - readReadableStream: (path, options) => { + readPullStream: (path, options = {}) => { return toPullStream.source(methods.read(path, options)) }, - rm: callbackify(methods.rm), - stat: callbackify(methods.stat), - write: callbackify(methods.write) + readReadableStream: (path, options = {}) => { + return toReadableStream(methods.read(path, options)) + }, + rm: callbackify.variadic(methods.rm), + stat: callbackify(async (path, options = {}) => { + const stats = await methods.stat(path, options) + + stats.hash = stats.cid.toBaseEncodedString(options && options.cidBase) + delete stats.cid + + return stats + }), + write: callbackify.variadic(async (path, content, options = {}) => { + if (isPullStream.isSource(content)) { + content = pullStreamToAsyncIterator(content) + } + + await methods.write(path, content, options) + }) } -} \ No newline at end of file +} diff --git a/src/core/components/files-regular/add-pull-stream.js b/src/core/components/files-regular/add-pull-stream.js index fee02dccd4..cd89230e75 100644 --- a/src/core/components/files-regular/add-pull-stream.js +++ b/src/core/components/files-regular/add-pull-stream.js @@ -26,8 +26,6 @@ const { parseChunkerString } = require('./utils') const streamFromFileReader = require('ipfs-utils/src/streams/stream-from-filereader') const { supportsFileReader } = require('ipfs-utils/src/supports') -const WRAPPER = 'wrapper/' - function noop () {} function prepareFile (file, self, opts, callback) { @@ -98,10 +96,6 @@ function normalizeContent (content, opts) { throw new Error('Must provide a path when wrapping with a directory') } - //if (opts.wrapWithDirectory) { - // data.path = WRAPPER + data.path - //} - return data }) } @@ -149,8 +143,8 @@ module.exports = function (self) { ? 1000 : Infinity }, options, { - ...chunkerOptions.chunkerOptions, - chunker: chunkerOptions.chunker + chunker: chunkerOptions.chunker, + chunkerOptions: chunkerOptions.chunkerOptions }) // CID v0 is for multihashes encoded with sha2-256 diff --git a/src/core/components/init.js b/src/core/components/init.js index 7953e51eb9..5b1a1b7cbe 100644 --- a/src/core/components/init.js +++ b/src/core/components/init.js @@ -130,7 +130,13 @@ module.exports = function init (self) { const tasks = [ (cb) => { waterfall([ - (cb) => DAGNode.create(new UnixFs('directory').marshal(), cb), + (cb) => { + try { + cb(null, DAGNode.create(new UnixFs('directory').marshal())) + } catch (err) { + cb(err) + } + }, (node, cb) => self.dag.put(node, { version: 0, format: multicodec.DAG_PB, diff --git a/src/core/components/object.js b/src/core/components/object.js index d33f1ffa09..a2ae7329fb 100644 --- a/src/core/components/object.js +++ b/src/core/components/object.js @@ -59,13 +59,48 @@ function parseJSONBuffer (buf, callback) { return callback(new Error('failed to parse JSON: ' + err)) } - DAGNode.create(data, links, callback) + try { + callback(null, DAGNode.create(data, links)) + } catch (err) { + callback(err) + } } function parseProtoBuffer (buf, callback) { dagPB.util.deserialize(buf, callback) } +function findLinks (node, links = []) { + for (let key in node) { + const val = node[key] + + if (key === '/' && Object.keys(node).length === 1) { + try { + links.push(new DAGLink('', 0, new CID(val))) + continue + } catch (_) { + // not a CID + } + } + + if (CID.isCID(val)) { + links.push(new DAGLink('', 0, val)) + + continue + } + + if (Array.isArray(val)) { + findLinks(val, links) + } + + if (typeof val === 'object' && !(val instanceof String)) { + findLinks(val, links) + } + } + + return links +} + module.exports = function object (self) { function editAndSave (edit) { return (multihash, options, callback) => { @@ -90,7 +125,7 @@ module.exports = function object (self) { self._ipld.put(node, multicodec.DAG_PB, { cidVersion: 0, - hashAlg: multicodec.SHA2_256, + hashAlg: multicodec.SHA2_256 }).then( (cid) => { if (options.preload !== false) { @@ -133,25 +168,27 @@ module.exports = function object (self) { data = Buffer.alloc(0) } - DAGNode.create(data, (err, node) => { - if (err) { - return callback(err) - } + let node - self._ipld.put(node, multicodec.DAG_PB, { - cidVersion: 0, - hashAlg: multicodec.SHA2_256, - }).then( - (cid) => { - if (options.preload !== false) { - self._preload(cid) - } + try { + node = DAGNode.create(data) + } catch (err) { + return callback(err) + } - callback(null, cid) - }, - (error) => callback(error) - ) - }) + self._ipld.put(node, multicodec.DAG_PB, { + cidVersion: 0, + hashAlg: multicodec.SHA2_256 + }).then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } + + callback(null, cid) + }, + (error) => callback(error) + ) }), put: promisify((obj, options, callback) => { if (typeof options === 'function') { @@ -174,26 +211,26 @@ module.exports = function object (self) { next() }) } else { - DAGNode.create(obj, (err, _node) => { - if (err) { - return callback(err) - } - node = _node - next() - }) + try { + node = DAGNode.create(obj) + } catch (err) { + return callback(err) + } + + next() } } else if (DAGNode.isDAGNode(obj)) { // already a dag node node = obj next() } else if (typeof obj === 'object') { - DAGNode.create(obj.Data, obj.Links, (err, _node) => { - if (err) { - return callback(err) - } - node = _node - next() - }) + try { + node = DAGNode.create(obj.Data, obj.Links) + } catch (err) { + return callback(err) + } + + next() } else { return callback(new Error('obj not recognized')) } @@ -201,7 +238,7 @@ module.exports = function object (self) { function next () { self._ipld.put(node, multicodec.DAG_PB, { cidVersion: 0, - hashAlg: multicodec.SHA2_256, + hashAlg: multicodec.SHA2_256 }).then( (cid) => { if (options.preload !== false) { @@ -262,7 +299,7 @@ module.exports = function object (self) { return callback(err) } - callback(null, node.data) + callback(null, node.Data) }) }), @@ -272,12 +309,28 @@ module.exports = function object (self) { options = {} } - self.object.get(multihash, options, (err, node) => { + const cid = new CID(multihash) + + self.dag.get(cid, options, (err, result) => { if (err) { return callback(err) } - callback(null, node.links) + if (cid.codec === 'raw') { + return callback(null, []) + } + + if (cid.codec === 'dag-pb') { + return callback(null, result.value.Links) + } + + if (cid.codec === 'dag-cbor') { + const links = findLinks(result) + + callback(null, links) + } + + callback(new Error(`Cannot resolve links from codec ${cid.codec}`)) }) }), @@ -292,9 +345,17 @@ module.exports = function object (self) { waterfall([ (cb) => self.object.get(multihash, options, cb), (node, cb) => { + cb(null, { + node, + serialized: dagPB.util.serialize(node) + }) + }, + ({ node, serialized }, cb) => { parallel({ - serialized: (next) => dagPB.util.serialize(node, next), - cid: (next) => dagPB.util.cid(node, next), + serialized: (next) => next(null, serialized), + cid: (next) => dagPB.util.cid(serialized, { + cidVersion: 0 + }).then((cid) => next(null, cid), next), node: (next) => next(null, node) }, cb) } @@ -304,14 +365,14 @@ module.exports = function object (self) { } const blockSize = result.serialized.length - const linkLength = result.node.links.reduce((a, l) => a + l.size, 0) + const linkLength = result.node.Links.reduce((a, l) => a + l.Tsize, 0) callback(null, { Hash: result.cid.toBaseEncodedString(), - NumLinks: result.node.links.length, + NumLinks: result.node.Links.length, BlockSize: blockSize, - LinksSize: blockSize - result.node.data.length, - DataSize: result.node.data.length, + LinksSize: blockSize - result.node.Data.length, + DataSize: result.node.Data.length, CumulativeSize: blockSize + linkLength }) }) @@ -320,31 +381,49 @@ module.exports = function object (self) { patch: promisify({ addLink (multihash, link, options, callback) { editAndSave((node, cb) => { - DAGNode.addLink(node, link, cb) + DAGNode.addLink(node, link).then((node) => { + cb(null, node) + }, cb) })(multihash, options, callback) }, rmLink (multihash, linkRef, options, callback) { editAndSave((node, cb) => { - if (DAGLink.isDAGLink(linkRef)) { - linkRef = linkRef._name - } else if (linkRef && linkRef.name) { - linkRef = linkRef.name + linkRef = linkRef.Name || linkRef.name + + try { + node = DAGNode.rmLink(node, linkRef) + } catch (err) { + return cb(err) } - DAGNode.rmLink(node, linkRef, cb) + + cb(null, node) })(multihash, options, callback) }, appendData (multihash, data, options, callback) { editAndSave((node, cb) => { - const newData = Buffer.concat([node.data, data]) - DAGNode.create(newData, node.links, cb) + const newData = Buffer.concat([node.Data, data]) + + try { + node = DAGNode.create(newData, node.Links) + } catch (err) { + return cb(err) + } + + cb(null, node) })(multihash, options, callback) }, setData (multihash, data, options, callback) { editAndSave((node, cb) => { - DAGNode.create(data, node.links, cb) + try { + node = DAGNode.create(data, node.Links) + } catch (err) { + return cb(err) + } + + cb(null, node) })(multihash, options, callback) } }) diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js index c178801c61..a4ce0aa510 100644 --- a/src/core/components/pin-set.js +++ b/src/core/components/pin-set.js @@ -25,7 +25,7 @@ function toB58String (hash) { function readHeader (rootNode) { // rootNode.data should be a buffer of the format: // < varint(headerLength) | header | itemData... > - const rootData = rootNode.data + const rootData = rootNode.Data const hdrLength = varint.decode(rootData) const vBytes = varint.decode.bytes if (vBytes <= 0) { @@ -39,7 +39,7 @@ function readHeader (rootNode) { if (header.version !== 1) { throw new Error(`Unsupported Set version: ${header.version}`) } - if (header.fanout > rootNode.links.length) { + if (header.fanout > rootNode.Links.length) { throw new Error('Impossibly large fanout') } return { @@ -147,17 +147,22 @@ exports = module.exports = function (dag) { }) }) // sorting makes any ordering of `pins` produce the same DAGNode - .sort((a, b) => Buffer.compare(a.link.cid.buffer, b.link.cid.buffer)) + .sort((a, b) => Buffer.compare(a.link.Hash.buffer, b.link.Hash.buffer)) const rootLinks = fanoutLinks.concat(nodes.map(item => item.link)) const rootData = Buffer.concat( [headerBuf].concat(nodes.map(item => item.data)) ) - DAGNode.create(rootData, rootLinks, (err, rootNode) => { - if (err) { return storePinsCb(err) } - return storePinsCb(null, rootNode) - }) + let rootNode + + try { + rootNode = DAGNode.create(rootData, rootLinks) + } catch (err) { + return storePinsCb(err) + } + + return storePinsCb(null, rootNode) } else { // If the array of pins is > maxItems, we: // - distribute the pins among `defaultFanout` bins @@ -183,10 +188,16 @@ exports = module.exports = function (dag) { ) }, err => { if (err) { return storePinsCb(err) } - DAGNode.create(headerBuf, fanoutLinks, (err, rootNode) => { - if (err) { return storePinsCb(err) } - return storePinsCb(null, rootNode) - }) + + let rootNode + + try { + rootNode = DAGNode.create(headerBuf, fanoutLinks) + } catch (err) { + return storePinsCb(err) + } + + return storePinsCb(null, rootNode) }) } @@ -210,15 +221,15 @@ exports = module.exports = function (dag) { }, loadSet: (rootNode, name, callback) => { - const link = rootNode.links.find(l => l.name === name) + const link = rootNode.Links.find(l => l.Name === name) if (!link) { return callback(new Error('No link found with name ' + name)) } - dag.get(link.cid, '', { preload: false }, (err, res) => { + dag.get(link.Hash, '', { preload: false }, (err, res) => { if (err) { return callback(err) } const keys = [] - const step = link => keys.push(link.cid.buffer) + const step = link => keys.push(link.Hash.buffer) pinSet.walkItems(res.value, step, err => { if (err) { return callback(err) } return callback(null, keys) @@ -234,11 +245,11 @@ exports = module.exports = function (dag) { return callback(err) } - eachOfSeries(node.links, (link, idx, eachCb) => { + eachOfSeries(node.Links, (link, idx, eachCb) => { if (idx < pbh.header.fanout) { // the first pbh.header.fanout links are fanout bins // if a fanout bin is not 'empty', dig into and walk its DAGLinks - const linkHash = link.cid.buffer + const linkHash = link.Hash.buffer if (!emptyKey.equals(linkHash)) { // walk the links of this fanout bin diff --git a/src/core/components/pin.js b/src/core/components/pin.js index b6b1b53439..876a680daa 100644 --- a/src/core/components/pin.js +++ b/src/core/components/pin.js @@ -60,13 +60,15 @@ module.exports = (self) => { return cb(err) } - map(nodes, (node, cb) => util.cid(node, cb), (err, cids) => { + map(nodes, (node, cb) => util.cid(util.serialize(node), { + cidVersion: 0 + }).then(cid => cb(null, cid), cb), (err, cids) => { if (err) { return cb(err) } cids - .map(cids => cids.toBaseEncodedString()) + .map(cids => cids.toBaseEncodedString('base58btc')) // recursive pins pre-empt indirect pins .filter(key => !recursivePins.has(key)) .forEach(key => indirectKeys.add(key)) @@ -89,31 +91,57 @@ module.exports = (self) => { // create a DAGLink to the node with direct pins cb => waterfall([ cb => pinset.storeSet(directKeys(), cb), - ({ node, cid }, cb) => DAGLink.create(types.direct, node.size, cid, cb), + ({ node, cid }, cb) => { + try { + cb(null, new DAGLink(types.direct, node.size, cid)) + } catch (err) { + cb(err) + } + }, (link, cb) => { dLink = link; cb(null) } ], cb), // create a DAGLink to the node with recursive pins cb => waterfall([ cb => pinset.storeSet(recursiveKeys(), cb), - ({ node, cid }, cb) => DAGLink.create(types.recursive, node.size, cid, cb), + ({ node, cid }, cb) => { + try { + cb(null, new DAGLink(types.recursive, node.size, cid)) + } catch (err) { + cb(err) + } + }, (link, cb) => { rLink = link; cb(null) } ], cb), // the pin-set nodes link to a special 'empty' node, so make sure it exists - cb => DAGNode.create(Buffer.alloc(0), (err, empty) => { - if (err) { return cb(err) } + cb => { + let empty + + try { + empty = DAGNode.create(Buffer.alloc(0)) + } catch (err) { + return cb(err) + } + dag.put(empty, { version: 0, format: multicodec.DAG_PB, hashAlg: multicodec.SHA2_256, preload: false }, cb) - }), + }, // create a root node with DAGLinks to the direct and recursive DAGs - cb => DAGNode.create(Buffer.alloc(0), [dLink, rLink], (err, node) => { - if (err) { return cb(err) } + cb => { + let node + + try { + node = DAGNode.create(Buffer.alloc(0), [dLink, rLink]) + } catch (err) { + return cb(err) + } + root = node dag.put(root, { version: 0, @@ -126,7 +154,7 @@ module.exports = (self) => { } cb(err) }) - }), + }, // hack for CLI tests cb => repo.closed ? repo.open(cb) : cb(null, null), diff --git a/src/core/components/resolve.js b/src/core/components/resolve.js index f863e75d1c..db9039551e 100644 --- a/src/core/components/resolve.js +++ b/src/core/components/resolve.js @@ -3,7 +3,6 @@ const promisify = require('promisify-es6') const isIpfs = require('is-ipfs') const setImmediate = require('async/setImmediate') -const doUntil = require('async/doUntil') const CID = require('cids') const { cidToString } = require('../../utils/cid') @@ -17,7 +16,7 @@ module.exports = (self) => { opts = opts || {} if (!isIpfs.path(name)) { - return setImmediate(() => cb(new Error('invalid argument'))) + return setImmediate(() => cb(new Error('invalid argument ' + name))) } // TODO remove this and update subsequent code when IPNS is implemented @@ -45,6 +44,12 @@ module.exports = (self) => { value = result.value remainderPath = '' } + + if (result.value && CID.isCID(result.value.Hash)) { + value = result.value.Hash + remainderPath = '' + } + break } diff --git a/src/core/runtime/ipld-nodejs.js b/src/core/runtime/ipld-nodejs.js index aa2172fe82..0a26ac1c75 100644 --- a/src/core/runtime/ipld-nodejs.js +++ b/src/core/runtime/ipld-nodejs.js @@ -1,39 +1,37 @@ 'use strict' const mergeOptions = require('merge-options') -const ipldDagCbor = require('ipld-dag-cbor') -const ipldDagPb = require('ipld-dag-pb') -const ipldRaw = require('ipld-raw') +const multicodec = require('multicodec') // All known (non-default) IPLD formats const IpldFormats = { - get 'bitcoin-block' () { + get [multicodec.BITCOIN_BLOCK] () { return require('ipld-bitcoin') }, - get 'eth-account-snapshot' () { + get [multicodec.ETH_ACCOUNT_SNAPSHOT] () { return require('ipld-ethereum').ethAccountSnapshot }, - get 'eth-block' () { + get [multicodec.ETH_BLOCK] () { return require('ipld-ethereum').ethBlock }, - get 'eth-block-list' () { + get [multicodec.ETH_BLOCK_LIST] () { return require('ipld-ethereum').ethBlockList }, - get 'eth-state-trie' () { + get [multicodec.ETH_STATE_TRIE] () { return require('ipld-ethereum').ethStateTrie }, - get 'eth-storage-trie' () { + get [multicodec.ETH_STORAGE_TRIE] () { return require('ipld-ethereum').ethStorageTrie }, - get 'eth-tx' () { + get [multicodec.ETH_TX] () { return require('ipld-ethereum').ethTx }, - get 'eth-tx-trie' () { + get [multicodec.ETH_TX_TRIE] () { return require('ipld-ethereum').ethTxTrie }, - get 'git-raw' () { + get [multicodec.GIT_RAW] () { return require('ipld-git') }, - get 'zcash-block' () { + get [multicodec.ZCASH_BLOCK] () { return require('ipld-zcash') } } @@ -44,11 +42,13 @@ module.exports = (blockService, options = {}, log) => { { concatArrays: true }, { blockService: blockService, - formats: [ipldDagCbor, ipldDagPb, ipldRaw], - loadFormat: (codec, callback) => { + loadFormat: (codec) => { log('Loading IPLD format', codec) - if (IpldFormats[codec]) return callback(null, IpldFormats[codec]) - callback(new Error(`Missing IPLD format "${codec}"`)) + if (IpldFormats[codec]) { + return IpldFormats[codec] + } else { + throw new Error(`Missing IPLD format "${codec}"`) + } } }, options) } diff --git a/src/http/api/resources/dag.js b/src/http/api/resources/dag.js index 71a3f04340..1084113cf3 100644 --- a/src/http/api/resources/dag.js +++ b/src/http/api/resources/dag.js @@ -1,6 +1,5 @@ 'use strict' -const promisify = require('promisify-es6') const CID = require('cids') const multipart = require('ipfs-multipart') const mh = require('multihashes') @@ -110,8 +109,14 @@ exports.get = { throw Boom.badRequest(err) } + let value = result.value + + if (!Buffer.isBuffer(result.value) && result.value.toJSON) { + value = result.value.toJSON() + } + try { - result.value = encodeBufferKeys(result.value, dataEncoding) + result.value = encodeBufferKeys(value, dataEncoding) } catch (err) { throw Boom.boomify(err) } @@ -177,12 +182,8 @@ exports.put = { // IPLD expects the format and hashAlg as constants const codecConstant = format.toUpperCase().replace(/-/g, '_') - node = await ipfs._ipld._deserialize({ - cid: { - codec: multicodec[codecConstant] - }, - data: data - }) + const ipldFormat = await ipfs._ipld._getFormat(multicodec[codecConstant]) + node = await ipldFormat.util.deserialize(data) } return { diff --git a/src/http/api/resources/object.js b/src/http/api/resources/object.js index 6480141510..bb55ac6938 100644 --- a/src/http/api/resources/object.js +++ b/src/http/api/resources/object.js @@ -1,13 +1,9 @@ 'use strict' -const promisify = require('promisify-es6') const CID = require('cids') const multipart = require('ipfs-multipart') const dagPB = require('ipld-dag-pb') const { DAGNode, DAGLink } = dagPB -const calculateCid = promisify(dagPB.util.cid) -const deserialize = promisify(dagPB.util.deserialize) -const createDagNode = promisify(DAGNode.create) const Joi = require('@hapi/joi') const multibase = require('multibase') const Boom = require('boom') @@ -87,7 +83,7 @@ exports.get = { let node, cid try { node = await ipfs.object.get(key, { enc: enc }) - cid = await calculateCid(node) + cid = await dagPB.util.cid(dagPB.util.serialize(node)) } catch (err) { throw Boom.boomify(err, { message: 'Failed to get object' }) } @@ -145,7 +141,7 @@ exports.put = { if (enc === 'protobuf') { try { - return { node: await deserialize(data) } + return { node: await dagPB.util.deserialize(data) } } catch (err) { throw Boom.badRequest('Failed to deserialize: ' + err) } @@ -159,7 +155,7 @@ exports.put = { } try { - return { node: await createDagNode(nodeJson.Data, nodeJson.Links) } + return { node: DAGNode.create(nodeJson.Data, nodeJson.Links) } } catch (err) { throw Boom.badRequest('Failed to create DAG node: ' + err) } @@ -258,26 +254,22 @@ exports.links = { async handler (request, h) { const { ipfs } = request.server.app const { key } = request.pre.args - - let node - try { - node = await ipfs.object.get(key) - } catch (err) { - throw Boom.boomify(err, { message: 'Failed to get object links' }) + const response = { + Hash: cidToString(key, { base: request.query['cid-base'], upgrade: false }) } + const links = await ipfs.object.links(key) - const nodeJSON = node.toJSON() - - return h.response({ - Hash: cidToString(key, { base: request.query['cid-base'], upgrade: false }), - Links: nodeJSON.links.map((l) => { + if (links) { + response.Links = links.map((l) => { return { - Name: l.name, - Size: l.size, - Hash: cidToString(l.cid, { base: request.query['cid-base'], upgrade: false }) + Name: l.Name, + Size: l.Tsize, + Hash: cidToString(l.Hash, { base: request.query['cid-base'], upgrade: false }) } }) - }) + } + + return h.response(response) } } diff --git a/src/utils/cid.js b/src/utils/cid.js index adf16963d0..2c4cb4b23f 100644 --- a/src/utils/cid.js +++ b/src/utils/cid.js @@ -19,7 +19,7 @@ const CID = require('cids') */ exports.cidToString = (cid, options) => { options = options || {} - options.base = options.base || null + options.base = options.base || 'base58btc' options.upgrade = options.upgrade !== false if (!CID.isCID(cid)) { diff --git a/test/cli/dag.js b/test/cli/dag.js index 9cd6a3a586..9e572f7ab1 100644 --- a/test/cli/dag.js +++ b/test/cli/dag.js @@ -3,6 +3,7 @@ const expect = require('chai').expect const runOnAndOff = require('../utils/on-and-off') +const path = require('path') describe('dag', () => runOnAndOff.off((thing) => { let ipfs @@ -15,7 +16,7 @@ describe('dag', () => runOnAndOff.off((thing) => { this.timeout(20 * 1000) // put test eth-block - return ipfs('block put --format eth-block --mhtype keccak-256 test/fixtures/test-data/eth-block').then((out) => { + return ipfs(`block put --format eth-block --mhtype keccak-256 ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/eth-block`).then((out) => { expect(out).to.eql('z43AaGF23fmvRnDP56Ub9WcJCfzSfqtmzNCCvmz5eudT8dtdCDS\n') // lookup path on eth-block return ipfs('dag get z43AaGF23fmvRnDP56Ub9WcJCfzSfqtmzNCCvmz5eudT8dtdCDS/parentHash') diff --git a/test/cli/dht.js b/test/cli/dht.js index 45f13cadc3..f772ebc6af 100644 --- a/test/cli/dht.js +++ b/test/cli/dht.js @@ -9,14 +9,14 @@ chai.use(dirtyChai) const series = require('async/series') const parallel = require('async/parallel') - +const path = require('path') const DaemonFactory = require('ipfsd-ctl') const df = DaemonFactory.create({ type: 'js' }) const ipfsExec = require('../utils/ipfs-exec') const daemonOpts = { - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config: { Bootstrap: [], Discovery: { diff --git a/test/cli/name-pubsub.js b/test/cli/name-pubsub.js index fd125948a5..fef64295de 100644 --- a/test/cli/name-pubsub.js +++ b/test/cli/name-pubsub.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const parallel = require('async/parallel') const series = require('async/series') const ipfsExec = require('../utils/ipfs-exec') @@ -19,7 +19,7 @@ const emptyDirCid = 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' const spawnDaemon = (callback) => { df.spawn({ - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), args: ['--enable-namesys-pubsub'], initOptions: { bits: 512 }, config: { @@ -171,7 +171,7 @@ describe('name-pubsub', () => { before(function (done) { this.timeout(50 * 1000) - ipfsA('add src/init-files/init-docs/readme') + ipfsA(`add ${path.resolve(`${__dirname}/../../src/init-files/init-docs/readme`)}`) .then((out) => { cidAdded = out.split(' ')[1] done() @@ -225,7 +225,7 @@ describe('name-pubsub', () => { this.timeout(80 * 1000) df.spawn({ - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config: {}, initOptions: { bits: 512 } }, (err, node) => { diff --git a/test/cli/name.js b/test/cli/name.js index c34ae70ef4..1c0638503f 100644 --- a/test/cli/name.js +++ b/test/cli/name.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const hat = require('hat') const ipfsExec = require('../utils/ipfs-exec') @@ -32,7 +32,7 @@ describe('name', () => { this.timeout(80 * 1000) df.spawn({ - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config: { Bootstrap: [] }, @@ -56,7 +56,7 @@ describe('name', () => { expect(id).to.have.property('id') nodeId = id.id - return ipfs('add src/init-files/init-docs/readme') + return ipfs(`add ${path.resolve(`${__dirname}/../../src/init-files/init-docs/readme`)}`) }) .then((out) => { cidAdded = out.split(' ')[1] @@ -197,7 +197,7 @@ describe('name', () => { this.timeout(80 * 1000) df.spawn({ - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config: { Bootstrap: [], Discovery: { @@ -229,7 +229,7 @@ describe('name', () => { expect(id).to.have.property('id') nodeId = id.id - return ipfs('add src/init-files/init-docs/readme') + return ipfs(`add ${path.resolve(`${__dirname}/../../src/init-files/init-docs/readme`)}`) }) .then((out) => { cidAdded = out.split(' ')[1] diff --git a/test/cli/object.js b/test/cli/object.js index ceb9f24627..efaaf6cc5e 100644 --- a/test/cli/object.js +++ b/test/cli/object.js @@ -56,7 +56,7 @@ describe('object', () => runOnAndOff((thing) => { return ipfs('object new') .then((out) => out.trim()) - .then((hash) => ipfs(`object patch set-data ${hash} test/fixtures/test-data/hello`)) + .then((hash) => ipfs(`object patch set-data ${hash} ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/hello`)) .then((out) => out.trim()) .then((hash) => ipfs(`object get ${hash}`)) .then((out) => { @@ -70,7 +70,7 @@ describe('object', () => runOnAndOff((thing) => { return ipfs('object new') .then((out) => out.trim()) - .then((hash) => ipfs(`object patch set-data ${hash} test/fixtures/test-data/hello`)) + .then((hash) => ipfs(`object patch set-data ${hash} ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/hello`)) .then((out) => out.trim()) .then((hash) => ipfs(`object get --data-encoding=utf8 ${hash}`)) .then((out) => { @@ -80,7 +80,7 @@ describe('object', () => runOnAndOff((thing) => { }) it('should get and print CIDs encoded in specified base', () => { - return ipfs('add test/fixtures/planets -r --cid-version=1') + return ipfs(`add ${path.resolve(path.join(__dirname, '..'))}/fixtures/planets -r --cid-version=1`) .then(out => { const lines = out.trim().split('\n') return lines[lines.length - 1].split(' ')[1] @@ -96,7 +96,7 @@ describe('object', () => runOnAndOff((thing) => { }) it('put', () => { - return ipfs('object put test/fixtures/test-data/node.json').then((out) => { + return ipfs(`object put ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/node.json`).then((out) => { expect(out).to.eql( 'added QmZZmY4KCu9r3e7M2Pcn46Fc5qbn6NpzaAGaYb22kbfTqm\n' ) @@ -105,7 +105,7 @@ describe('object', () => runOnAndOff((thing) => { // TODO: unskip after switch to v1 CIDs by default it.skip('should put and print CID encoded in specified base', () => { - return ipfs('object put test/fixtures/test-data/node.json --cid-base=base64') + return ipfs(`object put ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/node.json --cid-base=base64`) .then((out) => { expect(out).to.eql( 'added mAXASIKbM02Neyt6L1RRLYVEOuNlqDOzTvBboo3cI/u6f/+Vk\n' @@ -163,7 +163,7 @@ describe('object', () => runOnAndOff((thing) => { }) it('should get links and print CIDs encoded in specified base', () => { - return ipfs('add test/fixtures/planets -r --cid-version=1') + return ipfs(`add ${path.resolve(path.join(__dirname, '..'))}/fixtures/planets -r --cid-version=1`) .then(out => { const lines = out.trim().split('\n') return lines[lines.length - 1].split(' ')[1] @@ -180,8 +180,12 @@ describe('object', () => runOnAndOff((thing) => { describe('patch', function () { this.timeout(40 * 1000) + before(async () => { + await ipfs('object new') + }) + it('append-data', () => { - return ipfs('object patch append-data QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n test/fixtures/test-data/badconfig').then((out) => { + return ipfs(`object patch append-data QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/badconfig`).then((out) => { expect(out).to.eql( 'QmfY37rjbPCZRnhvvJuQ46htW3VCAWziVB991P79h6WSv6\n' ) @@ -190,7 +194,7 @@ describe('object', () => runOnAndOff((thing) => { // TODO: unskip after switch to v1 CIDs by default it.skip('should append-data and print CID encoded in specified base', () => { - return ipfs('object patch append-data QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n test/fixtures/test-data/badconfig --cid-base=base64').then((out) => { + return ipfs(`object patch append-data QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/badconfig --cid-base=base64`).then((out) => { expect(out).to.eql( 'mAXASIP+BZ7jGtaTyLGOs0xYcQvH7K9kVKEbyzXAkwLoZwrRj\n' ) @@ -198,7 +202,7 @@ describe('object', () => runOnAndOff((thing) => { }) it('set-data', () => { - return ipfs('object patch set-data QmfY37rjbPCZRnhvvJuQ46htW3VCAWziVB991P79h6WSv6 test/fixtures/test-data/badconfig').then((out) => { + return ipfs(`object patch set-data QmfY37rjbPCZRnhvvJuQ46htW3VCAWziVB991P79h6WSv6 ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/badconfig`).then((out) => { expect(out).to.eql( 'QmfY37rjbPCZRnhvvJuQ46htW3VCAWziVB991P79h6WSv6\n' ) @@ -207,7 +211,7 @@ describe('object', () => runOnAndOff((thing) => { // TODO: unskip after switch to v1 CIDs by default it.skip('should set-data and print CID encoded in specified base', () => { - return ipfs('object patch set-data QmfY37rjbPCZRnhvvJuQ46htW3VCAWziVB991P79h6WSv6 test/fixtures/test-data/badconfig --cid-base=base64').then((out) => { + return ipfs(`object patch set-data QmfY37rjbPCZRnhvvJuQ46htW3VCAWziVB991P79h6WSv6 ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/badconfig --cid-base=base64`).then((out) => { expect(out).to.eql( 'mAXASIP+BZ7jGtaTyLGOs0xYcQvH7K9kVKEbyzXAkwLoZwrRj\n' ) diff --git a/test/cli/pin.js b/test/cli/pin.js index 5313e3dbd3..15458c12cb 100644 --- a/test/cli/pin.js +++ b/test/cli/pin.js @@ -5,13 +5,15 @@ const expect = require('chai').expect const CID = require('cids') const runOnAndOff = require('../utils/on-and-off') +const path = require('path') // fixture structure: // planets/ // solar-system.md // mercury/ // wiki.md -const fixturePath = 'test/fixtures/planets' +const fixturePath = path.resolve(path.join(__dirname, '..'), 'fixtures/planets') + const pins = { root: 'QmTAMavb995EHErSrKo7mB8dYkpaSJxu6ys1a6XJyB2sys', solarWiki: 'QmTMbkDfvHwq3Aup6Nxqn3KKw9YnoKzcZvuArAfQ9GF3QG', diff --git a/test/cli/ping.js b/test/cli/ping.js index 170a1fc3b9..05d9b709ab 100644 --- a/test/cli/ping.js +++ b/test/cli/ping.js @@ -7,7 +7,7 @@ const dirtyChai = require('dirty-chai') const series = require('async/series') const DaemonFactory = require('ipfsd-ctl') const ipfsExec = require('../utils/ipfs-exec') - +const path = require('path') const df = DaemonFactory.create({ type: 'js' }) const expect = chai.expect chai.use(dirtyChai) @@ -35,7 +35,7 @@ describe('ping', function () { series([ (cb) => { df.spawn({ - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config, initOptions: { bits: 512 } }, (err, _ipfsd) => { @@ -59,7 +59,7 @@ describe('ping', function () { this.timeout(60 * 1000) df.spawn({ - exec: './src/cli/bin.js', + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config, initoptions: { bits: 512 } }, (err, _ipfsd) => { diff --git a/test/cli/pubsub.js b/test/cli/pubsub.js index b80eed7f09..fcc231331c 100644 --- a/test/cli/pubsub.js +++ b/test/cli/pubsub.js @@ -10,7 +10,7 @@ const delay = require('delay') const series = require('async/series') const ipfsExec = require('../utils/ipfs-exec') const IPFS = require('../../src') - +const path = require('path') const DaemonFactory = require('ipfsd-ctl') const df = DaemonFactory.create({ type: 'js' }) @@ -61,7 +61,7 @@ describe('pubsub', function () { df.spawn({ initOptions: { bits: 512 }, args: ['--enable-pubsub-experiment'], - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config }, (err, _ipfsd) => { expect(err).to.not.exist() diff --git a/test/cli/resolve.js b/test/cli/resolve.js index 3aa7bcb70c..190eba0c66 100644 --- a/test/cli/resolve.js +++ b/test/cli/resolve.js @@ -16,7 +16,7 @@ describe('resolve', () => runOnAndOff((thing) => { }) it('should resolve an IPFS hash', () => { - const filePath = path.join(process.cwd(), '/src/init-files/init-docs/readme') + const filePath = path.join(path.resolve(__dirname, '..', '..'), 'src/init-files/init-docs/readme') let hash return ipfs(`add ${filePath}`) @@ -33,7 +33,7 @@ describe('resolve', () => runOnAndOff((thing) => { it('should resolve an IPFS hash and print CID encoded in specified base', function () { this.timeout(10 * 1000) - const filePath = path.join(process.cwd(), '/src/init-files/init-docs/readme') + const filePath = path.join(path.resolve(__dirname, '..', '..'), 'src/init-files/init-docs/readme') let b58Hash let b64Hash @@ -52,7 +52,7 @@ describe('resolve', () => runOnAndOff((thing) => { it('should resolve an IPFS path link', function () { this.timeout(10 * 1000) - const filePath = path.join(process.cwd(), '/src/init-files/init-docs/readme') + const filePath = path.join(path.resolve(__dirname, '..', '..'), 'src/init-files/init-docs/readme') let fileHash, rootHash return ipfs(`add ${filePath} --wrap-with-directory`) diff --git a/test/cli/swarm.js b/test/cli/swarm.js index 889b309681..805bd1affe 100644 --- a/test/cli/swarm.js +++ b/test/cli/swarm.js @@ -8,7 +8,7 @@ const expect = chai.expect chai.use(dirtyChai) const series = require('async/series') const ipfsExec = require('../utils/ipfs-exec') - +const path = require('path') const parallel = require('async/parallel') const DaemonFactory = require('ipfsd-ctl') @@ -37,7 +37,7 @@ describe('swarm', () => { series([ (cb) => { df.spawn({ - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config, initOptions: { bits: 512 } }, (err, node) => { @@ -49,7 +49,7 @@ describe('swarm', () => { }, (cb) => { df.spawn({ - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config, initOptions: { bits: 512 } }, (err, node) => { diff --git a/test/core/bitswap.spec.js b/test/core/bitswap.spec.js index b3261d4d88..befff9b2bf 100644 --- a/test/core/bitswap.spec.js +++ b/test/core/bitswap.spec.js @@ -15,7 +15,7 @@ const multiaddr = require('multiaddr') const isNode = require('detect-node') const multihashing = require('multihashing-async') const CID = require('cids') - +const path = require('path') const IPFSFactory = require('ipfsd-ctl') const IPFS = require('../../src/core') @@ -64,7 +64,7 @@ let nodes = [] function addNode (fDaemon, inProcNode, callback) { fDaemon.spawn({ - exec: './src/cli/bin.js', + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), initOptions: { bits: 512 }, config: { Addresses: { diff --git a/test/core/files.spec.js b/test/core/files.spec.js index 77f99739ca..6570480006 100644 --- a/test/core/files.spec.js +++ b/test/core/files.spec.js @@ -91,7 +91,7 @@ describe('files', () => { }, (err, files) => { expect(err).to.not.exist() expect(files.length).to.equal(1) - expect(files[0].hash).to.equal('zb2rhiNedvrkpYhcrgtpmhKk5UPzcgizgSXaQLYXNY745BmYP') + expect(files[0].hash).to.equal('bafkreifojmzibzlof6xyh5auu3r5vpu5l67brf3fitaf73isdlglqw2t7q') expect(files[0].size).to.equal(3) done() }) @@ -104,7 +104,7 @@ describe('files', () => { }, (err, files) => { expect(err).to.not.exist() expect(files.length).to.equal(1) - expect(files[0].hash).to.equal('zdj7WcDSFNSsZkdkbpSDGeLsBtHbYKyvPQsaw6PpeeYdGqoAx') + expect(files[0].hash).to.equal('bafybeide2caf5we5a7izifzwzz5ds2gla67vsfgrzvbzpnyyirnfzgwf5e') expect(files[0].size).to.equal(11) done() }) diff --git a/test/core/kad-dht.node.js b/test/core/kad-dht.node.js index 1d7452c596..d07be33f20 100644 --- a/test/core/kad-dht.node.js +++ b/test/core/kad-dht.node.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const parallel = require('async/parallel') const IPFSFactory = require('ipfsd-ctl') @@ -26,7 +26,7 @@ const config = { function createNode (callback) { f.spawn({ - exec: './src/cli/bin.js', + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), config, initOptions: { bits: 512 } }, callback) diff --git a/test/core/pin-set.js b/test/core/pin-set.js index c1a31a1319..39fc12bbe2 100644 --- a/test/core/pin-set.js +++ b/test/core/pin-set.js @@ -11,7 +11,8 @@ const parallelLimit = require('async/parallelLimit') const series = require('async/series') const { util: { - cid + cid, + serialize }, DAGNode } = require('ipld-dag-pb') @@ -35,7 +36,7 @@ function createNodes (num, callback) { const items = [] for (let i = 0; i < num; i++) { items.push(cb => - createNode(String(i), (err, res) => cb(err, res.cid.toBaseEncodedString())) + createNode(String(i), (err, res) => cb(err, !err && res.cid.toBaseEncodedString())) ) } @@ -48,18 +49,20 @@ function createNode (data, links = [], callback) { links = [] } - DAGNode.create(data, links, (err, node) => { - if (err) { - return callback(err) - } + let node - cid(node, (err, result) => { - callback(err, { - node, - cid: result - }) + try { + node = DAGNode.create(data, links) + } catch (err) { + return callback(err) + } + + cid(serialize(node)).then(cid => { + callback(null, { + node, + cid }) - }) + }, err => callback(err)) } describe('pinSet', function () { diff --git a/test/core/ping.spec.js b/test/core/ping.spec.js index 3b477e05ee..6d04792013 100644 --- a/test/core/ping.spec.js +++ b/test/core/ping.spec.js @@ -9,10 +9,10 @@ const parallel = require('async/parallel') const series = require('async/series') const DaemonFactory = require('ipfsd-ctl') const isNode = require('detect-node') - +const path = require('path') const expect = chai.expect chai.use(dirtyChai) -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) const dfProc = DaemonFactory.create({ exec: require('../../'), type: 'proc' diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index edb3c2153b..52e0441cb9 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -205,7 +205,7 @@ describe('preload', () => { ipfs.object.patch.addLink(result.parent.cid, { name: 'link', - cid: result.link.cid, + cid: result.link.Hash, size: result.link.node.size }, (err, cid) => { expect(err).to.not.exist() diff --git a/test/http-api/block.js b/test/http-api/block.js index afa8933833..1efce99d05 100644 --- a/test/http-api/block.js +++ b/test/http-api/block.js @@ -5,12 +5,12 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const multihash = require('multihashes') const waterfall = require('async/waterfall') const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) describe('block endpoint', () => { let ipfs = null diff --git a/test/http-api/bootstrap.js b/test/http-api/bootstrap.js index 7fb59d6474..b53ba29fdf 100644 --- a/test/http-api/bootstrap.js +++ b/test/http-api/bootstrap.js @@ -6,9 +6,9 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) describe('bootstrap endpoint', () => { let ipfs = null diff --git a/test/http-api/config.js b/test/http-api/config.js index 87c6b4d35d..51eb3e6e01 100644 --- a/test/http-api/config.js +++ b/test/http-api/config.js @@ -17,7 +17,7 @@ const fs = require('fs') const path = require('path') const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) skipOnWindows('config endpoint', () => { const repoExample = path.join(__dirname, '../fixtures/go-ipfs-repo') diff --git a/test/http-api/dns.js b/test/http-api/dns.js index b90802a621..f27e5ee217 100644 --- a/test/http-api/dns.js +++ b/test/http-api/dns.js @@ -5,9 +5,9 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) describe('dns endpoint', () => { let ipfs = null diff --git a/test/http-api/files.js b/test/http-api/files.js index 59bd05e118..66a508c34e 100644 --- a/test/http-api/files.js +++ b/test/http-api/files.js @@ -10,8 +10,8 @@ chai.use(dirtyChai) const { FILE_TYPES } = require('ipfs-mfs') - -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const path = require('path') +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) describe('.files', () => { let ipfs = null diff --git a/test/http-api/id.js b/test/http-api/id.js index ea54e158ee..05052d83fe 100644 --- a/test/http-api/id.js +++ b/test/http-api/id.js @@ -15,7 +15,7 @@ const isWindows = require('../utils/platforms').isWindows const skipOnWindows = isWindows() ? describe.skip : describe const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) skipOnWindows('id endpoint', () => { const repoExample = path.join(__dirname, '../fixtures/go-ipfs-repo') diff --git a/test/http-api/inject/dag.js b/test/http-api/inject/dag.js index 605cb076dd..a86c5258e5 100644 --- a/test/http-api/inject/dag.js +++ b/test/http-api/inject/dag.js @@ -6,9 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const promisify = require('promisify-es6') const DAGNode = require('ipld-dag-pb').DAGNode -const createDAGPBNode = promisify(DAGNode.create) const Readable = require('stream').Readable const FormData = require('form-data') const streamToPromise = require('stream-to-promise') @@ -58,7 +56,7 @@ module.exports = (http) => { }) it('returns value', async () => { - const node = await createDAGPBNode(Buffer.from([]), []) + const node = DAGNode.create(Buffer.from([]), []) const cid = await http.api._ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' @@ -75,7 +73,7 @@ module.exports = (http) => { }) it('uses text encoding for data by default', async () => { - const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), []) + const node = DAGNode.create(Buffer.from([0, 1, 2, 3]), []) const cid = await http.api._ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' @@ -93,7 +91,7 @@ module.exports = (http) => { }) it('overrides data encoding', async () => { - const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), []) + const node = DAGNode.create(Buffer.from([0, 1, 2, 3]), []) const cid = await http.api._ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' @@ -128,7 +126,7 @@ module.exports = (http) => { }) it('returns value with a path as part of the cid for dag-pb nodes', async () => { - const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), []) + const node = DAGNode.create(Buffer.from([0, 1, 2, 3]), []) const cid = await http.api._ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' @@ -235,8 +233,8 @@ module.exports = (http) => { const added = await http.api._ipfs.dag.get(cid) - expect(added.value.data).to.be.empty() - expect(added.value.links).to.be.empty() + expect(added.value.Data).to.be.empty() + expect(added.value.Links).to.be.empty() }) it('adds a raw node', async () => { diff --git a/test/http-api/interface.js b/test/http-api/interface.js index eea370a4b7..43c6ff6a12 100644 --- a/test/http-api/interface.js +++ b/test/http-api/interface.js @@ -3,10 +3,11 @@ const tests = require('interface-ipfs-core') const CommonFactory = require('../utils/interface-common-factory') +const path = require('path') describe('interface-ipfs-core over ipfs-http-client tests', () => { const defaultCommonFactory = CommonFactory.create({ - factoryOptions: { exec: 'src/cli/bin.js' } + factoryOptions: { exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) } }) tests.bitswap(defaultCommonFactory) diff --git a/test/http-api/object.js b/test/http-api/object.js index 811aad6d72..8ba9eaa352 100644 --- a/test/http-api/object.js +++ b/test/http-api/object.js @@ -6,13 +6,13 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const fs = require('fs') const dagPB = require('ipld-dag-pb') const DAGLink = dagPB.DAGLink const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) function asJson (cb) { return (err, result) => { diff --git a/test/http-api/version.js b/test/http-api/version.js index 9fc0c3acc0..3b92a30b6c 100644 --- a/test/http-api/version.js +++ b/test/http-api/version.js @@ -5,9 +5,9 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ exec: 'src/cli/bin.js' }) +const df = DaemonFactory.create({ exec: path.resolve(`${__dirname}/../../src/cli/bin.js`) }) describe('version endpoint', () => { let ipfs = null diff --git a/test/utils/ipfs-exec.js b/test/utils/ipfs-exec.js index 3f9572f2a3..f4be357c02 100644 --- a/test/utils/ipfs-exec.js +++ b/test/utils/ipfs-exec.js @@ -5,7 +5,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) - +const path = require('path') const _ = require('lodash') // This is our new test utility to easily check and execute ipfs cli commands. @@ -27,9 +27,8 @@ module.exports = (repoPath, opts) => { env: env, timeout: 60 * 1000 }, opts) - - const exec = (args) => execa(`${process.cwd()}/src/cli/bin.js`, args, config) - const execRaw = (args) => execa(`${process.cwd()}/src/cli/bin.js`, args, Object.assign({}, config, { + const exec = (args) => execa(path.resolve(`${__dirname}/../../src/cli/bin.js`), args, config) + const execRaw = (args) => execa(path.resolve(`${__dirname}/../../src/cli/bin.js`), args, Object.assign({}, config, { encoding: null })) diff --git a/test/utils/on-and-off.js b/test/utils/on-and-off.js index d315491b90..840d7bbe8e 100644 --- a/test/utils/on-and-off.js +++ b/test/utils/on-and-off.js @@ -13,6 +13,7 @@ const os = require('os') const DaemonFactory = require('ipfsd-ctl') const df = DaemonFactory.create() +const path = require('path') function off (tests) { describe('daemon off (directly to core)', () => { @@ -51,7 +52,7 @@ function on (tests) { df.spawn({ type: 'js', - exec: `./src/cli/bin.js`, + exec: path.resolve(`${__dirname}/../../src/cli/bin.js`), initOptions: { bits: 512 }, config: { Bootstrap: [] } }, (err, node) => {