diff --git a/src/cli/commands/refs.js b/src/cli/commands/refs.js new file mode 100644 index 0000000000..5548133896 --- /dev/null +++ b/src/cli/commands/refs.js @@ -0,0 +1,100 @@ +'use strict' + +const { print } = require('../utils') + +const Format = { + default: '', + edges: ' -> ' +} + +module.exports = { + command: 'refs ', + + describe: 'List links (references) from an object', + + builder: { + r: { + alias: 'recursive', + desc: 'Recursively list links of child nodes.', + type: 'boolean', + default: false + }, + format: { + desc: 'Output edges with given format. Available tokens: .', + type: 'string', + default: Format.default + }, + e: { + alias: 'edges', + desc: 'Output edge format: ` -> `', + type: 'boolean', + default: false + }, + u: { + alias: 'unique', + desc: 'Omit duplicate refs from output.', + type: 'boolean', + default: false + }, + 'max-depth': { + desc: 'Only for recursive refs, limits fetch and listing to the given depth.', + type: 'number' + } + }, + + handler ({ getIpfs, key, recursive, format, e, u, resolve, maxDepth }) { + resolve((async () => { + if (format !== Format.default && e) { + throw new Error('Cannot set edges to true and also specify format') + } + + if (maxDepth === 0) { + return + } + + const ipfs = await getIpfs() + let links = await ipfs.refs(key, { recursive, maxDepth }) + if (!links.length) { + return + } + + const linkDAG = getLinkDAG(links) + format = e ? Format.edges : format || Format.default + printLinks(linkDAG, links[0], format, u && new Set()) + })()) + } +} + +function getLinkDAG (links) { + const linkNames = {} + for (const link of links) { + linkNames[link.name] = link + } + + const linkDAG = {} + for (const link of links) { + const parentName = link.path.substring(0, link.path.lastIndexOf('/')) + linkDAG[parentName] = linkDAG[parentName] || [] + linkDAG[parentName].push(link) + } + return linkDAG +} + +function printLinks (linkDAG, link, format, uniques) { + const children = linkDAG[link.path] || [] + for (const child of children) { + if (!uniques || !uniques.has(child.hash)) { + uniques && uniques.add(child.hash) + printLink(link, child, format) + printLinks(linkDAG, child, format, uniques) + } + } +} + +function printLink (src, dst, format) { + let out = format.replace(//g, src.hash) + out = out.replace(//g, dst.hash) + out = out.replace(//g, dst.name) + print(out) +} + diff --git a/src/core/components/files-regular/index.js b/src/core/components/files-regular/index.js index 058d07d618..43308118d8 100644 --- a/src/core/components/files-regular/index.js +++ b/src/core/components/files-regular/index.js @@ -15,5 +15,7 @@ module.exports = self => ({ getReadableStream: require('./get-readable-stream')(self), ls: require('./ls')(self), lsPullStream: require('./ls-pull-stream')(self), - lsReadableStream: require('./ls-readable-stream')(self) + lsReadableStream: require('./ls-readable-stream')(self), + refs: require('./refs')(self), + refsPullStream: require('./refs-pull-stream')(self) }) diff --git a/src/core/components/files-regular/refs-pull-stream.js b/src/core/components/files-regular/refs-pull-stream.js new file mode 100644 index 0000000000..14e78a6cd8 --- /dev/null +++ b/src/core/components/files-regular/refs-pull-stream.js @@ -0,0 +1,32 @@ +'use strict' + +const exporter = require('ipfs-unixfs-exporter') +const pull = require('pull-stream') +const { normalizePath } = require('./utils') + +module.exports = function (self) { + return function (ipfsPath, options = {}) { + const path = normalizePath(ipfsPath) + const pathComponents = path.split('/') + const pathDepth = pathComponents.length + if (options.maxDepth === undefined) { + options.maxDepth = options.recursive ? global.Infinity : pathDepth + } else { + options.maxDepth = options.maxDepth + pathDepth - 1 + } + + if (options.preload !== false) { + self._preload(pathComponents[0]) + } + + return pull( + exporter(ipfsPath, self._ipld, options), + pull.map(node => { + node.hash = node.cid.toString() + delete node.cid + delete node.content + return node + }) + ) + } +} diff --git a/src/core/components/files-regular/refs.js b/src/core/components/files-regular/refs.js new file mode 100644 index 0000000000..64982030bb --- /dev/null +++ b/src/core/components/files-regular/refs.js @@ -0,0 +1,25 @@ +'use strict' + +const promisify = require('promisify-es6') +const pull = require('pull-stream') + +module.exports = function (self) { + return promisify((ipfsPath, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + + options = options || {} + + pull( + self.refsPullStream(ipfsPath, options), + pull.collect((err, values) => { + if (err) { + return callback(err) + } + callback(null, values) + }) + ) + }) +} diff --git a/test/cli/refs.js b/test/cli/refs.js new file mode 100644 index 0000000000..e25f6861f8 --- /dev/null +++ b/test/cli/refs.js @@ -0,0 +1,263 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const runOnAndOff = require('../utils/on-and-off') + +// describe('refs', () => runOnAndOff((thing) => { +describe('refs', () => runOnAndOff.off((thing) => { + let ipfs + + before(() => { + ipfs = thing.ipfs + return ipfs('add -r test/fixtures/test-data/refs') + }) + + it('prints added files', function () { + this.timeout(20 * 1000) + return ipfs('refs QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then((out) => { + expect(out).to.eql( + 'QmdUmXjesQaPAk7NNw7epwcU1uytoJrH1qaaHHVAeZQvJJ\n' + + 'QmcSVZRN5E814KkPy4EHnftNAR7htbFvVhRKKqFs4FBwDG\n' + + 'QmXcybpFANuQw1VqvTAvB3gGNZp3fZtfzRfq7R7MNZvUBA\n' + + 'QmVwtsLUHurA6wUirPSdGeEW5tfBEqenXpeRaqr8XN7bNY\n' + ) + }) + }) + + it('prints files in edges format', function () { + this.timeout(20 * 1000) + return ipfs('refs -e QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then((out) => { + expect(out).to.eql( + 'QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 -> QmdUmXjesQaPAk7NNw7epwcU1uytoJrH1qaaHHVAeZQvJJ\n' + + 'QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 -> QmcSVZRN5E814KkPy4EHnftNAR7htbFvVhRKKqFs4FBwDG\n' + + 'QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 -> QmXcybpFANuQw1VqvTAvB3gGNZp3fZtfzRfq7R7MNZvUBA\n' + + 'QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 -> QmVwtsLUHurA6wUirPSdGeEW5tfBEqenXpeRaqr8XN7bNY\n' + ) + }) + }) + + it('prints files in custom format', function () { + this.timeout(20 * 1000) + return ipfs('refs --format ": => " QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then((out) => { + expect(out).to.eql( + 'animals: QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 => QmdUmXjesQaPAk7NNw7epwcU1uytoJrH1qaaHHVAeZQvJJ\n' + + 'atlantic-animals: QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 => QmcSVZRN5E814KkPy4EHnftNAR7htbFvVhRKKqFs4FBwDG\n' + + 'fruits: QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 => QmXcybpFANuQw1VqvTAvB3gGNZp3fZtfzRfq7R7MNZvUBA\n' + + 'mushroom.txt: QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4 => QmVwtsLUHurA6wUirPSdGeEW5tfBEqenXpeRaqr8XN7bNY\n' + ) + }) + }) + + it('follows a path, /', function () { + this.timeout(20 * 1000) + + return ipfs('refs --format="" /ipfs/QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4/animals') + .then((out) => { + expect(out).to.eql( + 'land\n' + + 'sea\n' + ) + }) + }) + + it('follows a path, //', function () { + this.timeout(20 * 1000) + + return ipfs('refs --format="" /ipfs/QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4/animals/land') + .then((out) => { + expect(out).to.eql( + 'african.txt\n' + + 'americas.txt\n' + + 'australian.txt\n' + ) + }) + }) + + it('follows a path with recursion, /', function () { + this.timeout(20 * 1000) + + return ipfs('refs -r --format="" /ipfs/QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4/animals') + .then((out) => { + expect(out).to.eql( + 'land\n' + + 'african.txt\n' + + 'americas.txt\n' + + 'australian.txt\n' + + 'sea\n' + + 'atlantic.txt\n' + + 'indian.txt\n' + ) + }) + }) + + // + // Directory structure: + // + // animals + // land + // african.txt + // americas.txt + // australian.txt + // sea + // atlantic.txt + // indian.txt + // fruits + // tropical.txt + // mushroom.txt + // + + it('recursively follows folders, -r', function () { + this.slow(2000) + this.timeout(20 * 1000) + + return ipfs('refs -r --format="" QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then(out => { + expect(out).to.eql( + 'animals\n' + + 'land\n' + + 'african.txt\n' + + 'americas.txt\n' + + 'australian.txt\n' + + 'sea\n' + + 'atlantic.txt\n' + + 'indian.txt\n' + + 'atlantic-animals\n' + + 'fruits\n' + + 'tropical.txt\n' + + 'mushroom.txt\n' + ) + }) + }) + + it('recursive with unique option', function () { + this.slow(2000) + this.timeout(20 * 1000) + + return ipfs('refs -u -r --format="" QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then(out => { + expect(out).to.eql( + 'animals\n' + + 'land\n' + + 'african.txt\n' + + 'americas.txt\n' + + 'australian.txt\n' + + 'sea\n' + + 'atlantic.txt\n' + + 'indian.txt\n' + + 'fruits\n' + + 'tropical.txt\n' + + 'mushroom.txt\n' + ) + }) + }) + + it('max depth of 1', function () { + this.timeout(20 * 1000) + return ipfs('refs -r --max-depth=1 --format="" QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then((out) => { + expect(out).to.eql( + 'animals\n' + + 'atlantic-animals\n' + + 'fruits\n' + + 'mushroom.txt\n' + ) + }) + }) + + it('max depth of 2', function () { + this.timeout(20 * 1000) + return ipfs('refs -r --max-depth=2 --format="" QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then((out) => { + expect(out).to.eql( + 'animals\n' + + 'land\n' + + 'sea\n' + + 'atlantic-animals\n' + + 'fruits\n' + + 'tropical.txt\n' + + 'mushroom.txt\n' + ) + }) + }) + + it('max depth of 3', function () { + this.timeout(20 * 1000) + return ipfs('refs -r --max-depth=3 --format="" QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then((out) => { + expect(out).to.eql( + 'animals\n' + + 'land\n' + + 'african.txt\n' + + 'americas.txt\n' + + 'australian.txt\n' + + 'sea\n' + + 'atlantic.txt\n' + + 'indian.txt\n' + + 'atlantic-animals\n' + + 'fruits\n' + + 'tropical.txt\n' + + 'mushroom.txt\n' + ) + }) + }) + + it('max depth of 0', function () { + this.timeout(20 * 1000) + return ipfs('refs -r --max-depth=0 --format="" QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4') + .then((out) => expect(out).to.eql('')) + }) + + it('follows a path with max depth 1, /', function () { + this.timeout(20 * 1000) + + return ipfs('refs -r --max-depth=1 --format="" /ipfs/QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4/animals') + .then((out) => { + expect(out).to.eql( + 'land\n' + + 'sea\n' + ) + }) + }) + + it('follows a path with max depth 2, /', function () { + this.timeout(20 * 1000) + + return ipfs('refs -r --max-depth=2 --format="" /ipfs/QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4/animals') + .then((out) => { + expect(out).to.eql( + 'land\n' + + 'african.txt\n' + + 'americas.txt\n' + + 'australian.txt\n' + + 'sea\n' + + 'atlantic.txt\n' + + 'indian.txt\n' + ) + }) + }) + + it('cannot specify edges and format', function () { + this.timeout(20 * 1000) + // If the daemon is off, ls should fail + // If the daemon is on, ls should search until it hits a timeout + return Promise.race([ + ipfs.fail('refs --format="" -e QmXW5PJso8qkBzavt7ZDXjmXAzJUwKi8d6AZxoSqG6rLJ4'), + new Promise((resolve, reject) => setTimeout(resolve, 4000)) + ]) + .catch(() => expect.fail(0, 1, 'Should have thrown or timedout')) + }) + + it('prints nothing for non-existent hashes', function () { + // If the daemon is off, ls should fail + // If the daemon is on, ls should search until it hits a timeout + return Promise.race([ + ipfs.fail('refs QmYmW4HiZhotsoSqnv2o1oSssvkRM8b9RweBoH7ao5nki2'), + new Promise((resolve, reject) => setTimeout(resolve, 4000)) + ]) + .catch(() => expect.fail(0, 1, 'Should have thrown or timedout')) + }) +})) diff --git a/test/fixtures/test-data/refs/animals/land/african.txt b/test/fixtures/test-data/refs/animals/land/african.txt new file mode 100644 index 0000000000..29decfcd50 --- /dev/null +++ b/test/fixtures/test-data/refs/animals/land/african.txt @@ -0,0 +1,2 @@ +elephant +rhinocerous \ No newline at end of file diff --git a/test/fixtures/test-data/refs/animals/land/americas.txt b/test/fixtures/test-data/refs/animals/land/americas.txt new file mode 100644 index 0000000000..21368a871d --- /dev/null +++ b/test/fixtures/test-data/refs/animals/land/americas.txt @@ -0,0 +1,2 @@ +ñandu +tapir \ No newline at end of file diff --git a/test/fixtures/test-data/refs/animals/land/australian.txt b/test/fixtures/test-data/refs/animals/land/australian.txt new file mode 100644 index 0000000000..a78c7bc9c3 --- /dev/null +++ b/test/fixtures/test-data/refs/animals/land/australian.txt @@ -0,0 +1,2 @@ +emu +kangaroo \ No newline at end of file diff --git a/test/fixtures/test-data/refs/animals/sea/atlantic.txt b/test/fixtures/test-data/refs/animals/sea/atlantic.txt new file mode 100644 index 0000000000..f77ffe5119 --- /dev/null +++ b/test/fixtures/test-data/refs/animals/sea/atlantic.txt @@ -0,0 +1,2 @@ +dolphin +whale \ No newline at end of file diff --git a/test/fixtures/test-data/refs/animals/sea/indian.txt b/test/fixtures/test-data/refs/animals/sea/indian.txt new file mode 100644 index 0000000000..c455106f6c --- /dev/null +++ b/test/fixtures/test-data/refs/animals/sea/indian.txt @@ -0,0 +1,2 @@ +cuttlefish +octopus \ No newline at end of file diff --git a/test/fixtures/test-data/refs/atlantic-animals b/test/fixtures/test-data/refs/atlantic-animals new file mode 120000 index 0000000000..670958bfa8 --- /dev/null +++ b/test/fixtures/test-data/refs/atlantic-animals @@ -0,0 +1 @@ +animals/sea/atlantic.txt \ No newline at end of file diff --git a/test/fixtures/test-data/refs/fruits/tropical.txt b/test/fixtures/test-data/refs/fruits/tropical.txt new file mode 100644 index 0000000000..4f331bc7d2 --- /dev/null +++ b/test/fixtures/test-data/refs/fruits/tropical.txt @@ -0,0 +1,2 @@ +banana +pineapple \ No newline at end of file diff --git a/test/fixtures/test-data/refs/mushroom.txt b/test/fixtures/test-data/refs/mushroom.txt new file mode 100644 index 0000000000..8b248aa9c8 --- /dev/null +++ b/test/fixtures/test-data/refs/mushroom.txt @@ -0,0 +1 @@ +mushroom \ No newline at end of file diff --git a/test/utils/ipfs-exec.js b/test/utils/ipfs-exec.js index 3f9572f2a3..4f634e6ff9 100644 --- a/test/utils/ipfs-exec.js +++ b/test/utils/ipfs-exec.js @@ -7,6 +7,7 @@ const expect = chai.expect chai.use(dirtyChai) const _ = require('lodash') +const yargs = require('yargs') // This is our new test utility to easily check and execute ipfs cli commands. // @@ -34,11 +35,7 @@ module.exports = (repoPath, opts) => { })) const execute = (exec, args) => { - if (args.length === 1) { - args = args[0].split(' ') - } - - const cp = exec(args) + const cp = exec(yargs('-- ' + args[0]).argv._) const res = cp.then((res) => { // We can't escape the os.tmpdir warning due to: // https://github.com/shelljs/shelljs/blob/master/src/tempdir.js#L43