diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 9e7d192106..9435c95fc5 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -112,6 +112,15 @@ module.exports = { 'shard-split-threshold': { type: 'integer', default: 1000 + }, + 'raw-leaves': { + type: 'boolean', + default: undefined, + describe: 'Use raw blocks for leaf nodes. (experimental)' + }, + 'cid-version': { + type: 'integer', + describe: 'Cid version. Non-zero value will change default of \'raw-leaves\' to true. (experimental)' } }, @@ -120,7 +129,25 @@ module.exports = { const index = inPath.lastIndexOf('/') + 1 const options = { strategy: argv.trickle ? 'trickle' : 'balanced', - shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity + shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity, + 'cid-version': argv['cid-version'], + 'raw-leaves': argv['raw-leaves'] + } + + // Temporary restriction on raw-leaves: + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + if (argv['cid-version'] > 0 && argv['raw-leaves'] !== false) { + throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0') + } + + if (argv['raw-leaves']) { + throw new Error('Not implemented: raw-leaves') } if (argv.enableShardingExperiment && utils.isDaemonOn()) { diff --git a/src/core/components/files.js b/src/core/components/files.js index 3d650e74bc..6a9f5fbced 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -4,7 +4,6 @@ const unixfsEngine = require('ipfs-unixfs-engine') const importer = unixfsEngine.importer const exporter = unixfsEngine.exporter const promisify = require('promisify-es6') -const multihashes = require('multihashes') const pull = require('pull-stream') const sort = require('pull-sort') const pushable = require('pull-pushable') @@ -13,6 +12,7 @@ const toPull = require('stream-to-pull-stream') const waterfall = require('async/waterfall') const isStream = require('is-stream') const Duplex = require('stream').Duplex +const CID = require('cids') module.exports = function files (self) { const createAddPullStream = (options) => { @@ -24,7 +24,7 @@ module.exports = function files (self) { pull.map(normalizeContent), pull.flatten(), importer(self._ipldResolver, opts), - pull.asyncMap(prepareFile.bind(null, self)) + pull.asyncMap(prepareFile.bind(null, self, opts)) ) } @@ -68,7 +68,7 @@ module.exports = function files (self) { pull( pull.values(normalizeContent(data)), importer(self._ipldResolver, options), - pull.asyncMap(prepareFile.bind(null, self)), + pull.asyncMap(prepareFile.bind(null, self, options)), sort((a, b) => { if (a.path < b.path) return 1 if (a.path > b.path) return -1 @@ -114,15 +114,23 @@ module.exports = function files (self) { } } -function prepareFile (self, file, callback) { - const bs58mh = multihashes.toB58String(file.multihash) +function prepareFile (self, opts, file, callback) { + opts = opts || {} waterfall([ (cb) => self.object.get(file.multihash, cb), (node, cb) => { + let cid = new CID(node.multihash) + + if (opts['cid-version'] === 1) { + cid = cid.toV1() + } + + const b58Hash = cid.toBaseEncodedString() + cb(null, { - path: file.path || bs58mh, - hash: bs58mh, + path: file.path || b58Hash, + hash: b58Hash, size: node.size }) } diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 68ecc797bd..aa410f974b 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -11,6 +11,7 @@ const toPull = require('stream-to-pull-stream') const pushable = require('pull-pushable') const EOL = require('os').EOL const toStream = require('pull-stream-to-stream') +const Joi = require('joi') exports = module.exports @@ -140,6 +141,28 @@ exports.get = { } exports.add = { + validate: { + query: Joi.object() + .keys({ + 'cid-version': Joi.number().integer().min(0).max(1), + // Temporary restriction on raw-leaves: + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + 'raw-leaves': Joi.any().when('cid-version', { + is: 1, + then: Joi.boolean().valid(false).required(), + otherwise: Joi.boolean().valid(false) + }) + }) + // TODO: Necessary until validate "recursive", "stream-channels" etc. + .options({ allowUnknown: true }) + }, + handler: (request, reply) => { if (!request.payload) { return reply({ @@ -181,9 +204,14 @@ exports.add = { fileAdder.end() }) + const options = { + 'cid-version': request.query['cid-version'], + 'raw-leaves': request.query['raw-leaves'] + } + pull( fileAdder, - ipfs.files.createAddPullStream(), + ipfs.files.createAddPullStream(options), pull.map((file) => { return { Name: file.path ? file.path : file.hash, diff --git a/src/http-api/routes/files.js b/src/http-api/routes/files.js index da57b3f2f1..e7ce4f456d 100644 --- a/src/http-api/routes/files.js +++ b/src/http-api/routes/files.js @@ -38,7 +38,8 @@ module.exports = (server) => { parse: false, output: 'stream' }, - handler: resources.files.add.handler + handler: resources.files.add.handler, + validate: resources.files.add.validate } }) } diff --git a/test/cli/files.js b/test/cli/files.js index 7dc01599c6..4d5ca1a0a0 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -136,6 +136,60 @@ describe('files', () => runOnAndOff((thing) => { }) }) + it('add with cid-version=0', () => { + return ipfs('add src/init-files/init-docs/readme --cid-version=0').then((out) => { + expect(out) + .to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n') + }) + }) + + // Temporarily expect to fail as raw-leaves not yet implemented. + // + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + it('add with cid-version=1', () => { + return new Promise((resolve, reject) => { + ipfs('add src/init-files/init-docs/readme --cid-version=1') + .then(() => reject(new Error('Raw leaves not expected to be implemented'))) + .catch((err) => { + expect(err).to.exist() + resolve() + }) + }) + }) + + it('add with cid-version=1 and raw-leaves=false', () => { + return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false').then((out) => { + expect(out) + .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n') + }) + }) + + // Temporarily expect to fail as raw-leaves not yet implemented + // + // When cid-version=1 then raw-leaves MUST be present and false. + // + // This is because raw-leaves is not yet implemented in js-ipfs, + // and go-ipfs changes the value of raw-leaves to true when + // cid-version > 0 unless explicitly set to false. + // + // This retains feature parity without having to implement raw-leaves. + it('add with cid-version=1 and raw-leaves=true', () => { + return new Promise((resolve, reject) => { + ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true') + .then(() => reject(new Error('Raw leaves not expected to be implemented'))) + .catch((err) => { + expect(err).to.exist() + resolve() + }) + }) + }) + it('cat', () => { return ipfs('files cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') .then((out) => {