diff --git a/package.json b/package.json index 25459d5..1c325c8 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "devDependencies": { "aegir": "^15.0.0", "chai": "^4.1.2", + "detect-node": "^2.0.3", "detect-webworker": "^1.0.0", "dirty-chai": "^2.0.1", "ipfs": "~0.30.0", @@ -48,15 +49,13 @@ "dependencies": { "async": "^2.6.1", "blob": "~0.0.4", - "bs58": "^4.0.1", "cids": "~0.5.3", "debug": "^3.1.0", - "detect-node": "^2.0.3", "file-api": "~0.10.4", "filereader-stream": "^2.0.0", "interface-datastore": "~0.4.2", "ipfs-unixfs": "~0.1.15", - "ipfs-unixfs-engine": "~0.30.0", + "ipfs-unixfs-engine": "~0.31.1", "is-pull-stream": "~0.0.0", "is-stream": "^1.1.0", "joi": "^13.4.0", @@ -67,7 +66,7 @@ "pull-cat": "^1.1.11", "pull-paramap": "^1.2.2", "pull-pushable": "^2.2.0", - "pull-stream": "^3.6.7", + "pull-stream": "^3.6.8", "pull-stream-to-stream": "^1.3.4", "pull-traverse": "^1.0.3", "stream-to-pull-stream": "^1.7.2" diff --git a/src/cli/write.js b/src/cli/write.js index b76c370..1d50b1f 100644 --- a/src/cli/write.js +++ b/src/cli/write.js @@ -16,7 +16,7 @@ module.exports = { describe: 'Create any non-existent intermediate directories' }, create: { - alias: 'c', + alias: 'e', type: 'boolean', default: false, coerce: asBoolean, @@ -42,7 +42,7 @@ module.exports = { rawLeaves: { alias: 'r', type: 'boolean', - default: true, + default: false, coerce: asBoolean, describe: 'Whether to write leaf nodes as raw UnixFS nodes' }, @@ -65,9 +65,9 @@ module.exports = { default: 'balanced' }, cidVersion: { - alias: 'c', + alias: ['cid-ver', 'cid-version'], type: 'number', - default: 1 + default: 0 }, hashAlg: { alias: 'h', @@ -105,7 +105,7 @@ module.exports = { length, create, truncate, - rawLeafNodes: rawLeaves, + rawLeaves, reduceSingleLeafToSelf, cidVersion, hashAlg, diff --git a/src/core/mkdir.js b/src/core/mkdir.js index 4bbad89..40a201c 100644 --- a/src/core/mkdir.js +++ b/src/core/mkdir.js @@ -12,7 +12,7 @@ const { const defaultOptions = { parents: false, hash: undefined, - cidVersion: undefined + cidVersion: 0 } module.exports = (ipfs) => { diff --git a/src/core/utils/load-node.js b/src/core/utils/load-node.js index c1440e6..4d64f3b 100644 --- a/src/core/utils/load-node.js +++ b/src/core/utils/load-node.js @@ -3,10 +3,9 @@ const waterfall = require('async/waterfall') const CID = require('cids') const log = require('debug')('ipfs:mfs:utils:load-node') -const bs58 = require('bs58') -const loadNode = (ipfs, cid, callback) => { - const multihash = cid && (cid.multihash || cid.hash) +const loadNode = (ipfs, object, callback) => { + const multihash = object && (object.multihash || object.hash) if (!multihash) { log(`No multihash passed so cannot load DAGNode`) @@ -14,10 +13,12 @@ const loadNode = (ipfs, cid, callback) => { return callback() } - log(`Loading DAGNode for child ${bs58.encode(multihash)}`) + const cid = new CID(multihash) + + log(`Loading DAGNode for child ${cid.toBaseEncodedString()}`) waterfall([ - (cb) => ipfs.dag.get(new CID(multihash), cb), + (cb) => ipfs.dag.get(cid, cb), (result, cb) => cb(null, result.value) ], callback) } diff --git a/src/core/utils/traverse-to.js b/src/core/utils/traverse-to.js index aa98425..f4dfd51 100644 --- a/src/core/utils/traverse-to.js +++ b/src/core/utils/traverse-to.js @@ -1,6 +1,5 @@ 'use strict' -const bs58 = require('bs58') const CID = require('cids') const log = require('debug')('ipfs:mfs:utils:traverse-to') const UnixFS = require('ipfs-unixfs') @@ -77,8 +76,6 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { node: rootNode, parent: null }, (parent, {pathSegment, index}, done) => { - log(`Looking for ${pathSegment} in ${parent.name} ${bs58.encode(parent.node.multihash)}`) - const existingLink = parent.node.links.find(link => link.name === pathSegment) if (!existingLink) { @@ -114,6 +111,7 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { next(error, { name: pathSegment, node: emptyDirectory, + cid: new CID(emptyDirectory.multihash), parent: parent }) }) @@ -126,43 +124,31 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { } let hash = existingLink.hash || existingLink.multihash + const cid = new CID(hash) // child existed, fetch it - ipfs.dag.get(new CID(hash), (error, result) => { - log(`Loaded ${bs58.encode(result.value.multihash)} from ${bs58.encode(hash)}`) + ipfs.dag.get(cid, (error, result) => { + if (error) { + return done(error) + } + + const node = result.value + const child = { name: pathSegment, - node: result && result.value, + node, parent: parent } trail.push(child) - done(error, child) + done(null, child) }) }, cb) } ], done) } - ], (error, trail) => { - if (!error) { - let node = trail - let path = [] - - while (node) { - path.push(`${node.name} ${bs58.encode(node.node.multihash)}`) - node = node.parent - } - - log('Path:') - - path - .reverse() - .forEach((segment) => log(segment)) - } - - callback(error, trail) - }) + ], callback) } module.exports = traverseTo diff --git a/src/core/utils/update-mfs-root.js b/src/core/utils/update-mfs-root.js index 2d33b92..3dffdf5 100644 --- a/src/core/utils/update-mfs-root.js +++ b/src/core/utils/update-mfs-root.js @@ -1,8 +1,8 @@ 'use strict' -const bs58 = require('bs58') const log = require('debug')('ipfs:mfs:utils:update-mfs:root') const waterfall = require('async/waterfall') +const CID = require('cids') const { MFS_ROOT_KEY } = require('./constants') @@ -15,11 +15,9 @@ const updateMfsRoot = (ipfs, buffer, callback) => { return callback(new Error('Please run jsipfs init first')) } - if (typeof buffer === 'string' || buffer instanceof String) { - buffer = bs58.decode(buffer) - } + const cid = new CID(buffer) - log(`New MFS root will be ${bs58.encode(buffer)}`) + log(`New MFS root will be ${cid.toBaseEncodedString()}`) waterfall([ (cb) => { @@ -31,8 +29,8 @@ const updateMfsRoot = (ipfs, buffer, callback) => { log('Datastore was already open') cb() }, - (cb) => datastore.put(MFS_ROOT_KEY, buffer, (error) => cb(error)) - ], (error) => callback(error, buffer)) + (cb) => datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => cb(error)) + ], (error) => callback(error, cid)) } module.exports = updateMfsRoot diff --git a/src/core/utils/with-mfs-root.js b/src/core/utils/with-mfs-root.js index b71737f..4235a32 100644 --- a/src/core/utils/with-mfs-root.js +++ b/src/core/utils/with-mfs-root.js @@ -1,6 +1,5 @@ 'use strict' -const bs58 = require('bs58') const CID = require('cids') const log = require('debug')('ipfs:mfs:utils:with-mfs-root') const waterfall = require('async/waterfall') @@ -32,21 +31,21 @@ const withMfsRoot = (ipfs, callback) => { path: '/' }, next), // Turn the hash into a Buffer - ([{hash}], next) => next(null, bs58.decode(hash)), - (buffer, next) => repo.closed ? datastore.open((error) => next(error, buffer)) : next(null, buffer), + ([{hash}], next) => next(null, new CID(hash)), + (cid, next) => repo.closed ? datastore.open((error) => next(error, cid)) : next(null, cid), // Store the Buffer in the datastore - (buffer, next) => datastore.put(MFS_ROOT_KEY, buffer, (error) => next(error, buffer)) + (cid, next) => datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => next(error, cid)) ], cb) } - cb(error, result) + cb(error, new CID(result)) }) }, // Turn the Buffer into a CID - (hash, cb) => { - log(`Fetched MFS root ${bs58.encode(hash)}`) + (cid, cb) => { + log(`Fetched MFS root ${cid.toBaseEncodedString()}`) - cb(null, new CID(hash)) + cb(null, cid) } // Invoke the API function with the root CID ], callback) diff --git a/src/core/write/import-node.js b/src/core/write/import-node.js index ee4affe..cac7d9c 100644 --- a/src/core/write/import-node.js +++ b/src/core/write/import-node.js @@ -5,8 +5,6 @@ const pull = require('pull-stream/pull') const values = require('pull-stream/sources/values') const collect = require('pull-stream/sinks/collect') const importer = require('ipfs-unixfs-engine').importer -const bs58 = require('bs58') -const log = require('debug')('ipfs:mfs:import-node') const { loadNode } = require('../utils') @@ -19,19 +17,16 @@ const importStream = (ipfs, source, options, callback) => { }]), importer(ipfs._ipld, { progress: options.progress, - hashAlg: options.hash, + hashAlg: options.hashAlg, cidVersion: options.cidVersion, strategy: options.strategy, - rawLeafNodes: options.rawLeafNodes, - reduceSingleLeafToSelf: options.reduceSingleLeafToSelf + rawLeaves: options.rawLeaves, + reduceSingleLeafToSelf: options.reduceSingleLeafToSelf, + leafType: options.leafType }), collect(cb) ), - (results, cb) => { - log(`Imported file ${bs58.encode(results[0].multihash)}`) - - return loadNode(ipfs, results[0], cb) - } + (results, cb) => loadNode(ipfs, results[0], cb) ], callback) } diff --git a/src/core/write/index.js b/src/core/write/index.js index 52189e7..01e807a 100644 --- a/src/core/write/index.js +++ b/src/core/write/index.js @@ -14,37 +14,31 @@ const { } = require('../utils') const values = require('pull-stream/sources/values') const log = require('debug')('ipfs:mfs:write') -const bs58 = require('bs58') const importNode = require('./import-node') const updateNode = require('./update-node') const toPull = require('stream-to-pull-stream') const isStream = require('is-stream') -const isNode = require('detect-node') const fileReaderStream = require('filereader-stream') const isPullStream = require('is-pull-stream') const cat = require('pull-cat') const pull = require('pull-stream/pull') - -let fs - -if (isNode) { - fs = require('fs') -} +const fs = require('fs') const defaultOptions = { offset: 0, // the offset in the file to begin writing length: undefined, // how many bytes from the incoming buffer to write create: false, // whether to create the file if it does not exist truncate: false, // whether to truncate the file first - rawLeafNodes: true, + rawLeaves: false, reduceSingleLeafToSelf: false, - cidVersion: undefined, + cidVersion: 0, hashAlg: 'sha2-256', format: 'dag-pb', parents: false, // whether to create intermediate directories if they do not exist progress: undefined, strategy: 'trickle', - flush: true + flush: true, + leafType: 'raw' } const toPullSource = (content, options, callback) => { @@ -160,10 +154,11 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback) }, null) if (existingChild) { - log('Updating linked DAGNode', bs58.encode(existingChild.multihash)) + const cid = new CID(existingChild.multihash) + log(`Updating linked DAGNode ${cid.toBaseEncodedString()}`) // overwrite the existing file or part of it, possibly truncating what's left - updateNode(ipfs, new CID(existingChild.multihash), source, options, next) + updateNode(ipfs, cid, source, options, next) } else { if (!options.create) { return next(new Error('file does not exist')) @@ -185,10 +180,7 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback) ) log('Importing file', path.name) - importNode(ipfs, source, options, (error, result) => { - log(`Imported file ${path.name} ${bs58.encode(result.multihash)}`) - next(error, result) - }) + importNode(ipfs, source, options, next) } }, @@ -205,12 +197,6 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback) // Store new containing folder CID containingFolder.node = newContaingFolder - log(`New CID for the containing folder is ${bs58.encode(newContaingFolder.multihash)}`) - - newContaingFolder.links.forEach(link => { - log(`${link.name} ${bs58.encode(link.multihash)}`) - }) - next(error) }), diff --git a/src/core/write/truncate-node.js b/src/core/write/truncate-node.js index 73094ff..e2a306b 100644 --- a/src/core/write/truncate-node.js +++ b/src/core/write/truncate-node.js @@ -29,8 +29,9 @@ const truncateNode = (ipfs, dagNode, newLength, options, callback) => { hashAlg: options.hash, cidVersion: options.cidVersion, strategy: options.strategy, - rawLeafNodes: true, - reduceSingleLeafToSelf: false + rawLeaves: options.rawLeaves, + reduceSingleLeafToSelf: options.reduceSingleLeafToSelf, + leafType: options.leafType }), collect(cb) ) diff --git a/src/core/write/update-tree.js b/src/core/write/update-tree.js index ba02614..75f8315 100644 --- a/src/core/write/update-tree.js +++ b/src/core/write/update-tree.js @@ -9,10 +9,10 @@ const collect = require('pull-stream/sinks/collect') const paramap = require('pull-paramap') const log = require('debug')('ipfs:mfs:write:update-tree') const UnixFs = require('ipfs-unixfs') +const CID = require('cids') const { unmarshal } = UnixFs -const bs58 = require('bs58') const { leafFirst } = require('pull-traverse') @@ -83,7 +83,7 @@ const updateTree = (ipfs, root, fileSize, streamStart, streamEnd, source, option updatedRoot = updatedRoot.node } - log(`Updated root is ${bs58.encode(updatedRoot.multihash)}`) + log(`Updated root is ${new CID(updatedRoot.multihash).toBaseEncodedString()}`) } cb(error, updatedRoot) @@ -186,8 +186,6 @@ const updateTree = (ipfs, root, fileSize, streamStart, streamEnd, source, option // Create a DAGNode with the new data (nodeData, cb) => createNode(ipfs, nodeData, [], options, cb), (newNode, cb) => { - log(`Created DAGNode with new data with hash ${bs58.encode(newNode.multihash)} to replace ${bs58.encode(node.multihash)}`) - // Store the CID and friends so we can update it's parent's links cb(null, { parent: parent, diff --git a/src/http/write.js b/src/http/write.js index 40429e0..6e3b905 100644 --- a/src/http/write.js +++ b/src/http/write.js @@ -103,7 +103,7 @@ const mfsWrite = (api) => { length: Joi.number().integer().min(0), create: Joi.boolean().default(false), truncate: Joi.boolean().default(false), - rawLeaves: Joi.boolean().default(true), + rawLeaves: Joi.boolean().default(false), cidVersion: Joi.number().integer().valid([ 0, 1 diff --git a/test/cp.spec.js b/test/cp.spec.js index b1ab116..85c002b 100644 --- a/test/cp.spec.js +++ b/test/cp.spec.js @@ -4,11 +4,11 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const bufferStream = require('./fixtures/buffer-stream') const { - createMfs -} = require('./fixtures') + createMfs, + bufferStream +} = require('./helpers') describe('cp', function () { this.timeout(30000) diff --git a/test/flush.spec.js b/test/flush.spec.js index ee1e441..25f375a 100644 --- a/test/flush.spec.js +++ b/test/flush.spec.js @@ -3,7 +3,7 @@ const { createMfs -} = require('./fixtures') +} = require('./helpers') describe('flush', function () { this.timeout(30000) diff --git a/test/fixtures/buffer-stream.js b/test/helpers/buffer-stream.js similarity index 100% rename from test/fixtures/buffer-stream.js rename to test/helpers/buffer-stream.js diff --git a/test/helpers/collect-leaf-cids.js b/test/helpers/collect-leaf-cids.js new file mode 100644 index 0000000..c4c876b --- /dev/null +++ b/test/helpers/collect-leaf-cids.js @@ -0,0 +1,40 @@ +'use strict' + +const pull = require('pull-stream') +const traverse = require('pull-traverse') +const CID = require('cids') + +module.exports = (ipfs, multihash) => { + return new Promise((resolve, reject) => { + pull( + traverse.depthFirst(new CID(multihash), (cid) => { + return pull( + pull.values([cid]), + pull.asyncMap((cid, callback) => { + ipfs.dag.get(cid, (error, result) => { + callback(error, !error && result.value) + }) + }), + pull.asyncMap((node, callback) => { + if (!node.links) { + return callback() + } + + return callback( + null, node.links.map(link => new CID(link.multihash)) + ) + }), + pull.filter(Boolean), + pull.flatten() + ) + }), + pull.collect((error, cids) => { + if (error) { + return reject(error) + } + + resolve(cids) + }) + ) + }) +} diff --git a/test/fixtures/index.js b/test/helpers/index.js similarity index 91% rename from test/fixtures/index.js rename to test/helpers/index.js index 9508a63..84e761f 100644 --- a/test/fixtures/index.js +++ b/test/helpers/index.js @@ -38,5 +38,7 @@ const createMfs = promisify((cb) => { module.exports = { createMfs, + bufferStream: require('./buffer-stream'), + collectLeafCids: require('./collect-leaf-cids'), EMPTY_DIRECTORY_HASH: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' } diff --git a/test/ls.spec.js b/test/ls.spec.js index 743b761..3236184 100644 --- a/test/ls.spec.js +++ b/test/ls.spec.js @@ -10,7 +10,7 @@ const { const { createMfs -} = require('./fixtures') +} = require('./helpers') describe('ls', function () { this.timeout(30000) diff --git a/test/mkdir.spec.js b/test/mkdir.spec.js index eb701fa..e3f7ea8 100644 --- a/test/mkdir.spec.js +++ b/test/mkdir.spec.js @@ -6,7 +6,7 @@ chai.use(require('dirty-chai')) const expect = chai.expect const { createMfs -} = require('./fixtures') +} = require('./helpers') describe('mkdir', function () { this.timeout(30000) diff --git a/test/mv.spec.js b/test/mv.spec.js index ea39601..c5dfb95 100644 --- a/test/mv.spec.js +++ b/test/mv.spec.js @@ -4,11 +4,10 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const bufferStream = require('./fixtures/buffer-stream') - const { - createMfs -} = require('./fixtures') + createMfs, + bufferStream +} = require('./helpers') describe('mv', function () { this.timeout(30000) diff --git a/test/read.spec.js b/test/read.spec.js index 8db372e..c319ea9 100644 --- a/test/read.spec.js +++ b/test/read.spec.js @@ -6,13 +6,14 @@ chai.use(require('dirty-chai')) const expect = chai.expect const path = require('path') const loadFixture = require('aegir/fixtures') -const bufferStream = require('./fixtures/buffer-stream') +const { + bufferStream +} = require('./helpers') const pull = require('pull-stream/pull') const collect = require('pull-stream/sinks/collect') - const { createMfs -} = require('./fixtures') +} = require('./helpers') describe('read', function () { this.timeout(30000) @@ -63,7 +64,7 @@ describe('read', function () { data = Buffer.concat([data, buffer]) }) - stream.on('end', (buffer) => { + stream.on('end', () => { resolve(data) }) diff --git a/test/rm.spec.js b/test/rm.spec.js index 33ee283..b61e8e2 100644 --- a/test/rm.spec.js +++ b/test/rm.spec.js @@ -4,12 +4,10 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const bufferStream = require('./fixtures/buffer-stream') - const { - createMfs -} = require('./fixtures') - + createMfs, + bufferStream +} = require('./helpers') const { FILE_SEPARATOR } = require('../src/core/utils') diff --git a/test/stat.spec.js b/test/stat.spec.js index 01d6bcd..7c902ca 100644 --- a/test/stat.spec.js +++ b/test/stat.spec.js @@ -10,7 +10,7 @@ const loadFixture = require('aegir/fixtures') const { createMfs, EMPTY_DIRECTORY_HASH -} = require('./fixtures') +} = require('./helpers') describe('stat', function () { this.timeout(30000) diff --git a/test/write.spec.js b/test/write.spec.js index 8a8fa92..dd934f4 100644 --- a/test/write.spec.js +++ b/test/write.spec.js @@ -8,7 +8,11 @@ const path = require('path') const loadFixture = require('aegir/fixtures') const isNode = require('detect-node') const values = require('pull-stream/sources/values') -const bufferStream = require('./fixtures/buffer-stream') +const { + bufferStream, + collectLeafCids, + createMfs +} = require('./helpers') let fs @@ -16,10 +20,6 @@ if (isNode) { fs = require('fs') } -const { - createMfs -} = require('./fixtures') - describe('write', function () { this.timeout(30000) @@ -378,6 +378,23 @@ describe('write', function () { }) }) + runTest(({type, path, content}) => { + it(`writes a file with raw blocks for newly created leaf nodes (${type})`, () => { + return mfs.write(path, content, { + create: true, + rawLeaves: true + }) + .then(() => mfs.stat(path)) + .then((stats) => collectLeafCids(mfs.node, stats.hash)) + .then((cids) => { + const rawNodes = cids + .filter(cid => cid.codec === 'raw') + + expect(rawNodes).to.not.be.empty() + }) + }) + }) + it('supports concurrent writes', function () { const files = [] @@ -443,10 +460,6 @@ describe('write', function () { }) }) - it.skip('writes a file with raw blocks for newly created leaf nodes', () => { - - }) - it.skip('writes a file with a different CID version to the parent', () => { })