diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index e0fa46448b..fde5b2c526 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -49,8 +49,8 @@ "err-code": "^3.0.1", "interface-blockstore": "^1.0.0", "ipfs-core-types": "^0.7.0", - "ipfs-unixfs": "^5.0.0", - "ipfs-unixfs-importer": "^8.0.2", + "ipfs-unixfs": "^6.0.3", + "ipfs-unixfs-importer": "^9.0.3", "ipfs-utils": "^8.1.4", "ipns": "^0.13.3", "is-ipfs": "^6.0.1", diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 080f9d61da..351ee2744a 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -46,7 +46,7 @@ "browser-readablestream-to-it": "^1.0.1", "err-code": "^3.0.1", "ipfs-core-types": "^0.7.0", - "ipfs-unixfs": "^5.0.0", + "ipfs-unixfs": "^6.0.3", "ipfs-utils": "^8.1.4", "it-all": "^1.0.4", "it-map": "^1.0.4", diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index dcd7a5065f..65a05f6266 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -60,6 +60,7 @@ "@ipld/car": "^3.1.0", "@ipld/dag-cbor": "^6.0.5", "@ipld/dag-pb": "^2.1.3", + "@multiformats/murmur3": "^1.0.1", "abort-controller": "^3.0.0", "array-shuffle": "^2.0.0", "blockstore-datastore-adapter": "^1.0.0", @@ -79,9 +80,9 @@ "ipfs-core-utils": "^0.10.1", "ipfs-http-client": "^52.0.1", "ipfs-repo": "^11.0.1", - "ipfs-unixfs": "^5.0.0", - "ipfs-unixfs-exporter": "^6.0.2", - "ipfs-unixfs-importer": "^8.0.2", + "ipfs-unixfs": "^6.0.3", + "ipfs-unixfs-exporter": "^7.0.3", + "ipfs-unixfs-importer": "^9.0.3", "ipfs-utils": "^8.1.4", "ipns": "^0.13.3", "is-domain-name": "^1.0.1", diff --git a/packages/ipfs-core/src/components/files/chmod.js b/packages/ipfs-core/src/components/files/chmod.js index 92be83bedf..eea9f87a3c 100644 --- a/packages/ipfs-core/src/components/files/chmod.js +++ b/packages/ipfs-core/src/components/files/chmod.js @@ -17,8 +17,7 @@ const { recursive } = require('ipfs-unixfs-exporter') const last = require('it-last') const cp = require('./cp') const rm = require('./rm') -// @ts-ignore - TODO: refactor this so it does not require a deep require -const persist = require('ipfs-unixfs-importer/src/utils/persist') +const persist = require('./utils/persist') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') /** diff --git a/packages/ipfs-core/src/components/files/cp.js b/packages/ipfs-core/src/components/files/cp.js index 4549fc9736..b9106fd8ad 100644 --- a/packages/ipfs-core/src/components/files/cp.js +++ b/packages/ipfs-core/src/components/files/cp.js @@ -208,7 +208,8 @@ const addSourceToParent = async (context, source, childName, parent, options) => const sourceBlock = await context.repo.blocks.get(source.cid) const { node, - cid + cid, + size } = await addLink(context, { parentCid: parent.cid, size: sourceBlock.length, @@ -222,7 +223,7 @@ const addSourceToParent = async (context, source, childName, parent, options) => parent.node = node parent.cid = cid - parent.size = node.size + parent.size = size return parent } diff --git a/packages/ipfs-core/src/components/files/utils/add-link.js b/packages/ipfs-core/src/components/files/utils/add-link.js index a5568466c2..121c1220f2 100644 --- a/packages/ipfs-core/src/components/files/utils/add-link.js +++ b/packages/ipfs-core/src/components/files/utils/add-link.js @@ -1,15 +1,10 @@ 'use strict' -// @ts-ignore const dagPb = require('@ipld/dag-pb') -const { sha256, sha512 } = require('multiformats/hashes/sha2') const { CID } = require('multiformats/cid') const log = require('debug')('ipfs:mfs:core:utils:add-link') const { UnixFS } = require('ipfs-unixfs') -// @ts-ignore - refactor this to not need deep require -const DirSharded = require('ipfs-unixfs-importer/src/dir-sharded') -// @ts-ignore - refactor this to not need deep require -const defaultImporterOptions = require('ipfs-unixfs-importer/src/options') +const DirSharded = require('./dir-sharded') const { updateHamtDirectory, recreateHamtLevel, @@ -223,6 +218,11 @@ const addToShardedDirectory = async (context, options) => { shard, path } = await addFileToShardedDirectory(context, options) const result = await last(shard.flush(context.repo.blocks)) + + if (!result) { + throw new Error('No result from flushing shard') + } + const block = await context.repo.blocks.get(result.cid) const node = dagPb.decode(block) @@ -269,44 +269,24 @@ const addFileToShardedDirectory = async (context, options) => { // start at the root bucket and descend, loading nodes as we go const rootBucket = await recreateInitialHamtLevel(options.parent.Links) const node = UnixFS.unmarshal(options.parent.Data) - const importerOptions = defaultImporterOptions() - - // NOTE vmx 2021-04-01: in ipfs the hash algorithm is a constant in unixfs - // it's an implementation. Do the option conversion at the boundary between - // ipfs and unixfs. - let hasher - switch (options.hashAlg) { - case 'sha2-256': - hasher = sha256 - break - case 'sha2-512': - hasher = sha512 - break - default: - throw new Error(`TODO vmx 2021-03-31: Proper error message for unsupported hash algorithms like ${options.hashAlg}`) - } const shard = new DirSharded({ root: true, dir: true, - parent: null, - parentKey: null, + parent: undefined, + parentKey: undefined, path: '', dirty: true, flat: false, mode: node.mode - }, { - hamtHashFn: importerOptions.hamtHashFn, - hamtHashCode: importerOptions.hamtHashCode, - hamtBucketBits: importerOptions.hamtBucketBits, - hasher, - ...options - }) + }, options) shard._bucket = rootBucket if (node.mtime) { // update mtime if previously set - shard.mtime = new Date() + shard.mtime = { + secs: Math.round(Date.now() / 1000) + } } // load subshards until the bucket & position no longer changes diff --git a/packages/ipfs-core/src/components/files/utils/dir-sharded.js b/packages/ipfs-core/src/components/files/utils/dir-sharded.js new file mode 100644 index 0000000000..606e17bd30 --- /dev/null +++ b/packages/ipfs-core/src/components/files/utils/dir-sharded.js @@ -0,0 +1,259 @@ +'use strict' + +const { encode, prepare } = require('@ipld/dag-pb') +const { UnixFS } = require('ipfs-unixfs') +const persist = require('./persist') +const { createHAMT, Bucket } = require('hamt-sharding') +const { + hamtHashCode, + hamtHashFn, + hamtBucketBits +} = require('./hamt-constants') + +/** + * @typedef {import('ipfs-unixfs-importer').ImporterOptions} ImporterOptions + * @typedef {import('interface-blockstore').Blockstore} Blockstore + * @typedef {import('multiformats/cid').CID} CID + * @typedef {import('ipfs-unixfs').Mtime} Mtime + * + * @typedef {object} ImportResult + * @property {CID} cid + * @property {import('@ipld/dag-pb').PBNode} node + * @property {number} size + * + * @typedef {object} DirContents + * @property {CID} [cid] + * @property {number} [size] + * + * @typedef {object} DirOptions + * @property {Mtime} [mtime] + * @property {number} [mode] + * @property {import('multiformats/codecs/interface').BlockCodec} [codec] + * @property {import('multiformats/cid').CIDVersion} [cidVersion] + * @property {boolean} [onlyHash] + * @property {AbortSignal} [signal] + */ + +/** + * @typedef {object} DirProps + * @property {boolean} root + * @property {boolean} dir + * @property {string} path + * @property {boolean} dirty + * @property {boolean} flat + * @property {Dir} [parent] + * @property {string} [parentKey] + * @property {import('ipfs-unixfs').UnixFS} [unixfs] + * @property {number} [mode] + * @property {import('ipfs-unixfs').Mtime} [mtime] + */ +class Dir { + /** + * @param {DirProps} props + * @param {DirOptions} options + */ + constructor (props, options) { + this.options = options || {} + this.root = props.root + this.dir = props.dir + this.path = props.path + this.dirty = props.dirty + this.flat = props.flat + this.parent = props.parent + this.parentKey = props.parentKey + this.unixfs = props.unixfs + this.mode = props.mode + this.mtime = props.mtime + /** @type {CID | undefined} */ + this.cid = undefined + /** @type {number | undefined} */ + this.size = undefined + } + + /** + * @param {string} name + * @param {DirContents} value + */ + async put (name, value) { } + /** + * @param {string} name + * @returns {Promise} + */ + get (name) { + return Promise.resolve(this) + } + + /** + * @returns {AsyncIterable<{ key: string, child: DirContents}>} + */ + async * eachChildSeries () { } + /** + * @param {Blockstore} blockstore + * @returns {AsyncIterable} + */ + async * flush (blockstore) { } +} + +class DirSharded extends Dir { + /** + * @param {DirProps} props + * @param {DirOptions} options + */ + constructor (props, options) { + super(props, options) + + /** @type {Bucket} */ + this._bucket = createHAMT({ + hashFn: hamtHashFn, + bits: hamtBucketBits + }) + } + + /** + * @param {string} name + * @param {DirContents} value + */ + async put (name, value) { + await this._bucket.put(name, value) + } + + /** + * @param {string} name + */ + get (name) { + return this._bucket.get(name) + } + + childCount () { + return this._bucket.leafCount() + } + + directChildrenCount () { + return this._bucket.childrenCount() + } + + onlyChild () { + return this._bucket.onlyChild() + } + + async * eachChildSeries () { + for await (const { key, value } of this._bucket.eachLeafSeries()) { + yield { + key, + child: value + } + } + } + + /** + * @param {Blockstore} blockstore + * @returns {AsyncIterable} + */ + async * flush (blockstore) { + yield * flush(this._bucket, blockstore, this, this.options) + } +} + +module.exports = DirSharded + +/** + * @param {Bucket} bucket + * @param {Blockstore} blockstore + * @param {*} shardRoot + * @param {DirOptions} options + * @returns {AsyncIterable} + */ +async function * flush (bucket, blockstore, shardRoot, options) { + const children = bucket._children + const links = [] + let childrenSize = 0 + + for (let i = 0; i < children.length; i++) { + const child = children.get(i) + + if (!child) { + continue + } + + const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0') + + if (child instanceof Bucket) { + let shard + + for await (const subShard of await flush(child, blockstore, null, options)) { + shard = subShard + } + + if (!shard) { + throw new Error('Could not flush sharded directory, no subshard found') + } + + links.push({ + Name: labelPrefix, + Tsize: shard.size, + Hash: shard.cid + }) + childrenSize += shard.size + } else if (typeof child.value.flush === 'function') { + const dir = child.value + let flushedDir + + for await (const entry of dir.flush(blockstore)) { + flushedDir = entry + + yield flushedDir + } + + const label = labelPrefix + child.key + links.push({ + Name: label, + Tsize: flushedDir.size, + Hash: flushedDir.cid + }) + + childrenSize += flushedDir.size + } else { + const value = child.value + + if (!value.cid) { + continue + } + + const label = labelPrefix + child.key + const size = value.size + + links.push({ + Name: label, + Tsize: size, + Hash: value.cid + }) + childrenSize += size + } + } + + // go-ipfs uses little endian, that's why we have to + // reverse the bit field before storing it + const data = Uint8Array.from(children.bitField().reverse()) + const dir = new UnixFS({ + type: 'hamt-sharded-directory', + data, + fanout: bucket.tableSize(), + hashType: hamtHashCode, + mtime: shardRoot && shardRoot.mtime, + mode: shardRoot && shardRoot.mode + }) + + const node = { + Data: dir.marshal(), + Links: links + } + const buffer = encode(prepare(node)) + const cid = await persist(buffer, blockstore, options) + const size = buffer.length + childrenSize + + yield { + cid, + node, + size + } +} diff --git a/packages/ipfs-core/src/components/files/utils/hamt-constants.js b/packages/ipfs-core/src/components/files/utils/hamt-constants.js new file mode 100644 index 0000000000..28a13a3a3e --- /dev/null +++ b/packages/ipfs-core/src/components/files/utils/hamt-constants.js @@ -0,0 +1,20 @@ +'use strict' + +const { murmur3128 } = require('@multiformats/murmur3') + +module.exports = { + hamtHashCode: murmur3128.code, + hamtBucketBits: 8, + /** + * @param {Uint8Array} buf + */ + async hamtHashFn (buf) { + return (await murmur3128.encode(buf)) + // Murmur3 outputs 128 bit but, accidentally, IPFS Go's + // implementation only uses the first 64, so we must do the same + // for parity.. + .slice(0, 8) + // Invert buffer because that's how Go impl does it + .reverse() + } +} diff --git a/packages/ipfs-core/src/components/files/utils/hamt-utils.js b/packages/ipfs-core/src/components/files/utils/hamt-utils.js index 5fa18989c1..026773a8c1 100644 --- a/packages/ipfs-core/src/components/files/utils/hamt-utils.js +++ b/packages/ipfs-core/src/components/files/utils/hamt-utils.js @@ -5,14 +5,16 @@ const { Bucket, createHAMT } = require('hamt-sharding') -// @ts-ignore - refactor this to not need deep require -const DirSharded = require('ipfs-unixfs-importer/src/dir-sharded') -// @ts-ignore - refactor this to not need deep require -const defaultImporterOptions = require('ipfs-unixfs-importer/src/options') +const DirSharded = require('./dir-sharded') const log = require('debug')('ipfs:mfs:core:utils:hamt-utils') const { UnixFS } = require('ipfs-unixfs') const last = require('it-last') const { CID } = require('multiformats/cid') +const { + hamtHashCode, + hamtHashFn, + hamtBucketBits +} = require('./hamt-constants') /** * @typedef {import('multiformats/cid').CIDVersion} CIDVersion @@ -33,8 +35,6 @@ const { CID } = require('multiformats/cid') * @param {string} options.hashAlg */ const updateHamtDirectory = async (context, links, bucket, options) => { - const importerOptions = defaultImporterOptions() - if (!options.parent.Data) { throw new Error('Could not update HAMT directory because parent had no data') } @@ -46,7 +46,7 @@ const updateHamtDirectory = async (context, links, bucket, options) => { type: 'hamt-sharded-directory', data, fanout: bucket.tableSize(), - hashType: importerOptions.hamtHashCode, + hashType: hamtHashCode, mode: node.mode, mtime: node.mtime }) @@ -94,10 +94,9 @@ const recreateHamtLevel = async (links, rootBucket, parentBucket, positionAtPare * @param {PBLink[]} links */ const recreateInitialHamtLevel = async (links) => { - const importerOptions = defaultImporterOptions() const bucket = createHAMT({ - hashFn: importerOptions.hamtHashFn, - bits: importerOptions.hamtBucketBits + hashFn: hamtHashFn, + bits: hamtBucketBits }) await addLinksToHamtBucket(links, bucket, bucket) @@ -253,26 +252,17 @@ const generatePath = async (context, fileName, rootNode) => { * @param {number} [options.mode] */ const createShard = async (context, contents, options = {}) => { - const importerOptions = defaultImporterOptions() - const shard = new DirSharded({ root: true, dir: true, - parent: null, - parentKey: null, + parent: undefined, + parentKey: undefined, path: '', dirty: true, flat: false, mtime: options.mtime, mode: options.mode - }, { - hamtHashFn: importerOptions.hamtHashFn, - hamtHashCode: importerOptions.hamtHashCode, - hamtBucketBits: importerOptions.hamtBucketBits, - hasher: importerOptions.hasher, - ...options, - codec: dagPb - }) + }, options) for (let i = 0; i < contents.length; i++) { await shard._bucket.put(contents[i].name, { @@ -281,7 +271,13 @@ const createShard = async (context, contents, options = {}) => { }) } - return last(shard.flush(context.repo.blocks)) + const res = await last(shard.flush(context.repo.blocks)) + + if (!res) { + throw new Error('Flushing shard yielded no result') + } + + return res } module.exports = { diff --git a/packages/ipfs-core/src/components/files/utils/persist.js b/packages/ipfs-core/src/components/files/utils/persist.js new file mode 100644 index 0000000000..98bbeeb754 --- /dev/null +++ b/packages/ipfs-core/src/components/files/utils/persist.js @@ -0,0 +1,50 @@ +'use strict' + +const { CID } = require('multiformats/cid') +const dagPb = require('@ipld/dag-pb') +const { sha256 } = require('multiformats/hashes/sha2') + +/** + * @typedef {object} PersistOptions + * @property {import('multiformats/codecs/interface').BlockCodec} [codec] + * @property {import('multiformats/hashes/interface').MultihashHasher} [hasher] + * @property {import('multiformats/cid').CIDVersion} [cidVersion] + * @property {boolean} [onlyHash] + * @property {AbortSignal} [signal] + */ + +/** + * @param {Uint8Array} buffer + * @param {import('interface-blockstore').Blockstore} blockstore + * @param {PersistOptions} options + */ +const persist = async (buffer, blockstore, options) => { + if (!options.codec) { + options.codec = dagPb + } + + if (!options.hasher) { + options.hasher = sha256 + } + + if (options.cidVersion === undefined) { + options.cidVersion = 1 + } + + if (options.codec === dagPb && options.hasher !== sha256) { + options.cidVersion = 1 + } + + const multihash = await options.hasher.digest(buffer) + const cid = CID.create(options.cidVersion, options.codec.code, multihash) + + if (!options.onlyHash) { + await blockstore.put(cid, buffer, { + signal: options.signal + }) + } + + return cid +} + +module.exports = persist diff --git a/packages/ipfs-daemon/src/index.js b/packages/ipfs-daemon/src/index.js index 4cd255cabe..436a127167 100644 --- a/packages/ipfs-daemon/src/index.js +++ b/packages/ipfs-daemon/src/index.js @@ -73,7 +73,7 @@ class Daemon { /** * @type {import('ipfs-core').Libp2pFactoryFn} */ -async function getLibp2p ({ libp2pOptions, options, config, peerId }) { +async function getLibp2p ({ libp2pOptions }) { // Attempt to use any of the WebRTC versions available globally let electronWebRTC let wrtc diff --git a/packages/ipfs-grpc-client/package.json b/packages/ipfs-grpc-client/package.json index 6f9bd571d9..31a61d5218 100644 --- a/packages/ipfs-grpc-client/package.json +++ b/packages/ipfs-grpc-client/package.json @@ -39,7 +39,7 @@ "ipfs-core-types": "^0.7.0", "ipfs-core-utils": "^0.10.1", "ipfs-grpc-protocol": "^0.4.0", - "ipfs-unixfs": "^5.0.0", + "ipfs-unixfs": "^6.0.3", "it-first": "^1.0.4", "it-pushable": "^1.4.2", "multiaddr": "^10.0.0", diff --git a/packages/ipfs-http-server/package.json b/packages/ipfs-http-server/package.json index 61d83117ce..5f6556eda1 100644 --- a/packages/ipfs-http-server/package.json +++ b/packages/ipfs-http-server/package.json @@ -42,7 +42,7 @@ "ipfs-core-types": "^0.7.0", "ipfs-core-utils": "^0.10.1", "ipfs-http-gateway": "^0.6.0", - "ipfs-unixfs": "^5.0.0", + "ipfs-unixfs": "^6.0.3", "it-all": "^1.0.4", "it-drain": "^1.0.3", "it-filter": "^1.0.2", diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index 38126ec9eb..1eea9c66f6 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -36,7 +36,7 @@ "browser-readablestream-to-it": "^1.0.1", "ipfs-core-types": "^0.7.0", "ipfs-message-port-protocol": "^0.9.0", - "ipfs-unixfs": "^5.0.0", + "ipfs-unixfs": "^6.0.3", "multiformats": "^9.4.1" }, "devDependencies": {