From 38679294a5baa3c69a9da1b9c0915f9977525461 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 24 Nov 2021 20:12:51 +1100 Subject: [PATCH 1/2] chore(dag): replace custom dag walk with multiformats/traversal Ref: https://github.com/multiformats/js-multiformats/pull/127 --- packages/interface-ipfs-core/package.json | 2 +- packages/ipfs-cli/package.json | 2 +- packages/ipfs-core-types/package.json | 2 +- packages/ipfs-core-utils/package.json | 2 +- packages/ipfs-core/package.json | 2 +- .../ipfs-core/src/components/dag/export.js | 73 +++++++------------ packages/ipfs-core/src/version.js | 4 +- packages/ipfs-grpc-client/package.json | 2 +- packages/ipfs-http-client/package.json | 2 +- packages/ipfs-http-gateway/package.json | 2 +- packages/ipfs-http-server/package.json | 2 +- .../ipfs-message-port-client/package.json | 2 +- .../ipfs-message-port-protocol/package.json | 2 +- .../ipfs-message-port-server/package.json | 2 +- 14 files changed, 40 insertions(+), 61 deletions(-) diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index e980ee4b25..de16123bc9 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -93,7 +93,7 @@ "libp2p-crypto": "^0.19.7", "libp2p-websockets": "^0.16.2", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "nanoid": "^3.1.23", "native-abort-controller": "^1.0.3", "p-map": "^4.0.0", diff --git a/packages/ipfs-cli/package.json b/packages/ipfs-cli/package.json index 62931f433d..ed546950fd 100644 --- a/packages/ipfs-cli/package.json +++ b/packages/ipfs-cli/package.json @@ -86,7 +86,7 @@ "mafmt": "^10.0.0", "multiaddr": "^10.0.0", "multiaddr-to-uri": "^8.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "parse-duration": "^1.0.0", "pretty-bytes": "^5.4.1", "progress": "^2.0.3", diff --git a/packages/ipfs-core-types/package.json b/packages/ipfs-core-types/package.json index 7eafbbda21..520122ec22 100644 --- a/packages/ipfs-core-types/package.json +++ b/packages/ipfs-core-types/package.json @@ -46,7 +46,7 @@ "dependencies": { "interface-datastore": "^6.0.2", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1", diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index dbd9517792..98a7e3b76f 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -127,7 +127,7 @@ "merge-options": "^3.0.4", "multiaddr": "^10.0.0", "multiaddr-to-uri": "^8.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "nanoid": "^3.1.23", "parse-duration": "^1.0.0", "timeout-abort-controller": "^1.1.1", diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 0c2bec7bf7..d6beba4b8e 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -121,7 +121,7 @@ "mortice": "^2.0.0", "multiaddr": "^10.0.0", "multiaddr-to-uri": "^8.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "native-abort-controller": "^1.0.3", "pako": "^1.0.2", "parse-duration": "^1.0.0", diff --git a/packages/ipfs-core/src/components/dag/export.js b/packages/ipfs-core/src/components/dag/export.js index f601dbbade..b69a495959 100644 --- a/packages/ipfs-core/src/components/dag/export.js +++ b/packages/ipfs-core/src/components/dag/export.js @@ -1,11 +1,11 @@ import { CID } from 'multiformats/cid' import { createUnsafe } from 'multiformats/block' -import { base58btc } from 'multiformats/bases/base58' import { CarWriter } from '@ipld/car/writer' import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option' import debug from 'debug' import * as raw from 'multiformats/codecs/raw' import * as json from 'multiformats/codecs/json' +import { walk } from 'multiformats/traversal' const log = debug('ipfs:components:dag:import') @@ -23,6 +23,11 @@ const NO_LINKS_CODECS = [ * @typedef {import('ipfs-core-types/src/utils').AbortOptions} AbortOptions */ +/** + * @template T + * @typedef {import('multiformats/block').Block} Block + */ + /** * @param {Object} config * @param {IPFSRepo} config.repo @@ -53,13 +58,9 @@ export function createExport ({ repo, preload, codecs }) { let err = null ;(async () => { try { - await traverseWrite( - repo, - { signal: options.signal, timeout: options.timeout }, - cid, - writer, - codecs) - writer.close() + const load = makeLoader(repo, writer, { signal: options.signal, timeout: options.timeout }, codecs) + await walk({ cid, load }) + await writer.close() } catch (/** @type {any} */ e) { err = e } @@ -80,52 +81,30 @@ export function createExport ({ repo, preload, codecs }) { } /** + * @template T * @param {IPFSRepo} repo - * @param {AbortOptions} options - * @param {CID} cid * @param {BlockWriter} writer + * @param {AbortOptions} options * @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs - * @param {Set} seen - * @returns {Promise} + * @returns {(cid:CID)=>Promise|null>} */ -async function traverseWrite (repo, options, cid, writer, codecs, seen = new Set()) { - const b58Cid = cid.toString(base58btc) - if (seen.has(b58Cid)) { - return - } +function makeLoader (repo, writer, options, codecs) { + return async (cid) => { + const codec = await codecs.getCodec(cid.code) - const block = await getBlock(repo, options, cid, codecs) + if (!codec) { + throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`) + } - log(`Adding block ${cid} to car`) - await writer.put(block) - seen.add(b58Cid) + const bytes = await repo.blocks.get(cid, options) - // recursive traversal of all links - for (const link of block.links) { - await traverseWrite(repo, options, link, writer, codecs, seen) - } -} + log(`Adding block ${cid} to car`) + await writer.put({ cid, bytes }) -/** - * @param {IPFSRepo} repo - * @param {AbortOptions} options - * @param {CID} cid - * @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs - * @returns {Promise<{cid:CID, bytes:Uint8Array, links:CID[]}>} - */ -async function getBlock (repo, options, cid, codecs) { - const bytes = await repo.blocks.get(cid, options) - - /** @type {CID[]} */ - let links = [] - const codec = await codecs.getCodec(cid.code) - - if (codec) { - const block = createUnsafe({ bytes, cid, codec }) - links = [...block.links()].map((l) => l[1]) - } else if (!NO_LINKS_CODECS.includes(cid.code)) { - throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`) - } + if (NO_LINKS_CODECS.includes(cid.code)) { + return null // skip this block, no need to look inside + } - return { cid, bytes, links } + return createUnsafe({ bytes, cid, codec }) + } } diff --git a/packages/ipfs-core/src/version.js b/packages/ipfs-core/src/version.js index 422ff78580..55230f8d72 100644 --- a/packages/ipfs-core/src/version.js +++ b/packages/ipfs-core/src/version.js @@ -1,4 +1,4 @@ -export const ipfsCore = '0.12.1' +export const ipfsCore = '0.12.2' export const commit = '' -export const interfaceIpfsCore = '^0.152.1' +export const interfaceIpfsCore = '^0.152.2' diff --git a/packages/ipfs-grpc-client/package.json b/packages/ipfs-grpc-client/package.json index 55d92d9acd..6a57d616c4 100644 --- a/packages/ipfs-grpc-client/package.json +++ b/packages/ipfs-grpc-client/package.json @@ -61,7 +61,7 @@ "it-first": "^1.0.4", "it-pushable": "^1.4.2", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "p-defer": "^3.0.0", "protobufjs": "^6.10.2", "wherearewe": "^1.0.0", diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 8f52369b0f..aea7c3783b 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -67,7 +67,7 @@ "it-last": "^1.0.4", "merge-options": "^3.0.4", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "native-abort-controller": "^1.0.3", "parse-duration": "^1.0.0", "stream-to-it": "^0.2.2", diff --git a/packages/ipfs-http-gateway/package.json b/packages/ipfs-http-gateway/package.json index 3d3dfeef87..5cc3254950 100644 --- a/packages/ipfs-http-gateway/package.json +++ b/packages/ipfs-http-gateway/package.json @@ -69,7 +69,7 @@ "it-last": "^1.0.4", "it-to-stream": "^1.0.0", "joi": "^17.2.1", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "uint8arrays": "^3.0.0", "uri-to-multiaddr": "^6.0.0" }, diff --git a/packages/ipfs-http-server/package.json b/packages/ipfs-http-server/package.json index 81d7dc4025..80faeb0c75 100644 --- a/packages/ipfs-http-server/package.json +++ b/packages/ipfs-http-server/package.json @@ -72,7 +72,7 @@ "joi": "^17.2.1", "just-safe-set": "^2.2.1", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "native-abort-controller": "^1.0.3", "parse-duration": "^1.0.0", "stream-to-it": "^0.2.2", diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index d92a5e2233..3059578397 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -51,7 +51,7 @@ "ipfs-message-port-protocol": "^0.10.4", "ipfs-unixfs": "^6.0.3", "it-peekable": "^1.0.2", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1", diff --git a/packages/ipfs-message-port-protocol/package.json b/packages/ipfs-message-port-protocol/package.json index 4d7f164648..7304d2505c 100644 --- a/packages/ipfs-message-port-protocol/package.json +++ b/packages/ipfs-message-port-protocol/package.json @@ -74,7 +74,7 @@ }, "dependencies": { "ipfs-core-types": "^0.8.4", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1", diff --git a/packages/ipfs-message-port-server/package.json b/packages/ipfs-message-port-server/package.json index 3f8e46a5dd..b095efd15f 100644 --- a/packages/ipfs-message-port-server/package.json +++ b/packages/ipfs-message-port-server/package.json @@ -70,7 +70,7 @@ "ipfs-core-types": "^0.8.4", "ipfs-message-port-protocol": "^0.10.4", "it-all": "^1.0.4", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1", From ac4f5ca395597d02dd2caf3106ddff07c3d0ce27 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 28 Jan 2022 16:08:33 +0000 Subject: [PATCH 2/2] chore: remove extra close --- packages/ipfs-core/src/components/dag/export.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/ipfs-core/src/components/dag/export.js b/packages/ipfs-core/src/components/dag/export.js index 77876fa77f..9d559bf7b5 100644 --- a/packages/ipfs-core/src/components/dag/export.js +++ b/packages/ipfs-core/src/components/dag/export.js @@ -63,7 +63,6 @@ export function createExport ({ repo, preload, codecs }) { timeout: options.timeout }, codecs) await walk({ cid, load }) - await writer.close() } catch (/** @type {any} */ e) { err = e } finally {