From befb8e6c81e70e9d0f740ba6d948a9b8ff36bce5 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 10 May 2021 14:44:15 +0100 Subject: [PATCH 1/6] fix: ipfs get with raw blocks - tweak ipfs-cli get to handle `raw` - tweak ipfs-core mapFile to handle `raw` fixes #3682 License: MIT Signed-off-by: Oli Evans --- packages/ipfs-cli/src/commands/get.js | 10 +++++++--- packages/ipfs-cli/test/get.js | 22 +++++++++++++++++++++ packages/ipfs-core/src/utils.js | 28 +++++++++++++++------------ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/packages/ipfs-cli/src/commands/get.js b/packages/ipfs-cli/src/commands/get.js index 2d09c5c05f..4108ae0490 100644 --- a/packages/ipfs-cli/src/commands/get.js +++ b/packages/ipfs-cli/src/commands/get.js @@ -53,16 +53,20 @@ module.exports = { throw new Error(`File prefix invalid, would write to files outside of ${output}, pass --force to override`) } - if (file.type === 'file') { + if (file.type === 'file' || file.type === 'raw' ) { await fs.promises.mkdir(path.join(output, path.dirname(file.path)), { recursive: true }) await pipe( file.content, map(chunk => chunk.slice()), // BufferList to Buffer toIterable.sink(fs.createWriteStream(fullFilePath)) ) - } else { - // this is a dir + + } else if (file.type === 'dir') { await fs.promises.mkdir(fullFilePath, { recursive: true }) + + } else { + // file.type === object | identity not supported yet + throw new Error(`Unknown node type ${file.type}`) } } } diff --git a/packages/ipfs-cli/test/get.js b/packages/ipfs-cli/test/get.js index 0677d4cfb4..a6dc508cb4 100644 --- a/packages/ipfs-cli/test/get.js +++ b/packages/ipfs-cli/test/get.js @@ -46,6 +46,28 @@ describe('get', () => { await clean(outPath) }) + it('should get file from a raw block', async () => { + const cid = new CID('bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e') + ipfs.get.withArgs(cid.toString(), defaultOptions).returns([{ + type: 'raw', + path: 'bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e', + content: function * () { + yield buf + } + }]) + + const outPath = path.join(process.cwd(), cid.toString()) + await clean(outPath) + + const out = await cli(`get ${cid}`, { ipfs }) + expect(out) + .to.equal(`Saving file(s) ${cid}\n`) + + expect(fs.readFileSync(outPath)).to.deep.equal(buf) + + await clean(outPath) + }) + it('get file with output option', async () => { ipfs.get.withArgs(cid.toString(), defaultOptions).returns([{ type: 'file', diff --git a/packages/ipfs-core/src/utils.js b/packages/ipfs-core/src/utils.js index 39494dc222..9982b79227 100644 --- a/packages/ipfs-core/src/utils.js +++ b/packages/ipfs-core/src/utils.js @@ -103,28 +103,32 @@ const mapFile = (file, options = {}) => { name: file.name, depth: file.path.split('/').length, size: 0, - type: 'file' + type: file.type } - if (file.type === 'file' || file.type === 'directory') { - // @ts-ignore - TS type can't be changed from File to Directory - output.type = file.type === 'directory' ? 'dir' : 'file' - - if (file.type === 'file') { - output.size = file.unixfs.fileSize() + // @ts-ignore - TS type can't be changed from File to Directory + if (file.type === 'directory') { + output.type = 'dir' + } - if (options.includeContent) { - // @ts-expect-error - content is readonly - output.content = file.content() - } - } + if (file.type === 'file') { + output.size = file.unixfs.fileSize() + } + if (file.type === 'file' || file.type === 'directory') { output.mode = file.unixfs.mode if (file.unixfs.mtime !== undefined) { output.mtime = file.unixfs.mtime } } + + if (options.includeContent) { + if (file.type === 'file' || file.type === 'raw') { + // @ts-expect-error - content is readonly + output.content = file.content() + } + } return output } From 90f353bcc4a30535785fbd8b81985b5bcc811bad Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 10 May 2021 16:13:17 +0100 Subject: [PATCH 2/6] chore: fix type warnings License: MIT Signed-off-by: Oli Evans --- packages/ipfs-core/src/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipfs-core/src/utils.js b/packages/ipfs-core/src/utils.js index 9982b79227..c3c670ef0e 100644 --- a/packages/ipfs-core/src/utils.js +++ b/packages/ipfs-core/src/utils.js @@ -103,11 +103,11 @@ const mapFile = (file, options = {}) => { name: file.name, depth: file.path.split('/').length, size: 0, - type: file.type + type: 'file' } - // @ts-ignore - TS type can't be changed from File to Directory if (file.type === 'directory') { + // @ts-ignore - TS type can't be changed from File to Directory output.type = 'dir' } From a2c86d95acb6fcb9adce546bde8f47f6be5d40f5 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 10 May 2021 16:35:38 +0100 Subject: [PATCH 3/6] chore: satisfy the type constraints - mapFile doesn't handle all mapping object or identity types to an IPFSEntry, so throw if it finds one. This makes the cli behaviour match go-ipfs for ipfs get License: MIT Signed-off-by: Oli Evans --- packages/ipfs-cli/src/commands/get.js | 10 +++------- packages/ipfs-core/src/utils.js | 5 +++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/ipfs-cli/src/commands/get.js b/packages/ipfs-cli/src/commands/get.js index 4108ae0490..2d09c5c05f 100644 --- a/packages/ipfs-cli/src/commands/get.js +++ b/packages/ipfs-cli/src/commands/get.js @@ -53,20 +53,16 @@ module.exports = { throw new Error(`File prefix invalid, would write to files outside of ${output}, pass --force to override`) } - if (file.type === 'file' || file.type === 'raw' ) { + if (file.type === 'file') { await fs.promises.mkdir(path.join(output, path.dirname(file.path)), { recursive: true }) await pipe( file.content, map(chunk => chunk.slice()), // BufferList to Buffer toIterable.sink(fs.createWriteStream(fullFilePath)) ) - - } else if (file.type === 'dir') { - await fs.promises.mkdir(fullFilePath, { recursive: true }) - } else { - // file.type === object | identity not supported yet - throw new Error(`Unknown node type ${file.type}`) + // this is a dir + await fs.promises.mkdir(fullFilePath, { recursive: true }) } } } diff --git a/packages/ipfs-core/src/utils.js b/packages/ipfs-core/src/utils.js index c3c670ef0e..d4497a69de 100644 --- a/packages/ipfs-core/src/utils.js +++ b/packages/ipfs-core/src/utils.js @@ -96,6 +96,11 @@ const resolvePath = async function (ipld, ipfsPath, options = {}) { * @param {boolean} [options.includeContent] */ const mapFile = (file, options = {}) => { + if (file.type !== 'file' && file.type !== 'directory' && file.type !== 'raw') { + // file.type === object | identity not supported yet + throw new Error(`Unknown node type '${file.type}'`) + } + /** @type {import('ipfs-core-types/src/root').IPFSEntry} */ const output = { cid: file.cid, From 587510218ea834b661499c3fa930205dbddd13e1 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 10 May 2021 21:13:14 +0100 Subject: [PATCH 4/6] chore: add test for get with rawLeaves - add interface-core test for get with rawLeaves: true - remove cli test for "raw" as core normalises the type to "file" so it was not testing a valid conditon License: MIT Signed-off-by: Oli Evans --- packages/interface-ipfs-core/src/get.js | 13 +++++++++++++ packages/ipfs-cli/test/get.js | 22 ---------------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/packages/interface-ipfs-core/src/get.js b/packages/interface-ipfs-core/src/get.js index 4f345954d1..c1f6e7671c 100644 --- a/packages/interface-ipfs-core/src/get.js +++ b/packages/interface-ipfs-core/src/get.js @@ -86,6 +86,19 @@ module.exports = (common, options) => { expect(uint8ArrayConcat(await all(output[0].content))).to.eql(input) }) + it('should get a file added as CIDv1 with rawLeaves', async () => { + const input = uint8ArrayFromString(`TEST${Math.random()}`) + + const res = await all(importer([{ content: input }], ipfs.block, { cidVersion: 1, rawLeaves: true })) + + const cidv1 = res[0].cid + expect(cidv1.version).to.equal(1) + + const output = await all(ipfs.get(cidv1)) + expect(output[0].type).to.eql('file') + expect(uint8ArrayConcat(await all(output[0].content))).to.eql(input) + }) + it('should get a BIG file', async () => { for await (const file of ipfs.get(fixtures.bigFile.cid)) { expect(file.path).to.equal(fixtures.bigFile.cid) diff --git a/packages/ipfs-cli/test/get.js b/packages/ipfs-cli/test/get.js index a6dc508cb4..0677d4cfb4 100644 --- a/packages/ipfs-cli/test/get.js +++ b/packages/ipfs-cli/test/get.js @@ -46,28 +46,6 @@ describe('get', () => { await clean(outPath) }) - it('should get file from a raw block', async () => { - const cid = new CID('bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e') - ipfs.get.withArgs(cid.toString(), defaultOptions).returns([{ - type: 'raw', - path: 'bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e', - content: function * () { - yield buf - } - }]) - - const outPath = path.join(process.cwd(), cid.toString()) - await clean(outPath) - - const out = await cli(`get ${cid}`, { ipfs }) - expect(out) - .to.equal(`Saving file(s) ${cid}\n`) - - expect(fs.readFileSync(outPath)).to.deep.equal(buf) - - await clean(outPath) - }) - it('get file with output option', async () => { ipfs.get.withArgs(cid.toString(), defaultOptions).returns([{ type: 'file', From 42f166b3da97ed53b4e583e3344d96cc13fc764a Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Tue, 11 May 2021 10:03:05 +0100 Subject: [PATCH 5/6] chore: delint License: MIT Signed-off-by: Oli Evans --- packages/ipfs-core/src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-core/src/utils.js b/packages/ipfs-core/src/utils.js index d4497a69de..94b2f8a90d 100644 --- a/packages/ipfs-core/src/utils.js +++ b/packages/ipfs-core/src/utils.js @@ -127,7 +127,7 @@ const mapFile = (file, options = {}) => { output.mtime = file.unixfs.mtime } } - + if (options.includeContent) { if (file.type === 'file' || file.type === 'raw') { // @ts-expect-error - content is readonly From d445b91f06b0aa1669eb5bc020db537697d8ac77 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Tue, 11 May 2021 17:14:19 +0100 Subject: [PATCH 6/6] fix: provide file.size for raw blocks License: MIT Signed-off-by: Oli Evans --- packages/ipfs-core/src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-core/src/utils.js b/packages/ipfs-core/src/utils.js index 94b2f8a90d..8dbb131694 100644 --- a/packages/ipfs-core/src/utils.js +++ b/packages/ipfs-core/src/utils.js @@ -107,7 +107,7 @@ const mapFile = (file, options = {}) => { path: file.path, name: file.name, depth: file.path.split('/').length, - size: 0, + size: file.size, type: 'file' }