From 89b0d7f2ff5d493ac908749014a597ba409c9a84 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 11 Feb 2022 16:24:16 +0100 Subject: [PATCH] fix: os-native add-to-ipfs on Windows and macOS (#1976) * fix: use new glob source API See https://github.com/ipfs/js-ipfs/pull/3889 * feat: add tests and improve add-to-ipfs --- package-lock.json | 13 +-- package.json | 1 + src/add-to-ipfs.js | 93 +++++++++++++------- test/unit/add-to-ipfs.spec.js | 71 +++++++++++++++ test/unit/fixtures/dir/within/hello-ipfs.txt | 1 + test/unit/fixtures/hello-world.txt | 1 + test/unit/mocks/electron.js | 5 +- test/unit/mocks/logger.js | 5 +- test/unit/mocks/notify.js | 8 ++ 9 files changed, 158 insertions(+), 40 deletions(-) create mode 100644 test/unit/add-to-ipfs.spec.js create mode 100644 test/unit/fixtures/dir/within/hello-ipfs.txt create mode 100644 test/unit/fixtures/hello-world.txt create mode 100644 test/unit/mocks/notify.js diff --git a/package-lock.json b/package-lock.json index 7cac384b7..3a70cf966 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "is-ipfs": "6.0.2", "it-all": "^1.0.6", "it-concat": "^2.0.0", + "it-last": "^1.0.6", "multiaddr": "10.0.1", "multiaddr-to-uri": "8.0.0", "portfinder": "^1.0.28", @@ -7708,9 +7709,9 @@ } }, "node_modules/it-last": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/it-last/-/it-last-1.0.5.tgz", - "integrity": "sha512-PV/2S4zg5g6dkVuKfgrQfN2rUN4wdTI1FzyAvU+i8RV96syut40pa2s9Dut5X7SkjwA3P0tOhLABLdnOJ0Y/4Q==" + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/it-last/-/it-last-1.0.6.tgz", + "integrity": "sha512-aFGeibeiX/lM4bX3JY0OkVCFkAw8+n9lkukkLNivbJRvNz8lI3YXv5xcqhFUV2lDJiraEK3OXRDbGuevnnR67Q==" }, "node_modules/it-map": { "version": "1.0.6", @@ -18955,9 +18956,9 @@ } }, "it-last": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/it-last/-/it-last-1.0.5.tgz", - "integrity": "sha512-PV/2S4zg5g6dkVuKfgrQfN2rUN4wdTI1FzyAvU+i8RV96syut40pa2s9Dut5X7SkjwA3P0tOhLABLdnOJ0Y/4Q==" + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/it-last/-/it-last-1.0.6.tgz", + "integrity": "sha512-aFGeibeiX/lM4bX3JY0OkVCFkAw8+n9lkukkLNivbJRvNz8lI3YXv5xcqhFUV2lDJiraEK3OXRDbGuevnnR67Q==" }, "it-map": { "version": "1.0.6", diff --git a/package.json b/package.json index d1d6097b9..9e353c479 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "is-ipfs": "6.0.2", "it-all": "^1.0.6", "it-concat": "^2.0.0", + "it-last": "^1.0.6", "multiaddr": "10.0.1", "multiaddr-to-uri": "8.0.0", "portfinder": "^1.0.28", diff --git a/src/add-to-ipfs.js b/src/add-to-ipfs.js index 57837e6ce..a8ea97069 100644 --- a/src/add-to-ipfs.js +++ b/src/add-to-ipfs.js @@ -1,14 +1,16 @@ const { extname, basename } = require('path') const { clipboard } = require('electron') +const { globSource } = require('ipfs-http-client') const i18n = require('i18next') +const last = require('it-last') +const fs = require('fs-extra') const logger = require('./common/logger') const { notify, notifyError } = require('./common/notify') -const { globSource } = require('ipfs-http-client') -async function copyFile (ipfs, cid, name) { +async function copyFileToMfs (ipfs, cid, filename) { let i = 0 - const ext = extname(name) - const base = basename(name, ext) + const ext = extname(filename) + const base = basename(filename, ext) while (true) { const newName = (i === 0 ? base : `${base} (${i})`) + ext @@ -16,50 +18,56 @@ async function copyFile (ipfs, cid, name) { try { await ipfs.files.stat(`/${newName}`) } catch (err) { - name = newName + filename = newName break } i++ } - return ipfs.files.cp(`/ipfs/${cid.toString()}`, `/${name}`) + return ipfs.files.cp(`/ipfs/${cid.toString()}`, `/${filename}`) } -async function makeShareableObject (ipfs, results) { - if (results.length === 1) { +async function getShareableCid (ipfs, files) { + if (files.length === 1) { // If it's just one object, we link it directly. - return results[0] + return files[0] } - let baseCID = await ipfs.object.new({ template: 'unixfs-dir' }) + // Note: we don't use 'object patch' here, it was deprecated. + // We are using MFS for creating CID of an ephemeral directory + // because it handles HAMT-sharding of big directories automatically + // See: https://github.com/ipfs/go-ipfs/issues/8106 + const dirpath = `/zzzz_${Date.now()}` + await ipfs.files.mkdir(dirpath, {}) - for (const { cid, path, size } of results) { - baseCID = (await ipfs.object.patch.addLink(baseCID, { - name: path, - size, - cid - })) + for (const { cid, filename } of files) { + await ipfs.files.cp(`/ipfs/${cid}`, `${dirpath}/${filename}`) } - return { cid: baseCID, path: '' } + const stat = await ipfs.files.stat(dirpath) + + // Do not wait for this + ipfs.files.rm(dirpath, { recursive: true }) + + return { cid: stat.cid, filename: '' } } -function sendNotification (failures, successes, launchWebUI, path) { +function sendNotification (launchWebUI, hasFailures, successCount, filename) { let link, title, body, fn - if (failures.length === 0) { + if (!hasFailures) { // All worked well! fn = notify - if (successes.length === 1) { - link = `/files/${path}` + if (successCount === 1) { + link = `/files/${filename}` title = i18n.t('itemAddedNotification.title') body = i18n.t('itemAddedNotification.message') } else { link = '/files' title = i18n.t('itemsAddedNotification.title') - body = i18n.t('itemsAddedNotification.message', { count: successes.length }) + body = i18n.t('itemsAddedNotification.message', { count: successCount }) } } else { // Some/all failed! @@ -75,9 +83,27 @@ function sendNotification (failures, successes, launchWebUI, path) { }) } +async function addFileOrDirectory (ipfs, filepath) { + const stat = fs.statSync(filepath) + let cid = null + + if (stat.isDirectory()) { + const files = globSource(filepath, '**/*', { recursive: true }) + const res = await last(ipfs.addAll(files, { pin: false, wrapWithDirectory: true })) + cid = res.cid + } else { + const readStream = fs.createReadStream(filepath) + const res = await ipfs.add(readStream, { pin: false }) + cid = res.cid + } + + const filename = basename(filepath) + await copyFileToMfs(ipfs, cid, filename) + return { cid, filename } +} + module.exports = async function ({ getIpfsd, launchWebUI }, files) { const ipfsd = await getIpfsd() - if (!ipfsd) { return } @@ -89,23 +115,26 @@ module.exports = async function ({ getIpfsd, launchWebUI }, files) { await Promise.all(files.map(async file => { try { - const result = await ipfsd.api.add(globSource(file, { recursive: true }), { pin: false }) - await copyFile(ipfsd.api, result.cid, result.path) - successes.push(result) + const res = await addFileOrDirectory(ipfsd.api, file) + successes.push(res) } catch (e) { - failures.push(e) + failures.push(e.toString()) } })) if (failures.length > 0) { - log.fail(new Error(failures.reduce((prev, curr) => `${prev} ${curr.toString()}`, ''))) + log.fail(new Error(failures.join('\n'))) } else { log.end() } - const { cid, path } = await makeShareableObject(ipfsd.api, successes) - sendNotification(failures, successes, launchWebUI, path) - const filename = path ? `?filename=${encodeURIComponent(path.split('/').pop())}` : '' - const url = `https://dweb.link/ipfs/${cid.toString()}${filename}` + const { cid, filename } = await getShareableCid(ipfsd.api, successes) + sendNotification(launchWebUI, failures.length !== 0, successes.length, filename) + + const query = filename ? `?filename=${encodeURIComponent(filename)}` : '' + const url = `https://dweb.link/ipfs/${cid.toString()}${query}` + clipboard.writeText(url) + + return cid } diff --git a/test/unit/add-to-ipfs.spec.js b/test/unit/add-to-ipfs.spec.js new file mode 100644 index 000000000..d22cee628 --- /dev/null +++ b/test/unit/add-to-ipfs.spec.js @@ -0,0 +1,71 @@ +/* eslint-env mocha */ + +const chai = require('chai') +const path = require('path') +const { expect } = chai +const dirtyChai = require('dirty-chai') + +const mockElectron = require('./mocks/electron') +const mockLogger = require('./mocks/logger') +const mockNotify = require('./mocks/notify') + +const proxyquire = require('proxyquire').noCallThru() + +const { makeRepository } = require('./../e2e/utils/ipfsd') + +chai.use(dirtyChai) + +const getFixtures = (...files) => files.map(f => path.join(__dirname, 'fixtures', f)) + +describe('Add To Ipfs', function () { + this.timeout(5000) + + let electron, notify, addToIpfs, ipfsd, ctx + + before(async () => { + const repo = await makeRepository({ start: true }) + ipfsd = repo.ipfsd + ctx = { + getIpfsd: () => ipfsd, + launchWebUI: () => {} + } + }) + + after(() => { + if (ipfsd) ipfsd.stop() + }) + + beforeEach(async () => { + electron = mockElectron() + notify = mockNotify() + addToIpfs = proxyquire('../../src/add-to-ipfs', { + electron: electron, + './common/notify': notify, + './common/logger': mockLogger() + }) + }) + + it('add to ipfs single file', async () => { + const cid = await addToIpfs(ctx, getFixtures('hello-world.txt')) + expect(electron.clipboard.writeText.callCount).to.equal(1) + expect(notify.notifyError.callCount).to.equal(0) + expect(notify.notify.callCount).to.equal(1) + expect(cid.toString()).to.equal('QmWGeRAEgtsHW3ec7U4qW2CyVy7eA2mFRVbk1nb24jFyks') + }) + + it('add to ipfs single directory', async () => { + const cid = await addToIpfs(ctx, getFixtures('dir')) + expect(electron.clipboard.writeText.callCount).to.equal(1) + expect(notify.notifyError.callCount).to.equal(0) + expect(notify.notify.callCount).to.equal(1) + expect(cid.toString()).to.equal('QmVuxXkWEyCKvQiMqVnDiwyJUUyDQZ7VsKhQDCZzPj1Yq8') + }) + + it('add to ipfs multiple files', async () => { + const cid = await addToIpfs(ctx, getFixtures('dir', 'hello-world.txt')) + expect(electron.clipboard.writeText.callCount).to.equal(1) + expect(notify.notifyError.callCount).to.equal(0) + expect(notify.notify.callCount).to.equal(1) + expect(cid.toString()).to.equal('QmdYASNGKMVK4HL1uzi3VCZyjQGg3M6VuLsgX5xTKL1gvH') + }) +}) diff --git a/test/unit/fixtures/dir/within/hello-ipfs.txt b/test/unit/fixtures/dir/within/hello-ipfs.txt new file mode 100644 index 000000000..e1c6787e0 --- /dev/null +++ b/test/unit/fixtures/dir/within/hello-ipfs.txt @@ -0,0 +1 @@ +Hello, IPFS! \ No newline at end of file diff --git a/test/unit/fixtures/hello-world.txt b/test/unit/fixtures/hello-world.txt new file mode 100644 index 000000000..5dd01c177 --- /dev/null +++ b/test/unit/fixtures/hello-world.txt @@ -0,0 +1 @@ +Hello, world! \ No newline at end of file diff --git a/test/unit/mocks/electron.js b/test/unit/mocks/electron.js index 6576e22f0..3ab471cb4 100644 --- a/test/unit/mocks/electron.js +++ b/test/unit/mocks/electron.js @@ -9,7 +9,10 @@ module.exports = function mockElectron (opts = {}) { BrowserWindow: { getAllWindows: sinon.stub() }, - app: {} + app: {}, + clipboard: { + writeText: sinon.spy() + } } if (opts.withDock) { diff --git a/test/unit/mocks/logger.js b/test/unit/mocks/logger.js index a4de9d604..ad9c7e792 100644 --- a/test/unit/mocks/logger.js +++ b/test/unit/mocks/logger.js @@ -1,6 +1,9 @@ module.exports = function () { return { - start: () => {}, + start: () => ({ + fail: () => {}, + end: () => {} + }), info: () => {}, error: () => {} } diff --git a/test/unit/mocks/notify.js b/test/unit/mocks/notify.js new file mode 100644 index 000000000..f8f4480b1 --- /dev/null +++ b/test/unit/mocks/notify.js @@ -0,0 +1,8 @@ +const sinon = require('sinon') + +module.exports = function mockNotify () { + return { + notify: sinon.spy(), + notifyError: sinon.spy() + } +}