From 0d9ede48b7f5f7c867cad30ac3743c7a5d7dacf0 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Thu, 25 Aug 2016 16:27:55 +0200 Subject: [PATCH] refactor: move to pull-streams internally Closes #403 --- package.json | 49 ++-- src/cli/commands/files/add.js | 105 ++++----- src/cli/commands/init.js | 4 +- src/core/default-repo/browser.js | 4 +- src/core/default-repo/node.js | 4 +- src/core/ipfs/block.js | 9 +- src/core/ipfs/files.js | 209 +++++++++--------- src/core/ipfs/go-online.js | 2 +- src/core/ipfs/init.js | 81 ++++--- src/core/ipfs/object.js | 6 +- src/http-api/index.js | 4 +- test/cli/test-bootstrap.js | 2 +- test/cli/test-object.js | 2 +- test/cli/test-swarm.js | 1 + test/core/both/index.js | 6 +- test/core/both/test-bitswap.js | 2 +- test/core/both/test-init.js | 2 +- test/core/browser.js | 14 +- test/core/node-only/test-init.js | 2 +- ...OCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data} | 0 ...JNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data} | Bin ...ZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data} | 0 ...74EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data} | Bin ...CX5XMZLMGTYVODWPEDIW7JYEG7YXBIA7IUWY.data} | 0 ...KVN6ALXC6QRHK2X4R6EUFRMBB5OSFO2FUYDQ.data} | 0 ...DCW7R77FRIVHQ3V5OIPPS3XQBX34KRPNOIRQ.data} | 0 ...SCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data} | 0 ...NT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data} | Bin ...DTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data} | 0 ...BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data} | 0 ...OQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data} | 0 ...TQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data} | Bin ...XLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data} | 0 ...VY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data} | Bin ...P4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data} | 0 ...6VA64TGWP67KYA3AIMBUMVWGZ5AQN2L2HSWQ.data} | Bin ...JNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data} | 0 ...LTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data} | Bin ...JNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data} | 0 ...JNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data} | 0 ...FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data} | Bin ...E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data} | 0 ...TG5JS3JCUJDQM2DRB5RVF33DCUUOFJNGVDUI.data} | Bin ...BXFD5QN6MTI5YBYMCVQJDXPKCOVR6RMLHZFQ.data} | Bin ...5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data} | Bin ...7JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data} | 0 ...YYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data} | 0 ...H7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data} | Bin ...IM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data} | 0 ...3ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data} | 0 ...NRNMP2VDKWBWGAEDDEJDACM3SGG3VIANDDXI.data} | Bin test/go-ipfs-repo/datastore/000010.ldb | Bin 0 -> 1143 bytes test/go-ipfs-repo/datastore/CURRENT | 2 +- test/go-ipfs-repo/datastore/LOG | 39 +++- test/go-ipfs-repo/datastore/LOG.old | 18 +- test/go-ipfs-repo/datastore/MANIFEST-000004 | Bin 141 -> 0 bytes test/go-ipfs-repo/datastore/MANIFEST-000007 | Bin 293 -> 0 bytes test/go-ipfs-repo/datastore/MANIFEST-000014 | Bin 0 -> 314 bytes test/go-ipfs-repo/version | 2 +- test/http-api/inject/test-block.js | 2 +- test/http-api/inject/test-object.js | 2 +- test/http-api/inject/test-swarm.js | 2 +- test/http-api/ipfs-api/test-repo.js | 2 - test/http-api/ipfs-api/test-swarm.js | 2 +- test/utils/factory-core/index.js | 4 +- test/utils/factory-http/index.js | 2 +- test/utils/temp-node.js | 19 +- test/utils/temp-repo.js | 4 +- 68 files changed, 300 insertions(+), 308 deletions(-) rename test/go-ipfs-repo/blocks/{1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data => CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data} (100%) rename test/go-ipfs-repo/blocks/{122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data => CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data} (100%) rename test/go-ipfs-repo/blocks/{122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data => CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data} (100%) rename test/go-ipfs-repo/blocks/{12203628/12203628a4a19525dd84bbbffe132ec0b0d3be3528fbbcc814feb7f2fbf71ca06162.data => CIQDM/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data} (100%) rename test/go-ipfs-repo/blocks/{12203aa9/12203aa913048b2ef582d4ee2bf6eccad869e2ae1d9e41a2df4e086fe2e1403e8a5b.data => CIQDV/CIQDVKITASFS55MC2TXCX5XMZLMGTYVODWPEDIW7JYEG7YXBIA7IUWY.data} (100%) rename test/go-ipfs-repo/blocks/{122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data => CIQEN/CIQENVCICS44LLYUDQ5KVN6ALXC6QRHK2X4R6EUFRMBB5OSFO2FUYDQ.data} (100%) rename test/go-ipfs-repo/blocks/{122048b2/122048b220016f4051d7dd74315bf1ffcb1454f0dd7ae43df2dde01bef8a8bdae446.data => CIQER/CIQERMRAAFXUAUOX3V2DCW7R77FRIVHQ3V5OIPPS3XQBX34KRPNOIRQ.data} (100%) rename test/go-ipfs-repo/blocks/{12204a5a/12204a5a95586f52e25811cf214677160e64383755a8c5163ba3c053c3b65777ed16.data => CIQEU/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data} (100%) rename test/go-ipfs-repo/blocks/{12205200/12205200cc6b6f79e1588480d9f9016ccadfda3d8bc7d2f8cdf302aa51dae8dae9da.data => CIQFE/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data} (100%) rename test/go-ipfs-repo/blocks/{122052c6/122052c63c7775396b3f82c639977a7223c2d96a9f70b5fd8b1d513f8c5b69dcaed4.data => CIQFF/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data} (100%) rename test/go-ipfs-repo/blocks/{12205994/122059948439065f29619ef41280cbb932be52c56d99c5966b65e0111239f098bbef.data => CIQFT/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data} (100%) rename test/go-ipfs-repo/blocks/{122062ce/122062ce1f2c91a13a97b596e873b5a3346a644605d7614dca53cd3a59f7385a8abb.data => CIQGF/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data} (100%) rename test/go-ipfs-repo/blocks/{12206781/122067817186b8ff365c758f387e3ae7f28fa9367ee167c312e6d65a2e02e81ab815.data => CIQGP/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data} (100%) rename test/go-ipfs-repo/blocks/{12207fb8/12207fb898b5d7be46d85feb75d894e16cfa9a7ae5533f8e997cdab2ebadd7506340.data => CIQH7/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data} (100%) rename test/go-ipfs-repo/blocks/{12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data => CIQHA/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data} (100%) rename test/go-ipfs-repo/blocks/{1220709b/1220709b2dcc5f6a90ad64d6fe3a5d202a72b057d1c7f2e682d0776a5363e2cca974.data => CIQHB/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data} (100%) rename test/go-ipfs-repo/blocks/{122077d2/122077d2a2b0fc907805a547ea83dc99acff7d5806c086068cad8d9e820dd2f4795a.data => CIQHP/CIQHPUVCWD6JA6AFUVD6VA64TGWP67KYA3AIMBUMVWGZ5AQN2L2HSWQ.data} (100%) rename test/go-ipfs-repo/blocks/{12208b87/12208b872ca4ee517608331696dd6b3e5cf3497a7845ee8f94456ccf4d1d2f6602b5.data => CIQIX/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data} (100%) rename test/go-ipfs-repo/blocks/{12209d6c/12209d6c2be50f706953479ab9df2ce3edca90b68053c00b3004b7f0accbe1e8eedf.data => CIQJ2/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data} (100%) rename test/go-ipfs-repo/blocks/{122090c0/122090c07a7795c1193510a696d1fdfc0f1e4947cff8e422610996e609dbcb976598.data => CIQJB/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data} (100%) rename test/go-ipfs-repo/blocks/{1220929a/1220929a303c39da8a0b67c09697462f687a00c638bcb580feae06452e0c1f20b4.data => CIQJF/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data} (100%) rename test/go-ipfs-repo/blocks/{1220933b/1220933b41d37fd4508cdff45930dff56baef91c7dc345e73d049ab570abe10dfbb9.data => CIQJG/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data} (100%) rename test/go-ipfs-repo/blocks/{1220a52c/1220a52c3602030cb912edfe4de97002fdadf9d45666c3be122a2efb5db93c1d5fa6.data => CIQKK/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data} (100%) rename test/go-ipfs-repo/blocks/{1220b0ab/1220b0abba9f487a9f38ed23337532da45448e0cd0e21ec6a5dec62a51c54b4d51d1.data => CIQLB/CIQLBK52T5EHVHZY5URTG5JS3JCUJDQM2DRB5RVF33DCUUOFJNGVDUI.data} (100%) rename test/go-ipfs-repo/blocks/{1220b0cb/1220b0cba7371f11461f77081b947d837cc9a3b80e182ac123bbd427563e8b167c96.data => CIQLB/CIQLBS5HG4PRCRQ7O4EBXFD5QN6MTI5YBYMCVQJDXPKCOVR6RMLHZFQ.data} (100%) rename test/go-ipfs-repo/blocks/{1220c0fc/1220c0fc6b49543d7bf04e83d2a5a7cbe72a83e80f9c7bca1abcaa42298a57a33ff5.data => CIQMB/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data} (100%) rename test/go-ipfs-repo/blocks/{1220e3b0/1220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.data => CIQOH/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data} (100%) rename test/go-ipfs-repo/blocks/{1220e586/1220e586199640e1a4c63fa38c5434b9e72dc99d23a391d3db5e1e42d00527241671.data => CIQOL/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data} (100%) rename test/go-ipfs-repo/blocks/{1220e605/1220e605408ac3f78113ac9a7fd486441317afc9f967abe2aa05e7c641f7bbe98a37.data => CIQOM/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data} (100%) rename test/go-ipfs-repo/blocks/{1220e6a0/1220e6a045864ff8569e43e4866c0af807def07b0db2f018808620eb30b980e94011.data => CIQON/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data} (100%) rename test/go-ipfs-repo/blocks/{1220ec5b/1220ec5b533a3218991f4377b8b8c2538b95dd29d31eac6433af0fb6fcd83dd80778.data => CIQOY/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data} (100%) rename test/go-ipfs-repo/blocks/{1220f1c1/1220f1c121360796b1c031b0d8b58fd546ab06c601063224601337231b75401a31dd.data => CIQPD/CIQPDQJBGYDZNMOAGGYNRNMP2VDKWBWGAEDDEJDACM3SGG3VIANDDXI.data} (100%) create mode 100644 test/go-ipfs-repo/datastore/000010.ldb delete mode 100644 test/go-ipfs-repo/datastore/MANIFEST-000004 delete mode 100644 test/go-ipfs-repo/datastore/MANIFEST-000007 create mode 100644 test/go-ipfs-repo/datastore/MANIFEST-000014 diff --git a/package.json b/package.json index 537778901c..a47ca35645 100644 --- a/package.json +++ b/package.json @@ -40,16 +40,18 @@ }, "homepage": "https://github.com/ipfs/js-ipfs#readme", "devDependencies": { - "aegir": "^8.0.0", + "aegir": "^8.0.1", "buffer-loader": "0.0.1", "chai": "^3.5.0", "expose-loader": "^0.7.1", - "form-data": "^1.0.0-rc4", + "form-data": "^1.0.1", + "fs-pull-blob-store": "^0.3.0", "gulp": "^3.9.1", "idb-plus-blob-store": "^1.1.2", - "interface-ipfs-core": "^0.14.3", + "idb-pull-blob-store": "^0.4.0", + "interface-ipfs-core": "^0.14.4", "left-pad": "^1.1.1", - "lodash": "^4.14.1", + "lodash": "^4.15.0", "ncp": "^2.0.0", "nexpect": "^0.5.0", "pre-commit": "^1.1.3", @@ -60,45 +62,50 @@ "dependencies": { "babel-runtime": "^6.11.6", "bl": "^1.1.2", - "boom": "^3.2.2", + "boom": "^4.0.0", "bs58": "^3.0.0", "debug": "^2.2.0", "detect-node": "^2.0.3", - "fs-blob-store": "^5.2.1", - "glob": "^7.0.5", + "glob": "^7.0.6", "hapi": "^15.0.3", "ipfs-api": "^8.0.3", - "ipfs-bitswap": "^0.6.0", + "ipfs-bitswap": "^0.7.0", "ipfs-block": "^0.3.0", - "ipfs-block-service": "^0.4.0", - "ipfs-merkle-dag": "^0.6.2", + "ipfs-block-service": "^0.5.0", + "ipfs-merkle-dag": "^0.7.0", "ipfs-multipart": "^0.1.0", - "ipfs-repo": "^0.8.0", + "ipfs-repo": "^0.9.0", "ipfs-unixfs": "^0.1.4", - "ipfs-unixfs-engine": "^0.10.1", + "ipfs-unixfs-engine": "^0.11.2", "isstream": "^0.1.2", "joi": "^9.0.4", - "libp2p-ipfs": "^0.12.1", - "libp2p-ipfs-browser": "^0.12.1", - "lodash.get": "^4.4.1", + "libp2p-ipfs": "^0.13.0", + "libp2p-ipfs-browser": "^0.14.0", + "lodash.get": "^4.4.2", "lodash.has": "^4.5.2", - "lodash.set": "^4.3.1", - "lodash.sortby": "^4.6.1", - "mafmt": "^2.1.1", + "lodash.set": "^4.3.2", + "lodash.sortby": "^4.7.0", + "mafmt": "^2.1.2", "map-limit": "0.0.1", - "multiaddr": "^2.0.2", + "multiaddr": "^2.0.3", "multihashes": "^0.2.2", "path-exists": "^3.0.0", "peer-book": "^0.3.0", "peer-id": "^0.7.0", - "peer-info": "^0.7.0", + "peer-info": "^0.7.1", "promisify-es6": "^1.0.1", + "pull-file": "^1.0.0", + "pull-paramap": "^1.1.6", + "pull-sort": "^1.0.0", + "pull-stream": "^3.4.5", + "pull-stream-to-stream": "^1.3.3", + "pull-zip": "^2.0.0", "read-pkg-up": "^1.0.1", - "readable-stream": "1.1.13", "run-parallel": "^1.1.6", "run-parallel-limit": "^1.0.3", "run-series": "^1.1.4", "run-waterfall": "^1.1.3", + "stream-to-pull-stream": "^1.7.0", "temp": "^0.8.3", "through2": "^2.0.1", "update-notifier": "^1.0.2", diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index bb0c10fc08..eb13ac1cb9 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,7 +8,10 @@ const fs = require('fs') const path = require('path') const glob = require('glob') const sortBy = require('lodash.sortby') -const mapLimit = require('map-limit') +const pull = require('pull-stream') +const pullFile = require('pull-file') +const paramap = require('pull-paramap') +const zip = require('pull-zip') function checkPath (inPath, recursive) { // This function is to check for the following possible inputs @@ -48,76 +51,48 @@ module.exports = { }, handler (argv) { - let inPath = checkPath(argv.file, argv.recursive) + const inPath = checkPath(argv.file, argv.recursive) + const index = inPath.lastIndexOf('/') + 1 + + utils.getIPFS((err, ipfs) => { + if (err) throw err + + glob(path.join(inPath, '/**/*'), (err, list) => { + if (err) throw err + if (list.length === 0) list = [inPath] + + pull( + zip( + pull.values(list), + pull( + pull.values(list), + paramap(fs.stat.bind(fs), 50) + ) + ), + pull.map((pair) => ({ + path: pair[0], + isDirectory: pair[1].isDirectory() + })), + pull.filter((file) => !file.isDirectory), + pull.map((file) => ({ + path: file.path.substring(index, file.path.length), + content: pullFile(file.path) + })), + ipfs.files.createAddPullStream(), + pull.map((file) => ({ + hash: file.hash, + path: file.path + })), + pull.collect((err, added) => { + if (err) throw err - utils.getIPFS(gotIPFS) - - function gotIPFS (err, ipfs) { - if (err) { - throw err - } - - glob(path.join(inPath, '/**/*'), (err, res) => { - if (err) { - throw err - } - - ipfs.files.createAddStream((err, i) => { - if (err) { - throw err - } - const added = [] - - i.on('data', (file) => { - added.push({ - hash: file.hash, - path: file.path - }) - }) - - i.on('end', () => { sortBy(added, 'path') .reverse() .map((file) => `added ${file.hash} ${file.path}`) .forEach((msg) => console.log(msg)) }) - - if (res.length === 0) { - res = [inPath] - } - - const writeToStream = (stream, element) => { - const index = inPath.lastIndexOf('/') + 1 - i.write({ - path: element.substring(index, element.length), - content: fs.createReadStream(element) - }) - } - - mapLimit(res, 50, (file, cb) => { - fs.stat(file, (err, stat) => { - if (err) { - return cb(err) - } - return cb(null, { - path: file, - isDirectory: stat.isDirectory() - }) - }) - }, (err, res) => { - if (err) { - throw err - } - - res - .filter((elem) => !elem.isDirectory) - .map((elem) => elem.path) - .forEach((elem) => writeToStream(i, elem)) - - i.end() - }) - }) + ) }) - } + }) } } diff --git a/src/cli/commands/init.js b/src/cli/commands/init.js index 24313cc217..809824a8e9 100644 --- a/src/cli/commands/init.js +++ b/src/cli/commands/init.js @@ -2,7 +2,7 @@ const IpfsRepo = require('ipfs-repo') const Ipfs = require('../../core') -const fsBlobStore = require('fs-blob-store') +const Store = require('fs-pull-blob-store') const utils = require('../utils') module.exports = { @@ -33,7 +33,7 @@ module.exports = { const path = utils.getRepoPath() const repo = new IpfsRepo(path, { - stores: fsBlobStore + stores: Store }) const ipfs = new Ipfs(repo) diff --git a/src/core/default-repo/browser.js b/src/core/default-repo/browser.js index cc568f06c5..00741dbe0d 100644 --- a/src/core/default-repo/browser.js +++ b/src/core/default-repo/browser.js @@ -1,9 +1,9 @@ 'use strict' -const idb = require('idb-plus-blob-store') +const Store = require('idb-pull-blob-store') const IPFSRepo = require('ipfs-repo') module.exports = (dir) => { const repoPath = dir || 'ipfs' - return new IPFSRepo(repoPath, {stores: idb}) + return new IPFSRepo(repoPath, {stores: Store}) } diff --git a/src/core/default-repo/node.js b/src/core/default-repo/node.js index 8d59a1db50..95afd9b92c 100644 --- a/src/core/default-repo/node.js +++ b/src/core/default-repo/node.js @@ -1,11 +1,11 @@ 'use strict' const os = require('os') -const fs = require('fs-blob-store') +const Store = require('fs-pull-blob-store') const IPFSRepo = require('ipfs-repo') const path = require('path') module.exports = (dir) => { const repoPath = dir || path.join(os.homedir(), '.ipfs') - return new IPFSRepo(repoPath, {stores: fs}) + return new IPFSRepo(repoPath, {stores: Store}) } diff --git a/src/core/ipfs/block.js b/src/core/ipfs/block.js index a3e87a6c96..751fcf0efe 100644 --- a/src/core/ipfs/block.js +++ b/src/core/ipfs/block.js @@ -7,8 +7,7 @@ module.exports = function block (self) { return { get: (hash, callback) => { hash = cleanHash(hash) - - self._blockS.getBlock(hash, callback) + self._blockS.get(hash, callback) }, put: (block, callback) => { if (Array.isArray(block)) { @@ -18,18 +17,18 @@ module.exports = function block (self) { block = new Block(block) } - self._blockS.addBlock(block, (err) => { + self._blockS.put(block, (err) => { callback(err, block) }) }, del: (hash, callback) => { hash = cleanHash(hash) - self._blockS.deleteBlock(hash, callback) + self._blockS.delete(hash, callback) }, stat: (hash, callback) => { hash = cleanHash(hash) - self._blockS.getBlock(hash, (err, block) => { + self._blockS.get(hash, (err, block) => { if (err) { return callback(err) } diff --git a/src/core/ipfs/files.js b/src/core/ipfs/files.js index 03e8689fcc..5070398b17 100644 --- a/src/core/ipfs/files.js +++ b/src/core/ipfs/files.js @@ -1,134 +1,135 @@ 'use strict' const unixfsEngine = require('ipfs-unixfs-engine') -const Importer = unixfsEngine.Importer -const Exporter = unixfsEngine.Exporter +const importer = unixfsEngine.importer +const exporter = unixfsEngine.exporter const UnixFS = require('ipfs-unixfs') -const through = require('through2') const isStream = require('isstream') const promisify = require('promisify-es6') -const Duplex = require('stream').Duplex const multihashes = require('multihashes') +const pull = require('pull-stream') +const sort = require('pull-sort') +const toStream = require('pull-stream-to-stream') +const toPull = require('stream-to-pull-stream') module.exports = function files (self) { + const createAddPullStream = () => pull( + pull.map(normalizeContent), + pull.flatten(), + importer(self._dagS), + pull.asyncMap(prepareFile.bind(null, self)) + ) + return { createAddStream: (callback) => { - const i = new Importer(self._dagS) - const ds = new Duplex({ objectMode: true }) - - ds._read = (n) => {} - ds._write = (file, enc, next) => { - i.write(file) - next() - } - - ds.end = () => { - i.end() - } - - let counter = 0 + callback(null, toStream(createAddPullStream())) + }, - i.on('data', (file) => { - counter++ - const bs58mh = multihashes.toB58String(file.multihash) - self.object.get(file.multihash, (err, node) => { - if (err) { - return ds.emit('error', err) - } - ds.push({ - path: file.path, - hash: bs58mh, - size: node.size() - }) - counter-- - }) - }) - - i.on('end', () => { - function canFinish () { - if (counter === 0) { - ds.push(null) - } else { - setTimeout(canFinish, 100) - } - } - canFinish() - }) + createAddPullStream: createAddPullStream, - callback(null, ds) - }, add: promisify((data, callback) => { - // Buffer input - if (Buffer.isBuffer(data)) { - data = [{ - path: '', - content: data - }] - } - // Readable stream input - if (isStream.isReadable(data)) { - data = [{ - path: '', - content: data - }] - } if (!callback || typeof callback !== 'function') { callback = function noop () {} } - if (!Array.isArray(data)) { - return callback(new Error('"data" must be an array of { path: string, content: Buffer|Readable } or Buffer or Readable')) - } - const i = new Importer(self._dagS) - const res = [] - - // Transform file info tuples to DAGNodes - i.pipe(through.obj((info, enc, next) => { - const bs58mh = multihashes.toB58String(info.multihash) - self._dagS.get(bs58mh, (err, node) => { - if (err) return callback(err) - var obj = { - path: info.path || bs58mh, - hash: bs58mh, - size: node.size() - } - res.push(obj) - next() - }) - }, (done) => { - callback(null, res) - })) - - data.forEach((tuple) => { - i.write(tuple) - }) - - i.end() + pull( + pull.values(normalizeContent(data)), + importer(self._dagS), + pull.asyncMap(prepareFile.bind(null, self)), + sort((a, b) => { + if (a.path < b.path) return 1 + if (a.path > b.path) return -1 + return 0 + }), + pull.collect(callback) + ) }), cat: promisify((hash, callback) => { if (typeof hash === 'function') { return callback(new Error('You must supply a multihash')) } - self._dagS.get(hash, (err, fetchedNode) => { - if (err) { - return callback(err) - } - const data = UnixFS.unmarshal(fetchedNode.data) - if (data.type === 'directory') { - callback(new Error('This dag node is a directory')) - } else { - const exportStream = Exporter(hash, self._dagS) - exportStream.once('data', (object) => { - callback(null, object.content) - }) - } - }) + + pull( + pull.values([hash]), + pull.asyncMap(self._dagS.get.bind(self._dagS)), + pull.map((node) => { + const data = UnixFS.unmarshal(node.data) + if (data.type === 'directory') { + return pull.error(new Error('This dag node is a directory')) + } + + return exporter(hash, self._dagS) + }), + pull.flatten(), + pull.collect((err, files) => { + if (err) return callback(err) + callback(null, toStream.source(files[0].content)) + }) + ) }), get: promisify((hash, callback) => { - const exportFile = Exporter(hash, self._dagS) - callback(null, exportFile) + callback(null, toStream.source(pull( + exporter(hash, self._dagS), + pull.map((file) => { + if (file.content) { + file.content = toStream.source(file.content) + file.content.pause() + } + + return file + }) + ))) + }) + } +} + +function prepareFile (self, file, cb) { + const bs58mh = multihashes.toB58String(file.multihash) + self.object.get(file.multihash, (err, node) => { + if (err) return cb(err) + + cb(null, { + path: file.path || bs58mh, + hash: bs58mh, + size: node.size() }) + }) +} + +function normalizeContent (content) { + if (!Array.isArray(content)) { + content = [content] } + + return content.map((data) => { + // Buffer input + if (Buffer.isBuffer(data)) { + data = { + path: '', + content: pull.values([data]) + } + } + + // Readable stream input + if (isStream.isReadable(data)) { + data = { + path: '', + content: toPull.source(data) + } + } + + if (data && data.content && typeof data.content !== 'function') { + if (Buffer.isBuffer(data.content)) { + data.content = pull.values([data.content]) + } + + if (isStream.isReadable(data.content)) { + data.content = toPull.source(data.content) + } + } + + return data + }) } diff --git a/src/core/ipfs/go-online.js b/src/core/ipfs/go-online.js index 9df94b6c48..985f307e27 100644 --- a/src/core/ipfs/go-online.js +++ b/src/core/ipfs/go-online.js @@ -16,7 +16,7 @@ module.exports = function goOnline (self) { self._bitswap = new Bitswap( self._libp2pNode.peerInfo, self._libp2pNode, - self._repo.datastore, + self._repo.blockstore, self._libp2pNode.peerBook ) self._bitswap.start() diff --git a/src/core/ipfs/init.js b/src/core/ipfs/init.js index 43792d3a0a..d251800b00 100644 --- a/src/core/ipfs/init.js +++ b/src/core/ipfs/init.js @@ -5,10 +5,10 @@ const BlockService = require('ipfs-block-service') const DagService = require('ipfs-merkle-dag').DAGService const path = require('path') const glob = require('glob') -const parallelLimit = require('run-parallel-limit') -const Readable = require('stream').Readable const fs = require('fs') -const Importer = require('ipfs-unixfs-engine').importer +const importer = require('ipfs-unixfs-engine').importer +const pull = require('pull-stream') +const file = require('pull-file') module.exports = function init (self) { return (opts, callback) => { @@ -20,8 +20,17 @@ module.exports = function init (self) { opts.emptyRepo = opts.emptyRepo || false opts.bits = opts.bits || 2048 + let config // Pre-set config values. - const config = JSON.parse(fs.readFileSync(path.join(__dirname, '../../init-files/default-config.json')).toString()) + try { + config = JSON.parse( + fs.readFileSync( + path.join(__dirname, '../../init-files/default-config.json') + ).toString() + ) + } catch (err) { + return callback(err) + } // Verify repo does not yet exist. self._repo.exists((err, exists) => { @@ -53,7 +62,7 @@ module.exports = function init (self) { const version = '3' self._repo.version.set(version, (err) => { - if (err) { return callback(err) } + if (err) return callback(err) writeConfig() }) @@ -62,7 +71,7 @@ module.exports = function init (self) { // Write the config to the repo. function writeConfig () { self._repo.config.set(config, (err) => { - if (err) { return callback(err) } + if (err) return callback(err) addDefaultAssets() }) @@ -73,52 +82,38 @@ module.exports = function init (self) { // Skip this step on the browser, or if emptyRepo was supplied. const isNode = require('detect-node') if (!isNode || opts.emptyRepo) { - return doneImport(null) + return callback(null, true) } const blocks = new BlockService(self._repo) const dag = new DagService(blocks) const initDocsPath = path.join(__dirname, '../../init-files/init-docs') - - const i = new Importer(dag) - i.resume() - - glob(path.join(initDocsPath, '/**/*'), (err, res) => { - if (err) { - throw err - } - const index = __dirname.lastIndexOf('/') - parallelLimit(res.map((element) => (callback) => { + const index = __dirname.lastIndexOf('/') + + pull( + pull.values([initDocsPath]), + pull.asyncMap((val, cb) => { + glob(path.join(val, '/**/*'), cb) + }), + pull.flatten(), + pull.map((element) => { const addPath = element.substring(index + 1, element.length) - if (!fs.statSync(element).isDirectory()) { - const rs = new Readable() - rs.push(fs.readFileSync(element)) - rs.push(null) - const filePair = {path: addPath, content: rs} - i.write(filePair) - } - callback() - }), 10, (err) => { - if (err) { - throw err - } - i.end() - }) - }) - - i.once('end', () => { - doneImport(null) - }) + if (fs.statSync(element).isDirectory()) return - function doneImport (err, stat) { - if (err) { - return callback(err) - } + return { + path: addPath, + content: file(element) + } + }), + pull.filter(Boolean), + importer(dag), + pull.onEnd((err) => { + if (err) return callback(err) - // All finished! - callback(null, true) - } + callback(null, true) + }) + ) } } } diff --git a/src/core/ipfs/object.js b/src/core/ipfs/object.js index f1ab133a88..85eac9033d 100644 --- a/src/core/ipfs/object.js +++ b/src/core/ipfs/object.js @@ -65,7 +65,7 @@ module.exports = function object (self) { waterfall([ (cb) => self.object.get(multihash, options, cb), (node, cb) => { - self._dagS.add(edit(node), (err) => { + self._dagS.put(edit(node), (err) => { cb(err, node) }) } @@ -82,7 +82,7 @@ module.exports = function object (self) { new: promisify((cb) => { const node = new DAGNode() - self._dagS.add(node, function (err) { + self._dagS.put(node, function (err) { if (err) { return cb(err) } @@ -119,7 +119,7 @@ module.exports = function object (self) { return cb(new Error('obj not recognized')) } - self._dagS.add(node, (err, block) => { + self._dagS.put(node, (err, block) => { if (err) { return cb(err) } diff --git a/src/http-api/index.js b/src/http-api/index.js index 91b93ed130..848d3a00ce 100644 --- a/src/http-api/index.js +++ b/src/http-api/index.js @@ -6,8 +6,8 @@ const debug = require('debug') const fs = require('fs') const path = require('path') const IPFSRepo = require('ipfs-repo') -const fsbs = require('fs-blob-store') const multiaddr = require('multiaddr') +const Store = require('fs-pull-blob-store') const log = debug('api') log.error = debug('api:error') @@ -20,7 +20,7 @@ exports = module.exports = function HttpApi (repo) { this.start = (callback) => { if (typeof repo === 'string') { - repo = new IPFSRepo(repo, {stores: fsbs}) + repo = new IPFSRepo(repo, {stores: Store}) } this.ipfs = new IPFS(repo) diff --git a/test/cli/test-bootstrap.js b/test/cli/test-bootstrap.js index ba5746b160..c58cb4c78a 100644 --- a/test/cli/test-bootstrap.js +++ b/test/cli/test-bootstrap.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/cli/test-object.js b/test/cli/test-object.js index 4be994bb49..fd4972fc87 100644 --- a/test/cli/test-object.js +++ b/test/cli/test-object.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/cli/test-swarm.js b/test/cli/test-swarm.js index c1092ea116..d673c1c170 100644 --- a/test/cli/test-swarm.js +++ b/test/cli/test-swarm.js @@ -1,3 +1,4 @@ +/* eslint max-nested-callbacks: ["error", 8] */ /* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' diff --git a/test/core/both/index.js b/test/core/both/index.js index 9c5c515fcf..dc6e823fba 100644 --- a/test/core/both/index.js +++ b/test/core/both/index.js @@ -7,11 +7,7 @@ describe('--both', () => { const tests = fs.readdirSync(__dirname) tests.filter((file) => { - if (file === 'index.js') { - return false - } else { - return true - } + return !(file === 'index.js' || file.indexOf('.') === 0) }).forEach((file) => { require('./' + file) }) diff --git a/test/core/both/test-bitswap.js b/test/core/both/test-bitswap.js index adc634b18f..a5f3bccb1a 100644 --- a/test/core/both/test-bitswap.js +++ b/test/core/both/test-bitswap.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/core/both/test-init.js b/test/core/both/test-init.js index f6a669d94a..fb566b670d 100644 --- a/test/core/both/test-init.js +++ b/test/core/both/test-init.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/core/browser.js b/test/core/browser.js index 9f5e55f6f3..02eced8342 100644 --- a/test/core/browser.js +++ b/test/core/browser.js @@ -2,8 +2,9 @@ 'use strict' const series = require('run-series') -const store = require('idb-plus-blob-store') +const Store = require('idb-pull-blob-store') const _ = require('lodash') +const pull = require('pull-stream') const repoContext = require.context('buffer!./../go-ipfs-repo', true) @@ -26,8 +27,8 @@ describe('core', function () { }) }) - const mainBlob = store('ipfs') - const blocksBlob = store('ipfs/blocks') + const mainBlob = new Store('ipfs') + const blocksBlob = new Store('ipfs/blocks') series(repoData.map((file) => (cb) => { if (_.startsWith(file.key, 'datastore/')) { @@ -38,9 +39,10 @@ describe('core', function () { const blob = blocks ? blocksBlob : mainBlob const key = blocks ? file.key.replace(/^blocks\//, '') : file.key - blob.createWriteStream({ - key: key - }).end(file.value, cb) + pull( + pull.values([file.value]), + blob.write(key, cb) + ) }), done) }) diff --git a/test/core/node-only/test-init.js b/test/core/node-only/test-init.js index 09d2d7cfe8..01edafd006 100644 --- a/test/core/node-only/test-init.js +++ b/test/core/node-only/test-init.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/go-ipfs-repo/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data b/test/go-ipfs-repo/blocks/CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data rename to test/go-ipfs-repo/blocks/CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data diff --git a/test/go-ipfs-repo/blocks/122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data b/test/go-ipfs-repo/blocks/CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data similarity index 100% rename from test/go-ipfs-repo/blocks/122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data rename to test/go-ipfs-repo/blocks/CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data diff --git a/test/go-ipfs-repo/blocks/122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data b/test/go-ipfs-repo/blocks/CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data similarity index 100% rename from test/go-ipfs-repo/blocks/122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data rename to test/go-ipfs-repo/blocks/CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data diff --git a/test/go-ipfs-repo/blocks/12203628/12203628a4a19525dd84bbbffe132ec0b0d3be3528fbbcc814feb7f2fbf71ca06162.data b/test/go-ipfs-repo/blocks/CIQDM/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/12203628/12203628a4a19525dd84bbbffe132ec0b0d3be3528fbbcc814feb7f2fbf71ca06162.data rename to test/go-ipfs-repo/blocks/CIQDM/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data diff --git a/test/go-ipfs-repo/blocks/12203aa9/12203aa913048b2ef582d4ee2bf6eccad869e2ae1d9e41a2df4e086fe2e1403e8a5b.data b/test/go-ipfs-repo/blocks/CIQDV/CIQDVKITASFS55MC2TXCX5XMZLMGTYVODWPEDIW7JYEG7YXBIA7IUWY.data similarity index 100% rename from test/go-ipfs-repo/blocks/12203aa9/12203aa913048b2ef582d4ee2bf6eccad869e2ae1d9e41a2df4e086fe2e1403e8a5b.data rename to test/go-ipfs-repo/blocks/CIQDV/CIQDVKITASFS55MC2TXCX5XMZLMGTYVODWPEDIW7JYEG7YXBIA7IUWY.data diff --git a/test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data b/test/go-ipfs-repo/blocks/CIQEN/CIQENVCICS44LLYUDQ5KVN6ALXC6QRHK2X4R6EUFRMBB5OSFO2FUYDQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/122046d4/122046d44814b9c5af141c3aaab7c05dc5e844ead5f91f12858b021eba45768b4c0e.data rename to test/go-ipfs-repo/blocks/CIQEN/CIQENVCICS44LLYUDQ5KVN6ALXC6QRHK2X4R6EUFRMBB5OSFO2FUYDQ.data diff --git a/test/go-ipfs-repo/blocks/122048b2/122048b220016f4051d7dd74315bf1ffcb1454f0dd7ae43df2dde01bef8a8bdae446.data b/test/go-ipfs-repo/blocks/CIQER/CIQERMRAAFXUAUOX3V2DCW7R77FRIVHQ3V5OIPPS3XQBX34KRPNOIRQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/122048b2/122048b220016f4051d7dd74315bf1ffcb1454f0dd7ae43df2dde01bef8a8bdae446.data rename to test/go-ipfs-repo/blocks/CIQER/CIQERMRAAFXUAUOX3V2DCW7R77FRIVHQ3V5OIPPS3XQBX34KRPNOIRQ.data diff --git a/test/go-ipfs-repo/blocks/12204a5a/12204a5a95586f52e25811cf214677160e64383755a8c5163ba3c053c3b65777ed16.data b/test/go-ipfs-repo/blocks/CIQEU/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/12204a5a/12204a5a95586f52e25811cf214677160e64383755a8c5163ba3c053c3b65777ed16.data rename to test/go-ipfs-repo/blocks/CIQEU/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data diff --git a/test/go-ipfs-repo/blocks/12205200/12205200cc6b6f79e1588480d9f9016ccadfda3d8bc7d2f8cdf302aa51dae8dae9da.data b/test/go-ipfs-repo/blocks/CIQFE/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/12205200/12205200cc6b6f79e1588480d9f9016ccadfda3d8bc7d2f8cdf302aa51dae8dae9da.data rename to test/go-ipfs-repo/blocks/CIQFE/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data diff --git a/test/go-ipfs-repo/blocks/122052c6/122052c63c7775396b3f82c639977a7223c2d96a9f70b5fd8b1d513f8c5b69dcaed4.data b/test/go-ipfs-repo/blocks/CIQFF/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data similarity index 100% rename from test/go-ipfs-repo/blocks/122052c6/122052c63c7775396b3f82c639977a7223c2d96a9f70b5fd8b1d513f8c5b69dcaed4.data rename to test/go-ipfs-repo/blocks/CIQFF/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data diff --git a/test/go-ipfs-repo/blocks/12205994/122059948439065f29619ef41280cbb932be52c56d99c5966b65e0111239f098bbef.data b/test/go-ipfs-repo/blocks/CIQFT/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data similarity index 100% rename from test/go-ipfs-repo/blocks/12205994/122059948439065f29619ef41280cbb932be52c56d99c5966b65e0111239f098bbef.data rename to test/go-ipfs-repo/blocks/CIQFT/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data diff --git a/test/go-ipfs-repo/blocks/122062ce/122062ce1f2c91a13a97b596e873b5a3346a644605d7614dca53cd3a59f7385a8abb.data b/test/go-ipfs-repo/blocks/CIQGF/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data similarity index 100% rename from test/go-ipfs-repo/blocks/122062ce/122062ce1f2c91a13a97b596e873b5a3346a644605d7614dca53cd3a59f7385a8abb.data rename to test/go-ipfs-repo/blocks/CIQGF/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data diff --git a/test/go-ipfs-repo/blocks/12206781/122067817186b8ff365c758f387e3ae7f28fa9367ee167c312e6d65a2e02e81ab815.data b/test/go-ipfs-repo/blocks/CIQGP/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data similarity index 100% rename from test/go-ipfs-repo/blocks/12206781/122067817186b8ff365c758f387e3ae7f28fa9367ee167c312e6d65a2e02e81ab815.data rename to test/go-ipfs-repo/blocks/CIQGP/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data diff --git a/test/go-ipfs-repo/blocks/12207fb8/12207fb898b5d7be46d85feb75d894e16cfa9a7ae5533f8e997cdab2ebadd7506340.data b/test/go-ipfs-repo/blocks/CIQH7/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data similarity index 100% rename from test/go-ipfs-repo/blocks/12207fb8/12207fb898b5d7be46d85feb75d894e16cfa9a7ae5533f8e997cdab2ebadd7506340.data rename to test/go-ipfs-repo/blocks/CIQH7/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data diff --git a/test/go-ipfs-repo/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data b/test/go-ipfs-repo/blocks/CIQHA/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data similarity index 100% rename from test/go-ipfs-repo/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data rename to test/go-ipfs-repo/blocks/CIQHA/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data diff --git a/test/go-ipfs-repo/blocks/1220709b/1220709b2dcc5f6a90ad64d6fe3a5d202a72b057d1c7f2e682d0776a5363e2cca974.data b/test/go-ipfs-repo/blocks/CIQHB/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220709b/1220709b2dcc5f6a90ad64d6fe3a5d202a72b057d1c7f2e682d0776a5363e2cca974.data rename to test/go-ipfs-repo/blocks/CIQHB/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data diff --git a/test/go-ipfs-repo/blocks/122077d2/122077d2a2b0fc907805a547ea83dc99acff7d5806c086068cad8d9e820dd2f4795a.data b/test/go-ipfs-repo/blocks/CIQHP/CIQHPUVCWD6JA6AFUVD6VA64TGWP67KYA3AIMBUMVWGZ5AQN2L2HSWQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/122077d2/122077d2a2b0fc907805a547ea83dc99acff7d5806c086068cad8d9e820dd2f4795a.data rename to test/go-ipfs-repo/blocks/CIQHP/CIQHPUVCWD6JA6AFUVD6VA64TGWP67KYA3AIMBUMVWGZ5AQN2L2HSWQ.data diff --git a/test/go-ipfs-repo/blocks/12208b87/12208b872ca4ee517608331696dd6b3e5cf3497a7845ee8f94456ccf4d1d2f6602b5.data b/test/go-ipfs-repo/blocks/CIQIX/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data similarity index 100% rename from test/go-ipfs-repo/blocks/12208b87/12208b872ca4ee517608331696dd6b3e5cf3497a7845ee8f94456ccf4d1d2f6602b5.data rename to test/go-ipfs-repo/blocks/CIQIX/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data diff --git a/test/go-ipfs-repo/blocks/12209d6c/12209d6c2be50f706953479ab9df2ce3edca90b68053c00b3004b7f0accbe1e8eedf.data b/test/go-ipfs-repo/blocks/CIQJ2/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data similarity index 100% rename from test/go-ipfs-repo/blocks/12209d6c/12209d6c2be50f706953479ab9df2ce3edca90b68053c00b3004b7f0accbe1e8eedf.data rename to test/go-ipfs-repo/blocks/CIQJ2/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data diff --git a/test/go-ipfs-repo/blocks/122090c0/122090c07a7795c1193510a696d1fdfc0f1e4947cff8e422610996e609dbcb976598.data b/test/go-ipfs-repo/blocks/CIQJB/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data similarity index 100% rename from test/go-ipfs-repo/blocks/122090c0/122090c07a7795c1193510a696d1fdfc0f1e4947cff8e422610996e609dbcb976598.data rename to test/go-ipfs-repo/blocks/CIQJB/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data diff --git a/test/go-ipfs-repo/blocks/1220929a/1220929a303c39da8a0b67c09697462f687a00c638bcb580feae06452e0c1f20b4.data b/test/go-ipfs-repo/blocks/CIQJF/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220929a/1220929a303c39da8a0b67c09697462f687a00c638bcb580feae06452e0c1f20b4.data rename to test/go-ipfs-repo/blocks/CIQJF/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data diff --git a/test/go-ipfs-repo/blocks/1220933b/1220933b41d37fd4508cdff45930dff56baef91c7dc345e73d049ab570abe10dfbb9.data b/test/go-ipfs-repo/blocks/CIQJG/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220933b/1220933b41d37fd4508cdff45930dff56baef91c7dc345e73d049ab570abe10dfbb9.data rename to test/go-ipfs-repo/blocks/CIQJG/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data diff --git a/test/go-ipfs-repo/blocks/1220a52c/1220a52c3602030cb912edfe4de97002fdadf9d45666c3be122a2efb5db93c1d5fa6.data b/test/go-ipfs-repo/blocks/CIQKK/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220a52c/1220a52c3602030cb912edfe4de97002fdadf9d45666c3be122a2efb5db93c1d5fa6.data rename to test/go-ipfs-repo/blocks/CIQKK/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data diff --git a/test/go-ipfs-repo/blocks/1220b0ab/1220b0abba9f487a9f38ed23337532da45448e0cd0e21ec6a5dec62a51c54b4d51d1.data b/test/go-ipfs-repo/blocks/CIQLB/CIQLBK52T5EHVHZY5URTG5JS3JCUJDQM2DRB5RVF33DCUUOFJNGVDUI.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220b0ab/1220b0abba9f487a9f38ed23337532da45448e0cd0e21ec6a5dec62a51c54b4d51d1.data rename to test/go-ipfs-repo/blocks/CIQLB/CIQLBK52T5EHVHZY5URTG5JS3JCUJDQM2DRB5RVF33DCUUOFJNGVDUI.data diff --git a/test/go-ipfs-repo/blocks/1220b0cb/1220b0cba7371f11461f77081b947d837cc9a3b80e182ac123bbd427563e8b167c96.data b/test/go-ipfs-repo/blocks/CIQLB/CIQLBS5HG4PRCRQ7O4EBXFD5QN6MTI5YBYMCVQJDXPKCOVR6RMLHZFQ.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220b0cb/1220b0cba7371f11461f77081b947d837cc9a3b80e182ac123bbd427563e8b167c96.data rename to test/go-ipfs-repo/blocks/CIQLB/CIQLBS5HG4PRCRQ7O4EBXFD5QN6MTI5YBYMCVQJDXPKCOVR6RMLHZFQ.data diff --git a/test/go-ipfs-repo/blocks/1220c0fc/1220c0fc6b49543d7bf04e83d2a5a7cbe72a83e80f9c7bca1abcaa42298a57a33ff5.data b/test/go-ipfs-repo/blocks/CIQMB/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220c0fc/1220c0fc6b49543d7bf04e83d2a5a7cbe72a83e80f9c7bca1abcaa42298a57a33ff5.data rename to test/go-ipfs-repo/blocks/CIQMB/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data diff --git a/test/go-ipfs-repo/blocks/1220e3b0/1220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.data b/test/go-ipfs-repo/blocks/CIQOH/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220e3b0/1220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.data rename to test/go-ipfs-repo/blocks/CIQOH/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data diff --git a/test/go-ipfs-repo/blocks/1220e586/1220e586199640e1a4c63fa38c5434b9e72dc99d23a391d3db5e1e42d00527241671.data b/test/go-ipfs-repo/blocks/CIQOL/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220e586/1220e586199640e1a4c63fa38c5434b9e72dc99d23a391d3db5e1e42d00527241671.data rename to test/go-ipfs-repo/blocks/CIQOL/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data diff --git a/test/go-ipfs-repo/blocks/1220e605/1220e605408ac3f78113ac9a7fd486441317afc9f967abe2aa05e7c641f7bbe98a37.data b/test/go-ipfs-repo/blocks/CIQOM/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220e605/1220e605408ac3f78113ac9a7fd486441317afc9f967abe2aa05e7c641f7bbe98a37.data rename to test/go-ipfs-repo/blocks/CIQOM/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data diff --git a/test/go-ipfs-repo/blocks/1220e6a0/1220e6a045864ff8569e43e4866c0af807def07b0db2f018808620eb30b980e94011.data b/test/go-ipfs-repo/blocks/CIQON/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220e6a0/1220e6a045864ff8569e43e4866c0af807def07b0db2f018808620eb30b980e94011.data rename to test/go-ipfs-repo/blocks/CIQON/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data diff --git a/test/go-ipfs-repo/blocks/1220ec5b/1220ec5b533a3218991f4377b8b8c2538b95dd29d31eac6433af0fb6fcd83dd80778.data b/test/go-ipfs-repo/blocks/CIQOY/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220ec5b/1220ec5b533a3218991f4377b8b8c2538b95dd29d31eac6433af0fb6fcd83dd80778.data rename to test/go-ipfs-repo/blocks/CIQOY/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data diff --git a/test/go-ipfs-repo/blocks/1220f1c1/1220f1c121360796b1c031b0d8b58fd546ab06c601063224601337231b75401a31dd.data b/test/go-ipfs-repo/blocks/CIQPD/CIQPDQJBGYDZNMOAGGYNRNMP2VDKWBWGAEDDEJDACM3SGG3VIANDDXI.data similarity index 100% rename from test/go-ipfs-repo/blocks/1220f1c1/1220f1c121360796b1c031b0d8b58fd546ab06c601063224601337231b75401a31dd.data rename to test/go-ipfs-repo/blocks/CIQPD/CIQPDQJBGYDZNMOAGGYNRNMP2VDKWBWGAEDDEJDACM3SGG3VIANDDXI.data diff --git a/test/go-ipfs-repo/datastore/000010.ldb b/test/go-ipfs-repo/datastore/000010.ldb new file mode 100644 index 0000000000000000000000000000000000000000..92077582ca4c1094d81e1c35f8703550afefa5f7 GIT binary patch literal 1143 zcmZQDV9?Jj$Sc+tQjj$4&Wq#uX{9W(pHXd=&LNx2cdi6ocRIV%d~TcDw2tFO0@nWL9`h-bL3Yk+%*afr8@TbPk^RJchgVHA9N_5eTxD41QxO=J9Z^{56JZ); z7-4K;T3DFokyM;!W}Z>zXzZ6K)WGC&@4ao?>Nz(8|F>suXs_zIpd7w&-MpLjzTew7 zw;bnNa{uX~ml~7*<|J4d{FVA*zBjGA&O}w!=F`N7tE3H@g#_I96y)_DdRxPzt0SfDVr1iIeMXL&AqQ0=U2^7|FGm(P2B50#;ggx{k5MHf;A&mO%C&EUfb~C z(50BO4E<;J@;iOi|8-%O$GnSduXnK?Eh$zxasPps6ob@BXVWP+4NN^+mzirX^ndu{ z#t*;Sx0dNGNfK9{8ZLg&Y3{=GymywrWge8Ax zb8_8$u}O-_UoIvnSEU^1zHqkmzk}Jo{lB**@XgG7Tp+=qBxht`XsT;yq-$sxVrXh* zVrgY+u4iCoXlQ9}W@#R!!6>CfIN}*~849wANg!O1*k!T*#RNJcTULFAv{=p%R0p5|p&LRGejGWK}qNY!jl|rkT zI2eSMF&Q*5DjD#yacZ@Bw0-AgWMpJzU~XdMX8?+GF*PwVGHml$WiN9uvO;p{6~^*7 zjnC_)?mL!yo-^-fogiH<)#V;6^U8XnY{lXir8n#>9VYuaw!{YA*;%2_v890VVH5k0 z=ev2D=Y>S8w`o4rzw}&TkCx-1kjg?{vrD=>%@*e)9$i}<^>(k?*#eGjDX!_SpO^_A zd~u&uKT_JeV$DC^DKqPN`vna4@g$$Li@X1c@ALF|GfrhLJ5!(9IqB?r3;R{-856yK z?v9F;+ODbh_-v(d^JI4|4f(#n-Cg|`C$%;*ad_N1xygC@-QBO+mg?V7WV_2So6|Ky~%*a4g zWP`$=5k#;l^#s5<2R<#_z`)6D!1x~m7-n+7<=&p`pgoS$2eSd>_jU&O@B%*w*d&cJk;Q9?f_ kKRGc+KP@vSwYVrhzl4#A0Sttp5(Sxg#f*#)G17Ga00?Cx+yDRo diff --git a/test/go-ipfs-repo/datastore/MANIFEST-000007 b/test/go-ipfs-repo/datastore/MANIFEST-000007 deleted file mode 100644 index 6af3b5450fb46c2d16258f242969f0d88dfa5892..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 293 zcmbOjutG$Rk&#gJcTULFAv{=p%R0p5|p&LRGe zjGPcN*cn(K5N|UhD@2zb$Y#P0Wn_a{!gQHYLO&-zIWb2+Ei)&zxF|orgpmm%D-4w= M$jmEdWQ2$T0KrCJEdT%j literal 0 HcmV?d00001 diff --git a/test/go-ipfs-repo/version b/test/go-ipfs-repo/version index 00750edc07..b8626c4cff 100644 --- a/test/go-ipfs-repo/version +++ b/test/go-ipfs-repo/version @@ -1 +1 @@ -3 +4 diff --git a/test/http-api/inject/test-block.js b/test/http-api/inject/test-block.js index 29dee5ed4c..c3bfda135a 100644 --- a/test/http-api/inject/test-block.js +++ b/test/http-api/inject/test-block.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/http-api/inject/test-object.js b/test/http-api/inject/test-object.js index 47d43bfcb7..af3e36e033 100644 --- a/test/http-api/inject/test-object.js +++ b/test/http-api/inject/test-object.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/http-api/inject/test-swarm.js b/test/http-api/inject/test-swarm.js index 5312a65eec..399c90e3f0 100644 --- a/test/http-api/inject/test-swarm.js +++ b/test/http-api/inject/test-swarm.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/http-api/ipfs-api/test-repo.js b/test/http-api/ipfs-api/test-repo.js index 2a78787a0b..13e1434b22 100644 --- a/test/http-api/ipfs-api/test-repo.js +++ b/test/http-api/ipfs-api/test-repo.js @@ -1,6 +1,4 @@ /* eslint-env mocha */ -/* eslint max-nested-callbacks: ["error", 8] */ - 'use strict' // const expect = require('chai').expect diff --git a/test/http-api/ipfs-api/test-swarm.js b/test/http-api/ipfs-api/test-swarm.js index a925f8d4dc..edfa2d79ef 100644 --- a/test/http-api/ipfs-api/test-swarm.js +++ b/test/http-api/ipfs-api/test-swarm.js @@ -1,5 +1,5 @@ -/* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ 'use strict' const expect = require('chai').expect diff --git a/test/utils/factory-core/index.js b/test/utils/factory-core/index.js index 463c36b8a2..03b95d89ad 100644 --- a/test/utils/factory-core/index.js +++ b/test/utils/factory-core/index.js @@ -46,7 +46,7 @@ function Factory () { let teardown if (isNode) { - store = require('fs-blob-store') + store = require('fs-pull-blob-store') teardown = (done) => { cleanRepo(repoPath) done() @@ -56,7 +56,7 @@ function Factory () { window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB - store = require('idb-plus-blob-store') + store = require('idb-pull-blob-store') teardown = (done) => { idb.deleteDatabase(repoPath) idb.deleteDatabase(repoPath + '/blocks') diff --git a/test/utils/factory-http/index.js b/test/utils/factory-http/index.js index 58cf2b211f..bcafd3b2a2 100644 --- a/test/utils/factory-http/index.js +++ b/test/utils/factory-http/index.js @@ -44,7 +44,7 @@ function Factory () { // set up the repo const repo = new IPFSRepo(repoPath, { - stores: require('fs-blob-store') + stores: require('fs-pull-blob-store') }) repo.teardown = (done) => { cleanRepo(repoPath) diff --git a/test/utils/temp-node.js b/test/utils/temp-node.js index 5c0321a865..adabcc0155 100644 --- a/test/utils/temp-node.js +++ b/test/utils/temp-node.js @@ -3,6 +3,7 @@ const expect = require('chai').expect const leftPad = require('left-pad') +const series = require('run-series') const IPFS = require('../../src/core') const createTempRepo = require('./temp-repo') @@ -29,16 +30,14 @@ function createTempNode (num, callback) { num = leftPad(num, 3, 0) - ipfs.init({ emptyRepo: true }, (err) => { - expect(err).to.not.exist - setAddresses(repo, num, (err) => { - expect(err).to.not.exist - - ipfs.load((err) => { - expect(err).to.not.exist - callback(null, ipfs) - }) - }) + series([ + (cb) => ipfs.init({ emptyRepo: true }, cb), + (cb) => setAddresses(repo, num, cb), + (cb) => ipfs.load(cb) + ], (err) => { + if (err) return callback(err) + callback(null, ipfs) }) } + module.exports = createTempNode diff --git a/test/utils/temp-repo.js b/test/utils/temp-repo.js index addb581ec9..47d39de6e0 100644 --- a/test/utils/temp-repo.js +++ b/test/utils/temp-repo.js @@ -12,7 +12,7 @@ function createTempRepo () { const isNode = require('detect-node') if (isNode) { - store = require('fs-blob-store') + store = require('fs-pull-blob-store') teardown = (done) => { clean(repoPath) done() @@ -22,7 +22,7 @@ function createTempRepo () { window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB - store = require('idb-plus-blob-store') + store = require('idb-pull-blob-store') teardown = (done) => { idb.deleteDatabase(repoPath) idb.deleteDatabase(repoPath + '/blocks')