WIP: make things work

package.json

@@ -250,5 +250,8 @@
     "Jade Meskill <jade.meskill@gmail.com>",
     "Jacob Karlsson <jacob.karlsson95@gmail.com>",
     "noah the goodra <peterpan0413@live.com>"
-  ]
+  ],
+  "dependencies": {
+    "@mapbox/node-pre-gyp": "^1.0.0"
+  }
 }

packages/interface-ipfs-core/package.json

@@ -39,8 +39,8 @@
     "delay": "^4.4.0",
     "dirty-chai": "^2.0.1",
     "err-code": "^2.0.3",
-    "ipfs-unixfs": "^2.0.3",
-    "ipfs-unixfs-importer": "^5.0.0",
+    "ipfs-unixfs": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs?js-dag-pb",
+    "ipfs-unixfs-importer": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs-importer?js-dag-pb",
     "ipfs-utils": "^6.0.1",
     "ipld-block": "^0.11.0",
     "ipld-dag-cbor": "^0.17.0",

packages/ipfs-cli/package.json

@@ -20,7 +20,7 @@
   },
   "scripts": {
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "test": "npm run test:node",
     "test:node": "aegir test -t node",
     "coverage": "nyc --reporter=text --reporter=lcov npm run test:node",

packages/ipfs-client/package.json

@@ -32,7 +32,7 @@
   "scripts": {
     "test": "aegir test",
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "prepublishOnly": "aegir build",
     "coverage": "npx nyc -r html npm run test:node -- --bail",
     "clean": "rimraf ./dist",

packages/ipfs-core-utils/package.json

@@ -30,12 +30,13 @@
     "test:electron-renderer": "aegir test -t electron-renderer",
     "test:node": "aegir test -t node",
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "clean": "rimraf ./dist",
     "dep-check": "aegir dep-check -i rimraf -i ipfs-core-types"
   },
   "license": "MIT",
   "dependencies": {
+    "multiformats": "^4.0.0",
     "any-signal": "^2.1.2",
     "blob-to-it": "^1.0.1",
     "browser-readablestream-to-it": "^1.0.1",

packages/ipfs-core/package.json

@@ -39,7 +39,7 @@
   },
   "scripts": {
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "prepublishOnly": "aegir build",
     "test": "aegir test",
     "test:node": "aegir test -t node",
@@ -53,6 +53,7 @@
     "dep-check": "aegir dep-check -i interface-ipfs-core -i ipfs-core-types -i abort-controller"
   },
   "dependencies": {
+    "@ipld/dag-pb": "0.0.1",
     "abort-controller": "^3.0.0",
     "array-shuffle": "^2.0.0",
     "bignumber.js": "^9.0.0",
@@ -73,9 +74,9 @@
     "ipfs-core-types": "^0.3.0",
     "ipfs-core-utils": "^0.7.1",
     "ipfs-repo": "^8.0.0",
-    "ipfs-unixfs": "^2.0.3",
-    "ipfs-unixfs-exporter": "^3.0.4",
-    "ipfs-unixfs-importer": "^5.0.0",
+    "ipfs-unixfs": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs?js-dag-pb",
+    "ipfs-unixfs-exporter": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs-exporter?js-dag-pb",
+    "ipfs-unixfs-importer": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs-importer?js-dag-pb",
     "ipfs-utils": "^6.0.1",
     "ipld": "^0.28.0",
     "ipld-block": "^0.11.0",

packages/ipfs-core/src/components/add-all/index.js

@@ -6,6 +6,7 @@ const { parseChunkerString } = require('./utils')
 const { pipe } = require('it-pipe')
 const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option')
 const mergeOptions = require('merge-options').bind({ ignoreUndefined: true })
+const asLegacyCid = require('ipfs-core-utils/src/as-legacy-cid')
 
 /**
  * @typedef {Object} Context
@@ -124,7 +125,7 @@ function transformFile (opts) {
 
       yield {
         path,
-        cid,
+        cid: asLegacyCid(cid),
         size: file.size,
         mode: file.unixfs && file.unixfs.mode,
         mtime: file.unixfs && file.unixfs.mtime

packages/ipfs-core/src/components/add.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const last = require('it-last')
+const asLegacyCid = require('ipfs-core-utils/src/as-legacy-cid')
 
 /**
  * @typedef {Object} Context
@@ -27,7 +28,10 @@ module.exports = ({ addAll }) => {
       throw Error('Failed to add a file, if you see this please report a bug')
     }
 
-    return result
+    let legacyResult = result
+    legacyResult.cid = asLegacyCid(result.cid)
+
+    return legacyResult
   }
 
   return add

packages/ipfs-core/src/components/cat.js

@@ -6,12 +6,12 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option')
 
 /**
  * @typedef {Object} Context
- * @property {import('.').IPLD} ipld
+ * @property {import('.').BlockService} blockService
  * @property {import('.').Preload} preload
  *
  * @param {Context} context
  */
-module.exports = function ({ ipld, preload }) {
+module.exports = function ({ blockService, preload }) {
   /**
    * Returns content of the file addressed by a valid IPFS Path or CID.
    *
@@ -27,7 +27,7 @@ module.exports = function ({ ipld, preload }) {
       preload(pathComponents[0])
     }
 
-    const file = await exporter(ipfsPath, ipld, options)
+    const file = await exporter(ipfsPath, blockService, options)
 
     // File may not have unixfs prop if small & imported with rawLeaves true
     if (file.unixfs && file.unixfs.type.includes('dir')) {

packages/ipfs-core/src/components/files/chmod.js

@@ -9,7 +9,13 @@ const toTrail = require('./utils/to-trail')
 const addLink = require('./utils/add-link')
 const updateTree = require('./utils/update-tree')
 const updateMfsRoot = require('./utils/update-mfs-root')
-const { DAGNode } = require('ipld-dag-pb')
+const dagPb = require('@ipld/dag-pb')
+// @ts-ignore
+const { sha256 } = require('multiformats/hashes/sha2')
+const Block = require('multiformats/block')
+// @ts-ignore
+const IpldBlock = require('ipld-block')
+const CID = require('cids')
 const mc = require('multicodec')
 const mh = require('multihashing-async').multihash
 const { pipe } = require('it-pipe')
@@ -193,10 +199,14 @@ module.exports = (context) => {
       // but do not reimport files, only manipulate dag-pb nodes
       const root = await pipe(
         async function * () {
-          for await (const entry of exporter.recursive(cid, context.ipld)) {
-            let node = await context.ipld.get(entry.cid)
+          for await (const entry of exporter.recursive(cid, context.blockService)) {
+            const block = await context.blockService(entry.cid)
+            let node = dagPb.decode(block.data)
             entry.unixfs.mode = calculateMode(mode, entry.unixfs)
-            node = new DAGNode(entry.unixfs.marshal(), node.Links)
+            node = prepare({
+              Data: entry.unixfs.marshal(),
+              Links: node.Links
+            })
 
             yield {
               path: entry.path,
@@ -234,20 +244,32 @@ module.exports = (context) => {
       return
     }
 
-    let node = await context.ipld.get(cid)
+    const block = await context.blockService.get(cid)
+    let node = dagPb.decode(block.data)
     const metadata = UnixFS.unmarshal(node.Data)
     metadata.mode = calculateMode(mode, metadata)
-    node = new DAGNode(metadata.marshal(), node.Links)
+    node = prepare({
+      Data: metadata.marshal(),
+      Links: node.Links
+    })
+
 
-    const updatedCid = await context.ipld.put(node, mc.DAG_PB, {
-      cidVersion: cid.version,
-      hashAlg: mh.names[opts.hashAlg || defaultOptions.hashAlg],
-      onlyHash: !opts.flush
+    const updatedBlock = await Block.encode({
+      value: node,
+      codec: dagPb,
+      // TODO vmx 2021-02-22: Add back support for other hashing algorithms
+      hasher: sha256
     })
+    updatedCid = updatedBlock.cid
+    if (settings.flush) {
+      const legacyCid = new CID(updatedBlock.cid.multihash.bytes)
+      await context.blockService.put(new IpldBlock(updatedBlock.bytes, legacyCid))
+    }
 
     const trail = await toTrail(context, mfsDirectory)
     const parent = trail[trail.length - 1]
-    const parentNode = await context.ipld.get(parent.cid)
+    const parentBlock = await context.blockService(parent.cid)
+    const parentNode = dagPb.decode(parentBlock.data)
 
     const result = await addLink(context, {
       parent: parentNode,

packages/ipfs-core/src/components/files/index.js

@@ -51,7 +51,6 @@ const wrap = ({
 
 const defaultOptions = {
   repoOwner: true,
-  ipld: null,
   repo: null
 }
 
@@ -93,17 +92,15 @@ function createMfs (options) {
 
 /**
  * @param {Object} context
- * @param {import('..').IPLD} context.ipld
  * @param {import('..').Block} context.block
  * @param {import('..').BlockService} context.blockService
  * @param {import('..').Repo} context.repo
  * @param {import('..').Preload} context.preload
  * @param {import('..').Options} context.options
  * @returns {MFS}
  */
-module.exports = ({ ipld, block, blockService, repo, preload, options: constructorOptions }) => {
+module.exports = ({ block, blockService, repo, preload, options: constructorOptions }) => {
   const methods = createMfs({
-    ipld,
     block,
     blocks: blockService,
     datastore: repo.root,

packages/ipfs-core/src/components/files/ls.js

@@ -64,7 +64,7 @@ module.exports = (context) => {
    */
   async function * mfsLs (path, options = {}) {
     const mfsPath = await toMfsPath(context, path, options)
-    const fsDir = await exporter(mfsPath.mfsPath, context.ipld)
+    const fsDir = await exporter(mfsPath.mfsPath, context.blocks)
 
     // single file/node
     if (!fsDir.unixfs || !fsDir.unixfs.type.includes('directory')) {

packages/ipfs-core/src/components/files/mkdir.js

@@ -75,7 +75,7 @@ module.exports = (context) => {
       const subPath = `/ipfs/${root}/${subPathComponents.join('/')}`
 
       try {
-        parent = await exporter(subPath, context.ipld)
+        parent = await exporter(subPath, context.blockService)
         log(`${subPath} existed`)
         log(`${subPath} had children ${parent.node.Links.map(link => link.Name)}`)
 

packages/ipfs-core/src/components/files/read.js

@@ -40,7 +40,7 @@ module.exports = (context) => {
     return {
       [Symbol.asyncIterator]: async function * read () {
         const mfsPath = await toMfsPath(context, path, options)
-        const result = await exporter(mfsPath.mfsPath, context.ipld)
+        const result = await exporter(mfsPath.mfsPath, context.blockService)
 
         if (result.unixfs.type !== 'file') {
           throw errCode(new Error(`${path} was not a file`), 'ERR_NOT_FILE')

packages/ipfs-core/src/components/files/stat.js

@@ -5,7 +5,9 @@ const toMfsPath = require('./utils/to-mfs-path')
 const exporter = require('ipfs-unixfs-exporter')
 const log = require('debug')('ipfs:mfs:stat')
 const errCode = require('err-code')
+const mc = require('multicodec')
 const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option')
+const asLegacyCid = require('ipfs-core-utils/src/as-legacy-cid')
 
 const defaultOptions = {
   withLocal: false,
@@ -14,7 +16,7 @@ const defaultOptions = {
 
 /**
  * @param {Object} context
- * @param {import('..').IPLD} context.ipld
+ * @param {import('..').BlockService} context.blockService
  */
 module.exports = (context) => {
   /**
@@ -39,7 +41,7 @@ module.exports = (context) => {
     let file
 
     try {
-      file = await exporter(exportPath, context.ipld)
+      file = await exporter(exportPath, context.blocks)
     } catch (err) {
       if (err.code === 'ERR_NOT_FOUND') {
         throw errCode(new Error(`${path} does not exist`), 'ERR_NOT_FOUND')
@@ -48,11 +50,11 @@ module.exports = (context) => {
       throw err
     }
 
-    if (!statters[file.cid.codec]) {
-      throw new Error(`Cannot stat codec ${file.cid.codec}`)
+    if (!statters[file.cid.code]) {
+      throw new Error(`Cannot stat codec ${mc.getNameFromCode(file.cid.code)}`)
     }
 
-    return statters[file.cid.codec](file)
+    return statters[file.cid.code](file)
   }
 
   return withTimeoutOption(mfsStat)
@@ -64,9 +66,9 @@ const statters = {
    * @param {any} file
    * @returns {Stat}
    */
-  raw: (file) => {
+  [mc.RAW]: (file) => {
     return {
-      cid: file.cid,
+      cid: asLegacyCid(file.cid),
       size: file.node.length,
       cumulativeSize: file.node.length,
       blocks: 0,
@@ -80,14 +82,14 @@ const statters = {
    * @param {any} file
    * @returns {Stat}
    */
-  'dag-pb': (file) => {
+  [mc.DAG_PB]: (file) => {
     const blocks = file.node.Links.length
     const size = file.node.size
     const cumulativeSize = file.node.size
 
     /** @type {Stat} */
     const output = {
-      cid: file.cid,
+      cid: asLegacyCid(file.cid),
       type: 'file',
       size: size,
       cumulativeSize: cumulativeSize,
@@ -129,11 +131,11 @@ const statters = {
    * @param {any} file
    * @returns {Stat}
    */
-  'dag-cbor': (file) => {
+  [mc.DAG_CBOR]: (file) => {
     // @ts-ignore - This is incompatible with Stat object
     // @TODO - https://github.com/ipfs/js-ipfs/issues/3325
     return {
-      cid: file.cid,
+      cid: asLegacyCid(file.cid),
       local: undefined,
       sizeLocal: undefined,
       withLocality: false
@@ -143,9 +145,9 @@ const statters = {
    * @param {any} file
    * @returns {Stat}
    */
-  identity: (file) => {
+  [mc.IDENTITY]: (file) => {
     return {
-      cid: file.cid,
+      cid: asLegacyCid(file.cid),
       size: file.node.digest.length,
       cumulativeSize: file.node.digest.length,
       blocks: 0,