feat: mfs implementation (#1360)

* feat: add mfs implementation

* fix: increase timeout for unadulterated data test (#1425)

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

README.md

@@ -74,6 +74,7 @@ You can check the development status at the [Waffle Board](https://waffle.io/ipf
       - [Network](#network)
       - [Node Management](#node-management)
       - [Domain data types](#domain-data-types)
+      - [Util](#util)
 - [FAQ](#faq)
 - [Running js-ipfs with Docker](#running-js-ipfs-with-docker)
 - [Packages](#packages)

examples/browser-script-tag/index.html

@@ -30,7 +30,7 @@ <h2>Some suggestions</h2>
   <code style="display:block; white-space:pre-wrap; background-color:#d7d6d6">
     node.files.add(new node.types.Buffer('Hello world!'), (err, filesAdded) => {
       if (err) {
-        return console.error('Error - ipfs files add', err, res)
+        return console.error('Error - ipfs add', err, res)
       }
 
       filesAdded.forEach((file) => console.log('successfully stored', file.hash))

examples/circuit-relaying/package.json

@@ -18,7 +18,7 @@
     "ipfs-pubsub-room": "~0.3.0"
   },
   "devDependencies": {
-    "aegir": "^13.0.5",
+    "aegir": "^14.0.0",
     "http-server": "~0.10.0",
     "ipfs-css": "~0.2.0",
     "parcel-bundler": "^1.6.2",

package.json

@@ -108,6 +108,7 @@
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
     "ipfs-http-response": "~0.1.2",
+    "ipfs-mfs": "~0.0.14",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.22.1",
     "ipfs-unixfs": "~0.1.15",
@@ -166,7 +167,8 @@
     "through2": "^2.0.3",
     "update-notifier": "^2.5.0",
     "yargs": "^12.0.1",
-    "yargs-parser": "^10.1.0"
+    "yargs-parser": "^10.1.0",
+    "yargs-promise": "^1.1.0"
   },
   "optionalDependencies": {
     "prom-client": "^11.1.1",

src/cli/bin.js

@@ -2,13 +2,14 @@
 
 'use strict'
 
+const YargsPromise = require('yargs-promise')
 const yargs = require('yargs')
 const updateNotifier = require('update-notifier')
 const readPkgUp = require('read-pkg-up')
-const fs = require('fs')
-const path = require('path')
 const utils = require('./utils')
 const print = utils.print
+const mfs = require('ipfs-mfs/cli')
+const debug = require('debug')('ipfs:cli')
 
 const pkg = readPkgUp.sync({cwd: __dirname}).pkg
 updateNotifier({
@@ -18,10 +19,6 @@ updateNotifier({
 
 const args = process.argv.slice(2)
 
-// Determine if the first argument is a sub-system command
-const commandNames = fs.readdirSync(path.join(__dirname, 'commands'))
-const isCommand = commandNames.includes(`${args[0]}.js`)
-
 const cli = yargs
   .option('silent', {
     desc: 'Write no output',
@@ -34,14 +31,6 @@ const cli = yargs
     type: 'string',
     default: ''
   })
-  .commandDir('commands', {
-    // Only include the commands for the sub-system we're using, or include all
-    // if no sub-system command has been passed.
-    include (path, filename) {
-      if (!isCommand) return true
-      return `${args[0]}.js` === filename
-    }
-  })
   .epilog(utils.ipfsPathHelp)
   .demandCommand(1)
   .fail((msg, err, yargs) => {
@@ -56,27 +45,15 @@ const cli = yargs
     yargs.showHelp()
   })
 
-// If not a sub-system command then load the top level aliases
-if (!isCommand) {
-  // NOTE: This creates an alias of
-  // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
-  // This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
-  const addCmd = require('./commands/files/add')
-  const catCmd = require('./commands/files/cat')
-  const getCmd = require('./commands/files/get')
-  const aliases = [addCmd, catCmd, getCmd]
-  aliases.forEach((alias) => {
-    cli.command(alias.command, alias.describe, alias.builder, alias.handler)
-  })
-}
-
 // Need to skip to avoid locking as these commands
 // don't require a daemon
 if (args[0] === 'daemon' || args[0] === 'init') {
   cli
     .help()
     .strict()
     .completion()
+    .command(require('./commands/daemon'))
+    .command(require('./commands/init'))
     .parse(args)
 } else {
   // here we have to make a separate yargs instance with
@@ -86,19 +63,61 @@ if (args[0] === 'daemon' || args[0] === 'init') {
     if (err) {
       throw err
     }
+
     utils.getIPFS(argv, (err, ipfs, cleanup) => {
-      if (err) { throw err }
+      if (err) {
+        throw err
+      }
+
+      // add mfs commands
+      mfs(cli)
+
+      // NOTE: This creates an alias of
+      // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
+      // This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
+      const addCmd = require('./commands/files/add')
+      const catCmd = require('./commands/files/cat')
+      const getCmd = require('./commands/files/get')
+      const aliases = [addCmd, catCmd, getCmd]
+      aliases.forEach((alias) => {
+        cli.command(alias)
+      })
 
       cli
+        .commandDir('commands')
         .help()
         .strict()
         .completion()
-        .parse(args, { ipfs: ipfs }, (err, argv, output) => {
-          if (output) { print(output) }
 
-          cleanup(() => {
-            if (err) { throw err }
-          })
+      let exitCode = 0
+
+      const parser = new YargsPromise(cli, { ipfs })
+      parser.parse(args)
+        .then(({ data, argv }) => {
+          if (data) {
+            print(data)
+          }
+        })
+        .catch((arg) => {
+          debug(arg)
+
+          // the argument can have a different shape depending on where the error came from
+          if (arg.message) {
+            print(arg.message)
+          } else if (arg.error && arg.error.message) {
+            print(arg.error.message)
+          } else {
+            print('Unknown error, please re-run the command with DEBUG=ipfs:cli to see debug output')
+          }
+
+          exitCode = 1
+        })
+        .then(() => cleanup())
+        .catch(() => {})
+        .then(() => {
+          if (exitCode !== 0) {
+            process.exit(exitCode)
+          }
         })
     })
   })

src/cli/commands/files/get.js

@@ -30,6 +30,7 @@ function fileHandler (dir) {
         callback(err)
       } else {
         const fullFilePath = path.join(dir, file.path)
+
         if (file.content) {
           file.content
             .pipe(fs.createWriteStream(fullFilePath))

src/cli/utils.js

@@ -9,6 +9,7 @@ const log = debug('cli')
 log.error = debug('cli:error')
 const Progress = require('progress')
 const byteman = require('byteman')
+const promisify = require('promisify-es6')
 
 exports = module.exports
 
@@ -40,7 +41,7 @@ function getAPICtl (apiAddr) {
 
 exports.getIPFS = (argv, callback) => {
   if (argv.api || isDaemonOn()) {
-    return callback(null, getAPICtl(argv.api), (cb) => cb())
+    return callback(null, getAPICtl(argv.api), promisify((cb) => cb()))
   }
 
   // Required inline to reduce startup time
@@ -55,13 +56,13 @@ exports.getIPFS = (argv, callback) => {
     }
   })
 
-  const cleanup = (cb) => {
+  const cleanup = promisify((cb) => {
     if (node && node._repo && !node._repo.closed) {
-      node._repo.close(() => cb())
+      node._repo.close((err) => cb(err))
     } else {
       cb()
     }
-  }
+  })
 
   node.on('error', (err) => {
     throw err

src/core/components/index.js

@@ -26,3 +26,4 @@ exports.dht = require('./dht')
 exports.dns = require('./dns')
 exports.key = require('./key')
 exports.stats = require('./stats')
+exports.mfs = require('ipfs-mfs/core')

src/core/config.js

@@ -7,6 +7,7 @@ const schema = Joi.object().keys({
     Joi.object(), // TODO: schema for IPFS repo
     Joi.string()
   ).allow(null),
+  repoOwner: Joi.boolean().default(true),
   init: Joi.alternatives().try(
     Joi.boolean(),
     Joi.object().keys({ bits: Joi.number().integer() })

src/core/index.js

@@ -133,6 +133,13 @@ class IPFS extends EventEmitter {
       isIPFS: isIPFS
     }
 
+    // ipfs.files
+    const mfs = components.mfs(this, this._options)
+
+    Object.keys(mfs).forEach(key => {
+      this.files[key] = mfs[key]
+    })
+
     boot(this)
   }
 }

src/http/api/resources/files.js

@@ -34,7 +34,8 @@ exports.parseKey = (request, reply) => {
   if (!request.query.arg) {
     return reply({
       Message: "Argument 'key' is required",
-      Code: 0
+      Code: 0,
+      Type: 'error'
     }).code(400).takeover()
   }
 
@@ -54,7 +55,8 @@ exports.parseKey = (request, reply) => {
     log.error(err)
     return reply({
       Message: 'invalid ipfs ref path',
-      Code: 0
+      Code: 0,
+      Type: 'error'
     }).code(500).takeover()
   }
 
@@ -81,9 +83,9 @@ exports.cat = {
       if (err) {
         log.error(err)
         if (err.message === 'No such file') {
-          reply({Message: 'No such file'}).code(500)
+          reply({Message: 'No such file', Code: 0, Type: 'error'}).code(500)
         } else {
-          reply({Message: 'Failed to cat file: ' + err, Code: 0}).code(500)
+          reply({Message: 'Failed to cat file: ' + err, Code: 0, Type: 'error'}).code(500)
         }
         return
       }
@@ -177,7 +179,8 @@ exports.add = {
     if (!request.payload) {
       return reply({
         Message: 'Array, Buffer, or String is required.',
-        code: 0
+        Code: 0,
+        Type: 'error'
       }).code(400).takeover()
     }
 
@@ -211,7 +214,8 @@ exports.add = {
       if (!filesParsed) {
         return reply({
           Message: "File argument 'data' is required.",
-          code: 0
+          Code: 0,
+          Type: 'error'
         }).code(400).takeover()
       }
       fileAdder.end()
@@ -302,7 +306,8 @@ exports.immutableLs = {
       if (err) {
         return reply({
           Message: 'Failed to list dir: ' + err.message,
-          Code: 0
+          Code: 0,
+          Type: 'error'
         }).code(500).takeover()
       }
 

src/http/api/routes/files.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const resources = require('./../resources')
+const mfs = require('ipfs-mfs/http')
 
 module.exports = (server) => {
   const api = server.select('API')
@@ -54,4 +55,6 @@ module.exports = (server) => {
       handler: resources.files.immutableLs.handler
     }
   })
+
+  mfs(api)
 }

src/http/error-handler.js

@@ -14,9 +14,15 @@ module.exports = (api, server) => {
 
     let statusCode = 200
     let msg = 'Sorry, something went wrong, please retrace your steps.'
+    let code = 1
 
     if (res.isBoom) {
       statusCode = res.output.payload.statusCode
+      msg = res.output.payload.message
+
+      if (res.data && res.data.code !== undefined) {
+        code = res.data.code
+      }
 
       if (res.message && res.isDeveloperError) {
         msg = res.message.replace('Uncaught error: ', '')
@@ -36,7 +42,8 @@ module.exports = (api, server) => {
 
       reply({
         Message: msg,
-        Code: 1
+        Code: code,
+        Type: 'error'
       }).code(statusCode)
       return
     }