feat: jsipfs ls -r (Recursive list directory) (#1222)

package.json

@@ -106,9 +106,9 @@
     "hapi": "^16.6.2",
     "hapi-set-header": "^1.0.2",
     "hoek": "^5.0.3",
-    "ipfs-api": "^18.0.0",
-    "ipfs-bitswap": "~0.19.0",
     "human-to-milliseconds": "^1.0.0",
+    "ipfs-api": "^18.1.1",
+    "ipfs-bitswap": "~0.19.0",
     "ipfs-block": "~0.6.1",
     "ipfs-block-service": "~0.13.0",
     "ipfs-multipart": "~0.1.0",

src/cli/commands/file/ls.js

@@ -11,6 +11,8 @@ module.exports = {
 
   handler (argv) {
     let path = argv.key
+    // `ipfs file ls` is deprecated. See https://ipfs.io/docs/commands/#ipfs-file-ls
+    print(`This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.`)
     argv.ipfs.ls(path, (err, links) => {
       if (err) {
         throw err

src/cli/commands/files.js

@@ -1,5 +1,8 @@
 'use strict'
 
+const print = require('../utils').print
+const lsCmd = require('./ls')
+
 module.exports = {
   command: 'files <command>',
 
@@ -8,9 +11,10 @@ module.exports = {
   builder (yargs) {
     return yargs
       .commandDir('files')
+      .command(lsCmd)
   },
 
   handler (argv) {
-    console.log('Type `jsipfs bitswap --help` for more instructions')
+    print('Type `jsipfs files --help` for more instructions')
   }
 }

src/cli/commands/ls.js

@@ -14,6 +14,12 @@ module.exports = {
       type: 'boolean',
       default: false
     },
+    r: {
+      alias: 'recursive',
+      desc: 'List subdirectories recursively',
+      type: 'boolean',
+      default: false
+    },
     'resolve-type': {
       desc: 'Resolve linked objects to find out their types. (not implemented yet)',
       type: 'boolean',
@@ -27,7 +33,7 @@ module.exports = {
       path = path.replace('/ipfs/', '')
     }
 
-    argv.ipfs.ls(path, (err, links) => {
+    argv.ipfs.ls(path, { recursive: argv.recursive }, (err, links) => {
       if (err) {
         throw err
       }
@@ -36,20 +42,17 @@ module.exports = {
         links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links)
       }
 
-      links = links.filter((link) => link.path !== path)
-      links.forEach((link) => {
-        if (link.type === 'dir') {
-          // directory: add trailing "/"
-          link.name = (link.name || '') + '/'
-        }
-      })
       const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length))
       const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length))
 
-      links.forEach((file) => {
-        utils.print(utils.rightpad(file.hash, multihashWidth + 1) +
-          utils.rightpad(file.size || '', sizeWidth + 1) +
-            file.name)
+      links.forEach(link => {
+        const fileName = link.type === 'dir' ? `${link.name || ''}/` : link.name
+        const padding = link.depth - path.split('/').length
+        utils.print(
+          utils.rightpad(link.hash, multihashWidth + 1) +
+          utils.rightpad(link.size || '', sizeWidth + 1) +
+          '  '.repeat(padding) + fileName
+        )
       })
     })
   }

src/core/components/files.js

@@ -161,14 +161,20 @@ module.exports = function files (self) {
     return d
   }
 
-  function _lsPullStreamImmutable (ipfsPath) {
+  function _lsPullStreamImmutable (ipfsPath, options) {
     const path = normalizePath(ipfsPath)
-    const depth = path.split('/').length
+    const recursive = options && options.recursive
+    const pathDepth = path.split('/').length
+    const maxDepth = recursive ? global.Infinity : pathDepth
+
     return pull(
-      exporter(ipfsPath, self._ipldResolver, { maxDepth: depth }),
-      pull.filter((node) => node.depth === depth),
-      pull.map((node) => {
-        node = Object.assign({}, node, { hash: toB58String(node.hash) })
+      exporter(ipfsPath, self._ipldResolver, { maxDepth: maxDepth }),
+      pull.filter(node =>
+        recursive ? node.depth >= pathDepth : node.depth === pathDepth
+      ),
+      pull.map(node => {
+        const cid = new CID(node.hash)
+        node = Object.assign({}, node, { hash: cid.toBaseEncodedString() })
         delete node.content
         return node
       })
@@ -278,20 +284,26 @@ module.exports = function files (self) {
       return exporter(ipfsPath, self._ipldResolver)
     },
 
-    lsImmutable: promisify((ipfsPath, callback) => {
+    lsImmutable: promisify((ipfsPath, options, callback) => {
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
+      }
+
       pull(
-        _lsPullStreamImmutable(ipfsPath),
+        _lsPullStreamImmutable(ipfsPath, options),
         pull.collect((err, values) => {
           if (err) {
-            return callback(err)
+            callback(err)
+            return
           }
           callback(null, values)
         })
       )
     }),
 
-    lsReadableStreamImmutable: (ipfsPath) => {
-      return toStream.source(_lsPullStreamImmutable(ipfsPath))
+    lsReadableStreamImmutable: (ipfsPath, options) => {
+      return toStream.source(_lsPullStreamImmutable(ipfsPath, options))
     },
 
     lsPullStreamImmutable: _lsPullStreamImmutable

src/http/api/resources/files.js

@@ -272,13 +272,14 @@ exports.immutableLs = {
   handler: (request, reply) => {
     const key = request.pre.args.key
     const ipfs = request.server.app.ipfs
+    const recursive = request.query && request.query.recursive === 'true'
 
-    ipfs.ls(key, (err, files) => {
+    ipfs.ls(key, { recursive: recursive }, (err, files) => {
       if (err) {
-        reply({
+        return reply({
           Message: 'Failed to list dir: ' + err.message,
           Code: 0
-        }).code(500)
+        }).code(500).takeover()
       }
 
       reply({
@@ -288,7 +289,8 @@ exports.immutableLs = {
             Name: file.name,
             Hash: file.hash,
             Size: file.size,
-            Type: toTypeCode(file.type)
+            Type: toTypeCode(file.type),
+            Depth: file.depth
           }))
         }]
       })

test/cli/file.js

@@ -17,12 +17,16 @@ describe('file ls', () => runOnAndOff((thing) => {
 
   it('prints a filename', () => {
     return ipfs(`file ls ${file}`)
-      .then((out) => expect(out).to.eql(`${file}\n`))
+      .then((out) => expect(out).to.eql(
+        `This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.\n` +
+        `${file}\n`
+      ))
   })
 
   it('prints the filenames in a directory', () => {
     return ipfs(`file ls ${dir}`)
       .then((out) => expect(out).to.eql(
+        `This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.\n` +
         'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9\n' +
         'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN\n' +
         'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz\n' +

test/cli/files.js

@@ -296,73 +296,6 @@ describe('files', () => runOnAndOff((thing) => {
       })
   })
 
-  it('ls', function () {
-    this.timeout(20 * 1000)
-
-    return ipfs('ls QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2')
-      .then((out) => {
-        expect(out).to.eql(
-          'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' +
-          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939   config\n' +
-          'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503   datastore/\n' +
-          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397   init-docs/\n' +
-          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10     version\n')
-      })
-  })
-
-  it('ls -v', function () {
-    this.timeout(20 * 1000)
-
-    return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2 -v')
-      .then((out) => {
-        expect(out).to.eql(
-          'Hash                                           Size   Name\n' +
-          'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' +
-          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939   config\n' +
-          'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503   datastore/\n' +
-          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397   init-docs/\n' +
-          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10     version\n')
-      })
-  })
-
-  it('ls <subdir>', function () {
-    this.timeout(20 * 1000)
-
-    return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2/init-docs')
-      .then((out) => {
-        expect(out).to.eql(
-          'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' +
-          'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200  contact\n' +
-          'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65   docs/\n' +
-          'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322  help\n' +
-          'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' +
-          'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' +
-          'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' +
-          'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863  tour/\n')
-      })
-  })
-
-  it('ls --help', function () {
-    this.timeout(20 * 1000)
-
-    return ipfs('ls --help')
-      .then((out) => {
-        expect(out.split('\n').slice(1)).to.eql(['',
-          'List files for the given directory',
-          '',
-          'Options:',
-          '  --version       Show version number                                  [boolean]',
-          '  --silent        Write no output                     [boolean] [default: false]',
-          '  --pass          Pass phrase for the keys                [string] [default: ""]',
-          '  --help          Show help                                            [boolean]',
-          '  -v, --headers   Print table headers (Hash, Size, Name).',
-          '                                                      [boolean] [default: false]',
-          '  --resolve-type  Resolve linked objects to find out their types. (not',
-          '                  implemented yet)                    [boolean] [default: false]',
-          '', ''])
-      })
-  })
-
   it('get', function () {
     this.timeout(20 * 1000)
 

test/cli/ls.js

@@ -0,0 +1,92 @@
+/* eslint-env mocha */
+'use strict'
+
+const expect = require('chai').expect
+const runOnAndOff = require('../utils/on-and-off')
+
+describe('ls', () => runOnAndOff((thing) => {
+  let ipfs
+
+  before(() => {
+    ipfs = thing.ipfs
+    return ipfs('files add -r test/fixtures/test-data/recursive-get-dir')
+  })
+
+  it('prints added files', function () {
+    this.timeout(20 * 1000)
+    return ipfs('ls QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2')
+      .then((out) => {
+        expect(out).to.eql(
+          'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' +
+          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939   config\n' +
+          'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503   datastore/\n' +
+          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397   init-docs/\n' +
+          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10     version\n'
+        )
+      })
+  })
+
+  it('prints nothing for non-existant hashes', function () {
+    // If the daemon is off, ls should fail
+    // If the daemon is on, ls should search until it hits a timeout
+    return Promise.race([
+      ipfs.fail('ls QmYmW4HiZhotsoSqnv2o1oSssvkRM8b9RweBoH7ao5nki2'),
+      new Promise((res, rej) => setTimeout(res, 4000))
+    ])
+      .catch(() => expect.fail(0, 1, 'Should have thrown or timedout'))
+  })
+
+  it('adds a header, -v', function () {
+    this.timeout(20 * 1000)
+    return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2 -v')
+      .then((out) => {
+        expect(out).to.eql(
+          'Hash                                           Size   Name\n' +
+          'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' +
+          'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939   config\n' +
+          'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503   datastore/\n' +
+          'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397   init-docs/\n' +
+          'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10     version\n'
+        )
+      })
+  })
+
+  it('follows a path, <hash>/<subdir>', function () {
+    this.timeout(20 * 1000)
+
+    return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2/init-docs')
+      .then((out) => {
+        expect(out).to.eql(
+          'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' +
+          'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200  contact\n' +
+          'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65   docs/\n' +
+          'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322  help\n' +
+          'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' +
+          'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' +
+          'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' +
+          'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863  tour/\n'
+        )
+      })
+  })
+
+  it('recursively follows folders, -r', function () {
+    this.slow(2000)
+    this.timeout(20 * 1000)
+
+    return ipfs('ls -r /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2/init-docs')
+      .then(out => {
+        expect(out).to.eql(
+          'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' +
+          'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200  contact\n' +
+          'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65   docs/\n' +
+          'QmQN88TEidd3RY2u3dpib49fERTDfKtDpvxnvczATNsfKT 14     index\n' +
+          'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322  help\n' +
+          'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' +
+          'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' +
+          'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' +
+          'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863  tour/\n' +
+          'QmYE7xo6NxbHEVEHej1yzxijYaNY51BaeKxjXxn6Ssa6Bs 807    0.0-intro\n'
+        )
+      })
+  })
+}))