This repository has been archived by the owner on Feb 12, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This PR adds a new method [`addFromFs`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addfromfs) allowing users to more easily add files from their file system without having to specify every single file to add. In the browser the user will receive a "not available" error. I've pulled out a module `glob-source.js` - call it with some file paths and it returns a pull stream source that can be piped to `ipfs.addPullStream`. This PR comes with the following added benefits: * `ipfs add` on the CLI uses `glob-source.js` - **nice and DRY** * `glob-source.js` uses the events that the `glob` module provides allowing the globbing to be a `pull-pushable`, which means that matched paths can begin to be added before all the globbing is done - **faster** * `ipfs add` now supports adding multiple paths, fixes #1625 - **better** * `ipfs add --progress=false` doesn't calculate the total size of the files to be added anymore! It didn't need to do that as the total was completely discarded when progress was disabled. It means we can add BIGGER directories without running into memory issues - **stronger** License: MIT Signed-off-by: Alan Shaw <alan.shaw@protocol.ai>
- Loading branch information
Showing
10 changed files
with
216 additions
and
114 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| 'use strict' | ||
|
|
||
| module.exports = (self) => require('../../runtime/add-from-fs-nodejs')(self) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| 'use strict' | ||
|
|
||
| const promisify = require('promisify-es6') | ||
|
|
||
| module.exports = self => { | ||
| return promisify((...args) => { | ||
| const callback = args.pop() | ||
| callback(new Error('not available in the browser')) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| 'use strict' | ||
|
|
||
| const promisify = require('promisify-es6') | ||
| const pull = require('pull-stream') | ||
| const globSource = require('../../utils/files/glob-source') | ||
|
|
||
| module.exports = self => { | ||
| return promisify((...args) => { | ||
| const callback = args.pop() | ||
| const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() | ||
| const paths = args | ||
|
|
||
| pull( | ||
| globSource(...paths, options), | ||
| self.addPullStream(options), | ||
| pull.collect(callback) | ||
| ) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| 'use strict' | ||
|
|
||
| const fs = require('fs') | ||
| const Path = require('path') | ||
| const pull = require('pull-stream') | ||
| const glob = require('glob') | ||
| const cat = require('pull-cat') | ||
| const defer = require('pull-defer') | ||
| const pushable = require('pull-pushable') | ||
| const map = require('async/map') | ||
| const errCode = require('err-code') | ||
|
|
||
| /** | ||
| * Create a pull stream source that can be piped to ipfs.addPullStream for the | ||
| * provided file paths. | ||
| * | ||
| * @param {String} ...paths File system path(s) to glob from | ||
| * @param {Object} [options] Optional options | ||
| * @param {Boolean} [options.recursive] Recursively glob all paths in directories | ||
| * @param {Boolean} [options.hidden] Include .dot files in matched paths | ||
| * @param {Array<String>} [options.ignore] Glob paths to ignore | ||
| * @param {Boolean} [options.followSymlinks] follow symlinks | ||
| * @returns {Function} pull stream source | ||
| */ | ||
| module.exports = (...args) => { | ||
| const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() | ||
| const paths = args | ||
| const deferred = defer.source() | ||
|
|
||
| const globSourceOptions = { | ||
| recursive: options.recursive, | ||
| glob: { | ||
| dot: Boolean(options.hidden), | ||
| ignore: Array.isArray(options.ignore) ? options.ignore : [], | ||
| follow: options.followSymlinks != null ? options.followSymlinks : true | ||
| } | ||
| } | ||
|
|
||
| // Check the input paths comply with options.recursive and convert to glob sources | ||
| map(paths, pathAndType, (err, results) => { | ||
| if (err) return deferred.abort(err) | ||
|
|
||
| try { | ||
| const sources = results.map(res => toGlobSource(res, globSourceOptions)) | ||
| deferred.resolve(cat(sources)) | ||
| } catch (err) { | ||
| deferred.abort(err) | ||
| } | ||
| }) | ||
|
|
||
| return pull( | ||
| deferred, | ||
| pull.map(({ path, contentPath }) => ({ | ||
| path, | ||
| content: fs.createReadStream(contentPath) | ||
| })) | ||
| ) | ||
| } | ||
|
|
||
| function toGlobSource ({ path, type }, options) { | ||
| options = options || {} | ||
|
|
||
| const baseName = Path.basename(path) | ||
|
|
||
| if (type === 'file') { | ||
| return pull.values([{ path: baseName, contentPath: path }]) | ||
| } | ||
|
|
||
| if (type === 'dir' && !options.recursive) { | ||
| throw errCode( | ||
| new Error(`'${path}' is a directory and recursive option not set`), | ||
| 'ERR_DIR_NON_RECURSIVE', | ||
| { path } | ||
| ) | ||
| } | ||
|
|
||
| const globOptions = Object.assign({}, options.glob, { | ||
| cwd: path, | ||
| nodir: true, | ||
| realpath: false, | ||
| absolute: false | ||
| }) | ||
|
|
||
| // TODO: want to use pull-glob but it doesn't have the features... | ||
| const pusher = pushable() | ||
|
|
||
| glob('**/*', globOptions) | ||
| .on('match', m => pusher.push(m)) | ||
| .on('end', () => pusher.end()) | ||
| .on('abort', () => pusher.end()) | ||
| .on('error', err => pusher.end(err)) | ||
|
|
||
| return pull( | ||
| pusher, | ||
| pull.map(p => ({ | ||
| path: `${baseName}/${toPosix(p)}`, | ||
| contentPath: Path.join(path, p) | ||
| })) | ||
| ) | ||
| } | ||
|
|
||
| function pathAndType (path, cb) { | ||
| fs.stat(path, (err, stat) => { | ||
| if (err) return cb(err) | ||
| cb(null, { path, type: stat.isDirectory() ? 'dir' : 'file' }) | ||
| }) | ||
| } | ||
|
|
||
| const toPosix = path => path.replace(/\\/g, '/') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.