feat: pull in new globSource (#3889)

* feat: pull in new globSource

The new globSource module has a simpler api.

Old:

```js
await ipfs.addAll(globSource('*', {
  recursive: true,
  cwd: './somedir'
})
```

New:

```js
await ipfs.addAll(globSource('./somedir', '**/*'))
```

See the [minimatch](https://www.npmjs.com/package/minimatch) docs for
more on what the pattern supports.

BREAKING CHANGE: the globSource api has changed from `globSource(dir, opts)` to `globSource(dir, pattern, opts)`

.github/workflows/test.yml

@@ -15,6 +15,7 @@ jobs:
         with:
           node-version: 16
       - uses: actions/cache@v2
+        id: cache
         env:
           CACHE_NAME: cache-node-modules
         with:
@@ -41,6 +42,7 @@ jobs:
         with:
           node-version: 16
       - uses: actions/cache@v2
+        id: cache
         env:
           CACHE_NAME: cache-node-modules
         with:

docs/MODULE.md

@@ -23,7 +23,7 @@ Use the IPFS module as a dependency of your project to spawn in process instance
     - [`node.start()`](#nodestart)
 - [Static types and utils](#static-types-and-utils)
       - [Glob source](#glob-source)
-        - [`globSource(path, [options])`](#globsourcepath-options)
+        - [`globSource(path, pattern, [options])`](#globsourcepath-pattern-options)
         - [Example](#example)
       - [URL source](#url-source)
         - [`urlSource(url)`](#urlsourceurl)
@@ -398,12 +398,11 @@ import { CID } from 'ipfs'
 
 A utility to allow files on the file system to be easily added to IPFS.
 
-###### `globSource(path, [options])`
+###### `globSource(path, pattern, [options])`
 
 - `path`: A path to a single file or directory to glob from
+- `pattern`: A pattern to match files under `path`
 - `options`: Optional options
-- `options.recursive`: If `path` is a directory, use option `{ recursive: true }` to add the directory and all its sub-directories.
-- `options.ignore`: To exclude file globs from the directory, use option `{ ignore: ['ignore/this/folder/**', 'and/this/file'] }`.
 - `options.hidden`: Hidden/dot files (files or folders starting with a `.`, for example, `.git/`) are not included by default. To add them, use the option `{ hidden: true }`.
 
 Returns an async iterable that yields `{ path, content }` objects suitable for passing to `ipfs.add`.
@@ -415,7 +414,7 @@ import { create, globSource } from 'ipfs'
 
 const ipfs = await create()
 
-for await (const file of ipfs.addAll(globSource('./docs', { recursive: true }))) {
+for await (const file of ipfs.addAll(globSource('./docs', '**/*'))) {
   console.log(file)
 }
 /*

packages/interface-ipfs-core/package.json

@@ -44,8 +44,7 @@
   "browser": {
     "fs": false,
     "os": false,
-    "path": false,
-    "ipfs-utils/src/files/glob-source": false
+    "path": false
   },
   "scripts": {
     "clean": "rimraf ./dist",
@@ -76,7 +75,7 @@
     "ipfs-core-types": "^0.7.3",
     "ipfs-unixfs": "^6.0.3",
     "ipfs-unixfs-importer": "^9.0.3",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-utils": "^9.0.1",
     "ipns": "^0.14.0",
     "is-ipfs": "^6.0.1",
     "iso-random-stream": "^2.0.0",

packages/interface-ipfs-core/src/add-all.js

@@ -397,7 +397,7 @@ export function testAddAll (factory, options) {
       if (!isNode) this.skip()
       const filesPath = resolve('test/fixtures/test-folder', 'interface-ipfs-core')
 
-      const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true })))
+      const result = await all(ipfs.addAll(globSource(filesPath, '**/*')))
       expect(result.length).to.be.above(8)
     })
 
@@ -407,7 +407,7 @@ export function testAddAll (factory, options) {
 
       const filesPath = resolve('test/fixtures/weird name folder [v0]', 'interface-ipfs-core')
 
-      const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true })))
+      const result = await all(ipfs.addAll(globSource(filesPath, '**/*')))
       expect(result.length).to.be.above(8)
     })
 
@@ -417,17 +417,17 @@ export function testAddAll (factory, options) {
 
       const filesPath = resolve('test/fixtures/test-folder', 'interface-ipfs-core')
 
-      const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true, ignore: ['files/**'] })))
-      expect(result.length).to.be.below(9)
+      const result = await all(ipfs.addAll(globSource(filesPath, '@(!(files*))')))
+      expect(result.length).to.equal(6)
     })
 
     it('should add a file from the file system', async function () {
       // @ts-ignore this is mocha
       if (!isNode) this.skip()
 
-      const filePath = resolve('test/fixtures/test-folder/ipfs-add.js', 'interface-ipfs-core')
+      const filePath = resolve('test/fixtures/test-folder', 'interface-ipfs-core')
 
-      const result = await all(ipfs.addAll(globSource(filePath)))
+      const result = await all(ipfs.addAll(globSource(filePath, 'ipfs-add.js')))
       expect(result.length).to.equal(1)
       expect(result[0].path).to.equal('ipfs-add.js')
     })
@@ -436,10 +436,9 @@ export function testAddAll (factory, options) {
       // @ts-ignore this is mocha
       if (!isNode) this.skip()
 
-      const filesPath = resolve('test/fixtures/hidden-files-folder', 'interface-ipfs-core')
+      const filesPath = resolve('test/fixtures', 'interface-ipfs-core')
 
-      const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true, hidden: true })))
-      expect(result.length).to.be.above(10)
+      const result = await all(ipfs.addAll(globSource(filesPath, 'hidden-files-folder/**/*', { hidden: true })))
       expect(result.map(object => object.path)).to.include('hidden-files-folder/.hiddenTest.txt')
       expect(result.map(object => object.cid.toString())).to.include('QmdbAjVmLRdpFyi8FFvjPfhTGB2cVXvWLuK7Sbt38HXrtt')
     })

packages/ipfs-cli/package.json

@@ -71,12 +71,13 @@
     "ipfs-daemon": "^0.9.8",
     "ipfs-http-client": "^52.0.5",
     "ipfs-repo": "^12.0.0",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-utils": "^9.0.1",
     "it-all": "^1.0.4",
     "it-concat": "^2.0.0",
     "it-first": "^1.0.4",
     "it-glob": "1.0.0",
     "it-map": "^1.0.5",
+    "it-merge": "^1.0.3",
     "it-pipe": "^1.1.0",
     "it-split": "^1.0.0",
     "it-tar": "^4.0.0",

packages/ipfs-cli/src/commands/add.js

@@ -13,6 +13,7 @@ import {
 } from '../utils.js'
 import globSource from 'ipfs-utils/src/files/glob-source.js'
 import parseDuration from 'parse-duration'
+import merge from 'it-merge'
 
 const getFolderSize = promisify(getFolderSizeCb)
 
@@ -286,15 +287,20 @@ export default {
       date = { secs: mtime, nsecs: mtimeNsecs }
     }
 
+    let pattern = '*'
+
+    if (recursive) {
+      pattern = '**/*'
+    }
+
     const source = file
-      ? globSource(file, {
-        recursive,
+      ? merge(...file.map(file => globSource(file, pattern, {
         hidden,
         preserveMode,
         preserveMtime,
         mode,
         mtime: date
-      })
+      })))
       : [{
           content: getStdin(),
           mode,

packages/ipfs-core-config/package.json

@@ -88,7 +88,7 @@
     "err-code": "^3.0.1",
     "hashlru": "^2.3.0",
     "ipfs-repo": "^12.0.0",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-utils": "^9.0.1",
     "ipns": "^0.14.0",
     "is-ipfs": "^6.0.1",
     "it-all": "^1.0.4",

packages/ipfs-core-utils/package.json

@@ -112,7 +112,7 @@
     "err-code": "^3.0.1",
     "ipfs-core-types": "^0.7.3",
     "ipfs-unixfs": "^6.0.3",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-utils": "^9.0.1",
     "it-all": "^1.0.4",
     "it-map": "^1.0.4",
     "it-peekable": "^1.0.2",

packages/ipfs-core/package.json

@@ -45,9 +45,6 @@
       "import": "./src/components/config/profiles.js"
     }
   },
-  "browser": {
-    "ipfs-utils/src/files/glob-source": false
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ipfs/js-ipfs.git"
@@ -94,7 +91,7 @@
     "ipfs-unixfs": "^6.0.3",
     "ipfs-unixfs-exporter": "^7.0.3",
     "ipfs-unixfs-importer": "^9.0.3",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-utils": "^9.0.1",
     "ipns": "^0.14.0",
     "is-domain-name": "^1.0.1",
     "is-ipfs": "^6.0.1",

packages/ipfs-daemon/package.json

@@ -50,7 +50,7 @@
     "ipfs-grpc-server": "^0.6.6",
     "ipfs-http-gateway": "^0.6.5",
     "ipfs-http-server": "^0.7.6",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-utils": "^9.0.1",
     "just-safe-set": "^2.2.1",
     "libp2p": "^0.32.0",
     "libp2p-webrtc-star": "^0.23.0"

packages/ipfs-http-client/README.md

@@ -49,7 +49,7 @@
   - [Instance Utils](#instance-utils)
   - [Static Types and Utils](#static-types-and-utils)
     - [Glob source](#glob-source)
-      - [`globSource(path, [options])`](#globsourcepath-options)
+      - [`globSource(path, pattern, [options])`](#globsourcepath-pattern-options)
       - [Example](#example-1)
     - [URL source](#url-source)
       - [`urlSource(url)`](#urlsourceurl)
@@ -182,25 +182,25 @@ import { CID } from 'ipfs-http-client'
 
 A utility to allow files on the file system to be easily added to IPFS.
 
-##### `globSource(path, [options])`
+##### `globSource(path, pattern, [options])`
 
 - `path`: A path to a single file or directory to glob from
+- `pattern`: A pattern to match files under `path`
 - `options`: Optional options
-- `options.recursive`: If `path` is a directory, use option `{ recursive: true }` to add the directory and all its sub-directories.
-- `options.ignore`: To exclude file globs from the directory, use option `{ ignore: ['ignore/this/folder/**', 'and/this/file'] }`.
 - `options.hidden`: Hidden/dot files (files or folders starting with a `.`, for example, `.git/`) are not included by default. To add them, use the option `{ hidden: true }`.
 
 Returns an async iterable that yields `{ path, content }` objects suitable for passing to `ipfs.add`.
 
 ##### Example
 
 ```js
-import { create, globSource } from 'ipfs-http-client'
-const ipfs = create()
+import { create, globSource } from 'ipfs'
 
-const file = await ipfs.add(globSource('./docs', { recursive: true }))
-console.log(file)
+const ipfs = await create()
 
+for await (const file of ipfs.addAll(globSource('./docs', '**/*'))) {
+  console.log(file)
+}
 /*
 {
   path: 'docs/assets/anchor.js',

packages/ipfs-http-client/package.json

@@ -62,7 +62,7 @@
     "err-code": "^3.0.1",
     "ipfs-core-types": "^0.7.3",
     "ipfs-core-utils": "^0.10.5",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-utils": "^9.0.1",
     "it-first": "^1.0.6",
     "it-last": "^1.0.4",
     "merge-options": "^3.0.4",

packages/ipfs-http-client/src/index.js

@@ -1,4 +1,3 @@
-
 /* eslint-env browser */
 
 import { Multibases } from 'ipfs-core-utils/multibases'
@@ -42,6 +41,7 @@ import { createResolve } from './resolve.js'
 import { createStart } from './start.js'
 import { createStop } from './stop.js'
 import { createVersion } from './version.js'
+import globSourceImport from 'ipfs-utils/src/files/glob-source.js'
 
 /**
  * @typedef {import('./types').EndpointConfig} EndpointConfig
@@ -142,5 +142,5 @@ export function create (options = {}) {
 
 export { CID } from 'multiformats/cid'
 export { Multiaddr as multiaddr } from 'multiaddr'
-export { default as globSource } from 'ipfs-utils/src/files/glob-source.js'
 export { default as urlSource } from 'ipfs-utils/src/files/url-source.js'
+export const globSource = globSourceImport

packages/ipfs/package.json

@@ -88,8 +88,8 @@
     "ipfs-client": "^0.6.6",
     "ipfs-core-types": "^0.7.3",
     "ipfs-http-client": "^52.0.5",
-    "ipfs-interop": "^6.0.1",
-    "ipfs-utils": "^8.1.4",
+    "ipfs-interop": "ipfs/interop#chore/update-globsource",
+    "ipfs-utils": "^9.0.1",
     "ipfsd-ctl": "^10.0.3",
     "iso-url": "^1.0.0",
     "libp2p-webrtc-star": "^0.23.0",