New: Add 'uniqueBy' option (closes #76) (#91)

README.md

@@ -90,6 +90,14 @@ Type: `Boolean`
 
 Default: `false`
 
+##### `options.uniqueBy`
+
+Filters stream to remove duplicates based on the string property name or the result of function. When using a function, the function receives the streamed data (objects containing `cwd`, `base`, `path` properties) to compare against.
+
+Type: `String` or `Function`
+
+Default: `'path'`
+
 ##### other
 
 Any glob-related options are documented in [node-glob][node-glob-url]. Those options are forwarded verbatim, with the exception of `root` and `ignore`. `root` is pre-resolved and `ignore` is joined with all negative globs.

index.js

@@ -20,6 +20,8 @@ function globStream(globs, opt) {
   ourOpt.dot = typeof ourOpt.dot === 'boolean' ? ourOpt.dot : false;
   ourOpt.silent = typeof ourOpt.silent === 'boolean' ? ourOpt.silent : true;
   ourOpt.cwdbase = typeof ourOpt.cwdbase === 'boolean' ? ourOpt.cwdbase : false;
+  ourOpt.uniqueBy =  typeof ourOpt.uniqueBy === 'string' ||
+                    typeof ourOpt.uniqueBy === 'function' ? ourOpt.uniqueBy : 'path';
 
   if (ourOpt.cwdbase) {
     ourOpt.base = ourOpt.cwd;
@@ -66,7 +68,7 @@ function globStream(globs, opt) {
 
   // Then just pipe them to a single unique stream and return it
   var aggregate = new Combine(streams);
-  var uniqueStream = unique('path');
+  var uniqueStream = unique(ourOpt.uniqueBy);
 
   return pumpify.obj(aggregate, uniqueStream);
 

test/index.js

@@ -341,7 +341,7 @@ describe('glob-stream', function() {
     ], done);
   });
 
-  it('removes duplicate objects from the stream', function(done) {
+  it('removes duplicate objects from the stream using default (path) filter', function(done) {
     var expected = {
       cwd: dir,
       base: dir + '/fixtures',
@@ -359,6 +359,54 @@ describe('glob-stream', function() {
     ], done);
   });
 
+  it('removes duplicate objects from the stream using custom string filter', function(done) {
+    var expected = {
+      cwd: dir,
+      base: dir + '/fixtures/stuff',
+      path: dir + '/fixtures/stuff/run.dmc',
+    };
+
+    function assert(pathObjs) {
+      expect(pathObjs.length).toEqual(1);
+      expect(pathObjs[0]).toEqual(expected);
+    }
+
+    pipe([
+      globStream(['./fixtures/stuff/run.dmc', './fixtures/stuff/test.dmc'], { cwd: dir, uniqueBy: 'base' }),
+      concat(assert),
+    ], done);
+  });
+
+  it('removes duplicate objects from the stream using custom function filter', function(done) {
+    var expected = [
+      {
+        cwd: dir,
+        base: dir + '/fixtures/stuff',
+        path: dir + '/fixtures/stuff/run.dmc',
+      },
+      {
+        cwd: dir,
+        base: dir + '/fixtures/stuff',
+        path: dir + '/fixtures/stuff/test.dmc',
+      },
+    ];
+
+    var uniqueBy = function(data) {
+      return data.path;
+    };
+
+    function assert(pathObjs) {
+      expect(pathObjs.length).toEqual(2);
+      expect(pathObjs).toInclude(expected[0]);
+      expect(pathObjs).toInclude(expected[1]);
+    }
+
+    pipe([
+      globStream('./fixtures/stuff/*.dmc', { cwd: dir, uniqueBy: uniqueBy }),
+      concat(assert),
+    ], done);
+  });
+
   it('ignores dotfiles without dot option', function(done) {
     function assert(pathObjs) {
       expect(pathObjs.length).toEqual(0);