From e0a51c362aad35fd6aad07c4dc463ba330deb634 Mon Sep 17 00:00:00 2001 From: Dominic Barnes Date: Fri, 10 Aug 2012 14:52:35 -0500 Subject: [PATCH 1/5] adding forEachOf to iterate objects asynchronously --- lib/async.js | 29 ++++++++++++++++++++++++++++ test/test-async.js | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/lib/async.js b/lib/async.js index 5f2fb189e..179918e97 100755 --- a/lib/async.js +++ b/lib/async.js @@ -57,6 +57,14 @@ return memo; }; + var _forEachOf = function (object, iterator) { + for (key in object) { + if (object.hasOwnProperty(key)) { + iterator(object[key], key); + } + } + }; + var _keys = function (obj) { if (Object.keys) { return Object.keys(obj); @@ -111,6 +119,27 @@ }); }; + async.forEachOf = function (object, iterator, callback) { + callback = callback || function () {}; + var completed = 0, size = _keys(object).length, key; + if (!size) { + return callback(); + } + _forEachOf(object, function (value, key) { + iterator(object[key], key, function (err) { + if (err) { + callback(err); + callback = function () {}; + } else { + completed += 1; + if (completed === size) { + callback(null); + } + } + }); + }); + }; + async.forEachSeries = function (arr, iterator, callback) { callback = callback || function () {}; if (!arr.length) { diff --git a/test/test-async.js b/test/test-async.js index 323527a99..e26e936c4 100755 --- a/test/test-async.js +++ b/test/test-async.js @@ -17,6 +17,13 @@ function forEachIterator(args, x, callback) { }, x*25); } +function forEachOfIterator(args, x, key, callback) { + setTimeout(function(){ + args.push(key, x); + callback(); + }, x*25); +} + function mapIterator(call_order, x, callback) { setTimeout(function(){ call_order.push(x); @@ -43,6 +50,13 @@ function forEachNoCallbackIterator(test, x, callback) { test.done(); } +function forEachOfNoCallbackIterator(test, x, key, callback) { + test.equal(x, 1); + test.equal(key, "a"); + callback(); + test.done(); +} + function getFunctionsObject(call_order) { return { one: function(callback){ @@ -652,6 +666,39 @@ exports['forEach no callback'] = function(test){ async.forEach([1], forEachNoCallbackIterator.bind(this, test)); }; +exports['forEachOf'] = function(test){ + var args = []; + async.forEachOf({ a: 1, b: 2 }, forEachOfIterator.bind(this, args), function(err){ + test.same(args, ["a", 1, "b", 2]); + test.done(); + }); +}; + +exports['forEachOf empty object'] = function(test){ + test.expect(1); + async.forEachOf({}, function(value, key, callback){ + test.ok(false, 'iterator should not be called'); + callback(); + }, function(err) { + test.ok(true, 'should call callback'); + }); + setTimeout(test.done, 25); +}; + +exports['forEachOf error'] = function(test){ + test.expect(1); + async.forEachOf({ a: 1, b: 2 }, function(value, key, callback) { + callback('error'); + }, function(err){ + test.equals(err, 'error'); + }); + setTimeout(test.done, 50); +}; + +exports['forEachOf no callback'] = function(test){ + async.forEachOf({ a: 1 }, forEachOfNoCallbackIterator.bind(this, test)); +}; + exports['forEachSeries'] = function(test){ var args = []; async.forEachSeries([1,3,2], forEachIterator.bind(this, args), function(err){ From 2a13d0857682663e556b1a344d8c33d3a6c289bf Mon Sep 17 00:00:00 2001 From: Dominic Barnes Date: Wed, 6 Feb 2013 22:01:50 -0600 Subject: [PATCH 2/5] adding forEachOfSeries and forEachOfLimit along with tests --- lib/async.js | 88 +++++++++++++++++++++++++- test/test-async.js | 151 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 235 insertions(+), 4 deletions(-) diff --git a/lib/async.js b/lib/async.js index 179918e97..f1ef20934 100755 --- a/lib/async.js +++ b/lib/async.js @@ -121,7 +121,8 @@ async.forEachOf = function (object, iterator, callback) { callback = callback || function () {}; - var completed = 0, size = _keys(object).length, key; + var size = object.length || _keys(object).length; + var completed = 0 if (!size) { return callback(); } @@ -173,6 +174,42 @@ iterate(); }; + async.forEachOfSeries = function (obj, iterator, callback) { + callback = callback || function () {}; + var keys = _keys(obj); + var size = keys.length; + if (!size) { + return callback(); + } + var completed = 0; + var iterate = function () { + var sync = true; + var key = keys[completed]; + iterator(obj[key], key, function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + if (completed >= size) { + callback(null); + } + else { + if (sync) { + async.nextTick(iterate); + } + else { + iterate(); + } + } + } + }); + sync = false; + }; + iterate(); + }; + async.forEachLimit = function (arr, limit, iterator, callback) { var fn = _forEachLimit(limit); fn.apply(null, [arr, iterator, callback]); @@ -219,6 +256,55 @@ }; + async.forEachOfLimit = function (obj, limit, iterator, callback) { + var fn = obj.constructor === Array ? _forEachOfLimit(limit) : _forEachOfLimit(limit); + fn.apply(null, [obj, iterator, callback]); + }; + + var _forEachOfLimit = function (limit) { + + return function (obj, iterator, callback) { + callback = callback || function () {}; + var keys = _keys(obj); + var size = keys.length; + if (!size || limit <= 0) { + return callback(); + } + var completed = 0; + var started = 0; + var running = 0; + + (function replenish () { + if (completed >= size) { + return callback(); + } + + while (running < limit && started < size) { + started += 1; + running += 1; + var key = keys[started - 1]; + iterator(obj[key], key, function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + running -= 1; + if (completed >= size) { + callback(); + } + else { + replenish(); + } + } + }); + } + })(); + }; + }; + + var doParallel = function (fn) { return function () { var args = Array.prototype.slice.call(arguments); diff --git a/test/test-async.js b/test/test-async.js index e26e936c4..cad8bb0ea 100755 --- a/test/test-async.js +++ b/test/test-async.js @@ -17,11 +17,11 @@ function forEachIterator(args, x, callback) { }, x*25); } -function forEachOfIterator(args, x, key, callback) { +function forEachOfIterator(args, value, key, callback) { setTimeout(function(){ - args.push(key, x); + args.push(key, value); callback(); - }, x*25); + }, value*25); } function mapIterator(call_order, x, callback) { @@ -699,6 +699,14 @@ exports['forEachOf no callback'] = function(test){ async.forEachOf({ a: 1 }, forEachOfNoCallbackIterator.bind(this, test)); }; +exports['forEachOf with array'] = function(test){ + var args = []; + async.forEachOf([ "a", "b" ], forEachOfIterator.bind(this, args), function(err){ + test.same(args, [0, "a", 1, "b"]); + test.done(); + }); +}; + exports['forEachSeries'] = function(test){ var args = []; async.forEachSeries([1,3,2], forEachIterator.bind(this, args), function(err){ @@ -735,6 +743,50 @@ exports['forEachSeries no callback'] = function(test){ async.forEachSeries([1], forEachNoCallbackIterator.bind(this, test)); }; +exports['forEachOfSeries'] = function(test){ + var args = []; + async.forEachOfSeries({ a: 1, b: 2 }, forEachOfIterator.bind(this, args), function(err){ + test.same(args, [ "a", 1, "b", 2 ]); + test.done(); + }); +}; + +exports['forEachOfSeries empty object'] = function(test){ + test.expect(1); + async.forEachOfSeries({}, function(x, callback){ + test.ok(false, 'iterator should not be called'); + callback(); + }, function(err){ + test.ok(true, 'should call callback'); + }); + setTimeout(test.done, 25); +}; + +exports['forEachOfSeries error'] = function(test){ + test.expect(2); + var call_order = []; + async.forEachOfSeries({ a: 1, b: 2 }, function(value, key, callback){ + call_order.push(value, key); + callback('error'); + }, function(err){ + test.same(call_order, [ 1, "a" ]); + test.equals(err, 'error'); + }); + setTimeout(test.done, 50); +}; + +exports['forEachOfSeries no callback'] = function(test){ + async.forEachOfSeries({ a: 1 }, forEachOfNoCallbackIterator.bind(this, test)); +}; + +exports['forEachOfSeries with array'] = function(test){ + var args = []; + async.forEachOfSeries([ "a", "b" ], forEachOfIterator.bind(this, args), function(err){ + test.same(args, [ 0, "a", 1, "b" ]); + test.done(); + }); +}; + exports['forEachLimit'] = function(test){ var args = []; var arr = [0,1,2,3,4,5,6,7,8,9]; @@ -822,6 +874,99 @@ exports['forEachLimit synchronous'] = function(test){ }); }; +exports['forEachOfLimit'] = function(test){ + var args = []; + var obj = { a: 1, b: 2, c: 3, d: 4 }; + async.forEachOfLimit(obj, 2, function(value, key, callback){ + setTimeout(function(){ + args.push(value, key); + callback(); + }, value * 5); + }, function(err){ + test.same(args, [ 1, "a", 2, "b", 3, "c", 4, "d" ]); + test.done(); + }); +}; + +exports['forEachOfLimit empty object'] = function(test){ + test.expect(1); + async.forEachOfLimit({}, 2, function(value, key, callback){ + test.ok(false, 'iterator should not be called'); + callback(); + }, function(err){ + test.ok(true, 'should call callback'); + }); + setTimeout(test.done, 25); +}; + +exports['forEachOfLimit limit exceeds size'] = function(test){ + var args = []; + var obj = { a: 1, b: 2, c: 3, d: 4, e: 5 }; + async.forEachOfLimit(obj, 10, forEachOfIterator.bind(this, args), function(err){ + test.same(args, [ "a", 1, "b", 2, "c", 3, "d", 4, "e", 5 ]); + test.done(); + }); +}; + +exports['forEachOfLimit limit equal size'] = function(test){ + var args = []; + var obj = { a: 1, b: 2, c: 3, d: 4, e: 5 }; + async.forEachOfLimit(obj, 5, forEachOfIterator.bind(this, args), function(err){ + test.same(args, [ "a", 1, "b", 2, "c", 3, "d", 4, "e", 5 ]); + test.done(); + }); +}; + +exports['forEachOfLimit zero limit'] = function(test){ + test.expect(1); + async.forEachOfLimit({ a: 1, b: 2 }, 0, function(x, callback){ + test.ok(false, 'iterator should not be called'); + callback(); + }, function(err){ + test.ok(true, 'should call callback'); + }); + setTimeout(test.done, 25); +}; + +exports['forEachOfLimit error'] = function(test){ + test.expect(2); + var obj = { a: 1, b: 2, c: 3, d: 4, e: 5 }; + var call_order = []; + + async.forEachOfLimit(obj, 3, function(value, key, callback){ + call_order.push(value, key); + if (value === 2) { + callback('error'); + } + }, function(err){ + test.same(call_order, [ 1, "a", 2, "b" ]); + test.equals(err, 'error'); + }); + setTimeout(test.done, 25); +}; + +exports['forEachOfLimit no callback'] = function(test){ + async.forEachOfLimit({ a: 1 }, 1, forEachOfNoCallbackIterator.bind(this, test)); +}; + +exports['forEachOfLimit synchronous'] = function(test){ + var args = []; + var obj = { a: 1, b: 2 }; + async.forEachOfLimit(obj, 5, forEachOfIterator.bind(this, args), function(err){ + test.same(args, [ "a", 1, "b", 2 ]); + test.done(); + }); +}; + +exports['forEachOfLimit with array'] = function(test){ + var args = []; + var arr = [ "a", "b" ] + async.forEachOfLimit(arr, 1, forEachOfIterator.bind(this, args), function (err) { + test.same(args, [ 0, "a", 1, "b" ]); + test.done(); + }); +}; + exports['map'] = function(test){ var call_order = []; async.map([1,3,2], mapIterator.bind(this, call_order), function(err, results){ From 6d612c8bceff7a88ebcc263a139f336adec79e9e Mon Sep 17 00:00:00 2001 From: Alexander Early Date: Mon, 19 Jan 2015 23:53:49 -0800 Subject: [PATCH 3/5] adding docs for forEachOf methods --- README.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/README.md b/README.md index 420cd6a4c..0f2e41bc3 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,9 @@ Usage: * [`each`](#each) * [`eachSeries`](#eachSeries) * [`eachLimit`](#eachLimit) +* [`forEachOf`](#forEachOf) +* [`forEachOfSeries`](#forEachOfSeries) +* [`forEachOfLimit`](#forEachOfLimit) * [`map`](#map) * [`mapSeries`](#mapSeries) * [`mapLimit`](#mapLimit) @@ -280,6 +283,53 @@ async.eachLimit(documents, 20, requestApi, function(err){ }); ``` +--------------------------------------- + + + +### forEachOf(obj, iterator, callback) + +Like `each`, except that it iterates over objects, and passes the key as the second argument to the iterator. + +__Arguments__ + +* `obj` - An object or array to iterate over. +* `iterator(item, key, callback)` - A function to apply to each item in `obj`. +The `key` is the item's key, or index in the case of an array. The iterator is +passed a `callback(err)` which must be called once it has completed. If no +error has occurred, the callback should be run without arguments or with an +explicit `null` argument. +* `callback(err)` - A callback which is called when all `iterator` functions have finished, or an error occurs. + +__Example__ + +```js +var obj = {a: 1, b: 2, c: 3}; + +async.forEachOf(obj, function (value, key, callback) { + console.log(value + ":" + key) // 1:a, 2:b, 3:c, etc... + callback(); +}, function (err) { + +}) +``` + +--------------------------------------- + + + +### forEachOfSeries(obj, iterator, callback) + +Like [`forEachOf`](#forEachOf), except only one `iterator` is run at a time. The order of execution is not guaranteed for objects, but it will be for arrays. +--------------------------------------- + + + +### forEachOfLimit(obj, limit, iterator, callback) + +Like [`forEachOf`](#forEachOf), except only one the number of `iterator`s running at a time is controlled by `limit`. The order of execution is not guaranteed for objects, but it will be for arrays. + + --------------------------------------- From 52dd10d744a3c2d82b3f29466462e569adc8de39 Mon Sep 17 00:00:00 2001 From: Alexander Early Date: Tue, 20 Jan 2015 11:18:50 -0800 Subject: [PATCH 4/5] reordered functions, added aliases, simplified eachOfLimit logic --- lib/async.js | 104 +++++++++++++++++++++++---------------------- test/test-async.js | 97 +++++++++++++++++++++--------------------- 2 files changed, 102 insertions(+), 99 deletions(-) diff --git a/lib/async.js b/lib/async.js index c99024be4..185812fd9 100644 --- a/lib/async.js +++ b/lib/async.js @@ -171,7 +171,56 @@ }; async.forEachSeries = async.eachSeries; - async.forEachOf = function (object, iterator, callback) { + + async.eachLimit = function (arr, limit, iterator, callback) { + var fn = _eachLimit(limit); + fn.apply(null, [arr, iterator, callback]); + }; + async.forEachLimit = async.eachLimit; + + var _eachLimit = function (limit) { + + return function (arr, iterator, callback) { + callback = callback || function () {}; + if (!arr.length || limit <= 0) { + return callback(); + } + var completed = 0; + var started = 0; + var running = 0; + + (function replenish () { + if (completed >= arr.length) { + return callback(); + } + + while (running < limit && started < arr.length) { + started += 1; + running += 1; + iterator(arr[started - 1], function (err) { + if (err) { + callback(err); + callback = function () {}; + } + else { + completed += 1; + running -= 1; + if (completed >= arr.length) { + callback(); + } + else { + replenish(); + } + } + }); + } + })(); + }; + }; + + + + async.forEachOf = async.eachOf = function (object, iterator, callback) { callback = callback || function () {}; var size = object.length || _keys(object).length; var completed = 0 @@ -193,7 +242,7 @@ }); }; - async.forEachOfSeries = function (obj, iterator, callback) { + async.forEachOfSeries = async.eachOfSeries = function (obj, iterator, callback) { callback = callback || function () {}; var keys = _keys(obj); var size = keys.length; @@ -230,56 +279,9 @@ }; - async.eachLimit = function (arr, limit, iterator, callback) { - var fn = _eachLimit(limit); - fn.apply(null, [arr, iterator, callback]); - }; - async.forEachLimit = async.eachLimit; - - var _eachLimit = function (limit) { - - return function (arr, iterator, callback) { - callback = callback || function () {}; - if (!arr.length || limit <= 0) { - return callback(); - } - var completed = 0; - var started = 0; - var running = 0; - - (function replenish () { - if (completed >= arr.length) { - return callback(); - } - - while (running < limit && started < arr.length) { - started += 1; - running += 1; - iterator(arr[started - 1], function (err) { - if (err) { - callback(err); - callback = function () {}; - } - else { - completed += 1; - running -= 1; - if (completed >= arr.length) { - callback(); - } - else { - replenish(); - } - } - }); - } - })(); - }; - }; - - async.forEachOfLimit = function (obj, limit, iterator, callback) { - var fn = obj.constructor === Array ? _forEachOfLimit(limit) : _forEachOfLimit(limit); - fn.apply(null, [obj, iterator, callback]); + async.forEachOfLimit = async.eachOfLimit = function (obj, limit, iterator, callback) { + _forEachOfLimit(limit)(obj, iterator, callback); }; var _forEachOfLimit = function (limit) { diff --git a/test/test-async.js b/test/test-async.js index f5fc72ec3..73e712214 100755 --- a/test/test-async.js +++ b/test/test-async.js @@ -1256,54 +1256,6 @@ exports['eachSeries no callback'] = function(test){ async.eachSeries([1], eachNoCallbackIterator.bind(this, test)); }; -exports['forEachSeries alias'] = function (test) { - test.strictEqual(async.eachSeries, async.forEachSeries); - test.done(); -}; - -exports['forEachOfSeries'] = function(test){ - var args = []; - async.forEachOfSeries({ a: 1, b: 2 }, forEachOfIterator.bind(this, args), function(err){ - test.same(args, [ "a", 1, "b", 2 ]); - test.done(); - }); -}; - -exports['forEachOfSeries empty object'] = function(test){ - test.expect(1); - async.forEachOfSeries({}, function(x, callback){ - test.ok(false, 'iterator should not be called'); - callback(); - }, function(err){ - test.ok(true, 'should call callback'); - }); - setTimeout(test.done, 25); -}; - -exports['forEachOfSeries error'] = function(test){ - test.expect(2); - var call_order = []; - async.forEachOfSeries({ a: 1, b: 2 }, function(value, key, callback){ - call_order.push(value, key); - callback('error'); - }, function(err){ - test.same(call_order, [ 1, "a" ]); - test.equals(err, 'error'); - }); - setTimeout(test.done, 50); -}; - -exports['forEachOfSeries no callback'] = function(test){ - async.forEachOfSeries({ a: 1 }, forEachOfNoCallbackIterator.bind(this, test)); -}; - -exports['forEachOfSeries with array'] = function(test){ - var args = []; - async.forEachOfSeries([ "a", "b" ], forEachOfIterator.bind(this, args), function(err){ - test.same(args, [ 0, "a", 1, "b" ]); - test.done(); - }); -}; exports['eachLimit'] = function(test){ var args = []; @@ -1392,6 +1344,55 @@ exports['eachLimit synchronous'] = function(test){ }); }; +exports['forEachSeries alias'] = function (test) { + test.strictEqual(async.eachSeries, async.forEachSeries); + test.done(); +}; + +exports['forEachOfSeries'] = function(test){ + var args = []; + async.forEachOfSeries({ a: 1, b: 2 }, forEachOfIterator.bind(this, args), function(err){ + test.same(args, [ "a", 1, "b", 2 ]); + test.done(); + }); +}; + +exports['forEachOfSeries empty object'] = function(test){ + test.expect(1); + async.forEachOfSeries({}, function(x, callback){ + test.ok(false, 'iterator should not be called'); + callback(); + }, function(err){ + test.ok(true, 'should call callback'); + }); + setTimeout(test.done, 25); +}; + +exports['forEachOfSeries error'] = function(test){ + test.expect(2); + var call_order = []; + async.forEachOfSeries({ a: 1, b: 2 }, function(value, key, callback){ + call_order.push(value, key); + callback('error'); + }, function(err){ + test.same(call_order, [ 1, "a" ]); + test.equals(err, 'error'); + }); + setTimeout(test.done, 50); +}; + +exports['forEachOfSeries no callback'] = function(test){ + async.forEachOfSeries({ a: 1 }, forEachOfNoCallbackIterator.bind(this, test)); +}; + +exports['forEachOfSeries with array'] = function(test){ + var args = []; + async.forEachOfSeries([ "a", "b" ], forEachOfIterator.bind(this, args), function(err){ + test.same(args, [ 0, "a", 1, "b" ]); + test.done(); + }); +}; + exports['forEachLimit alias'] = function (test) { test.strictEqual(async.eachLimit, async.forEachLimit); test.done(); From ae5d25f9e139af86bc5ef1431380e6f20f8eeea9 Mon Sep 17 00:00:00 2001 From: Alexander Early Date: Tue, 20 Jan 2015 11:30:12 -0800 Subject: [PATCH 5/5] tightening up the documentation --- README.md | 29 ++++++++++++++++++++++------- lib/async.js | 2 +- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0f2e41bc3..887e386eb 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,8 @@ __Arguments__ * `iterator(item, callback)` - A function to apply to each item in `arr`. The iterator is passed a `callback(err)` which must be called once it has completed. If no error has occurred, the `callback` should be run without - arguments or with an explicit `null` argument. + arguments or with an explicit `null` argument. The array index is not passed + to the iterator. If you need the index, use [`forEachOf`](#forEachOf). * `callback(err)` - A callback which is called when all `iterator` functions have finished, or an error occurs. @@ -286,6 +287,7 @@ async.eachLimit(documents, 20, requestApi, function(err){ --------------------------------------- + ### forEachOf(obj, iterator, callback) @@ -304,30 +306,43 @@ explicit `null` argument. __Example__ ```js -var obj = {a: 1, b: 2, c: 3}; +var obj = {dev: "/dev.json", test: "/test.json", prod: "/prod.json"}; +var configs = {}; async.forEachOf(obj, function (value, key, callback) { - console.log(value + ":" + key) // 1:a, 2:b, 3:c, etc... - callback(); + fs.readFile(__dirname + value, "utf8", function (err, data) { + if (err) return callback(err); + try { + configs[key] = JSON.parse(data); + } catch (e) { + return callback(e); + } + callback(); + }) }, function (err) { - + if (err) console.error(err.message); + // configs is now a map of JSON data + doSomethingWith(configs); }) ``` --------------------------------------- + ### forEachOfSeries(obj, iterator, callback) -Like [`forEachOf`](#forEachOf), except only one `iterator` is run at a time. The order of execution is not guaranteed for objects, but it will be for arrays. +Like [`forEachOf`](#forEachOf), except only one `iterator` is run at a time. The order of execution is not guaranteed for objects, but it will be guaranteed for arrays. + --------------------------------------- + ### forEachOfLimit(obj, limit, iterator, callback) -Like [`forEachOf`](#forEachOf), except only one the number of `iterator`s running at a time is controlled by `limit`. The order of execution is not guaranteed for objects, but it will be for arrays. +Like [`forEachOf`](#forEachOf), except the number of `iterator`s running at a given time is controlled by `limit`. --------------------------------------- diff --git a/lib/async.js b/lib/async.js index 185812fd9..87ebe47a2 100644 --- a/lib/async.js +++ b/lib/async.js @@ -280,7 +280,7 @@ - async.forEachOfLimit = async.eachOfLimit = function (obj, limit, iterator, callback) { + async.forEachOfLimit = async.eachOfLimit = function (obj, limit, iterator, callback) { _forEachOfLimit(limit)(obj, iterator, callback); };