Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Backport PR #84 and #85 #86

Merged
merged 2 commits into from
Jun 1, 2015
Merged

Backport PR #84 and #85 #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad93ac8
add X-Requested-With header; fix bug that broke retry; had to turn of…
lingyan Jun 1, 2015
1a52919
move default constructGetUri function to its own file
lingyan Jun 1, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 37 additions & 79 deletions libs/fetcher.client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,27 @@
* batches requests into one request.
* @module Fetcher
*/
var REST = require('./util/http.client'),
debug = require('debug')('FetchrClient'),
lodash = {
var REST = require('./util/http.client');
var debug = require('debug')('FetchrClient');
var lodash = {
isArray: require('lodash/lang/isArray'),
isFunction: require('lodash/lang/isFunction'),
isObject: require('lodash/lang/isObject'),
forEach: require('lodash/collection/forEach'),
merge: require('lodash/object/merge'),
noop: require('lodash/utility/noop'),
pick: require('lodash/object/pick'),
some: require('lodash/collection/some'),
values: require('lodash/object/values')
},
CORE_REQUEST_FIELDS = ['resource', 'operation', 'params', 'body'],
DEFAULT_GUID = 'g0',
DEFAULT_XHR_PATH = '/api',
// By default, wait for 20ms to trigger sweep of the queue, after an item is added to the queue.
DEFAULT_BATCH_WINDOW = 20,
MAX_URI_LEN = 2048,
OP_READ = 'read',
NAME = 'FetcherClient';
};
var CORE_REQUEST_FIELDS = ['resource', 'operation', 'params', 'body'];
var DEFAULT_GUID = 'g0';
var DEFAULT_XHR_PATH = '/api';
// By default, wait for 20ms to trigger sweep of the queue, after an item is added to the queue.
var DEFAULT_BATCH_WINDOW = 20;
var MAX_URI_LEN = 2048;
var OP_READ = 'read';
var NAME = 'FetcherClient';
var defaultConstructGetUri = require('./util/defaultConstructGetUri');

function parseResponse(response) {
if (response && response.responseText) {
Expand All @@ -47,13 +47,6 @@ function parseResponse(response) {
return null;
}

function jsonifyComplexType(value) {
if (lodash.isArray(value) || lodash.isObject(value)) {
return JSON.stringify(value);
}
return value;
}

/**
* The queue sweeps and processs items in the queue when there are items in the queue.
* When a item is pushed into the queue, a timeout is set to guarantee the item will be processd soon.
Expand Down Expand Up @@ -115,13 +108,13 @@ Queue.prototype = {

// setup timer
if (!this._timer) {
this._timer = setTimeout(function () {
this._timer = setTimeout(function sweepInterval() {
var items = self._items;
self._items = [];
clearTimeout(self._timer);
self._timer = null;
items = self._sweep(items);
lodash.forEach(items, function (item) {
lodash.forEach(items, function eachItem(item) {
self._cb(item);
});
}, this.config.wait);
Expand Down Expand Up @@ -269,9 +262,9 @@ Queue.prototype = {
if (!this._q) {
this._q = new Queue(this.name, {
wait: Fetcher.batchWindow
}, function (requests) {
}, function afterWait(requests) {
return self.batch(requests);
}, function (batched) {
}, function afterBatch(batched) {
if (!batched) {
return;
}
Expand All @@ -296,41 +289,14 @@ Queue.prototype = {
* @param {Object} params Parameters to be serialized
* @param {Object} Configuration object
*/
_defaultConstructGetUri: function (uri, resource, params, config) {
var query = [], matrix = [], id_param = config.id_param, id_val, final_uri = uri + '/resource/' + resource;
lodash.forEach(params, function (v, k) {
if (k === id_param) {
id_val = encodeURIComponent(v);
} else {
try {
matrix.push(k + '=' + encodeURIComponent(jsonifyComplexType(v)));
} catch (err) {
debug('jsonifyComplexType failed: ' + err, 'info', NAME);
}
}
});

lodash.forEach(this.context, function (v, k) {
query.push(k + '=' + encodeURIComponent(jsonifyComplexType(v)));
});
if (id_val) {
final_uri += '/' + id_param + '/' + id_val;
}
if (matrix.length > 0) {
final_uri += ';' + matrix.sort().join(';');
}
if (query.length > 0) {
final_uri += '?' + query.sort().join('&');
}
return final_uri;
},
_defaultConstructGetUri: defaultConstructGetUri,
/**
* @method _constructGroupUri
* @private
*/
_constructGroupUri: function (uri) {
var query = [], final_uri = uri;
lodash.forEach(this.context, function (v, k) {
lodash.forEach(this.context, function eachContext(v, k) {
query.push(k + '=' + encodeURIComponent(v));
});
if (query.length > 0) {
Expand Down Expand Up @@ -363,17 +329,14 @@ Queue.prototype = {
return;
}

var config = request.config,
callback = request.callback || lodash.noop,
use_post,
allow_retry_post,
uri = config.uri,
get_uri,
requests,
params,
data;


var config = request.config;
var callback = request.callback || lodash.noop;
var use_post;
var allow_retry_post;
var uri = config.uri;
var requests;
var params;
var data;

if (!uri) {
uri = config.cors ? this.corsPath : this.xhrPath;
Expand All @@ -382,13 +345,8 @@ Queue.prototype = {
use_post = request.operation !== OP_READ || config.post_for_read;

if (!use_post) {
if (lodash.isFunction(config.constructGetUri)) {
get_uri = config.constructGetUri.call(this, uri, request.resource, request.params, config);
}

if (!get_uri) {
get_uri = this._defaultConstructGetUri(uri, request.resource, request.params, config);
}
var getUriFn = lodash.isFunction(config.constructGetUri) ? config.constructGetUri : defaultConstructGetUri;
var get_uri = getUriFn.call(this, uri, request.resource, request.params, config, this.context);
if (get_uri.length <= MAX_URI_LEN) {
uri = get_uri;
} else {
Expand All @@ -397,7 +355,7 @@ Queue.prototype = {
}

if (!use_post) {
return REST.get(uri, {}, config, function (err, response) {
return REST.get(uri, {}, config, function getDone(err, response) {
if (err) {
debug('Syncing ' + request.resource + ' failed: statusCode=' + err.statusCode, 'info', NAME);
return callback(err);
Expand All @@ -418,7 +376,7 @@ Queue.prototype = {
}; // TODO: remove. leave here for now for backward compatibility
uri = this._constructGroupUri(uri);
allow_retry_post = (request.operation === OP_READ);
REST.post(uri, {}, data, lodash.merge({unsafeAllowRetry: allow_retry_post}, config), function (err, response) {
REST.post(uri, {}, data, lodash.merge({unsafeAllowRetry: allow_retry_post}, config), function postDone(err, response) {
if (err) {
debug('Syncing ' + request.resource + ' failed: statusCode=' + err.statusCode, 'info', NAME);
return callback(err);
Expand Down Expand Up @@ -450,7 +408,7 @@ Queue.prototype = {
var batched,
groups = {};

lodash.forEach(requests, function (request) {
lodash.forEach(requests, function eachRequest(request) {
var uri, batch, group_id;
if (request.config) {
uri = request.config.uri;
Expand Down Expand Up @@ -493,7 +451,7 @@ Queue.prototype = {
allow_retry_post = true,
request_map = {};

lodash.some(requests, function (request) {
lodash.some(requests, function findConfig(request) {
if (request.config) {
config = request.config;
return true;
Expand All @@ -508,7 +466,7 @@ Queue.prototype = {
context: this.context
}; // TODO: remove. leave here for now for backward compatibility

lodash.forEach(requests, function (request, i) {
lodash.forEach(requests, function eachRequest(request, i) {
var guid = 'g' + i;
data.requests[guid] = lodash.pick(request, CORE_REQUEST_FIELDS);
request_map[guid] = request;
Expand All @@ -518,16 +476,16 @@ Queue.prototype = {
});

uri = this._constructGroupUri(uri);
REST.post(uri, {}, data, lodash.merge({unsafeAllowRetry: allow_retry_post}, config), function (err, response) {
REST.post(uri, {}, data, lodash.merge({unsafeAllowRetry: allow_retry_post}, config), function postDone(err, response) {
if (err) {
lodash.forEach(requests, function (request) {
lodash.forEach(requests, function passErrorToEachRequest(request) {
request.callback(err);
});
return;
}
var result = parseResponse(response);
// split result for requests, so that each request gets back only the data that was originally requested
lodash.forEach(request_map, function (request, guid) {
lodash.forEach(request_map, function passREsultToEachRequest(request, guid) {
var res = (result && result[guid]) || {};
if (request.callback) {
request.callback(res.err || null, res.data || null);
Expand Down
69 changes: 69 additions & 0 deletions libs/util/defaultConstructGetUri.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/**
* Copyright 2015, Yahoo! Inc.
* Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.
*/
'use strict';

var debug = require('debug')('Fetcher:defaultConstructGetUri');
var lodash = {
forEach: require('lodash/collection/forEach'),
isArray: require('lodash/lang/isArray'),
isObject: require('lodash/lang/isObject')
};

function jsonifyComplexType(value) {
if (lodash.isArray(value) || lodash.isObject(value)) {
return JSON.stringify(value);
}
return value;
}

/**
* Construct xhr GET URI.
* @method defaultConstructGetUri
* @param {String} uri base URI
* @param {String} resource Resource name
* @param {Object} params Parameters to be serialized
* @param {Object} config Configuration object
* @param {String} config.id_param Name of the id parameter
* @param {Object} context Context object, which will become query params
*/
module.exports = function defaultConstructGetUri(baseUri, resource, params, config, context) {
var query = [];
var matrix = [];
var id_param = config.id_param;
var id_val;
var final_uri = baseUri + '/resource/' + resource;

if (params) {
lodash.forEach(params, function eachParam(v, k) {
if (k === id_param) {
id_val = encodeURIComponent(v);
} else {
try {
matrix.push(k + '=' + encodeURIComponent(jsonifyComplexType(v)));
} catch (err) {
debug('jsonifyComplexType failed: ' + err);
}
}
});
}

if (context) {
lodash.forEach(context, function eachContext(v, k) {
query.push(k + '=' + encodeURIComponent(jsonifyComplexType(v)));
});
}

if (id_val) {
final_uri += '/' + id_param + '/' + id_val;
}
if (matrix.length > 0) {
final_uri += ';' + matrix.sort().join(';');
}
if (query.length > 0) {
final_uri += '?' + query.sort().join('&');
}
debug('constructed get uri:', final_uri);
return final_uri;
};
14 changes: 8 additions & 6 deletions libs/util/http.client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ var _ = require('lodash'),
timeout: 3000,
retry: {
interval: 200,
max_retries: 2
max_retries: 0
}
},
CONTENT_TYPE = 'Content-Type',
Expand All @@ -33,13 +33,15 @@ var _ = require('lodash'),

//trim polyfill, maybe pull from npm later
if (!String.prototype.trim) {
String.prototype.trim = function () {
return this.replace(/^\s+|\s+$/g, '');
};
String.prototype.trim = function () {
return this.replace(/^\s+|\s+$/g, '');
};
}

function normalizeHeaders(headers, method) {
var normalized = {};
var normalized = {
'X-Requested-With': 'XMLHttpRequest'
};
var needContentType = (method === METHOD_PUT || method === METHOD_POST);
_.forEach(headers, function (v, field) {
if (field.toLowerCase() === 'content-type') {
Expand Down Expand Up @@ -139,7 +141,7 @@ function doXhr(method, url, headers, data, config, callback) {
callback(NULL, response);
},
failure : function (err, response) {
if (!shouldRetry(method, config, response.status)) {
if (!shouldRetry(method, config, response.statusCode)) {
callback(err);
} else {
_.delay(
Expand Down
Loading