Skip to content

Commit

Permalink
refactor: use native URL & URLSearchParams
Browse files Browse the repository at this point in the history 
...and ditch `qs` and `url`

BREAKING CHANGE: Now requires Node 8.x and/or a browser with native URL and URLSearchParams support
  • Loading branch information
@dbushong
dbushong committed Sep 23, 2019
1 parent 0e77ee5 commit bbe30c1
Show file tree
Hide file tree
Showing 7 changed files with 112 additions and 142 deletions.
98 changes: 41 additions & 57 deletions lib/fetch.browser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,9 @@
'use strict';

/* eslint-env browser */
/* global URLSearchParams */
const Url = require('url');

const qsParser = require('qs');

const StatusCodeError = require('./errors').StatusCodeError;
const urlUtils = require('./url');

const applyBaseUrl = urlUtils.applyBaseUrl;
const replacePathParams = urlUtils.replacePathParams;
const { replacePathParams, updateSearch } = require('./url');

const DEFAULT_TIMEOUT = 10 * 1000;

Expand Down Expand Up @@ -78,7 +71,7 @@ function isValidBody(body) {
body === undefined ||
typeof body === 'string' ||
(typeof FormData !== 'undefined' && body instanceof FormData) ||
(typeof URLSearchParams !== 'undefined' && body instanceof URLSearchParams)
body instanceof URLSearchParams
);
}

Expand All @@ -89,22 +82,6 @@ function validateBody(body) {
return body;
}

function generateSearch(queryString, qs) {
const query = Object.assign(qsParser.parse(queryString), qs || {});
const filtered = {};
const queryKeys = Object.keys(query).filter(key => {
const value = query[key];
const isSet = value !== null && value !== undefined;
if (isSet) {
filtered[key] = value;
}
return isSet;
});

if (queryKeys.length === 0) return '';
return `?${qsParser.stringify(filtered)}`;
}

function filterHeaders(headers) {
const filtered = {};
Object.keys(headers).forEach(name => {
Expand Down Expand Up @@ -210,18 +187,35 @@ function defaultTimeout(value, defaultValue) {
}

function fetchUrl(url, options) {
if (typeof url !== 'string') {
throw new TypeError('url has to be a string');
}

options = options || {};
let urlObj = Url.parse(url);
if (options.baseUrl && typeof options.baseUrl === 'string') {
urlObj = applyBaseUrl(urlObj, options.baseUrl);

const {
auth,
json,
form,
headers,
method = 'GET',
redirect,
serviceName,
endpointName,
pathParams,
methodName,
} = options;

let { baseUrl } = options;
if (!baseUrl && url && (typeof url === 'string' || url instanceof URL)) {
baseUrl = location.href;
}
if (baseUrl) {
if (baseUrl.includes('?')) {
throw new Error('baseUrl may not contain a query string');
}
if (baseUrl.substr(-1) !== '/') baseUrl += '/';
if (typeof url === 'string' && url[0] === '/') url = url.substr(1);
}

const json = options.json;
const form = options.form;
const urlObj = new URL(url, baseUrl);

let body = validateBody(options.body);

const defaultHeaders = {};
Expand All @@ -237,10 +231,9 @@ function fetchUrl(url, options) {
}
defaultHeaders['Content-Type'] =
'application/x-www-form-urlencoded;charset=UTF-8';
body = qsParser.stringify(form);
body = updateSearch(new URLSearchParams(), form).toString();
}

const auth = options.auth;
if (typeof auth === 'string') {
defaultHeaders.Authorization = `Basic ${btoa(auth)}`;
} else if (auth !== null && typeof auth === 'object') {
Expand All @@ -251,33 +244,24 @@ function fetchUrl(url, options) {

const timeout = defaultTimeout(options.timeout, DEFAULT_TIMEOUT);

const method = options.method || 'GET';
const nativeOptions = {
// All official fetch options:
method,
headers: filterHeaders(Object.assign(defaultHeaders, options.headers)),
headers: filterHeaders({ ...defaultHeaders, ...headers }),
body,
redirect: options.redirect,
redirect,
// Some things we might want to expose for instrumentation to pick up:
serviceName: options.serviceName,
endpointName: options.endpointName,
methodName: options.methodName || method.toLowerCase(),
pathParams: options.pathParams,
serviceName,
endpointName,
methodName: methodName || method.toLowerCase(),
pathParams,
};
const patchedPathname = replacePathParams(
urlObj.pathname,
options.pathParams
);
const patchedSearch = generateSearch(urlObj.query, options.qs);
const patchedUrl = {
protocol: urlObj.protocol,
hostname: urlObj.hostname,
port: urlObj.port,
pathname: patchedPathname,
search: patchedSearch,
path: patchedPathname + patchedSearch,
};
const nativeUrl = Url.format(patchedUrl);

urlObj.pathname = replacePathParams(urlObj.pathname, options.pathParams);
updateSearch(urlObj.searchParams, options.qs);

const nativeUrl = urlObj.toString();

const result = new Promise((resolve, reject) => {
function onTimedOut() {
const error = new Error(`Fetching from ${urlObj.hostname} timed out`);
Expand Down
68 changes: 27 additions & 41 deletions lib/fetch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,11 @@

const http = require('http');
const https = require('https');
const parseUrl = require('url').parse;
const { URL, URLSearchParams } = require('url');

const qsParser = require('qs');

const urlUtils = require('./url');
const request = require('./request');
const wrapForCallback = require('./legacy');

const applyBaseUrl = urlUtils.applyBaseUrl;
const replacePathParams = urlUtils.replacePathParams;
const { replacePathParams, updateSearch } = require('./url');

const DEFAULT_CONNECT_TIMEOUT = 1000;
const DEFAULT_TIMEOUT = 10 * 1000;
Expand Down Expand Up @@ -93,22 +88,6 @@ function getAgent(options, urlObj) {
return agent;
}

function generateSearch(queryString, qs) {
const query = Object.assign(qsParser.parse(queryString), qs || {});
const filtered = {};
const queryKeys = Object.keys(query).filter(key => {
const value = query[key];
const isSet = value !== null && value !== undefined;
if (isSet) {
filtered[key] = value;
}
return isSet;
});

if (queryKeys.length === 0) return '';
return `?${qsParser.stringify(filtered)}`;
}

function filterHeaders(headers) {
const filtered = {};
Object.keys(headers).forEach(name => {
Expand All @@ -134,10 +113,10 @@ function unifyAuth(auth) {
return `${user}:${pass}`;
}

function buildUserAgent(options) {
return `${options.clientName || 'noServiceName'}/${options.clientVersion ||
'noServiceVersion'} (${options.appName || 'noAppName'}/${options.appSha ||
'noAppSha'}; ${options.fqdn || 'noFQDN'})`;
function buildUserAgent({ clientName, clientVersion, appName, appSha, fqdn }) {
return `${clientName || 'noServiceName'}/${clientVersion ||
'noServiceVersion'} (${appName || 'noAppName'}/${appSha ||
'noAppSha'}; ${fqdn || 'noFQDN'})`;
}

function defaultTimeout(value, defaultValue) {
Expand Down Expand Up @@ -186,17 +165,15 @@ function buildHostname(hostname, searchDomain) {
}

function fetchUrlObj(urlObj, options) {
if (options.baseUrl && typeof options.baseUrl === 'string') {
urlObj = applyBaseUrl(urlObj, options.baseUrl);
}
const { json, form, baseUrl, qs, headers, pathParams, auth } = options;

if (baseUrl && typeof baseUrl === 'string') urlObj = new URL(urlObj, baseUrl);

const defaultHeaders = {
'Accept-Encoding': 'gzip',
'User-Agent': buildUserAgent(options),
};
let body = validateBody(options.body);
const json = options.json;
const form = options.form;

if (json !== undefined && json !== null) {
defaultHeaders['Content-Type'] = 'application/json;charset=UTF-8';
Expand All @@ -209,7 +186,7 @@ function fetchUrlObj(urlObj, options) {
}
defaultHeaders['Content-Type'] =
'application/x-www-form-urlencoded;charset=UTF-8';
body = qsParser.stringify(form);
body = updateSearch(new URLSearchParams(), form).toString();
}

const hostname = buildHostname(urlObj.hostname, options.searchDomain);
Expand All @@ -236,6 +213,8 @@ function fetchUrlObj(urlObj, options) {
maxFreeSockets: options.maxFreeSockets || 256,
});

updateSearch(urlObj.searchParams, qs);

const method = options.method || 'GET';
return request({
agent,
Expand All @@ -244,11 +223,9 @@ function fetchUrlObj(urlObj, options) {
hostname,
port: urlObj.port,
method,
path:
replacePathParams(urlObj.pathname, options.pathParams) +
generateSearch(urlObj.query, options.qs),
headers: filterHeaders(Object.assign(defaultHeaders, options.headers)),
auth: unifyAuth(options.auth || urlObj.auth),
path: replacePathParams(urlObj.pathname, pathParams) + urlObj.search,
headers: filterHeaders({ ...defaultHeaders, ...headers }),
auth: unifyAuth(auth || (urlObj.username && urlObj)),
localAddress: options.localAddress,
body,
connectTimeout: defaultTimeout(
Expand Down Expand Up @@ -277,10 +254,19 @@ function nodeify(promise, callback) {
}

function fetch(url, options, callback) {
if (typeof url !== 'string') {
throw new TypeError('url has to be a string');
if (!options) options = {};
let { baseUrl } = options;
if (baseUrl) {
if (baseUrl.includes('?')) {
throw new Error('baseUrl may not contain a query string');
}
if (baseUrl.substr(-1) !== '/') baseUrl += '/';
if (typeof url === 'string' && url[0] === '/') url = url.substr(1);
}
return nodeify(fetchUrlObj(parseUrl(url), options || {}), callback);
return nodeify(
fetchUrlObj(new URL(url, baseUrl || undefined), options),
callback
);
}

module.exports = fetch;
10 changes: 4 additions & 6 deletions lib/request.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

const http = require('http');
const https = require('https');
const formatUrl = require('url').format;
const { URL } = require('url');

const debug = require('debug')('gofer');

Expand Down Expand Up @@ -135,13 +135,11 @@ const reqProperties = {

function buildFullUrl(options) {
const pathParts = options.path.split('?');
return formatUrl({
protocol: options.protocol,
hostname: options.hostname,
return Object.assign(new URL(`${options.protocol}//${options.hostname}`), {
port: options.port,
pathname: pathParts[0],
search: pathParts[1],
});
search: pathParts[1] || '',
}).toString();
}

function requestFunc(options, resolve, reject) {
Expand Down
52 changes: 26 additions & 26 deletions lib/url.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,38 +32,38 @@

'use strict';

const url = require('url');

function applyBaseUrl(urlObj, baseUrl) {
// If the url already is absolute, baseUrl isn't needed.
if (urlObj.hostname) return urlObj;

const base = url.parse(baseUrl);

if (base.query) {
throw new Error('baseUrl may not contain a query string');
/**
* @typedef {string | QSVal[] | { [name: string]: QSVal }} QSVal
*
* @param {URLSearchParams} query
* @param {{ [name: string]: QSVal }} qs
* @param {string} prefix
*/
function updateSearch(query, qs, path = []) {
for (const [key, val] of Object.entries(qs || {})) {
if (val == null) continue;
if (typeof val === 'object') updateSearch(query, val, [...path, key]);
else {
const [first, ...rest] = [...path, key];
query.set(`${first}${rest.map(r => `[${r}]`).join('')}`, val);
}
}

const basePath = base.pathname && base.pathname !== '/' ? base.pathname : '';

return {
// Protocol/auth/hostname/port always apply
protocol: base.protocol,
auth: base.auth,
host: base.host,
hostname: base.hostname,
port: base.port,

// For the pathname, we join. E.g. http://host/v2 + /my-resource
pathname: basePath + (urlObj.pathname || '') || '/',
query: urlObj.query,
};
return query;
}
exports.applyBaseUrl = applyBaseUrl;
exports.updateSearch = updateSearch;

/**
* @param {string} pathname
* @param {{ [name: string]: string }} pathParams
*/
function replacePathParams(pathname, pathParams) {
pathParams = pathParams || {};

/**
* @param {string} match
* @param {string} fromCurly
* @param {string} fromEscaped
*/
function onPlaceHolder(match, fromCurly, fromEscaped) {
const key = fromCurly || fromEscaped;
const value = pathParams[fromCurly || fromEscaped];
Expand Down
Loading

0 comments on commit bbe30c1

Please sign in to comment.