diff --git a/src.ts/utils/fetch.ts b/src.ts/utils/fetch.ts index 47bcc9e6fa..f540c4f698 100644 --- a/src.ts/utils/fetch.ts +++ b/src.ts/utils/fetch.ts @@ -23,7 +23,7 @@ import { assert, assertArgument } from "./errors.js"; import { defineProperties } from "./properties.js"; import { toUtf8Bytes, toUtf8String } from "./utf8.js" -import { getUrl } from "./geturl.js"; +import { createGetUrl } from "./geturl.js"; /** * An environments implementation of ``getUrl`` must return this type. @@ -77,7 +77,7 @@ const MAX_ATTEMPTS = 12; const SLOT_INTERVAL = 250; // The global FetchGetUrlFunc implementation. -let getUrlFunc: FetchGetUrlFunc = getUrl; +let defaultGetUrlFunc: FetchGetUrlFunc = createGetUrl(); const reData = new RegExp("^data:([^;:]*)?(;base64)?,(.*)$", "i"); const reIpfs = new RegExp("^ipfs:/\/(ipfs/)?(.*)$", "i"); @@ -201,6 +201,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> { #throttle: Required; + #getUrlFunc: null | FetchGetUrlFunc; + /** * The fetch URI to requrest. */ @@ -429,6 +431,28 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> { this.#retry = retry; } + /** + * This function is called to fetch content from HTTP and + * HTTPS URLs and is platform specific (e.g. nodejs vs + * browsers). + * + * This is by default the currently registered global getUrl + * function, which can be changed using [[registerGetUrl]]. + * If this has been set, setting is to ``null`` will cause + * this FetchRequest (and any future clones) to revert back to + * using the currently registered global getUrl function. + * + * Setting this is generally not necessary, but may be useful + * for developers that wish to intercept requests or to + * configurege a proxy or other agent. + */ + get getUrlFunc(): FetchGetUrlFunc { + return this.#getUrlFunc || defaultGetUrlFunc; + } + set getUrlFunc(value: null | FetchGetUrlFunc) { + this.#getUrlFunc = value; + } + /** * Create a new FetchRequest instance with default values. * @@ -448,6 +472,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> { slotInterval: SLOT_INTERVAL, maxAttempts: MAX_ATTEMPTS }; + + this.#getUrlFunc = null; } toString(): string { @@ -510,7 +536,7 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> { // We have a preflight function; update the request if (this.preflightFunc) { req = await this.preflightFunc(req); } - const resp = await getUrlFunc(req, checkSignal(_request.#signal)); + const resp = await this.getUrlFunc(req, checkSignal(_request.#signal)); let response = new FetchResponse(resp.statusCode, resp.statusMessage, resp.headers, resp.body, _request); if (response.statusCode === 301 || response.statusCode === 302) { @@ -641,6 +667,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> { clone.#process = this.#process; clone.#retry = this.#retry; + clone.#getUrlFunc = this.#getUrlFunc; + return clone; } @@ -686,7 +714,22 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> { */ static registerGetUrl(getUrl: FetchGetUrlFunc): void { if (locked) { throw new Error("gateways locked"); } - getUrlFunc = getUrl; + defaultGetUrlFunc = getUrl; + } + + /** + * Creates a getUrl function that fetches content from HTTP and + * HTTPS URLs. + * + * The available %%options%% are dependent on the platform + * implementation of the default getUrl function. + * + * This is not generally something that is needed, but is useful + * when trying to customize simple behaviour when fetching HTTP + * content. + */ + static createGetUrlFunc(options?: Record): FetchGetUrlFunc { + return createGetUrl(options); } /** diff --git a/src.ts/utils/geturl-browser.ts b/src.ts/utils/geturl-browser.ts index 9d528f6788..a71abb2553 100644 --- a/src.ts/utils/geturl-browser.ts +++ b/src.ts/utils/geturl-browser.ts @@ -1,6 +1,8 @@ import { assert } from "./errors.js"; -import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js"; +import type { + FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse +} from "./fetch.js"; declare global { @@ -27,46 +29,58 @@ declare global { // @TODO: timeout is completely ignored; start a Promise.any with a reject? -export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise { - const protocol = req.url.split(":")[0].toLowerCase(); - - assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", { - info: { protocol }, - operation: "request" - }); - - assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", { - operation: "request" - }); - - let signal: undefined | AbortSignal = undefined; - if (_signal) { - const controller = new AbortController(); - signal = controller.signal; - _signal.addListener(() => { controller.abort(); }); +export function createGetUrl(options?: Record): FetchGetUrlFunc { + + async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise { + const protocol = req.url.split(":")[0].toLowerCase(); + + assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", { + info: { protocol }, + operation: "request" + }); + + assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", { + operation: "request" + }); + + let signal: undefined | AbortSignal = undefined; + if (_signal) { + const controller = new AbortController(); + signal = controller.signal; + _signal.addListener(() => { controller.abort(); }); + } + + const init = { + method: req.method, + headers: new Headers(Array.from(req)), + body: req.body || undefined, + signal + }; + + const resp = await fetch(req.url, init); + + const headers: Record = { }; + resp.headers.forEach((value, key) => { + headers[key.toLowerCase()] = value; + }); + + const respBody = await resp.arrayBuffer(); + const body = (respBody == null) ? null: new Uint8Array(respBody); + + return { + statusCode: resp.status, + statusMessage: resp.statusText, + headers, body + }; } - const init = { - method: req.method, - headers: new Headers(Array.from(req)), - body: req.body || undefined, - signal - }; - - const resp = await fetch(req.url, init); - - const headers: Record = { }; - resp.headers.forEach((value, key) => { - headers[key.toLowerCase()] = value; - }); + return getUrl; +} - const respBody = await resp.arrayBuffer(); - const body = (respBody == null) ? null: new Uint8Array(respBody); +// @TODO: remove in v7; provided for backwards compat +const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ }); - return { - statusCode: resp.status, - statusMessage: resp.statusText, - headers, body - }; +export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise { + return defaultGetUrl(req, _signal); } diff --git a/src.ts/utils/geturl.ts b/src.ts/utils/geturl.ts index de024c6da2..b7220acd24 100644 --- a/src.ts/utils/geturl.ts +++ b/src.ts/utils/geturl.ts @@ -5,92 +5,112 @@ import { gunzipSync } from "zlib"; import { assert } from "./errors.js"; import { getBytes } from "./data.js"; -import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js"; +import type { + FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse +} from "./fetch.js"; /** * @_ignore: */ -export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise { +export function createGetUrl(options?: Record): FetchGetUrlFunc { - const protocol = req.url.split(":")[0].toLowerCase(); + async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise { - assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", { - info: { protocol }, - operation: "request" - }); + const protocol = req.url.split(":")[0].toLowerCase(); - assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", { - operation: "request" - }); + assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", { + info: { protocol }, + operation: "request" + }); - const method = req.method; - const headers = Object.assign({ }, req.headers); + assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", { + operation: "request" + }); - const options: any = { method, headers }; + const method = req.method; + const headers = Object.assign({ }, req.headers); - const request = ((protocol === "http") ? http: https).request(req.url, options); + const reqOptions: any = { method, headers }; + if (options) { + if (options.agent) { reqOptions.agent = options.agent; } + } - request.setTimeout(req.timeout); + const request = ((protocol === "http") ? http: https).request(req.url, reqOptions); - const body = req.body; - if (body) { request.write(Buffer.from(body)); } + request.setTimeout(req.timeout); - request.end(); + const body = req.body; + if (body) { request.write(Buffer.from(body)); } - return new Promise((resolve, reject) => { - // @TODO: Node 15 added AbortSignal; once we drop support for - // Node14, we can add that in here too + request.end(); - request.once("response", (resp: http.IncomingMessage) => { - const statusCode = resp.statusCode || 0; - const statusMessage = resp.statusMessage || ""; - const headers = Object.keys(resp.headers || {}).reduce((accum, name) => { - let value = resp.headers[name] || ""; - if (Array.isArray(value)) { - value = value.join(", "); - } - accum[name] = value; - return accum; - }, <{ [ name: string ]: string }>{ }); + return new Promise((resolve, reject) => { + // @TODO: Node 15 added AbortSignal; once we drop support for + // Node14, we can add that in here too - let body: null | Uint8Array = null; - //resp.setEncoding("utf8"); + request.once("response", (resp: http.IncomingMessage) => { + const statusCode = resp.statusCode || 0; + const statusMessage = resp.statusMessage || ""; + const headers = Object.keys(resp.headers || {}).reduce((accum, name) => { + let value = resp.headers[name] || ""; + if (Array.isArray(value)) { + value = value.join(", "); + } + accum[name] = value; + return accum; + }, <{ [ name: string ]: string }>{ }); + + let body: null | Uint8Array = null; + //resp.setEncoding("utf8"); + + resp.on("data", (chunk: Uint8Array) => { + if (signal) { + try { + signal.checkSignal(); + } catch (error) { + return reject(error); + } + } - resp.on("data", (chunk: Uint8Array) => { - if (signal) { - try { - signal.checkSignal(); - } catch (error) { - return reject(error); + if (body == null) { + body = chunk; + } else { + const newBody = new Uint8Array(body.length + chunk.length); + newBody.set(body, 0); + newBody.set(chunk, body.length); + body = newBody; } - } - - if (body == null) { - body = chunk; - } else { - const newBody = new Uint8Array(body.length + chunk.length); - newBody.set(body, 0); - newBody.set(chunk, body.length); - body = newBody; - } - }); + }); - resp.on("end", () => { - if (headers["content-encoding"] === "gzip" && body) { - body = getBytes(gunzipSync(body)); - } + resp.on("end", () => { + if (headers["content-encoding"] === "gzip" && body) { + body = getBytes(gunzipSync(body)); + } - resolve({ statusCode, statusMessage, headers, body }); - }); + resolve({ statusCode, statusMessage, headers, body }); + }); - resp.on("error", (error) => { - //@TODO: Should this just return nornal response with a server error? - (error).response = { statusCode, statusMessage, headers, body }; - reject(error); + resp.on("error", (error) => { + //@TODO: Should this just return nornal response with a server error? + (error).response = { statusCode, statusMessage, headers, body }; + reject(error); + }); }); + + request.on("error", (error) => { reject(error); }); }); + } + + return getUrl; +} + +// @TODO: remove in v7; provided for backwards compat +const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ }); - request.on("error", (error) => { reject(error); }); - }); +/** + * @_ignore: + */ +export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise { + return defaultGetUrl(req, signal); }