diff --git a/packages/web/src.ts/browser-geturl.ts b/packages/web/src.ts/browser-geturl.ts index 51c35777a0..03c7515090 100644 --- a/packages/web/src.ts/browser-geturl.ts +++ b/packages/web/src.ts/browser-geturl.ts @@ -1,17 +1,10 @@ "use strict"; -export type GetUrlResponse = { - statusCode: number, - statusMessage: string; - headers: { [ key: string] : string }; - body: string; -}; - -export type Options = { - method?: string, - body?: string - headers?: { [ key: string] : string }, -}; +import { arrayify } from "@ethersproject/bytes"; + +import type { GetUrlResponse, Options } from "./types"; + +export { GetUrlResponse, Options }; export async function getUrl(href: string, options?: Options): Promise { if (options == null) { options = { }; } @@ -29,7 +22,7 @@ export async function getUrl(href: string, options?: Options): Promise { return new Promise((resolve, reject) => { @@ -37,11 +30,11 @@ function getResponse(request: http.ClientRequest): Promise { }, <{ [ name: string ]: string }>{ }), body: null }; - resp.setEncoding("utf8"); + //resp.setEncoding("utf8"); - resp.on("data", (chunk: string) => { - if (response.body == null) { response.body = ""; } - response.body += chunk; + resp.on("data", (chunk: Uint8Array) => { + if (response.body == null) { response.body = new Uint8Array(0); } + response.body = concat([ response.body, chunk ]); }); resp.on("end", () => { @@ -100,7 +93,7 @@ export async function getUrl(href: string, options?: Options): Promise any): Promise { +export function fetchData(connection: string | ConnectionInfo, body?: Uint8Array, processFunc?: (value: Uint8Array, response: FetchJsonResponse) => T): Promise { // How many times to retry in the event of a throttle const attemptLimit = (typeof(connection) === "object" && connection.throttleLimit != null) ? connection.throttleLimit: 12; @@ -124,10 +124,12 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr } } - if (json) { + if (body) { options.method = "POST"; - options.body = json; - headers["content-type"] = { key: "Content-Type", value: "application/json" }; + options.body = body; + if (headers["content-type"] == null) { + headers["content-type"] = { key: "Content-Type", value: "application/octet-stream" }; + } } const flatHeaders: { [ key: string ]: string } = { }; @@ -139,7 +141,7 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr const runningTimeout = (function() { let timer: NodeJS.Timer = null; - const promise = new Promise(function(resolve, reject) { + const promise: Promise = new Promise(function(resolve, reject) { if (timeout) { timer = setTimeout(() => { if (timer == null) { return; } @@ -189,6 +191,7 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr stall = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt))); } + //console.log("Stalling 429"); await staller(stall); continue; } @@ -225,25 +228,12 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr }); } - let json: any = null; - if (body != null) { + if (processFunc) { try { - json = JSON.parse(body); - } catch (error) { + const result = await processFunc(body, response); runningTimeout.cancel(); - logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, { - body: body, - error: error, - requestBody: (options.body || null), - requestMethod: options.method, - url: url - }); - } - } + return result; - if (processFunc) { - try { - json = await processFunc(json, response); } catch (error) { // Allow the processFunc to trigger a throttle if (error.throttleRetry && attempt < attemptLimit) { @@ -254,6 +244,7 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr if (tryAgain) { const timeout = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt))); + //console.log("Stalling callback"); await staller(timeout); continue; } @@ -261,7 +252,7 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr runningTimeout.cancel(); logger.throwError("processing response error", Logger.errors.SERVER_ERROR, { - body: json, + body: body, error: error, requestBody: (options.body || null), requestMethod: options.method, @@ -271,13 +262,67 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr } runningTimeout.cancel(); - return json; + + // If we had a processFunc, it eitehr returned a T or threw above. + // The "body" is now a Uint8Array. + return (body); } + + return logger.throwError("failed response", Logger.errors.SERVER_ERROR, { + requestBody: (options.body || null), + requestMethod: options.method, + url: url + }); })(); return Promise.race([ runningTimeout.promise, runningFetch ]); } +export function fetchJson(connection: string | ConnectionInfo, json?: string, processFunc?: (value: any, response: FetchJsonResponse) => any): Promise { + let processJsonFunc = (value: Uint8Array, response: FetchJsonResponse) => { + let result: any = null; + if (value != null) { + try { + result = JSON.parse(toUtf8String(value)); + } catch (error) { + logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, { + body: value, + error: error + }); + } + } + + if (processFunc) { + result = processFunc(result, response); + } + + return result; + } + + // If we have json to send, we must + // - add content-type of application/json (unless already overridden) + // - convert the json to bytes + let body: Uint8Array = null; + if (json != null) { + body = toUtf8Bytes(json); + + // Create a connection with the content-type set for JSON + const updated: ConnectionInfo = (typeof(connection) === "string") ? ({ url: connection }): connection; + if (updated.headers) { + const hasContentType = (Object.keys(updated.headers).filter((k) => (k.toLowerCase() === "content-type")).length) !== 0; + if (!hasContentType) { + updated.headers = shallowCopy(updated.headers); + updated.headers["content-type"] = "application/json"; + } + } else { + updated.headers = { "content-type": "application/json" }; + } + connection = updated; + } + + return fetchData(connection, body, processJsonFunc); +} + export function poll(func: () => Promise, options?: PollOptions): Promise { if (!options) { options = {}; } options = shallowCopy(options); diff --git a/packages/web/src.ts/types.ts b/packages/web/src.ts/types.ts new file mode 100644 index 0000000000..8ab7bfb7a1 --- /dev/null +++ b/packages/web/src.ts/types.ts @@ -0,0 +1,15 @@ +"use strict"; + +export type GetUrlResponse = { + statusCode: number, + statusMessage: string; + headers: { [ key: string] : string }; + body: Uint8Array; +}; + +export type Options = { + method?: string, + body?: Uint8Array + headers?: { [ key: string] : string }, +}; +