diff --git a/.changeset/dry-guests-count.md b/.changeset/dry-guests-count.md new file mode 100644 index 0000000000..875908b599 --- /dev/null +++ b/.changeset/dry-guests-count.md @@ -0,0 +1,5 @@ +--- +"@nomiclabs/hardhat-etherscan": patch +--- + +Added support for the `http_proxy` environment variable. When this variable is set, `hardhat-etherscan` will use the given proxy to send the verification requests. diff --git a/.changeset/few-flies-drum.md b/.changeset/few-flies-drum.md new file mode 100644 index 0000000000..0a3dd82fe3 --- /dev/null +++ b/.changeset/few-flies-drum.md @@ -0,0 +1,11 @@ +--- +"hardhat": patch +--- + +Added support for the `http_proxy` environment variable. When this variable is set, Hardhat will send its requests through the given proxy for things like JSON-RPC requests, mainnet forking and downloading compilers. + +We also removed support for the `HTTP_PROXY` and `HTTPS_PROXY` environment variables, since `http_proxy` is the most commonly used environment variable for this kind of thing. Those variables could only be used for downloading compilers. + +Finally, we also added support for `no_proxy`, which accepts a comma separated list of hosts or `"*"`. Any host included in this list will not be proxied. + +Note that requests to `"localhost"` or `"127.0.0.1"` are never proxied. diff --git a/docs/src/content/hardhat-runner/docs/troubleshooting/common-problems.md b/docs/src/content/hardhat-runner/docs/troubleshooting/common-problems.md index d4c0b789bd..56eecfe38a 100644 --- a/docs/src/content/hardhat-runner/docs/troubleshooting/common-problems.md +++ b/docs/src/content/hardhat-runner/docs/troubleshooting/common-problems.md @@ -13,3 +13,9 @@ npx hardhat --max-memory 4096 compile ``` If you find yourself using this all the time, you can set it with an environment variable in your `.bashrc` (if using bash) or `.zshrc` (if using zsh): `export HARDHAT_MAX_MEMORY=4096`. + +## Using Hardhat with a proxy server + +Hardhat supports the `http_proxy` environment variable. When this variable is set, Hardhat will send its requests through the given proxy for things like JSON-RPC requests, mainnet forking and downloading compilers. + +There's also support for the `no_proxy` variable, which accepts a comma separated list of hosts or `"*"`. Any host included in this list will not be proxied. Note that requests to `"localhost"` or `"127.0.0.1"` are never proxied. diff --git a/packages/hardhat-core/package.json b/packages/hardhat-core/package.json index 4c7bba28fd..e143a42421 100644 --- a/packages/hardhat-core/package.json +++ b/packages/hardhat-core/package.json @@ -90,7 +90,6 @@ "ethers": "^5.0.0", "mocha": "^10.0.0", "prettier": "2.4.1", - "proxy": "^1.0.2", "rimraf": "^3.0.2", "sinon": "^9.0.0", "time-require": "^0.1.2", diff --git a/packages/hardhat-core/src/internal/core/providers/http.ts b/packages/hardhat-core/src/internal/core/providers/http.ts index fba1c4e27d..7ad9bdd77d 100644 --- a/packages/hardhat-core/src/internal/core/providers/http.ts +++ b/packages/hardhat-core/src/internal/core/providers/http.ts @@ -1,4 +1,4 @@ -import type { Dispatcher, Pool as PoolT } from "undici"; +import type * as Undici from "undici"; import { EventEmitter } from "events"; @@ -17,6 +17,7 @@ import { import { getHardhatVersion } from "../../util/packageInfo"; import { HardhatError } from "../errors"; import { ERRORS } from "../errors-list"; +import { shouldUseProxy } from "../../util/proxy"; import { ProviderError } from "./errors"; @@ -33,7 +34,7 @@ const hardhatVersion = getHardhatVersion(); export class HttpProvider extends EventEmitter implements EIP1193Provider { private _nextRequestId = 1; - private _dispatcher: Dispatcher; + private _dispatcher: Undici.Dispatcher; private _path: string; private _authHeader: string | undefined; @@ -42,11 +43,11 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider { private readonly _networkName: string, private readonly _extraHeaders: { [name: string]: string } = {}, private readonly _timeout = 20000, - client: Dispatcher | undefined = undefined + client: Undici.Dispatcher | undefined = undefined ) { super(); - const { Pool } = require("undici") as { Pool: typeof PoolT }; + const { Pool, ProxyAgent } = require("undici") as typeof Undici; const url = new URL(this._url); this._path = url.pathname; @@ -59,6 +60,10 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider { ).toString("base64")}`; try { this._dispatcher = client ?? new Pool(url.origin); + + if (process.env.http_proxy !== undefined && shouldUseProxy(url.origin)) { + this._dispatcher = new ProxyAgent(process.env.http_proxy); + } } catch (e) { if (e instanceof TypeError && e.message === "Invalid URL") { e.message += ` ${url.origin}`; @@ -163,10 +168,13 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider { request: JsonRpcRequest | JsonRpcRequest[], retryNumber = 0 ): Promise { + const { request: sendRequest } = await import("undici"); + const url = new URL(this._url); + try { - const response = await this._dispatcher.request({ + const response = await sendRequest(url, { + dispatcher: this._dispatcher, method: "POST", - path: this._path, body: JSON.stringify(request), maxRedirections: 10, headersTimeout: @@ -194,8 +202,6 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider { return await this._retry(request, seconds, retryNumber); } - const url = new URL(this._url); - // eslint-disable-next-line @nomiclabs/hardhat-internal-rules/only-hardhat-error throw new ProviderError( `Too Many Requests error received from ${url.hostname}`, @@ -255,12 +261,12 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider { return true; } - private _isRateLimitResponse(response: Dispatcher.ResponseData) { + private _isRateLimitResponse(response: Undici.Dispatcher.ResponseData) { return response.statusCode === TOO_MANY_REQUEST_STATUS; } private _getRetryAfterSeconds( - response: Dispatcher.ResponseData + response: Undici.Dispatcher.ResponseData ): number | undefined { const header = response.headers["retry-after"]; diff --git a/packages/hardhat-core/src/internal/util/download.ts b/packages/hardhat-core/src/internal/util/download.ts index 0e6035b41d..34b70f3e40 100644 --- a/packages/hardhat-core/src/internal/util/download.ts +++ b/packages/hardhat-core/src/internal/util/download.ts @@ -1,9 +1,12 @@ +import type { Dispatcher } from "undici"; + import fs from "fs"; import fsExtra from "fs-extra"; import path from "path"; import util from "util"; import { getHardhatVersion } from "./packageInfo"; +import { shouldUseProxy } from "./proxy"; const TEMP_FILE_PREFIX = "tmp-"; @@ -27,23 +30,18 @@ export async function download( const { getGlobalDispatcher, ProxyAgent, request } = await import("undici"); const streamPipeline = util.promisify(pipeline); - function chooseDispatcher() { - if (process.env.HTTPS_PROXY !== undefined) { - return new ProxyAgent(process.env.HTTPS_PROXY); - } - - if (process.env.HTTP_PROXY !== undefined) { - return new ProxyAgent(process.env.HTTP_PROXY); - } - - return getGlobalDispatcher(); + let dispatcher: Dispatcher; + if (process.env.http_proxy !== undefined && shouldUseProxy(url)) { + dispatcher = new ProxyAgent(process.env.http_proxy); + } else { + dispatcher = getGlobalDispatcher(); } const hardhatVersion = getHardhatVersion(); // Fetch the url const response = await request(url, { - dispatcher: chooseDispatcher(), + dispatcher, headersTimeout: timeoutMillis, maxRedirections: 10, method: "GET", @@ -59,6 +57,9 @@ export async function download( await streamPipeline(response.body, fs.createWriteStream(tmpFilePath)); return fsExtra.move(tmpFilePath, filePath, { overwrite: true }); + } else { + // undici's response bodies must always be consumed to prevent leaks + await response.body.text(); } // eslint-disable-next-line @nomiclabs/hardhat-internal-rules/only-hardhat-error diff --git a/packages/hardhat-core/src/internal/util/proxy.ts b/packages/hardhat-core/src/internal/util/proxy.ts new file mode 100644 index 0000000000..24a6aa230c --- /dev/null +++ b/packages/hardhat-core/src/internal/util/proxy.ts @@ -0,0 +1,18 @@ +export function shouldUseProxy(url: string): boolean { + const { hostname } = new URL(url); + const noProxy = process.env.NO_PROXY; + + if (hostname === "localhost" || hostname === "127.0.0.1" || noProxy === "*") { + return false; + } + + if (noProxy !== undefined && noProxy !== "") { + const noProxyList = noProxy.split(","); + + if (noProxyList.includes(hostname)) { + return false; + } + } + + return true; +} diff --git a/packages/hardhat-core/test/internal/util/download.ts b/packages/hardhat-core/test/internal/util/download.ts index ad11f4ffba..e89bfdb9f7 100644 --- a/packages/hardhat-core/test/internal/util/download.ts +++ b/packages/hardhat-core/test/internal/util/download.ts @@ -1,9 +1,6 @@ import { assert } from "chai"; import fsExtra from "fs-extra"; import path from "path"; -// @ts-ignore -// eslint-disable-next-line import/no-extraneous-dependencies -import Proxy from "proxy"; import { download } from "../../../src/internal/util/download"; import { useTmpDir } from "../../helpers/fs"; @@ -24,76 +21,3 @@ describe("Compiler List download", function () { }); }); }); - -describe("Compiler List download with proxy", function () { - let env: typeof process.env; - let proxy: any; - let proxyPort: number; - - useTmpDir("compiler-downloader"); - - before(function (done) { - // Setup Proxy Server - proxy = new Proxy(); - proxy.listen(function () { - proxyPort = proxy.address().port; - done(); - }); - }); - - describe("Compilers list download with HTTPS_PROXY", function () { - before(function () { - // Save the Environment Settings and Set - env = process.env; - process.env.HTTPS_PROXY = `http://127.0.0.1:${proxyPort}`; - }); - - it("Should call download with the right params", async function () { - const compilersDir = this.tmpDir; - const downloadPath = path.join(compilersDir, "downloadedCompilerProxy"); - const expectedUrl = `http://solc-bin.ethereum.org/wasm/list.json`; - - // download the file - await download(expectedUrl, downloadPath); - // Assert that the file exists - assert.isTrue(await fsExtra.pathExists(downloadPath)); - }); - - after(function () { - // restoring everything back to the environment - process.env = env; - }); - }); - - describe("Compilers list download with HTTP_PROXY", function () { - before(function () { - // Save the Environment Settings and Set - env = process.env; - process.env.HTTP_PROXY = `http://127.0.0.1:${proxyPort}`; - }); - - it("Should call download with the right params", async function () { - const compilersDir = this.tmpDir; - const downloadPath = path.join(compilersDir, "downloadedCompilerProxy"); - const expectedUrl = `http://solc-bin.ethereum.org/wasm/list.json`; - - // download the file - await download(expectedUrl, downloadPath); - // Assert that the file exists - assert.isTrue(await fsExtra.pathExists(downloadPath)); - }); - - after(function () { - // restoring everything back to the environment - process.env = env; - }); - }); - - after(function (done) { - // Shutdown Proxy Server - proxy.once("close", function () { - done(); - }); - proxy.close(); - }); -}); diff --git a/packages/hardhat-etherscan/src/etherscan/EtherscanService.ts b/packages/hardhat-etherscan/src/etherscan/EtherscanService.ts index 305bf42eb7..acc41b14a3 100644 --- a/packages/hardhat-etherscan/src/etherscan/EtherscanService.ts +++ b/packages/hardhat-etherscan/src/etherscan/EtherscanService.ts @@ -2,6 +2,7 @@ import { NomicLabsHardhatPluginError } from "hardhat/plugins"; import { Dispatcher } from "undici"; import { pluginName } from "../constants"; +import { sendGetRequest, sendPostRequest } from "../undici"; import { EtherscanCheckStatusRequest, @@ -19,18 +20,11 @@ export async function verifyContract( url: string, req: EtherscanVerifyRequest ): Promise { - const { request } = await import("undici"); const parameters = new URLSearchParams({ ...req }); - const method: Dispatcher.HttpMethod = "POST"; - const requestDetails = { - method, - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: parameters.toString(), - }; let response: Dispatcher.ResponseData; try { - response = await request(url, requestDetails); + response = await sendPostRequest(new URL(url), parameters.toString()); } catch (error: any) { throw new NomicLabsHardhatPluginError( pluginName, @@ -84,10 +78,9 @@ export async function getVerificationStatus( const urlWithQuery = new URL(url); urlWithQuery.search = parameters.toString(); - const { request } = await import("undici"); let response; try { - response = await request(urlWithQuery, { method: "GET" }); + response = await sendGetRequest(urlWithQuery); if (!(response.statusCode >= 200 && response.statusCode <= 299)) { // This could be always interpreted as JSON if there were any such guarantee in the Etherscan API. diff --git a/packages/hardhat-etherscan/src/solc/version.ts b/packages/hardhat-etherscan/src/solc/version.ts index 254c2e8d3d..41717041c8 100644 --- a/packages/hardhat-etherscan/src/solc/version.ts +++ b/packages/hardhat-etherscan/src/solc/version.ts @@ -1,6 +1,7 @@ import { NomicLabsHardhatPluginError } from "hardhat/plugins"; import { pluginName } from "../constants"; +import { sendGetRequest } from "../undici"; const COMPILERS_LIST_URL = "https://solc-bin.ethereum.org/bin/list.json"; @@ -29,9 +30,8 @@ export async function getLongVersion(shortVersion: string): Promise { export async function getVersions(): Promise { try { - const { request } = await import("undici"); // It would be better to query an etherscan API to get this list but there's no such API yet. - const response = await request(COMPILERS_LIST_URL, { method: "GET" }); + const response = await sendGetRequest(new URL(COMPILERS_LIST_URL)); if (!(response.statusCode >= 200 && response.statusCode <= 299)) { const responseText = await response.body.text(); diff --git a/packages/hardhat-etherscan/src/undici.ts b/packages/hardhat-etherscan/src/undici.ts new file mode 100644 index 0000000000..f7acb99b23 --- /dev/null +++ b/packages/hardhat-etherscan/src/undici.ts @@ -0,0 +1,38 @@ +import type * as Undici from "undici"; + +function getDispatcher(): Undici.Dispatcher { + const { ProxyAgent, getGlobalDispatcher } = + require("undici") as typeof Undici; + if (process.env.http_proxy !== undefined) { + return new ProxyAgent(process.env.http_proxy); + } + + return getGlobalDispatcher(); +} + +export async function sendGetRequest( + url: URL +): Promise { + const { request } = await import("undici"); + const dispatcher = getDispatcher(); + + return request(url, { + dispatcher, + method: "GET", + }); +} + +export async function sendPostRequest( + url: URL, + body: string +): Promise { + const { request } = await import("undici"); + const dispatcher = getDispatcher(); + + return request(url, { + dispatcher, + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body, + }); +} diff --git a/yarn.lock b/yarn.lock index 59a16d4696..2040e0bd2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2260,16 +2260,6 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -args@5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/args/-/args-5.0.1.tgz#4bf298df90a4799a09521362c579278cc2fdd761" - integrity sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ== - dependencies: - camelcase "5.0.0" - chalk "2.4.2" - leven "2.1.0" - mri "1.1.4" - arr-diff@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520" @@ -3042,11 +3032,6 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" -basic-auth-parser@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/basic-auth-parser/-/basic-auth-parser-0.0.2.tgz#ce9e71a77f23c1279eecd2659b2a46244c156e41" - integrity sha512-Y7OBvWn+JnW45JWHLY6ybYub2k9cXCMrtCyO1Hds2s6eqClqWhPnOQpgXUPjAiMHj+A8TEPIQQ1dYENnJoBOHQ== - bcrypt-pbkdf@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" @@ -3500,11 +3485,6 @@ camelcase-keys@^6.2.2: map-obj "^4.0.0" quick-lru "^4.0.1" -camelcase@5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42" - integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA== - camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" @@ -3575,15 +3555,6 @@ chai@^4.2.0: pathval "^1.1.1" type-detect "^4.0.5" -chalk@2.4.2, chalk@^2.0.0, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - chalk@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/chalk/-/chalk-0.4.0.tgz#5199a3ddcd0c1efe23bc08c1b027b06176e0c64f" @@ -3604,6 +3575,15 @@ chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" +chalk@^2.0.0, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" @@ -7743,11 +7723,6 @@ levelup@^1.2.1: semver "~5.4.1" xtend "~4.0.0" -leven@2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/leven/-/leven-2.1.0.tgz#c2e7a9f772094dee9d34202ae8acce4687875580" - integrity sha512-nvVPLpIHUxCUoRLrFqTgSxXJ614d8AgQoWl7zPe/2VadE8+1dpU3LBhowRuBAcuwruWtOdD8oYC9jDNJjXDPyA== - levn@^0.3.0, levn@~0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" @@ -8422,11 +8397,6 @@ module-error@^1.0.1, module-error@^1.0.2: resolved "https://registry.yarnpkg.com/module-error/-/module-error-1.0.2.tgz#8d1a48897ca883f47a45816d4fb3e3c6ba404d86" integrity sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA== -mri@1.1.4: - version "1.1.4" - resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a" - integrity sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w== - ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -9409,15 +9379,6 @@ proxy-addr@~2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/proxy/-/proxy-1.0.2.tgz#e0cfbe11c0a7a8b238fd2d7134de4e2867578e7f" - integrity sha512-KNac2ueWRpjbUh77OAFPZuNdfEqNynm9DD4xHT14CccGpW8wKZwEkN0yjlb7X9G9Z9F55N0Q+1z+WfgAhwYdzQ== - dependencies: - args "5.0.1" - basic-auth-parser "0.0.2" - debug "^4.1.1" - prr@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"