Proxy support for Node.js fetch (microsoft#228697)

build/lib/layersChecker.ts

@@ -68,6 +68,7 @@ const CORE_TYPES = [
 	'fetch',
 	'RequestInit',
 	'Headers',
+	'Request',
 	'Response',
 	'Body',
 	'__type',

eslint.config.js

@@ -825,6 +825,7 @@ export default tseslint.config(
 						'string_decoder',
 						'tas-client-umd',
 						'tls',
+						'undici-types',
 						'url',
 						'util',
 						'v8-inspect-profiler',

package.json

@@ -75,7 +75,7 @@
     "@vscode/deviceid": "^0.1.1",
     "@vscode/iconv-lite-umd": "0.7.0",
     "@vscode/policy-watcher": "^1.1.8",
-    "@vscode/proxy-agent": "^0.22.0",
+    "@vscode/proxy-agent": "^0.24.0",
     "@vscode/ripgrep": "^1.15.9",
     "@vscode/spdlog": "^0.15.0",
     "@vscode/sqlite3": "5.1.8-vscode",
@@ -105,6 +105,7 @@
     "node-pty": "^1.1.0-beta22",
     "open": "^8.4.2",
     "tas-client-umd": "0.2.0",
+    "undici": "^6.20.1",
     "v8-inspect-profiler": "^0.1.1",
     "vscode-oniguruma": "1.7.0",
     "vscode-regexpp": "^3.1.0",

remote/package.json

@@ -8,7 +8,7 @@
     "@parcel/watcher": "2.1.0",
     "@vscode/deviceid": "^0.1.1",
     "@vscode/iconv-lite-umd": "0.7.0",
-    "@vscode/proxy-agent": "^0.22.0",
+    "@vscode/proxy-agent": "^0.24.0",
     "@vscode/ripgrep": "^1.15.9",
     "@vscode/spdlog": "^0.15.0",
     "@vscode/tree-sitter-wasm": "^0.0.4",
@@ -33,6 +33,7 @@
     "native-watchdog": "^1.4.1",
     "node-pty": "^1.1.0-beta22",
     "tas-client-umd": "0.2.0",
+    "undici": "^6.20.1",
     "vscode-oniguruma": "1.7.0",
     "vscode-regexpp": "^3.1.0",
     "vscode-textmate": "9.1.0",

src/vs/platform/request/common/request.ts

@@ -209,6 +209,12 @@ function registerProxyConfigurations(scope: ConfigurationScope): void {
 				default: false,
 				description: localize('electronFetch', "Controls whether use of Electron's fetch implementation instead of Node.js' should be enabled. All local extensions will get Electron's fetch implementation for the global fetch API."),
 				restricted: true
+			},
+			'http.fetchAdditionalSupport': {
+				type: 'boolean',
+				default: true,
+				description: localize('fetchAdditionalSupport', "Controls whether Node.js' fetch implementation should be extended with additional support."),
+				restricted: true
 			}
 		}
 	};

src/vs/workbench/api/node/proxyResolver.ts

@@ -11,16 +11,20 @@ import { ExtHostExtensionService } from './extHostExtensionService.js';
 import { URI } from '../../../base/common/uri.js';
 import { ILogService, LogLevel as LogServiceLevel } from '../../../platform/log/common/log.js';
 import { IExtensionDescription } from '../../../platform/extensions/common/extensions.js';
-import { LogLevel, createHttpPatch, createProxyResolver, createTlsPatch, ProxySupportSetting, ProxyAgentParams, createNetPatch, loadSystemCertificates } from '@vscode/proxy-agent';
+import { LogLevel, createHttpPatch, createProxyResolver, createTlsPatch, ProxySupportSetting, ProxyAgentParams, createNetPatch, loadSystemCertificates, ResolveProxyWithRequest, getOrLoadAdditionalCertificates, LookupProxyAuthorization } from '@vscode/proxy-agent';
 import { AuthInfo } from '../../../platform/request/common/request.js';
 import { DisposableStore } from '../../../base/common/lifecycle.js';
 import { createRequire } from 'node:module';
+import type * as undiciType from 'undici-types';
+import type * as tlsType from 'tls';
+import type * as streamType from 'stream';
 
 const require = createRequire(import.meta.url);
 const http = require('http');
 const https = require('https');
-const tls = require('tls');
+const tls: typeof tlsType = require('tls');
 const net = require('net');
+const undici: typeof undiciType = require('undici');
 
 const systemCertificatesV2Default = false;
 const useElectronFetchDefault = false;
@@ -35,8 +39,6 @@ export function connectProxyResolver(
 	disposables: DisposableStore,
 ) {
 
-	patchGlobalFetch(configProvider, mainThreadTelemetry, initData, disposables);
-
 	const useHostProxy = initData.environment.useHostProxy;
 	const doUseHostProxy = typeof useHostProxy === 'boolean' ? useHostProxy : !initData.remote.isRemote;
 	const params: ProxyAgentParams = {
@@ -86,8 +88,11 @@ export function connectProxyResolver(
 		},
 		env: process.env,
 	};
-	const resolveProxy = createProxyResolver(params);
-	const lookup = createPatchedModules(params, resolveProxy);
+	const { resolveProxyWithRequest, resolveProxyURL } = createProxyResolver(params);
+
+	patchGlobalFetch(configProvider, mainThreadTelemetry, initData, resolveProxyURL, params.lookupProxyAuthorization!, getOrLoadAdditionalCertificates.bind(undefined, params), disposables);
+
+	const lookup = createPatchedModules(params, resolveProxyWithRequest);
 	return configureModuleLoading(extensionService, lookup);
 }
 
@@ -103,10 +108,12 @@ const unsafeHeaders = [
 	'set-cookie',
 ];
 
-function patchGlobalFetch(configProvider: ExtHostConfigProvider, mainThreadTelemetry: MainThreadTelemetryShape, initData: IExtensionHostInitData, disposables: DisposableStore) {
-	if (!initData.remote.isRemote && !(globalThis as any).__originalFetch) {
+function patchGlobalFetch(configProvider: ExtHostConfigProvider, mainThreadTelemetry: MainThreadTelemetryShape, initData: IExtensionHostInitData, resolveProxyURL: (url: string) => Promise<string | undefined>, lookupProxyAuthorization: LookupProxyAuthorization, loadAdditionalCertificates: () => Promise<string[]>, disposables: DisposableStore) {
+	if (!initData.remote.isRemote && !(globalThis as any).__vscodeOriginalFetch) {
 		const originalFetch = globalThis.fetch;
-		(globalThis as any).__originalFetch = originalFetch;
+		(globalThis as any).__vscodeOriginalFetch = originalFetch;
+		const patchedFetch = patchFetch(originalFetch, configProvider, resolveProxyURL, lookupProxyAuthorization, loadAdditionalCertificates);
+		(globalThis as any).__vscodePatchedFetch = patchedFetch;
 		let useElectronFetch = configProvider.getConfiguration('http').get<boolean>('electronFetch', useElectronFetchDefault);
 		disposables.add(configProvider.onDidChangeConfiguration(e => {
 			if (e.affectsConfiguration('http.electronFetch')) {
@@ -115,8 +122,8 @@ function patchGlobalFetch(configProvider: ExtHostConfigProvider, mainThreadTelem
 		}));
 		const electron = require('electron');
 		// https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API
-		globalThis.fetch = async function fetch(input: any /* RequestInfo */ | URL, init?: RequestInit) {
-			function getRequestProperty(name: keyof any /* Request */ & keyof RequestInit) {
+		globalThis.fetch = async function fetch(input: string | URL | Request, init?: RequestInit) {
+			function getRequestProperty(name: keyof Request & keyof RequestInit) {
 				return init && name in init ? init[name] : typeof input === 'object' && 'cache' in input ? input[name] : undefined;
 			}
 			// Limitations: https://github.com/electron/electron/pull/36733#issuecomment-1405615494
@@ -139,7 +146,7 @@ function patchGlobalFetch(configProvider: ExtHostConfigProvider, mainThreadTelem
 				recordFetchFeatureUse(mainThreadTelemetry, 'integrity');
 			}
 			if (!useElectronFetch || isDataUrl || isBlobUrl || isManualRedirect || integrity) {
-				const response = await originalFetch(input, init);
+				const response = await patchedFetch(input, init, urlString);
 				monitorResponseProperties(mainThreadTelemetry, response, urlString);
 				return response;
 			}
@@ -160,6 +167,121 @@ function patchGlobalFetch(configProvider: ExtHostConfigProvider, mainThreadTelem
 	}
 }
 
+function patchFetch(originalFetch: typeof globalThis.fetch, configProvider: ExtHostConfigProvider, resolveProxyURL: (url: string) => Promise<string | undefined>, lookupProxyAuthorization: LookupProxyAuthorization, loadAdditionalCertificates: () => Promise<string[]>) {
+	return async function patchedFetch(input: string | URL | Request, init?: RequestInit, urlString?: string) {
+		const config = configProvider.getConfiguration('http');
+		const enabled = config.get<boolean>('fetchAdditionalSupport');
+		if (!enabled) {
+			return originalFetch(input, init);
+		}
+		const proxySupport = config.get<ProxySupportSetting>('proxySupport') || 'off';
+		const doResolveProxy = proxySupport === 'override' || proxySupport === 'fallback' || (proxySupport === 'on' && ((init as any)?.dispatcher) === undefined);
+		const addCerts = config.get<boolean>('systemCertificates');
+		if (!doResolveProxy && !addCerts) {
+			return originalFetch(input, init);
+		}
+		if (!urlString) { // for testing
+			urlString = typeof input === 'string' ? input : 'cache' in input ? input.url : input.toString();
+		}
+		const proxyURL = doResolveProxy ? await resolveProxyURL(urlString) : undefined;
+		if (!proxyURL && !addCerts) {
+			return originalFetch(input, init);
+		}
+		const ca = addCerts ? [...tls.rootCertificates, ...await loadAdditionalCertificates()] : undefined;
+		if (!proxyURL) {
+			const modifiedInit = {
+				...init,
+				dispatcher: new undici.Agent({
+					allowH2: true,
+					connect: { ca },
+				})
+			};
+			return originalFetch(input, modifiedInit);
+		}
+
+		const state: Record<string, any> = {};
+		const proxyAuthorization = await lookupProxyAuthorization(proxyURL, undefined, state);
+		const modifiedInit = {
+			...init,
+			dispatcher: new undici.ProxyAgent({
+				uri: proxyURL,
+				allowH2: true,
+				headers: proxyAuthorization ? { 'Proxy-Authorization': proxyAuthorization } : undefined,
+				...(addCerts ? {
+					proxyTls: { ca },
+					requestTls: { ca },
+				} : {}),
+				clientFactory: (origin: URL, opts: object): undiciType.Dispatcher => (new undici.Pool(origin, opts) as any).compose((dispatch: undiciType.Dispatcher['dispatch']) => {
+					class ProxyAuthHandler extends undici.DecoratorHandler {
+						private abort: ((err?: Error) => void) | undefined;
+						constructor(private dispatch: undiciType.Dispatcher['dispatch'], private options: undiciType.Dispatcher.DispatchOptions, private handler: undiciType.Dispatcher.DispatchHandlers) {
+							super(handler);
+						}
+						onConnect(abort: (err?: Error) => void): void {
+							this.abort = abort;
+							this.handler.onConnect?.(abort);
+						}
+						onError(err: Error): void {
+							if (!(err instanceof ProxyAuthError)) {
+								return this.handler.onError?.(err);
+							}
+							(async () => {
+								try {
+									const proxyAuthorization = await lookupProxyAuthorization(proxyURL!, err.proxyAuthenticate, state);
+									if (proxyAuthorization) {
+										if (!this.options.headers) {
+											this.options.headers = ['Proxy-Authorization', proxyAuthorization];
+										} else if (Array.isArray(this.options.headers)) {
+											const i = this.options.headers.findIndex((value, index) => index % 2 === 0 && value.toLowerCase() === 'proxy-authorization');
+											if (i === -1) {
+												this.options.headers.push('Proxy-Authorization', proxyAuthorization);
+											} else {
+												this.options.headers[i + 1] = proxyAuthorization;
+											}
+										} else {
+											this.options.headers['Proxy-Authorization'] = proxyAuthorization;
+										}
+										this.dispatch(this.options, this);
+									} else {
+										this.handler.onError?.(new undici.errors.RequestAbortedError(`Proxy response (407) ?.== 200 when HTTP Tunneling`)); // Mimick undici's behavior
+									}
+								} catch (err) {
+									this.handler.onError?.(err);
+								}
+							})();
+						}
+						onUpgrade(statusCode: number, headers: Buffer[] | string[] | null, socket: streamType.Duplex): void {
+							if (statusCode === 407 && headers) {
+								const proxyAuthenticate: string[] = [];
+								for (let i = 0; i < headers.length; i += 2) {
+									if (headers[i].toString().toLowerCase() === 'proxy-authenticate') {
+										proxyAuthenticate.push(headers[i + 1].toString());
+									}
+								}
+								if (proxyAuthenticate.length) {
+									this.abort?.(new ProxyAuthError(proxyAuthenticate));
+									return;
+								}
+							}
+							this.handler.onUpgrade?.(statusCode, headers, socket);
+						}
+					}
+					return function proxyAuthDispatch(options: undiciType.Dispatcher.DispatchOptions, handler: undiciType.Dispatcher.DispatchHandlers) {
+						return dispatch(options, new ProxyAuthHandler(dispatch, options, handler));
+					};
+				}),
+			})
+		};
+		return originalFetch(input, modifiedInit);
+	};
+}
+
+class ProxyAuthError extends Error {
+	constructor(public proxyAuthenticate: string[]) {
+		super('Proxy authentication required');
+	}
+}
+
 function monitorResponseProperties(mainThreadTelemetry: MainThreadTelemetryShape, response: Response, urlString: string) {
 	const originalUrl = response.url;
 	Object.defineProperty(response, 'url', {
@@ -220,7 +342,7 @@ function recordFetchFeatureUse(mainThreadTelemetry: MainThreadTelemetryShape, fe
 	}
 }
 
-function createPatchedModules(params: ProxyAgentParams, resolveProxy: ReturnType<typeof createProxyResolver>) {
+function createPatchedModules(params: ProxyAgentParams, resolveProxy: ResolveProxyWithRequest) {
 
 	function mergeModules(module: any, patch: any) {
 		return Object.assign(module.default || module, patch);