✨ Client proxy support

.github/.cache-key

@@ -1 +1 @@
-2
+3

packages/cli-exec/src/commands/exec/ping.js

@@ -1,5 +1,5 @@
 import Command, { flags } from '@percy/cli-command';
-import { request } from '@percy/client/dist/utils';
+import request from '@percy/client/dist/request';
 import logger from '@percy/logger';
 import execFlags from '../../flags';
 

packages/cli-exec/src/commands/exec/stop.js

@@ -1,5 +1,5 @@
 import Command, { flags } from '@percy/cli-command';
-import { request } from '@percy/client/dist/utils';
+import request from '@percy/client/dist/request';
 import logger from '@percy/logger';
 import execFlags from '../../flags';
 

packages/client/src/client.js

@@ -2,13 +2,8 @@ import PercyEnvironment from '@percy/env';
 import { git } from '@percy/env/dist/utils';
 import pkg from '../package.json';
 
-import {
-  sha256hash,
-  base64encode,
-  pool,
-  httpAgentFor,
-  request
-} from './utils';
+import { sha256hash, base64encode, pool } from './utils';
+import request, { ProxyHttpsAgent } from './request';
 
 // PercyClient is used to communicate with the Percy API to create and finalize
 // builds and snapshot. Uses @percy/env to collect environment information used
@@ -26,9 +21,9 @@ export default class PercyClient {
     Object.assign(this, {
       token,
       apiUrl,
-      httpAgent: httpAgentFor(apiUrl),
       clientInfo: new Set([].concat(clientInfo)),
       environmentInfo: new Set([].concat(environmentInfo)),
+      httpsAgent: new ProxyHttpsAgent({ keepAlive: true, maxSockets: 5 }),
       env: new PercyEnvironment(process.env),
       // build info is stored for reference
       build: { id: null, number: null, url: null }
@@ -81,7 +76,7 @@ export default class PercyClient {
   get(path) {
     return request(`${this.apiUrl}/${path}`, {
       method: 'GET',
-      agent: this.httpAgent,
+      agent: this.httpsAgent,
       headers: this.headers()
     });
   }
@@ -90,7 +85,7 @@ export default class PercyClient {
   post(path, body = {}) {
     return request(`${this.apiUrl}/${path}`, {
       method: 'POST',
-      agent: this.httpAgent,
+      agent: this.httpsAgent,
       body: JSON.stringify(body),
       headers: this.headers({
         'Content-Type': 'application/vnd.api+json'

packages/client/src/request.js

@@ -0,0 +1,174 @@
+import url from 'url';
+import net from 'net';
+import tls from 'tls';
+import http from 'http';
+import https from 'https';
+import { retry, hostnameMatches } from './utils';
+
+const CRLF = '\r\n';
+const STATUS_REG = /^HTTP\/1.[01] (\d*)/;
+const RETRY_ERROR_CODES = [
+  'ECONNREFUSED', 'ECONNRESET', 'EPIPE',
+  'EHOSTUNREACH', 'EAI_AGAIN'
+];
+
+// Proxified https agent
+export class ProxyHttpsAgent extends https.Agent {
+  // enforce request options
+  addRequest(request, options) {
+    options.href ||= url.format({
+      protocol: options.protocol,
+      hostname: options.hostname,
+      port: options.port,
+      slashes: true
+    }) + options.path;
+
+    options.uri ||= new URL(options.href);
+
+    let proxyUrl = (options.uri.protocol === 'https:' &&
+      (process.env.https_proxy || process.env.HTTPS_PROXY)) ||
+      (process.env.http_proxy || process.env.HTTP_PROXY);
+
+    let shouldProxy = !!proxyUrl && !hostnameMatches((
+      process.env.no_proxy || process.env.NO_PROXY
+    ), options.href);
+
+    if (shouldProxy) options.proxy = new URL(proxyUrl);
+
+    // useful when testing
+    options.rejectUnauthorized ??= this.rejectUnauthorized;
+
+    return super.addRequest(request, options);
+  }
+
+  // proxy https requests using a TLS connection
+  createConnection(options, callback) {
+    let { uri, proxy } = options;
+    let isProxyHttps = proxy?.protocol === 'https:';
+
+    if (!proxy) {
+      return super.createConnection(options, callback);
+    } else if (proxy.protocol !== 'http:' && !isProxyHttps) {
+      throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
+    }
+
+    // setup socket and listeners
+    let socket = (isProxyHttps ? tls : net).connect({
+      ...options,
+      host: proxy.hostname,
+      port: proxy.port
+    });
+
+    let handleError = err => {
+      socket.destroy(err);
+      callback(err);
+    };
+
+    let handleClose = () => handleError(
+      new Error('Connection closed while sending request to upstream proxy')
+    );
+
+    let buffer = '';
+    let handleData = data => {
+      buffer += data.toString();
+      // haven't received end of headers yet, keep buffering
+      if (!buffer.includes(CRLF.repeat(2))) return;
+      // stop listening after end of headers
+      socket.off('data', handleData);
+
+      if (buffer.match(STATUS_REG)?.[1] !== '200') {
+        return handleError(new Error(
+          'Error establishing proxy connection. ' +
+            `Response from server was: ${buffer}`
+        ));
+      }
+
+      options.socket = socket;
+      options.servername = uri.hostname;
+      // callback not passed in so not to be added as a listener
+      callback(null, super.createConnection(options));
+    };
+
+    // write proxy connect message to the socket
+    let connectMessage = [
+      `CONNECT ${uri.host} HTTP/1.1`,
+      `Host: ${uri.host}`
+    ];
+
+    if (proxy.username) {
+      let auth = proxy.username;
+      if (proxy.password) auth += `:${proxy.password}`;
+
+      connectMessage.push(`Proxy-Authorization: basic ${
+        Buffer.from(auth).toString('base64')
+      }`);
+    }
+
+    connectMessage = connectMessage.join(CRLF);
+    connectMessage += CRLF.repeat(2);
+
+    socket
+      .on('error', handleError)
+      .on('close', handleClose)
+      .on('data', handleData)
+      .write(connectMessage);
+  }
+}
+
+// Returns true or false if an error should cause the request to be retried
+function shouldRetryRequest(error) {
+  if (error.response) {
+    return error.response.status >= 500 && error.response.status < 600;
+  } else if (error.code) {
+    return RETRY_ERROR_CODES.includes(error.code);
+  } else {
+    return false;
+  }
+}
+
+// Returns a promise that resolves when the request is successful and rejects
+// when a non-successful response is received. The rejected error contains
+// response data and any received error details. Server 500 errors are retried
+// up to 5 times at 50ms intervals.
+export default function request(url, { body, ...options }) {
+  /* istanbul ignore next: the client api is https only, but this helper is borrowed in some
+   * cli-exec commands for its retryability with the internal api */
+  let { request } = url.startsWith('https:') ? https : http;
+  let { protocol, hostname, port, pathname, search } = new URL(url);
+  options = { ...options, protocol, hostname, port, path: pathname + search };
+
+  return retry((resolve, reject, retry) => {
+    let handleError = error => {
+      return shouldRetryRequest(error)
+        ? retry(error) : reject(error);
+    };
+
+    request(options)
+      .on('response', res => {
+        let status = res.statusCode;
+        let raw = '';
+
+        res.setEncoding('utf8')
+          .on('data', chunk => (raw += chunk))
+          .on('error', handleError)
+          .on('end', () => {
+            let body = raw;
+            try { body = JSON.parse(raw); } catch (e) {}
+
+            if (status >= 200 && status < 300) {
+              resolve(body);
+            } else {
+              handleError(Object.assign(new Error(), {
+                response: { status, body },
+                // use first error detail or the status message
+                message: body?.errors?.find(e => e.detail)?.detail || (
+                  `${status} ${res.statusMessage || raw}`
+                )
+              }));
+            }
+          });
+      })
+      .on('error', handleError)
+      .end(body);
+  });
+}

packages/client/src/utils.js

@@ -1,5 +1,4 @@
 import crypto from 'crypto';
-import { URL } from 'url';
 
 // Returns a sha256 hash of a string.
 export function sha256hash(content) {
@@ -60,7 +59,7 @@ export function pool(generator, context, concurrency) {
 // will recursively call the function at the specified interval until retries
 // are exhausted, at which point the promise will reject with the last error
 // passed to `retry`.
-function retry(fn, { retries = 5, interval = 50 } = {}) {
+export function retry(fn, { retries = 5, interval = 50 } = {}) {
   return new Promise((resolve, reject) => {
     // run the function, decrement retries
     let run = () => {
@@ -82,78 +81,40 @@ function retry(fn, { retries = 5, interval = 50 } = {}) {
   });
 }
 
-// Returns the appropriate http or https module for a given URL.
-function httpModuleFor(url) {
-  return url.match(/^https:\/\//) ? require('https') : require('http');
-}
-
-// Returns the appropriate http or https Agent instance for a given URL.
-export function httpAgentFor(url) {
-  let { Agent } = httpModuleFor(url);
-
-  return new Agent({
-    keepAlive: true,
-    maxSockets: 5
-  });
-}
-
-const RETRY_ERROR_CODES = [
-  'ECONNREFUSED', 'ECONNRESET', 'EPIPE',
-  'EHOSTUNREACH', 'EAI_AGAIN'
-];
-
-// Returns true or false if an error should cause the request to be retried
-function shouldRetryRequest(error) {
-  if (error.response) {
-    return error.response.status >= 500 && error.response.status < 600;
-  } else if (error.code) {
-    return RETRY_ERROR_CODES.includes(error.code);
-  } else {
-    return false;
+// Returns true if the URL hostname matches any patterns
+export function hostnameMatches(patterns, url) {
+  let subject = new URL(url);
+
+  /* istanbul ignore next: only strings are provided internally by the client proxy; core (which
+   * borrows this util) sometimes provides an array of patterns or undefined */
+  patterns = typeof patterns === 'string'
+    ? patterns.split(/[\s,]+/)
+    : [].concat(patterns);
+
+  for (let pattern of patterns) {
+    if (pattern === '*') return true;
+    if (!pattern) continue;
+
+    // parse pattern
+    let { groups: rule } = pattern.match(
+      /^(?<hostname>.+?)(?::(?<port>\d+))?$/
+    );
+
+    // missing a hostname or ports do not match
+    if (!rule.hostname || (rule.port && rule.port !== subject.port)) {
+      continue;
+    }
+
+    // wildcards are treated the same as leading dots
+    rule.hostname = rule.hostname.replace(/^\*/, '');
+
+    // hostnames are equal or end with a wildcard rule
+    if (rule.hostname === subject.hostname ||
+        (rule.hostname.startsWith('.') &&
+         subject.hostname.endsWith(rule.hostname))) {
+      return true;
+    }
   }
-}
 
-// Returns a promise that resolves when the request is successful and rejects
-// when a non-successful response is received. The rejected error contains
-// response data and any received error details. Server 500 errors are retried
-// up to 5 times at 50ms intervals.
-export function request(url, { body, ...options }) {
-  let http = httpModuleFor(url);
-  let { protocol, hostname, port, pathname, search } = new URL(url);
-  options = { ...options, protocol, hostname, port, path: pathname + search };
-
-  return retry((resolve, reject, retry) => {
-    let handleError = error => {
-      return shouldRetryRequest(error)
-        ? retry(error) : reject(error);
-    };
-
-    http.request(options)
-      .on('response', res => {
-        let status = res.statusCode;
-        let raw = '';
-
-        res.setEncoding('utf8')
-          .on('data', chunk => (raw += chunk))
-          .on('error', handleError)
-          .on('end', () => {
-            let body = raw;
-            try { body = JSON.parse(raw); } catch (e) {}
-
-            if (status >= 200 && status < 300) {
-              resolve(body);
-            } else {
-              handleError(Object.assign(new Error(), {
-                response: { status, body },
-                // use first error detail or the status message
-                message: body?.errors?.find(e => e.detail)?.detail || (
-                  `${status} ${res.statusMessage || raw}`
-                )
-              }));
-            }
-          });
-      })
-      .on('error', handleError)
-      .end(body);
-  });
+  return false;
 }

packages/client/test/certs/test.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICpDCCAYwCCQCyX37Mj8zDtDANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+b2NhbGhvc3QwHhcNMjEwNDI4MTgzMjA5WhcNMzEwNDI2MTgzMjA5WjAUMRIwEAYD
+VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDT
+3bm7xMop/ZiyU1dRZNOhfd8vhEaIsT2bs0mtZ8CP0bgtMz0qqF9R/bnoOmg0mE3S
+C+eZEmsrCkqBUWN673SVkJH/B0umvMajEM0JZZxCE0mmx+MT5X5J60UEKipCxOR5
+i+fNObwoY0sly9mFGwpYZkzRLzxB2JLUwRyqkTvODcIIs2qDxUiVgT6pTFM4noMn
+u9ev+OoAgWhPoJ/dzf2w+U/dXDB3oWskbrYoN2deEKHfwcmkh4lFuuU3V2+eAaqG
+l2wdZvrjml2HmIeXF/Ae/BOLFIoORLVOrGzBYVU+Hoz+I6P3q9IplLKePzu8pAtI
+jPrWQF2PAtRSbcSh2K8FAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAKqkWAgpl3Qa
+7hUc2BaGmJcuVNI/QnuyRBjp1TlIprEFpe3JxsNLT0yN052hlTAU2d5yYPF7D+e4
+uY+opg+Z4t/rc/JrQoupihh523MIOLAXLzjXcfb16qQ3v78rMIdDZWuzW/8r+u3m
+vD7kFfInYy7jS4o5wNCcU7pFDKsbhd3FeaoueVqihLCBzvnoOuzlIjm/p83BP4gq
+8mo0sSZtYXId3SQ7szLu7PN6hqZm/gFrqplzwOGfoidepwEoG/ZZMWZtvoOg0cab
+7aS4PHUipjtzFW8CHHtA5IL9JOlKutP1Bv5U+sV/BRClHzcUnL4oJux9zZoxyZn9
+1E9tlgkXsSQ=
+-----END CERTIFICATE-----