Refactor proxy into own package, implement middleware pattern

circle.yml

@@ -270,7 +270,7 @@ jobs:
       - run: npm run all test -- --package https-proxy
       - run: npm run all test -- --package launcher
       - run: npm run all test -- --package network
-      # how to pass Mocha reporter through zunder?
+      - run: npm run all test -- --package proxy
       - run: npm run all test -- --package reporter
       - run: npm run all test -- --package runner
       - run: npm run all test -- --package socket

packages/network/README.md

@@ -0,0 +1,3 @@
+# network
+
+This package contains networking-related classes and utilities.

packages/network/lib/blacklist.ts

@@ -0,0 +1,18 @@
+import _ from 'lodash'
+import minimatch from 'minimatch'
+import { stripProtocolAndDefaultPorts } from './uri'
+
+export function matches (urlToCheck, blacklistHosts) {
+  // normalize into flat array
+  blacklistHosts = [].concat(blacklistHosts)
+
+  urlToCheck = stripProtocolAndDefaultPorts(urlToCheck)
+
+  // use minimatch against the url
+  // to see if any match
+  const matchUrl = (hostMatcher) => {
+    return minimatch(urlToCheck, hostMatcher)
+  }
+
+  return _.find(blacklistHosts, matchUrl)
+}

packages/network/lib/concat-stream.ts

@@ -0,0 +1,29 @@
+import _ from 'lodash'
+import _concatStream from 'concat-stream'
+
+type Callback = (buf: Buffer) => void
+type ConcatOpts = {
+  encoding?: string
+}
+
+/**
+ * Wrapper for `concat-stream` to handle empty streams.
+ */
+export const concatStream : typeof _concatStream = function (opts: Callback | ConcatOpts, cb?: Callback) {
+  let _cb : Callback = cb!
+
+  if (!_cb) {
+    _cb = opts as Callback
+    opts = {}
+  }
+
+  return _concatStream(opts as ConcatOpts, function (buf: Buffer) {
+    if (!_.get(buf, 'length')) {
+      // concat-stream can give an empty array if the stream has
+      // no data - just call the callback with an empty buffer
+      return _cb(Buffer.from(''))
+    }
+
+    return _cb(buf)
+  })
+}

packages/server/lib/util/cors.js → packages/network/lib/cors.ts

@@ -1,32 +1,43 @@
-const _ = require('lodash')
-const url = require('url')
-const uri = require('./uri')
-const debug = require('debug')('cypress:server:cors')
-const parseDomain = require('parse-domain')
+import _ from 'lodash'
+import * as uri from './uri'
+import debugModule from 'debug'
+import _parseDomain, { ParsedDomain } from 'parse-domain'
+
+const debug = debugModule('cypress:network:cors')
 
 const ipAddressRe = /^[\d\.]+$/
 
-function getSuperDomain (url) {
+type ParsedHost = {
+  port?: string
+  tld?: string
+  domain?: string
+}
+
+export function getSuperDomain (url) {
   const parsed = parseUrlIntoDomainTldPort(url)
 
   return _.compact([parsed.domain, parsed.tld]).join('.')
 }
 
-function _parseDomain (domain, options = {}) {
-  return parseDomain(domain, _.defaults(options, {
+export function parseDomain (domain: string, options = {}) {
+  return _parseDomain(domain, _.defaults(options, {
     privateTlds: true,
     customTlds: ipAddressRe,
   }))
 }
 
-function parseUrlIntoDomainTldPort (str) {
-  let { hostname, port, protocol } = url.parse(str)
+export function parseUrlIntoDomainTldPort (str) {
+  let { hostname, port, protocol } = uri.parse(str)
 
-  if (port == null) {
+  if (!hostname) {
+    hostname = ''
+  }
+
+  if (!port) {
     port = protocol === 'https:' ? '443' : '80'
   }
 
-  let parsed = _parseDomain(hostname)
+  let parsed : Partial<ParsedDomain> | null = parseDomain(hostname)
 
   // if we couldn't get a parsed domain
   if (!parsed) {
@@ -43,46 +54,33 @@ function parseUrlIntoDomainTldPort (str) {
     }
   }
 
-  const obj = {}
+  const obj: ParsedHost = {}
 
   obj.port = port
   obj.tld = parsed.tld
   obj.domain = parsed.domain
-  // obj.protocol = protocol
 
   debug('Parsed URL %o', obj)
 
   return obj
 }
 
-function urlMatchesOriginPolicyProps (urlStr, props) {
+export function urlMatchesOriginPolicyProps (urlStr, props) {
   // take a shortcut here in the case
   // where remoteHostAndPort is null
   if (!props) {
     return false
   }
 
-  const parsedUrl = this.parseUrlIntoDomainTldPort(urlStr)
+  const parsedUrl = parseUrlIntoDomainTldPort(urlStr)
 
   // does the parsedUrl match the parsedHost?
   return _.isEqual(parsedUrl, props)
 }
 
-function urlMatchesOriginProtectionSpace (urlStr, origin) {
+export function urlMatchesOriginProtectionSpace (urlStr, origin) {
   const normalizedUrl = uri.addDefaultPort(urlStr).format()
   const normalizedOrigin = uri.addDefaultPort(origin).format()
 
   return _.startsWith(normalizedUrl, normalizedOrigin)
 }
-
-module.exports = {
-  parseUrlIntoDomainTldPort,
-
-  parseDomain: _parseDomain,
-
-  getSuperDomain,
-
-  urlMatchesOriginPolicyProps,
-
-  urlMatchesOriginProtectionSpace,
-}

packages/network/lib/index.ts

@@ -1,9 +1,17 @@
 import agent from './agent'
+import * as blacklist from './blacklist'
 import * as connect from './connect'
-import { allowDestroy } from './allow-destroy'
+import * as cors from './cors'
+import * as uri from './uri'
 
 export {
   agent,
-  allowDestroy,
+  blacklist,
   connect,
+  cors,
+  uri,
 }
+
+export { allowDestroy } from './allow-destroy'
+
+export { concatStream } from './concat-stream'

packages/server/lib/util/uri.js → packages/network/lib/uri.ts

@@ -5,8 +5,8 @@
 // node's url formatting algorithm (which acts pretty unexpectedly)
 // - https://nodejs.org/api/url.html#url_url_format_urlobject
 
-const _ = require('lodash')
-const url = require('url')
+import _ from 'lodash'
+import url from 'url'
 
 // yup, protocol contains a: ':' colon
 // at the end of it (-______________-)
@@ -25,9 +25,9 @@ const parseClone = (urlObject) => {
   return url.parse(_.clone(urlObject))
 }
 
-const parse = url.parse
+export const parse = url.parse
 
-const stripProtocolAndDefaultPorts = function (urlToCheck) {
+export function stripProtocolAndDefaultPorts (urlToCheck) {
   // grab host which is 'hostname:port' only
   const { host, hostname, port } = url.parse(urlToCheck)
 
@@ -41,20 +41,18 @@ const stripProtocolAndDefaultPorts = function (urlToCheck) {
   return host
 }
 
-const removePort = (urlObject) => {
+export function removePort (urlObject) {
   const parsed = parseClone(urlObject)
 
-  // set host to null else
-  // url.format(...) will ignore
-  // the port property
+  // set host to undefined else url.format(...) will ignore the port property
   // https://nodejs.org/api/url.html#url_url_format_urlobject
-  parsed.host = null
-  parsed.port = null
+  parsed.host = undefined
+  parsed.port = undefined
 
   return parsed
 }
 
-const removeDefaultPort = function (urlToCheck) {
+export function removeDefaultPort (urlToCheck) {
   let parsed = parseClone(urlToCheck)
 
   if (portIsDefault(parsed.port)) {
@@ -64,33 +62,19 @@ const removeDefaultPort = function (urlToCheck) {
   return parsed
 }
 
-const addDefaultPort = function (urlToCheck) {
+export function addDefaultPort (urlToCheck) {
   const parsed = parseClone(urlToCheck)
 
   if (!parsed.port) {
     // unset host...
     // see above for reasoning
-    parsed.host = null
-    parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol]
+    parsed.host = undefined
+    parsed.port = DEFAULT_PROTOCOL_PORTS[parsed.protocol || 'http:']
   }
 
   return parsed
 }
 
-const getPath = (urlToCheck) => {
+export function getPath (urlToCheck) {
   return url.parse(urlToCheck).path
 }
-
-module.exports = {
-  parse,
-
-  getPath,
-
-  removePort,
-
-  addDefaultPort,
-
-  removeDefaultPort,
-
-  stripProtocolAndDefaultPorts,
-}

packages/network/package.json

@@ -10,12 +10,15 @@
   },
   "dependencies": {
     "bluebird": "3.5.3",
+    "concat-stream": "1.6.2",
     "debug": "4.1.1",
     "lodash": "4.17.15",
+    "parse-domain": "2.0.0",
     "proxy-from-env": "1.0.0"
   },
   "devDependencies": {
     "@cypress/debugging-proxy": "2.0.1",
+    "@types/concat-stream": "1.6.0",
     "bin-up": "1.2.2",
     "express": "4.16.4",
     "request": "2.88.0",

packages/network/test/mocha.opts

@@ -1,5 +1,5 @@
 test/unit
 test/integration
---compilers ts:@packages/ts/register
+--compilers ts:@packages/ts/register,coffee:@packages/coffee/register
 --timeout 10000
 --recursive

packages/server/test/unit/blacklist_spec.js → packages/network/test/unit/blacklist_spec.ts

@@ -1,6 +1,5 @@
-require('../spec_helper')
-
-const blacklist = require(`${root}lib/util/blacklist`)
+import { blacklist } from '../..'
+import { expect } from 'chai'
 
 const hosts = [
   '*.google.com',
@@ -26,7 +25,7 @@ const matchesHost = (url, host) => {
   expect(blacklist.matches(url, hosts)).to.eq(host)
 }
 
-describe('lib/util/blacklist', () => {
+describe('lib/blacklist', () => {
   it('handles hosts, ports, wildcards', () => {
     matchesArray('https://mail.google.com/foo', true)
     matchesArray('https://shop.apple.com/bar', true)

packages/server/test/unit/cors_spec.js → packages/network/test/unit/cors_spec.ts

@@ -1,8 +1,7 @@
-require('../spec_helper')
+import { cors } from '../../lib'
+import { expect } from 'chai'
 
-const cors = require(`${root}lib/util/cors`)
-
-describe('lib/util/cors', () => {
+describe('lib/cors', () => {
   context('.parseUrlIntoDomainTldPort', () => {
     beforeEach(function () {
       this.isEq = (url, obj) => {

packages/proxy/README.md

@@ -0,0 +1,3 @@
+# proxy
+
+This package contains the code for Cypress's HTTP interception proxy.

packages/proxy/index.js

@@ -0,0 +1,5 @@
+if (process.env.CYPRESS_ENV !== 'production') {
+  require('@packages/ts/register')
+}
+
+module.exports = require('./lib')

packages/proxy/lib/http/error-middleware.ts

@@ -0,0 +1,48 @@
+import _ from 'lodash'
+import debugModule from 'debug'
+import { HttpMiddleware } from '.'
+import { Readable } from 'stream'
+import { Request } from 'request'
+
+const debug = debugModule('cypress:proxy:http:error-middleware')
+
+export type ErrorMiddleware = HttpMiddleware<{
+  error: Error
+  incomingResStream?: Readable
+  outgoingReq?: Request
+}>
+
+const LogError : ErrorMiddleware = function () {
+  debug('error proxying request %o', _.pick(this, 'error', 'req', 'res', 'incomingRes', 'outgoingReq', 'incomingResStream'))
+  this.next()
+}
+
+export const AbortRequest : ErrorMiddleware = function () {
+  if (this.outgoingReq) {
+    debug('aborting outgoingReq')
+    this.outgoingReq.abort()
+  }
+
+  this.next()
+}
+
+export const UnpipeResponse : ErrorMiddleware = function () {
+  if (this.incomingResStream) {
+    debug('unpiping resStream from response')
+    this.incomingResStream.unpipe()
+  }
+
+  this.next()
+}
+
+export const DestroyResponse : ErrorMiddleware = function () {
+  this.res.destroy()
+  this.end()
+}
+
+export default {
+  LogError,
+  AbortRequest,
+  UnpipeResponse,
+  DestroyResponse,
+}