Add support for rewriting to external resources (#10041)

* Add support for rewriting to external resources

* Update rewrite proxying test

Co-authored-by: Tim Neutkens <tim@timneutkens.nl>
Co-authored-by: Joe Haddad <timer150@gmail.com>

package.json

@@ -40,6 +40,7 @@
     "@babel/preset-react": "7.7.0",
     "@fullhuman/postcss-purgecss": "1.3.0",
     "@mdx-js/loader": "0.18.0",
+    "@types/http-proxy": "1.17.3",
     "@types/jest": "24.0.13",
     "@types/string-hash": "1.1.1",
     "@typescript-eslint/eslint-plugin": "2.17.0",
@@ -53,7 +54,9 @@
     "babel-core": "7.0.0-bridge.0",
     "babel-eslint": "10.0.3",
     "babel-jest": "24.9.0",
+    "browserslist": "^4.8.3",
     "browserstack-local": "1.4.0",
+    "caniuse-lite": "^1.0.30001019",
     "cheerio": "0.22.0",
     "clone": "2.1.2",
     "coveralls": "3.0.3",
@@ -100,9 +103,7 @@
     "tree-kill": "1.2.1",
     "typescript": "3.7.3",
     "wait-port": "0.2.2",
-    "webpack-bundle-analyzer": "3.3.2",
-    "browserslist": "^4.8.3",
-    "caniuse-lite": "^1.0.30001019"
+    "webpack-bundle-analyzer": "3.3.2"
   },
   "resolutions": {
     "browserslist": "^4.8.3",

packages/next/lib/check-custom-routes.ts

@@ -127,8 +127,13 @@ export default function checkCustomRoutes(
         invalidParts.push('`destination` is missing')
       } else if (typeof _route.destination !== 'string') {
         invalidParts.push('`destination` is not a string')
-      } else if (type === 'rewrite' && !_route.destination.startsWith('/')) {
-        invalidParts.push('`destination` does not start with /')
+      } else if (
+        type === 'rewrite' &&
+        !_route.destination.match(/^(\/|https:\/\/|http:\/\/)/)
+      ) {
+        invalidParts.push(
+          '`destination` does not start with `/`, `http://`, or `https://`'
+        )
       }
     }
 

packages/next/next-server/server/next-server.ts

@@ -1,5 +1,6 @@
 import compression from 'compression'
 import fs from 'fs'
+import Proxy from 'http-proxy'
 import { IncomingMessage, ServerResponse } from 'http'
 import { join, resolve, sep } from 'path'
 import { compile as compilePathToRegex } from 'path-to-regexp'
@@ -451,7 +452,7 @@ export default class Server {
             type: route.type,
             statusCode: (route as Redirect).statusCode,
             name: `${route.type} ${route.source} route`,
-            fn: async (_req, res, params, _parsedUrl) => {
+            fn: async (req, res, params, _parsedUrl) => {
               const parsedDestination = parseUrl(route.destination, true)
               const destQuery = parsedDestination.query
               let destinationCompiler = compilePathToRegex(
@@ -485,15 +486,15 @@ export default class Server {
                 throw err
               }
 
-              if (route.type === 'redirect') {
-                const parsedNewUrl = parseUrl(newUrl)
-                const updatedDestination = formatUrl({
-                  ...parsedDestination,
-                  pathname: parsedNewUrl.pathname,
-                  hash: parsedNewUrl.hash,
-                  search: undefined,
-                })
+              const parsedNewUrl = parseUrl(newUrl)
+              const updatedDestination = formatUrl({
+                ...parsedDestination,
+                pathname: parsedNewUrl.pathname,
+                hash: parsedNewUrl.hash,
+                search: undefined,
+              })
 
+              if (route.type === 'redirect') {
                 res.setHeader('Location', updatedDestination)
                 res.statusCode = getRedirectStatus(route as Redirect)
 
@@ -508,7 +509,26 @@ export default class Server {
                   finished: true,
                 }
               } else {
-                ;(_req as any)._nextDidRewrite = true
+                // external rewrite, proxy it
+                if (parsedDestination.protocol) {
+                  const proxy = new Proxy({
+                    target: updatedDestination,
+                    changeOrigin: true,
+                    ignorePath: true,
+                  })
+                  proxy.web(req, res)
+
+                  proxy.on('error', (err: Error) => {
+                    console.error(
+                      `Error occurred proxying ${updatedDestination}`,
+                      err
+                    )
+                  })
+                  return {
+                    finished: true,
+                  }
+                }
+                ;(req as any)._nextDidRewrite = true
               }
 
               return {

packages/next/package.json

@@ -102,6 +102,7 @@
     "fork-ts-checker-webpack-plugin": "3.1.1",
     "fresh": "0.5.2",
     "gzip-size": "5.1.1",
+    "http-proxy": "1.18.0",
     "ignore-loader": "0.1.2",
     "is-docker": "2.0.0",
     "is-wsl": "2.1.1",

test/integration/custom-routes/next.config.js

@@ -55,6 +55,10 @@ module.exports = {
           source: '/hidden/_next/:path*',
           destination: '/_next/:path*',
         },
+        {
+          source: '/proxy-me/:path*',
+          destination: 'http://localhost:__EXTERNAL_PORT__/:path*',
+        },
         {
           source: '/api-hello',
           destination: '/api/hello',

test/integration/custom-routes/test/index.test.js

@@ -1,5 +1,6 @@
 /* eslint-env jest */
 /* global jasmine */
+import http from 'http'
 import url from 'url'
 import stripAnsi from 'strip-ansi'
 import fs from 'fs-extra'
@@ -24,9 +25,13 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 2
 
 let appDir = join(__dirname, '..')
 const nextConfigPath = join(appDir, 'next.config.js')
+let externalServerHits = new Set()
+let nextConfigRestoreContent
 let nextConfigContent
-let buildId
+let externalServerPort
+let externalServer
 let stdout = ''
+let buildId
 let appPort
 let app
 
@@ -228,6 +233,13 @@ const runTests = (isDev = false) => {
     expect(res.headers.get('x-second-header')).toBe('second')
   })
 
+  it('should support proxying to external resource', async () => {
+    const res = await fetchViaHTTP(appPort, '/proxy-me/first')
+    expect(res.status).toBe(200)
+    expect([...externalServerHits]).toEqual(['/first'])
+    expect(await res.text()).toContain('hi from external')
+  })
+
   it('should support unnamed parameters correctly', async () => {
     const res = await fetchViaHTTP(appPort, '/unnamed/first/final', undefined, {
       redirect: 'manual',
@@ -493,6 +505,13 @@ const runTests = (isDev = false) => {
             ),
             source: '/hidden/_next/:path*',
           },
+          {
+            destination: `http://localhost:${externalServerPort}/:path*`,
+            regex: normalizeRegEx(
+              '^\\/proxy-me(?:\\/((?:[^\\/]+?)(?:\\/(?:[^\\/]+?))*))?$'
+            ),
+            source: '/proxy-me/:path*',
+          },
           {
             destination: '/api/hello',
             regex: normalizeRegEx('^\\/api-hello$'),
@@ -550,6 +569,32 @@ const runTests = (isDev = false) => {
 }
 
 describe('Custom routes', () => {
+  beforeEach(() => {
+    externalServerHits = new Set()
+  })
+  beforeAll(async () => {
+    externalServerPort = await findPort()
+    externalServer = http.createServer((req, res) => {
+      externalServerHits.add(req.url)
+      res.end('hi from external')
+    })
+    await new Promise((resolve, reject) => {
+      externalServer.listen(externalServerPort, error => {
+        if (error) return reject(error)
+        resolve()
+      })
+    })
+    nextConfigRestoreContent = await fs.readFile(nextConfigPath, 'utf8')
+    await fs.writeFile(
+      nextConfigPath,
+      nextConfigRestoreContent.replace(/__EXTERNAL_PORT__/, externalServerPort)
+    )
+  })
+  afterAll(async () => {
+    externalServer.close()
+    await fs.writeFile(nextConfigPath, nextConfigRestoreContent)
+  })
+
   describe('dev mode', () => {
     beforeAll(async () => {
       appPort = await findPort()

test/integration/invalid-custom-routes/test/index.test.js

@@ -167,7 +167,7 @@ const runTests = () => {
     )
 
     expect(stderr).toContain(
-      `\`destination\` does not start with / for route {"source":"/hello","destination":"another"}`
+      `\`destination\` does not start with \`/\`, \`http://\`, or \`https://\` for route {"source":"/hello","destination":"another"}`
     )
 
     expect(stderr).toContain(

yarn.lock

@@ -2583,6 +2583,13 @@
     "@types/tough-cookie" "*"
     form-data "^2.5.0"
 
+"@types/http-proxy@1.17.3":
+  version "1.17.3"
+  resolved "https://registry.yarnpkg.com/@types/http-proxy/-/http-proxy-1.17.3.tgz#348e1b808ff9585423cb909e9992d89ccdbf4c14"
+  integrity sha512-wIPqXANye5BbORbuh74exbwNzj+UWCwWyeEFJzUQ7Fq3W2NSAy+7x7nX1fgbEypr2/TdKqpeuxLnXWgzN533/Q==
+  dependencies:
+    "@types/node" "*"
+
 "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0":
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz#42995b446db9a48a11a07ec083499a860e9138ff"
@@ -5802,7 +5809,7 @@ debug@3.1.0:
   dependencies:
     ms "2.0.0"
 
-debug@^3.1.0, debug@^3.2.6:
+debug@^3.0.0, debug@^3.1.0, debug@^3.2.6:
   version "3.2.6"
   resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b"
   integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==
@@ -6578,6 +6585,11 @@ eventemitter3@^3.1.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7"
   integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==
 
+eventemitter3@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb"
+  integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==
+
 events@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/events/-/events-3.0.0.tgz#9a0a0dfaf62893d92b875b8f2698ca4114973e88"
@@ -7193,6 +7205,13 @@ fn-annotate@^1.1.3:
   resolved "https://registry.yarnpkg.com/fn-annotate/-/fn-annotate-1.2.0.tgz#28da000117dea61842fe61f353f41cf4c93a7a7e"
   integrity sha1-KNoAARfephhC/mHzU/Qc9Mk6en4=
 
+follow-redirects@^1.0.0:
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.9.0.tgz#8d5bcdc65b7108fe1508649c79c12d732dcedb4f"
+  integrity sha512-CRcPzsSIbXyVDl0QI01muNDu69S8trU4jArW9LpOt2WtC6LyUJetcIrmfHsRBx7/Jb6GHJUiuqyYxPooFfNt6A==
+  dependencies:
+    debug "^3.0.0"
+
 for-in@^0.1.3:
   version "0.1.8"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-0.1.8.tgz#d8773908e31256109952b1fdb9b3fa867d2775e1"
@@ -8110,6 +8129,15 @@ http-proxy-agent@^2.1.0:
     agent-base "4"
     debug "3.1.0"
 
+http-proxy@1.18.0:
+  version "1.18.0"
+  resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.0.tgz#dbe55f63e75a347db7f3d99974f2692a314a6a3a"
+  integrity sha512-84I2iJM/n1d4Hdgc6y2+qY5mDaz2PUVjlg9znE9byl+q0uC3DeByqBGReQu5tpLK0TAqTIXScRUV+dg7+bUPpQ==
+  dependencies:
+    eventemitter3 "^4.0.0"
+    follow-redirects "^1.0.0"
+    requires-port "^1.0.0"
+
 http-signature@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
@@ -14124,6 +14152,11 @@ require-main-filename@^2.0.0:
   resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b"
   integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==
 
+requires-port@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
+  integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=
+
 reserved-words@^0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/reserved-words/-/reserved-words-0.1.2.tgz#00a0940f98cd501aeaaac316411d9adc52b31ab1"