Proxy / CDN Config Implementation w/ tests

index.js

@@ -30,6 +30,7 @@ const defaultOptions = {
   puppeteerExecutablePath: undefined,
   puppeteerIgnoreHTTPSErrors: false,
   publicPath: "/",
+  proxy: {},
   minifyCss: {},
   minifyHtml: {
     collapseBooleanAttributes: true,

src/puppeteer_utils.js

@@ -12,19 +12,33 @@ const errorToString = jsHandle =>
 const objectToJson = jsHandle => jsHandle.jsonValue();
 
 /**
- * @param {{page: Page, options: {skipThirdPartyRequests: true}, basePath: string }} opt
+ * @param {{page: Page, options: {skipThirdPartyRequests: true, proxy: {}}, basePath: string }} opt
  * @return {Promise<void>}
  */
-const skipThirdPartyRequests = async opt => {
+const handleThirdPartyRequests = async opt => {
   const { page, options, basePath } = opt;
-  if (!options.skipThirdPartyRequests) return;
+  if (!options.skipThirdPartyRequests && !options.proxy) return;
   await page.setRequestInterception(true);
   page.on("request", request => {
-    if (request.url().startsWith(basePath)) {
-      request.continue();
-    } else {
+    if (options.proxy) {
+      for (proxyUrl in options.proxy) {
+        if (request.url().startsWith(proxyUrl)) {
+          const requestChanges = {};
+          if (typeof options.proxy[proxyUrl] === 'string') {
+            requestChanges.url = request.url().replace(proxyUrl, options.proxy[proxyUrl]);
+          }
+          request.continue(requestChanges);
+          return;
+        }
+      }
+    }
+
+    if (options.skipThirdPartyRequests && !request.url().startsWith(basePath)) {
       request.abort();
+      return;
     }
+
+    request.continue();
   });
 };
 
@@ -213,8 +227,8 @@ const crawl = async opt => {
         const page = await browser.newPage();
         await page.setCacheEnabled(options.puppeteer.cache);
         if (options.viewport) await page.setViewport(options.viewport);
-        if (options.skipThirdPartyRequests)
-          await skipThirdPartyRequests({ page, options, basePath });
+        if (options.skipThirdPartyRequests || options.proxy)
+          await handleThirdPartyRequests({ page, options, basePath });
         enableLogging({
           page,
           options,
@@ -280,7 +294,7 @@ const crawl = async opt => {
   });
 };
 
-exports.skipThirdPartyRequests = skipThirdPartyRequests;
+exports.handleThirdPartyRequests = handleThirdPartyRequests;
 exports.enableLogging = enableLogging;
 exports.getLinks = getLinks;
 exports.crawl = crawl;

tests/__snapshots__/defaultOptions.test.js.snap

@@ -31,6 +31,7 @@ Object {
   "port": 45678,
   "preconnectThirdParty": true,
   "preloadImages": false,
+  "proxy": Object {},
   "publicPath": "/",
   "puppeteer": Object {
     "cache": true,

tests/run.test.js

@@ -65,6 +65,35 @@ describe("one page", () => {
   });
 });
 
+describe("proxy", () => {
+  const sourceF = "tests/examples/proxy/frontend";
+  const sourceB = "tests/examples/proxy/backend";
+  const portF = Math.floor(Math.random() * 1000 + 45000);
+  const portB = Math.floor(Math.random() * 1000 + 45000);
+  const {
+    fs,
+    createReadStreamMock,
+    createWriteStreamMock,
+    filesCreated,
+    content,
+    name
+  } = mockFs();
+  beforeAll(() => {
+      const snapB = snapRun(mockFs().fs, { source: sourceB, port: portB });
+      const proxy = {};
+      proxy[`http://localhost:${portF}/text.txt`] = `http://localhost:${portB}/text.txt`;
+      return snapB.then(function() {
+        console.log(arguments);
+        return snapRun(fs, { source: sourceF, proxy, port: portF });
+      });
+  });
+  test("crawls / and saves as index.html to the same folder", () => {
+    expect(filesCreated()).toEqual(1);
+    expect(name(0)).toEqual(`/${sourceF}/index.html`);
+    expect(content(0)).toEqual('This is a test of the proxy.');
+  });
+});
+
 describe("saveAs png", () => {
   const source = "tests/examples/one-page";
   const cwd = process.cwd();