tests(fixtures): introduce script to update the report fixtures

.eslintrc.js

@@ -68,7 +68,7 @@ module.exports = {
     'arrow-parens': 0,
   },
   parserOptions: {
-    ecmaVersion: 6,
+    ecmaVersion: 2017,
     ecmaFeatures: {
       globalReturn: true,
       jsx: false,

.gitignore

@@ -24,6 +24,9 @@ last-run-results.html
 *.report.pretty
 *.artifacts.log
 
+!lighthouse-cli/test/sample_artifacts/*.trace.json
+!lighthouse-cli/test/sample_artifacts/*.devtoolslog.json
+
 latest-run
 
 closure-error.log

lighthouse-cli/cli-flags.js

@@ -111,7 +111,7 @@ function getFlags(manualArgv) {
         'disable-storage-reset', 'disable-device-emulation', 'disable-cpu-throttling',
         'disable-network-throttling', 'save-assets', 'list-all-audits',
         'list-trace-categories', 'perf', 'view', 'verbose', 'quiet', 'help',
-        'gather-mode', 'audit-mode', 'mixed-content',
+        'mixed-content',
       ])
       .choices('output', printer.getValidOutputOptions())
       // force as an array

lighthouse-cli/test/fixtures/static-server.js

@@ -3,6 +3,7 @@
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
+// @ts-nocheck
 'use strict';
 
 /* eslint-disable no-console */
@@ -122,6 +123,16 @@ const serverForOffline = http.createServer(requestHandler);
 serverForOnline.on('error', e => console.error(e.code, e));
 serverForOffline.on('error', e => console.error(e.code, e));
 
-// Listen
-serverForOnline.listen(10200, 'localhost');
-serverForOffline.listen(10503, 'localhost');
+
+// If called via `node static-server.js` then start listening, otherwise, just expose the servers
+if (require.main === module) {
+  // Start listening
+  serverForOnline.listen(10200, 'localhost');
+  serverForOffline.listen(10503, 'localhost');
+} else {
+  module.exports = {
+    server: serverForOnline,
+    serverForOffline,
+  };
+}
+

lighthouse-cli/test/sample_artifacts/redirectPass.devtoolslog.json

@@ -0,0 +1 @@
+[{"method":"Network.requestWillBeSent","params":{"requestId":"B153A75C1DEF181D4A3B369563BFF20C","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/dbw_tester.html","method":"GET","headers":{"Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36","X-DevTools-Emulate-Network-Conditions-Client-Id":"A36E31E517F452A18C51AA32F4163A8C"},"mixedContentType":"none","initialPriority":"VeryHigh","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.176299,"wallTime":1519974796.775882,"initiator":{"type":"other"},"type":"Document","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.responseReceived","params":{"requestId":"B153A75C1DEF181D4A3B369563BFF20C","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","timestamp":185617.179082,"type":"Document","response":{"url":"http://localhost:10200/dobetterweb/dbw_tester.html","status":200,"statusText":"OK","headers":{"Date":"Fri, 02 Mar 2018 07:13:16 GMT","Connection":"keep-alive","Transfer-Encoding":"chunked"},"headersText":"HTTP/1.1 200 OK\r\nDate: Fri, 02 Mar 2018 07:13:16 GMT\r\nConnection: keep-alive\r\nTransfer-Encoding: chunked\r\n\r\n","mimeType":"text/html","requestHeaders":{"Host":"localhost:10200","Connection":"keep-alive","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8","Accept-Encoding":"gzip, deflate, br","Accept-Language":"en-US,en;q=0.9"},"requestHeadersText":"GET /dobetterweb/dbw_tester.html HTTP/1.1\r\nHost: localhost:10200\r\nConnection: keep-alive\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: en-US,en;q=0.9\r\n","connectionReused":true,"connectionId":221,"remoteIPAddress":"127.0.0.1","remotePort":10200,"fromDiskCache":false,"fromServiceWorker":false,"encodedDataLength":108,"timing":{"requestTime":185617.176621,"proxyStart":-1,"proxyEnd":-1,"dnsStart":-1,"dnsEnd":-1,"connectStart":-1,"connectEnd":-1,"sslStart":-1,"sslEnd":-1,"workerStart":-1,"workerReady":-1,"sendStart":0.838,"sendEnd":0.873,"pushStart":0,"pushEnd":0,"receiveHeadersEnd":1.759},"protocol":"http/1.1","securityState":"neutral"},"frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Page.frameStartedLoading","params":{"frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.dataReceived","params":{"requestId":"B153A75C1DEF181D4A3B369563BFF20C","timestamp":185617.180942,"dataLength":3982,"encodedDataLength":0}},{"method":"Page.frameNavigated","params":{"frame":{"id":"A36E31E517F452A18C51AA32F4163A8C","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","url":"http://localhost:10200/dobetterweb/dbw_tester.html","securityOrigin":"http://localhost:10200","mimeType":"text/html"}}},{"method":"Network.dataReceived","params":{"requestId":"B153A75C1DEF181D4A3B369563BFF20C","timestamp":185617.185044,"dataLength":8537,"encodedDataLength":0}},{"method":"Network.loadingFinished","params":{"requestId":"B153A75C1DEF181D4A3B369563BFF20C","timestamp":185617.181093,"encodedDataLength":12640,"blockedCrossSiteDocument":false}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.66","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/dbw_tester.css?delay=100","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html"},"mixedContentType":"none","initialPriority":"VeryHigh","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.186619,"wallTime":1519974796.7862,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":22},"type":"Stylesheet","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.66","timestamp":185617.186622,"type":"Stylesheet","errorText":"","canceled":false,"blockedReason":"inspector"}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.68","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/unknown404.css?delay=200","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html"},"mixedContentType":"none","initialPriority":"VeryHigh","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.186715,"wallTime":1519974796.7863,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":23},"type":"Stylesheet","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.68","timestamp":185617.186717,"type":"Stylesheet","errorText":"","canceled":false,"blockedReason":"inspector"}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.70","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/dbw_tester.css?delay=2200","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html"},"mixedContentType":"none","initialPriority":"VeryHigh","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.186799,"wallTime":1519974796.78638,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":24},"type":"Stylesheet","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.70","timestamp":185617.186802,"type":"Stylesheet","errorText":"","canceled":false,"blockedReason":"inspector"}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.72","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/dbw_partial_a.html?delay=200","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html","Origin":"http://localhost:10200"},"mixedContentType":"none","initialPriority":"High","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.186917,"wallTime":1519974796.7865,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":26},"type":"Document","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.72","timestamp":185617.186919,"type":"Document","errorText":"","canceled":false,"blockedReason":"csp"}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.74","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/dbw_partial_b.html?delay=200&isasync","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html","Origin":"http://localhost:10200"},"mixedContentType":"none","initialPriority":"High","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.187006,"wallTime":1519974796.78659,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":27},"type":"Document","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.74","timestamp":185617.187008,"type":"Document","errorText":"","canceled":false,"blockedReason":"csp"}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.76","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/dbw_tester.css?delay=2000&async=true","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html"},"mixedContentType":"none","initialPriority":"VeryHigh","referrerPolicy":"no-referrer-when-downgrade","isLinkPreload":true},"timestamp":185617.187098,"wallTime":1519974796.78668,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":30},"type":"Stylesheet","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.76","timestamp":185617.1871,"type":"Stylesheet","errorText":"","canceled":false,"blockedReason":"inspector"}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.78","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/lighthouse-480x318.jpg","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html"},"mixedContentType":"none","initialPriority":"Low","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.187496,"wallTime":1519974796.78708,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":133},"type":"Image","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.78","timestamp":185617.187499,"type":"Image","errorText":"","canceled":false,"blockedReason":"inspector"}},{"method":"Network.requestWillBeSent","params":{"requestId":"75994.80","loaderId":"B153A75C1DEF181D4A3B369563BFF20C","documentURL":"http://localhost:10200/dobetterweb/dbw_tester.html","request":{"url":"http://localhost:10200/dobetterweb/lighthouse-480x318.jpg","method":"GET","headers":{"Referer":"http://localhost:10200/dobetterweb/dbw_tester.html"},"mixedContentType":"none","initialPriority":"Low","referrerPolicy":"no-referrer-when-downgrade"},"timestamp":185617.187603,"wallTime":1519974796.78719,"initiator":{"type":"parser","url":"http://localhost:10200/dobetterweb/dbw_tester.html","lineNumber":135},"type":"Image","frameId":"A36E31E517F452A18C51AA32F4163A8C"}},{"method":"Network.loadingFailed","params":{"requestId":"75994.80","timestamp":185617.187605,"type":"Image","errorText":"","canceled":false,"blockedReason":"inspector"}},{"method":"Page.domContentEventFired","params":{"timestamp":185617.189111}},{"method":"Page.loadEventFired","params":{"timestamp":185617.189281}},{"method":"Page.frameStoppedLoading","params":{"frameId":"A36E31E517F452A18C51AA32F4163A8C"}}]

lighthouse-core/runner.js

@@ -18,8 +18,6 @@ const path = require('path');
 const URL = require('./lib/url-shim');
 const Sentry = require('./lib/sentry');
 
-const basePath = path.join(process.cwd(), 'latest-run');
-
 class Runner {
   static run(connection, opts) {
     // Clean opts input.
@@ -68,7 +66,7 @@ class Runner {
     // Gather phase
     // Either load saved artifacts off disk, from config, or get from the browser
     if (opts.flags.auditMode && !opts.flags.gatherMode) {
-      run = run.then(_ => Runner._loadArtifactsFromDisk());
+      run = run.then(_ => Runner._loadArtifactsFromDisk(Runner._getArtifactsPath(opts.flags)));
     } else if (opts.config.artifacts) {
       run = run.then(_ => opts.config.artifacts);
     } else {
@@ -123,10 +121,11 @@ class Runner {
 
   /**
    * No browser required, just load the artifacts from disk
+   * @param {string} path
    * @return {!Promise<!Artifacts>}
    */
-  static _loadArtifactsFromDisk() {
-    return assetSaver.loadArtifacts(basePath);
+  static _loadArtifactsFromDisk(path) {
+    return assetSaver.loadArtifacts(path);
   }
 
   /**
@@ -144,18 +143,20 @@ class Runner {
     return GatherRunner.run(opts.config.passes, opts).then(artifacts => {
       const flags = opts.flags;
       const shouldSave = flags.gatherMode;
-      const p = shouldSave ? Runner._saveArtifacts(artifacts): Promise.resolve();
+      const path = Runner._getArtifactsPath(flags);
+      const p = shouldSave ? Runner._saveArtifacts(artifacts, path): Promise.resolve();
       return p.then(_ => artifacts);
     });
   }
 
   /**
    * Save collected artifacts to disk
    * @param {!Artifacts} artifacts
+   * @param {string} path
    * @return {!Promise>}
    */
-  static _saveArtifacts(artifacts) {
-    return assetSaver.saveArtifacts(artifacts, basePath);
+  static _saveArtifacts(artifacts, path) {
+    return assetSaver.saveArtifacts(artifacts, path);
   }
 
   /**
@@ -410,6 +411,20 @@ class Runner {
       extraHeaders: flags.extraHeaders || {},
     };
   }
+
+  /**
+   * Get path to use for -G and -A modes. Defaults to $CWD/latest-run
+   * @param {*} flags
+   * @return {string}
+   */
+  static _getArtifactsPath(flags) {
+    // This enables usage like: -GA=./custom-folder
+    if (typeof flags.auditMode === 'string') return path.join(process.cwd(), flags.auditMode);
+    if (typeof flags.gatherMode === 'string') return path.join(process.cwd(), flags.gatherMode);
+    return path.join(process.cwd(), 'latest-run');
+  }
 }
 
 module.exports = Runner;
+
+

lighthouse-core/scripts/update-report-fixtures.js

@@ -0,0 +1,53 @@
+/**
+ * @license Copyright 2016 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const cli = require('../../lighthouse-cli/run');
+
+const {server} = require(
+    '../../lighthouse-cli/test/fixtures/static-server');
+
+/**
+ * Update the report fixtures, either the artifacts or the json
+ * @param {{gather?: boolean, audit?: boolean}} opts
+ */
+async function update(opts) {
+  // get an available port
+  server.listen(0, 'localhost');
+  const port = await new Promise(res => server.on('listening', () => res(server.address().port)));
+
+  const artifactsPath = 'lighthouse-cli/test/sample_artifacts';
+
+  const url = `http://localhost:${port}/dobetterweb/dbw_tester.html`;
+  const flags = {
+    output: 'json',
+    outputPath: 'lighthouse-core/test/results/sample_v2.json',
+    gatherMode: opts.gather ? artifactsPath : undefined,
+    auditMode: opts.audit ? artifactsPath : undefined,
+  };
+  // @ts-ignore Remove when we fix Flags typing
+  await cli.runLighthouse(url, flags, undefined);
+  await new Promise(res => server.close(res));
+}
+
+/* eslint-disable no-console */
+const args = process.argv.slice(2);
+if (!args.length) {
+  console.log('Run this with either -G (to refresh the artifacts) or -A (to refresh the json).');
+  process.exit(1);
+}
+
+(async function() {
+  if (args[0] === '-G') {
+    console.log('Refreshing the artifact fixtures…');
+    await update({gather: true});
+  } else if (args[0] === '-A') {
+    console.log('Refreshing the json fixture…');
+    await update({audit: true});
+  } else {
+    console.error('Unknown flags');
+  }
+})();