core(preload): use lantern to compute savings

lighthouse-core/audits/uses-rel-preload.js

@@ -50,56 +50,119 @@ class UsesRelPreloadAudit extends Audit {
     return requests;
   }
 
+  /**
+   * Computes the estimated effect of preloading all the resources.
+   * @param {Set<string>} urls The array of byte savings results per resource
+   * @param {Node} graph
+   * @param {Simulator} simulator
+   * @param {LH.WebInspector.NetworkRequest} mainResource
+   * @return {{wastedMs: number, results: Array<{url: string, wastedMs: number}>}}
+   */
+  static computeWasteWithGraph(urls, graph, simulator, mainResource) {
+    if (!urls.size) {
+      return {wastedMs: 0, results: []};
+    }
+
+    const simulationBeforeChanges = simulator.simulate(graph, {flexibleOrdering: true});
+
+    const modifiedGraph = graph.cloneWithRelationships();
+
+    const nodesToPreload = [];
+    /** @type {Node|null} */
+    let mainDocumentNode = null;
+    modifiedGraph.traverse(node => {
+      if (node.record && urls.has(node.record.url)) {
+        nodesToPreload.push(node);
+      }
+
+      if (node.record && node.record.url === mainResource.url) {
+        mainDocumentNode = node;
+      }
+    });
+
+    if (!mainDocumentNode) {
+      // Should always find the main document node
+      throw new Error('Could not find main document node');
+    }
+
+    // Preload has the effect of moving the resource's only dependency to the main HTML document
+    // Remove all dependencies of the nodes
+    for (const node of nodesToPreload) {
+      node.removeAllDependencies();
+      node.addDependency(mainDocumentNode);
+    }
+
+    const simulationAfterChanges = simulator.simulate(modifiedGraph, {flexibleOrdering: true});
+    const originalNodesByRecord = Array.from(simulationBeforeChanges.nodeTimings.keys())
+        .reduce((map, node) => map.set(node.record, node), new Map());
+
+    const results = [];
+    for (const node of nodesToPreload) {
+      const originalNode = originalNodesByRecord.get(node.record);
+      const timingAfter = simulationAfterChanges.nodeTimings.get(node);
+      const timingBefore = simulationBeforeChanges.nodeTimings.get(originalNode);
+      const wastedMs = Math.round(timingBefore.endTime - timingAfter.endTime);
+      if (wastedMs < THRESHOLD_IN_MS) continue;
+      results.push({url: node.record.url, wastedMs});
+    }
+
+    if (!results.length) {
+      return {wastedMs: 0, results};
+    }
+
+    return {
+      // Preload won't necessarily impact the deepest chain/overall time
+      // We'll use the maximum endTime improvement for now
+      wastedMs: Math.max(...results.map(item => item.wastedMs)),
+      results,
+    };
+  }
+
   /**
    * @param {!Artifacts} artifacts
    * @return {!AuditResult}
    */
-  static audit(artifacts) {
+  static audit(artifacts, context) {
+    const trace = artifacts.traces[UsesRelPreloadAudit.DEFAULT_PASS];
     const devtoolsLog = artifacts.devtoolsLogs[UsesRelPreloadAudit.DEFAULT_PASS];
     const URL = artifacts.URL;
+    const simulatorOptions = {trace, devtoolsLog, settings: context.settings};
 
     return Promise.all([
+      // TODO(phulce): eliminate dependency on CRC
       artifacts.requestCriticalRequestChains({devtoolsLog, URL}),
       artifacts.requestMainResource({devtoolsLog, URL}),
-    ]).then(([critChains, mainResource]) => {
-      const results = [];
-      let maxWasted = 0;
+      artifacts.requestPageDependencyGraph({trace, devtoolsLog}),
+      artifacts.requestLoadSimulator(simulatorOptions),
+    ]).then(([critChains, mainResource, graph, simulator]) => {
       // get all critical requests 2 + mainResourceIndex levels deep
       const mainResourceIndex = mainResource.redirects ? mainResource.redirects.length : 0;
 
       const criticalRequests = UsesRelPreloadAudit._flattenRequests(critChains,
         3 + mainResourceIndex, 2 + mainResourceIndex);
-      criticalRequests.forEach(request => {
-        const networkRecord = request;
+      const urls = new Set();
+      for (const networkRecord of criticalRequests) {
         if (!networkRecord._isLinkPreload && networkRecord.protocol !== 'data') {
-          // calculate time between mainresource.endTime and resource start time
-          const wastedMs = Math.min(request._startTime - mainResource._endTime,
-            request._endTime - request._startTime) * 1000;
-
-          if (wastedMs >= THRESHOLD_IN_MS) {
-            maxWasted = Math.max(wastedMs, maxWasted);
-            results.push({
-              url: request.url,
-              wastedMs: Util.formatMilliseconds(wastedMs),
-            });
-          }
+          urls.add(networkRecord._url);
         }
-      });
+      }
 
+      const {results, wastedMs} = UsesRelPreloadAudit.computeWasteWithGraph(urls, graph, simulator,
+          mainResource);
       // sort results by wastedTime DESC
       results.sort((a, b) => b.wastedMs - a.wastedMs);
 
       const headings = [
         {key: 'url', itemType: 'url', text: 'URL'},
-        {key: 'wastedMs', itemType: 'text', text: 'Potential Savings'},
+        {key: 'wastedMs', itemType: 'ms', text: 'Potential Savings', granularity: 10},
       ];
-      const summary = {wastedMs: maxWasted};
+      const summary = {wastedMs};
       const details = Audit.makeTableDetails(headings, results, summary);
 
       return {
-        score: UnusedBytes.scoreForWastedMs(maxWasted),
-        rawValue: maxWasted,
-        displayValue: Util.formatMilliseconds(maxWasted),
+        score: UnusedBytes.scoreForWastedMs(wastedMs),
+        rawValue: wastedMs,
+        displayValue: Util.formatMilliseconds(wastedMs),
         extendedInfo: {
           value: results,
         },

lighthouse-core/lib/dependency-graph/node.js

@@ -110,6 +110,31 @@ class Node {
     this._dependencies.push(node);
   }
 
+  /**
+   * @param {Node} node
+   */
+  removeDependent(node) {
+    node.removeDependency(this);
+  }
+
+  /**
+   * @param {Node} node
+   */
+  removeDependency(node) {
+    if (!this._dependencies.includes(node)) {
+      return;
+    }
+
+    node._dependents.splice(node._dependents.indexOf(this), 1);
+    this._dependencies.splice(this._dependencies.indexOf(node), 1);
+  }
+
+  removeAllDependencies() {
+    for (const node of this._dependencies.slice()) {
+      this.removeDependency(node);
+    }
+  }
+
   /**
    * Clones the node's information without adding any dependencies/dependents.
    * @return {Node}

lighthouse-core/lib/dependency-graph/simulator/connection-pool.js

@@ -82,15 +82,27 @@ module.exports = class ConnectionPool {
         throw new Error(`Could not find a connection for origin: ${origin}`);
       }
 
+      // Make sure each origin has 6 connections available
+      // https://cs.chromium.org/chromium/src/net/socket/client_socket_pool_manager.cc?type=cs&q="int+g_max_sockets_per_group"
+      while (connections.length < 6) connections.push(connections[0].clone());
+
       this._connectionsByOrigin.set(origin, connections);
     }
   }
 
   /**
+   * This method finds an available connection to the origin specified by the network record or null
+   * if no connection was available. If returned, connection will not be available for other network
+   * records until release is called.
+   *
+   * If ignoreConnectionReused is true, acquire will consider all connections not in use as available.
+   * Otherwise, only connections that have matching "warmth" are considered available.
+   *
    * @param {LH.WebInspector.NetworkRequest} record
+   * @param {{ignoreConnectionReused?: boolean}} options
    * @return {?TcpConnection}
    */
-  acquire(record) {
+  acquire(record, options = {}) {
     if (this._connectionsByRecord.has(record)) {
       // @ts-ignore
       return this._connectionsByRecord.get(record);
@@ -99,16 +111,26 @@ module.exports = class ConnectionPool {
     const origin = String(record.parsedURL.securityOrigin());
     /** @type {TcpConnection[]} */
     const connections = this._connectionsByOrigin.get(origin) || [];
-    const wasConnectionWarm = !!this._connectionReusedByRequestId.get(record.requestId);
-    const connection = connections.find(connection => {
-      const meetsWarmRequirement = wasConnectionWarm === connection.isWarm();
-      return meetsWarmRequirement && !this._connectionsInUse.has(connection);
-    });
+    // Sort connections by decreasing congestion window, i.e. warmest to coldest
+    const availableConnections = connections
+      .filter(connection => !this._connectionsInUse.has(connection))
+      .sort((a, b) => b.congestionWindow - a.congestionWindow);
+
+    const observedConnectionWasReused = !!this._connectionReusedByRequestId.get(record.requestId);
+
+    /** @type {TcpConnection|undefined} */
+    let connectionToUse = availableConnections[0];
+    if (!options.ignoreConnectionReused) {
+      connectionToUse = availableConnections.find(
+        connection => connection.isWarm() === observedConnectionWasReused
+      );
+    }
+
+    if (!connectionToUse) return null;
 
-    if (!connection) return null;
-    this._connectionsInUse.add(connection);
-    this._connectionsByRecord.set(record, connection);
-    return connection;
+    this._connectionsInUse.add(connectionToUse);
+    this._connectionsByRecord.set(record, connectionToUse);
+    return connectionToUse;
   }
 
   /**

lighthouse-core/lib/dependency-graph/simulator/simulator.js

@@ -52,6 +52,7 @@ class Simulator {
     this._layoutTaskMultiplier = this._cpuSlowdownMultiplier * this._options.layoutTaskMultiplier;
 
     // Properties reset on every `.simulate` call but duplicated here for type checking
+    this._flexibleOrdering = false;
     this._nodeTimings = new Map();
     this._numberInProgressByType = new Map();
     this._nodes = {};
@@ -147,6 +148,16 @@ class Simulator {
     }
   }
 
+  /**
+   * @param {LH.WebInspector.NetworkRequest} record
+   * @return {?TcpConnection}
+   */
+  _acquireConnection(record) {
+    return this._connectionPool.acquire(record, {
+      ignoreConnectionReused: this._flexibleOrdering,
+    });
+  }
+
   /**
    * @param {Node} node
    * @param {number} totalElapsedTime
@@ -167,7 +178,7 @@ class Simulator {
     // Start a network request if we're not at max requests and a connection is available
     const numberOfActiveRequests = this._numberInProgress(node.type);
     if (numberOfActiveRequests >= this._maximumConcurrentRequests) return;
-    const connection = this._connectionPool.acquire(/** @type {NetworkNode} */ (node).record);
+    const connection = this._acquireConnection(/** @type {NetworkNode} */ (node).record);
     if (!connection) return;
 
     this._markNodeAsInProgress(node, totalElapsedTime);
@@ -212,7 +223,7 @@ class Simulator {
 
     const record = /** @type {NetworkNode} */ (node).record;
     const timingData = this._nodeTimings.get(node);
-    const connection = /** @type {TcpConnection} */ (this._connectionPool.acquire(record));
+    const connection = /** @type {TcpConnection} */ (this._acquireConnection(record));
     const calculation = connection.simulateDownloadUntil(
       record.transferSize - timingData.bytesDownloaded,
       {timeAlreadyElapsed: timingData.timeElapsed, maximumTimeToElapse: Infinity}
@@ -255,7 +266,7 @@ class Simulator {
     if (node.type !== Node.TYPES.NETWORK) throw new Error('Unsupported');
 
     const record = /** @type {NetworkNode} */ (node).record;
-    const connection = /** @type {TcpConnection} */ (this._connectionPool.acquire(record));
+    const connection = /** @type {TcpConnection} */ (this._acquireConnection(record));
     const calculation = connection.simulateDownloadUntil(
       record.transferSize - timingData.bytesDownloaded,
       {
@@ -281,10 +292,13 @@ class Simulator {
   /**
    * Estimates the time taken to process all of the graph's nodes.
    * @param {Node} graph
+   * @param {{flexibleOrdering?: boolean}=} options
    * @return {LH.Gatherer.Simulation.Result}
    */
-  simulate(graph) {
+  simulate(graph, options) {
+    options = Object.assign({flexibleOrdering: false}, options);
     // initialize the necessary data containers
+    this._flexibleOrdering = options.flexibleOrdering;
     this._initializeConnectionPool(graph);
     this._initializeAuxiliaryData();
 
@@ -308,6 +322,10 @@ class Simulator {
         this._startNodeIfPossible(node, totalElapsedTime);
       }
 
+      if (!nodesInProgress.size) {
+        throw new Error('Failed to start a node, potential mismatch in original execution');
+      }
+
       // set the available throughput for all connections based on # inflight
       this._updateNetworkCapacity();
 

lighthouse-core/lib/dependency-graph/simulator/tcp-connection.js

@@ -78,6 +78,20 @@ class TcpConnection {
     return this._warmed;
   }
 
+  /**
+   * @return {boolean}
+   */
+  isH2() {
+    return this._h2;
+  }
+
+  /**
+   * @return {number}
+   */
+  get congestionWindow() {
+    return this._congestionWindow;
+  }
+
   /**
    * Sets the number of excess bytes that are available to this connection on future downloads, only
    * applies to H2 connections.
@@ -88,6 +102,14 @@ class TcpConnection {
     this._h2OverflowBytesDownloaded = bytes;
   }
 
+  /**
+   * @return {TcpConnection}
+   */
+  clone() {
+    // @ts-ignore
+    return Object.assign(new TcpConnection(), this);
+  }
+
   /**
    * Simulates a network download of a particular number of bytes over an optional maximum amount of time
    * and returns information about the ending state.