core(lantern): create network graph from trace (experimental) (#16026)

.github/workflows/smoke.yml

@@ -55,6 +55,14 @@ jobs:
         yarn c8 yarn smoke --debug -j=2 --retries=2 --shard=${{ matrix.smoke-test-shard }}/$SHARD_TOTAL
         yarn c8 report --reporter text-lcov > smoke-coverage.lcov
 
+    # TODO(15841): remove when using trace by default, and we want to remove CDT graph impl.
+    - name: Run smoke tests (lantern + trace)
+      if: matrix.smoke-test-shard == 1 && matrix.chrome-channel == 'ToT'
+      run: |
+        yarn smoke --debug -j=2 --retries=2 lantern
+      env:
+        INTERNAL_LANTERN_USE_TRACE: 1
+
     - name: Upload test coverage to Codecov
       if: matrix.chrome-channel == 'ToT'
       uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d

.github/workflows/unit.yml

@@ -62,6 +62,9 @@ jobs:
     - name: yarn unit
       if: ${{ matrix.node != env.LATEST_NODE }}
       run: yarn unit:ci
+    - name: yarn unit-lantern-trace
+      if: ${{ matrix.node != env.LATEST_NODE }}
+      run: yarn unit-lantern-trace:ci
 
     # Only gather coverage on latest node, where c8 is the most accurate.
     - name: yarn unit:cicoverage

core/audits/byte-efficiency/render-blocking-resources.js

@@ -53,7 +53,7 @@ function getNodesAndTimingByRequestId(nodeTimings) {
   for (const [node, nodeTiming] of nodeTimings) {
     if (node.type !== 'network') continue;
 
-    requestIdToNode.set(node.record.requestId, {node, nodeTiming});
+    requestIdToNode.set(node.request.requestId, {node, nodeTiming});
   }
 
   return requestIdToNode;

core/audits/dobetterweb/uses-http2.js

@@ -89,7 +89,7 @@ class UsesHTTP2Audit extends Audit {
       if (node.type !== 'network') return;
       if (!urlsToChange.has(node.record.url)) return;
 
-      originalProtocols.set(node.record.requestId, node.record.protocol);
+      originalProtocols.set(node.request.requestId, node.record.protocol);
       node.request.protocol = 'h2';
     });
 
@@ -98,7 +98,7 @@ class UsesHTTP2Audit extends Audit {
     // Restore the original protocol after we've done our simulation
     graph.traverse(node => {
       if (node.type !== 'network') return;
-      const originalProtocol = originalProtocols.get(node.record.requestId);
+      const originalProtocol = originalProtocols.get(node.request.requestId);
       if (originalProtocol === undefined) return;
       node.request.protocol = originalProtocol;
     });

core/audits/prioritize-lcp-image.js

@@ -69,7 +69,7 @@ class PrioritizeLcpImage extends Audit {
   static findLCPNode(graph, lcpRecord) {
     for (const {node} of graph.traverseGenerator()) {
       if (node.type !== 'network') continue;
-      if (node.record.requestId === lcpRecord.requestId) {
+      if (node.request.requestId === lcpRecord.requestId) {
         return node;
       }
     }

core/audits/redirects.js

@@ -98,7 +98,7 @@ class Redirects extends Audit {
     const nodeTimingsById = new Map();
     for (const [node, timing] of metricResult.pessimisticEstimate.nodeTimings.entries()) {
       if (node.type === 'network') {
-        nodeTimingsById.set(node.record.requestId, timing);
+        nodeTimingsById.set(node.request.requestId, timing);
       }
     }
 

core/computed/metrics/lantern-metric.js

@@ -5,16 +5,20 @@
  */
 
 import {LanternError} from '../../lib/lantern/lantern-error.js';
+import {PageDependencyGraph as LanternPageDependencyGraph} from '../../lib/lantern/page-dependency-graph.js';
 import {LighthouseError} from '../../lib/lh-error.js';
 import {LoadSimulator} from '../load-simulator.js';
-import {PageDependencyGraph} from '../page-dependency-graph.js';
 import {ProcessedNavigation} from '../processed-navigation.js';
+import {ProcessedTrace} from '../processed-trace.js';
+import {TraceEngineResult} from '../trace-engine-result.js';
+import {PageDependencyGraph} from '../page-dependency-graph.js';
 
+// TODO: we need to update all test traces (that use lantern) before this can be removed
 /**
  * @param {LH.Artifacts.MetricComputationDataInput} data
  * @param {LH.Artifacts.ComputedContext} context
  */
-async function getComputationDataParams(data, context) {
+async function getComputationDataParamsFromDevtoolsLog(data, context) {
   if (data.gatherContext.gatherMode !== 'navigation') {
     throw new Error(`Lantern metrics can only be computed on navigations`);
   }
@@ -26,6 +30,35 @@ async function getComputationDataParams(data, context) {
   return {simulator, graph, processedNavigation};
 }
 
+/**
+ * @param {LH.Artifacts.URL} theURL
+ * @param {LH.Trace} trace
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+async function createGraphFromTrace(theURL, trace, context) {
+  const {mainThreadEvents} = await ProcessedTrace.request(trace, context);
+  const traceEngineResult = await TraceEngineResult.request({trace}, context);
+  return LanternPageDependencyGraph.createGraphFromTrace(
+    mainThreadEvents, trace, traceEngineResult, theURL);
+}
+
+/**
+ * @param {LH.Artifacts.MetricComputationDataInput} data
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+async function getComputationDataParamsFromTrace(data, context) {
+  if (data.gatherContext.gatherMode !== 'navigation') {
+    throw new Error(`Lantern metrics can only be computed on navigations`);
+  }
+
+  const {trace, URL} = data;
+  const {graph} = await createGraphFromTrace(URL, trace, context);
+  const processedNavigation = await ProcessedNavigation.request(data.trace, context);
+  const simulator = data.simulator || (await LoadSimulator.request(data, context));
+
+  return {simulator, graph, processedNavigation};
+}
+
 /**
  * @param {unknown} err
  * @return {never}
@@ -43,7 +76,23 @@ function lanternErrorAdapter(err) {
   throw err;
 }
 
+/**
+ * @param {LH.Artifacts.MetricComputationDataInput} data
+ * @param {LH.Artifacts.ComputedContext} context
+ */
+function getComputationDataParams(data, context) {
+  // TODO(15841): remove when all tests use the trace, and we want to remove CDT graph impl.
+  if (process.env.INTERNAL_LANTERN_USE_TRACE !== undefined) {
+    return getComputationDataParamsFromTrace(data, context);
+  } else {
+    // This is the default behavior.
+    return getComputationDataParamsFromDevtoolsLog(data, context);
+  }
+}
+
 export {
+  getComputationDataParamsFromTrace,
+  getComputationDataParamsFromDevtoolsLog,
   getComputationDataParams,
   lanternErrorAdapter,
 };

core/computed/trace-engine-result.js

@@ -21,6 +21,7 @@ const ENABLED_HANDLERS = {
   Samples: TraceEngine.TraceHandlers.Samples,
   Screenshots: TraceEngine.TraceHandlers.Screenshots,
   PageLoadMetrics: TraceEngine.TraceHandlers.PageLoadMetrics,
+  Workers: TraceEngine.TraceHandlers.Workers,
 };
 
 /**

core/lib/lantern/metric.js

@@ -6,7 +6,7 @@
 
 import * as Lantern from './types/lantern.js';
 import {BaseNode} from '../../lib/lantern/base-node.js';
-import {NetworkRequest} from '../../lib/network-request.js';
+import {RESOURCE_TYPES} from '../../lib/network-request.js';
 
 /** @typedef {import('./base-node.js').Node} Node */
 /** @typedef {import('./network-node.js').NetworkNode} NetworkNode */
@@ -33,9 +33,9 @@ class Metric {
 
     dependencyGraph.traverse(node => {
       if (node.type !== BaseNode.TYPES.NETWORK) return;
-      if (node.record.resourceType !== NetworkRequest.TYPES.Script) return;
+      if (node.request.resourceType !== RESOURCE_TYPES.Script) return;
       if (treatNodeAsRenderBlocking?.(node)) {
-        scriptUrls.add(node.record.url);
+        scriptUrls.add(node.request.url);
       }
     });
 

core/lib/lantern/metrics/first-contentful-paint.js

@@ -153,7 +153,7 @@ class FirstContentfulPaint extends Metric {
         const endedAfterPaint = node.endTime > cutoffTimestamp || node.startTime > cutoffTimestamp;
         if (endedAfterPaint && !node.isMainDocument()) return false;
 
-        const url = node.record.url;
+        const url = node.request.url;
         // If the URL definitely wasn't render-blocking then we filter it out.
         if (definitelyNotRenderBlockingScriptUrls.has(url)) {
           return false;

core/lib/lantern/metrics/interactive.js

@@ -41,13 +41,13 @@ class Interactive extends Metric {
       }
 
       // Include all scripts and high priority requests, exclude all images
-      const isImage = node.record.resourceType === NetworkRequestTypes.Image;
-      const isScript = node.record.resourceType === NetworkRequestTypes.Script;
+      const isImage = node.request.resourceType === NetworkRequestTypes.Image;
+      const isScript = node.request.resourceType === NetworkRequestTypes.Script;
       return (
         !isImage &&
         (isScript ||
-          node.record.priority === 'High' ||
-          node.record.priority === 'VeryHigh')
+          node.request.priority === 'High' ||
+          node.request.priority === 'VeryHigh')
       );
     });
   }

core/lib/lantern/metrics/largest-contentful-paint.js

@@ -32,8 +32,8 @@ class LargestContentfulPaint extends Metric {
    */
   static isNotLowPriorityImageNode(node) {
     if (node.type !== 'network') return true;
-    const isImage = node.record.resourceType === 'Image';
-    const isLowPriority = node.record.priority === 'Low' || node.record.priority === 'VeryLow';
+    const isImage = node.request.resourceType === 'Image';
+    const isLowPriority = node.request.priority === 'Low' || node.request.priority === 'VeryLow';
     return !isImage || !isLowPriority;
   }