Skip to content

Commit

Permalink
SI
Browse files Browse the repository at this point in the history
  • Loading branch information
connorjclark committed Mar 25, 2024
1 parent 163e0f9 commit f0db7c3
Show file tree
Hide file tree
Showing 9 changed files with 193 additions and 146 deletions.
128 changes: 12 additions & 116 deletions core/computed/metrics/lantern-speed-index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,89 +5,22 @@
*/

import {makeComputedArtifact} from '../computed-artifact.js';
import {LanternMetric} from './lantern-metric.js';
import {BaseNode} from '../../lib/lantern/base-node.js';
import {getComputationDataParams} from './lantern-metric.js';
import {Speedline} from '../speedline.js';
import {LanternFirstContentfulPaint} from './lantern-first-contentful-paint.js';
import {throttling as defaultThrottling} from '../../config/constants.js';
import {SpeedIndex} from '../../lib/lantern/metrics/speed-index.js';

/** @typedef {import('../../lib/lantern/base-node.js').Node<LH.Artifacts.NetworkRequest>} Node */
/** @typedef {import('../../lib/lantern/metric.js').Extras} Extras */

class LanternSpeedIndex extends LanternMetric {
class LanternSpeedIndex extends SpeedIndex {
/**
* @return {LH.Gatherer.Simulation.MetricCoefficients}
*/
static get COEFFICIENTS() {
return {
// Negative intercept is OK because estimate is Math.max(FCP, Speed Index) and
// the optimistic estimate is based on the real observed speed index rather than a real
// lantern graph.
intercept: -250,
optimistic: 1.4,
pessimistic: 0.65,
};
}

/**
* @param {number} rttMs
* @return {LH.Gatherer.Simulation.MetricCoefficients}
*/
static getScaledCoefficients(rttMs) { // eslint-disable-line no-unused-vars
// We want to scale our default coefficients based on the speed of the connection.
// We will linearly interpolate coefficients for the passed-in rttMs based on two pre-determined points:
// 1. Baseline point of 30 ms RTT where Speed Index should be a ~50/50 blend of optimistic/pessimistic.
// 30 ms was based on a typical home WiFi connection's actual RTT.
// Coefficients here follow from the fact that the optimistic estimate should be very close
// to reality at this connection speed and the pessimistic estimate compensates for minor
// connection speed differences.
// 2. Default throttled point of 150 ms RTT where the default coefficients have been determined to be most accurate.
// Coefficients here were determined through thorough analysis and linear regression on the
// lantern test data set. See core/scripts/test-lantern.sh for more detail.
// While the coefficients haven't been analyzed at the interpolated points, it's our current best effort.
const defaultCoefficients = this.COEFFICIENTS;
const defaultRttExcess = defaultThrottling.mobileSlow4G.rttMs - 30;
const multiplier = Math.max((rttMs - 30) / defaultRttExcess, 0);

return {
intercept: defaultCoefficients.intercept * multiplier,
optimistic: 0.5 + (defaultCoefficients.optimistic - 0.5) * multiplier,
pessimistic: 0.5 + (defaultCoefficients.pessimistic - 0.5) * multiplier,
};
}

/**
* @param {Node} dependencyGraph
* @return {Node}
*/
static getOptimisticGraph(dependencyGraph) {
return dependencyGraph;
}

/**
* @param {Node} dependencyGraph
* @return {Node}
*/
static getPessimisticGraph(dependencyGraph) {
return dependencyGraph;
}

/**
* @param {LH.Gatherer.Simulation.Result} simulationResult
* @param {import('../../lib/lantern/metric.js').Extras} extras
* @return {LH.Gatherer.Simulation.Result}
* @param {LH.Artifacts.MetricComputationDataInput} data
* @param {LH.Artifacts.ComputedContext} context
* @param {Omit<Extras, 'optimistic'>=} extras
* @return {Promise<LH.Artifacts.LanternMetric>}
*/
static getEstimateFromSimulation(simulationResult, extras) {
if (!extras.fcpResult) throw new Error('missing fcpResult');
if (!extras.speedline) throw new Error('missing speedline');

const fcpTimeInMs = extras.fcpResult.pessimisticEstimate.timeInMs;
const estimate = extras.optimistic
? extras.speedline.speedIndex
: LanternSpeedIndex.computeLayoutBasedSpeedIndex(simulationResult.nodeTimings, fcpTimeInMs);
return {
timeInMs: estimate,
nodeTimings: simulationResult.nodeTimings,
};
static async computeMetricWithGraphs(data, context, extras) {
return this.compute(await getComputationDataParams(data, context), extras);
}

/**
Expand All @@ -96,50 +29,13 @@ class LanternSpeedIndex extends LanternMetric {
* @return {Promise<LH.Artifacts.LanternMetric>}
*/
static async compute_(data, context) {
// TODO(15841): lib/lantern should probably run Speedline
const speedline = await Speedline.request(data.trace, context);
const fcpResult = await LanternFirstContentfulPaint.request(data, context);
const metricResult = await this.computeMetricWithGraphs(data, context, {
return this.computeMetricWithGraphs(data, context, {
speedline,
fcpResult,
});
metricResult.timing = Math.max(metricResult.timing, fcpResult.timing);
return metricResult;
}

/**
* Approximate speed index using layout events from the simulated node timings.
* The layout-based speed index is the weighted average of the endTime of CPU nodes that contained
* a 'Layout' task. log(duration) is used as the weight to stand for "significance" to the page.
*
* If no layout events can be found or the endTime of a CPU task is too early, FCP is used instead.
*
* This approach was determined after evaluating the accuracy/complexity tradeoff of many
* different methods. Read more in the evaluation doc.
*
* @see https://docs.google.com/document/d/1qJWXwxoyVLVadezIp_Tgdk867G3tDNkkVRvUJSH3K1E/edit#
* @param {LH.Gatherer.Simulation.Result['nodeTimings']} nodeTimings
* @param {number} fcpTimeInMs
* @return {number}
*/
static computeLayoutBasedSpeedIndex(nodeTimings, fcpTimeInMs) {
/** @type {Array<{time: number, weight: number}>} */
const layoutWeights = [];
for (const [node, timing] of nodeTimings.entries()) {
if (node.type !== BaseNode.TYPES.CPU) continue;

if (node.childEvents.some(x => x.name === 'Layout')) {
const timingWeight = Math.max(Math.log2(timing.endTime - timing.startTime), 0);
layoutWeights.push({time: timing.endTime, weight: timingWeight});
}
}

const totalWeightedTime = layoutWeights
.map(evt => evt.weight * Math.max(evt.time, fcpTimeInMs))
.reduce((a, b) => a + b, 0);
const totalWeight = layoutWeights.map(evt => evt.weight).reduce((a, b) => a + b, 0);

if (!totalWeight) return fcpTimeInMs;
return totalWeightedTime / totalWeight;
}
}

Expand Down
2 changes: 1 addition & 1 deletion core/lib/lantern/metrics/first-meaningful-paint.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {Metric} from '../metric.js';
import {LighthouseError} from '../../lh-error.js';
import {FirstContentfulPaint} from './first-contentful-paint.js';

/** @typedef {import('../base-node.js').Node<LH.Artifacts.NetworkRequest>} Node */
/** @typedef {import('../base-node.js').Node} Node */

class FirstMeaningfulPaint extends Metric {
/**
Expand Down
2 changes: 1 addition & 1 deletion core/lib/lantern/metrics/interactive.js
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class Interactive extends Metric {
throw new Error('FMP is required to calculate the Interactive metric');
}

const metricResult = await super.compute(data, {fmpResult});
const metricResult = await super.compute(data, extras);
metricResult.timing = Math.max(metricResult.timing, fmpResult.timing);
return metricResult;
}
Expand Down
145 changes: 145 additions & 0 deletions core/lib/lantern/metrics/speed-index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/**
* @license
* Copyright 2024 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/

import * as Lantern from '../types/lantern.js';
import {Metric} from '../metric.js';
import {BaseNode} from '../base-node.js';
// TODO(15841): move this default config value into lib/lantern
import {throttling as defaultThrottling} from '../../../config/constants.js';

/** @typedef {import('../base-node.js').Node} Node */

class SpeedIndex extends Metric {
/**
* @return {LH.Gatherer.Simulation.MetricCoefficients}
*/
static get COEFFICIENTS() {
return {
// Negative intercept is OK because estimate is Math.max(FCP, Speed Index) and
// the optimistic estimate is based on the real observed speed index rather than a real
// lantern graph.
intercept: -250,
optimistic: 1.4,
pessimistic: 0.65,
};
}

/**
* @param {number} rttMs
* @return {LH.Gatherer.Simulation.MetricCoefficients}
*/
static getScaledCoefficients(rttMs) { // eslint-disable-line no-unused-vars
// We want to scale our default coefficients based on the speed of the connection.
// We will linearly interpolate coefficients for the passed-in rttMs based on two pre-determined points:
// 1. Baseline point of 30 ms RTT where Speed Index should be a ~50/50 blend of optimistic/pessimistic.
// 30 ms was based on a typical home WiFi connection's actual RTT.
// Coefficients here follow from the fact that the optimistic estimate should be very close
// to reality at this connection speed and the pessimistic estimate compensates for minor
// connection speed differences.
// 2. Default throttled point of 150 ms RTT where the default coefficients have been determined to be most accurate.
// Coefficients here were determined through thorough analysis and linear regression on the
// lantern test data set. See core/scripts/test-lantern.sh for more detail.
// While the coefficients haven't been analyzed at the interpolated points, it's our current best effort.
const defaultCoefficients = this.COEFFICIENTS;
const defaultRttExcess = defaultThrottling.mobileSlow4G.rttMs - 30;
const multiplier = Math.max((rttMs - 30) / defaultRttExcess, 0);

return {
intercept: defaultCoefficients.intercept * multiplier,
optimistic: 0.5 + (defaultCoefficients.optimistic - 0.5) * multiplier,
pessimistic: 0.5 + (defaultCoefficients.pessimistic - 0.5) * multiplier,
};
}

/**
* @param {Node} dependencyGraph
* @return {Node}
*/
static getOptimisticGraph(dependencyGraph) {
return dependencyGraph;
}

/**
* @param {Node} dependencyGraph
* @return {Node}
*/
static getPessimisticGraph(dependencyGraph) {
return dependencyGraph;
}

/**
* @param {LH.Gatherer.Simulation.Result} simulationResult
* @param {import('../metric.js').Extras} extras
* @return {LH.Gatherer.Simulation.Result}
*/
static getEstimateFromSimulation(simulationResult, extras) {
if (!extras.fcpResult) throw new Error('missing fcpResult');
if (!extras.speedline) throw new Error('missing speedline');

const fcpTimeInMs = extras.fcpResult.pessimisticEstimate.timeInMs;
const estimate = extras.optimistic
? extras.speedline.speedIndex
: SpeedIndex.computeLayoutBasedSpeedIndex(simulationResult.nodeTimings, fcpTimeInMs);
return {
timeInMs: estimate,
nodeTimings: simulationResult.nodeTimings,
};
}

/**
* @param {Lantern.Simulation.MetricComputationDataInput} data
* @param {Omit<import('../metric.js').Extras, 'optimistic'>=} extras
* @return {Promise<LH.Artifacts.LanternMetric>}
*/
static async compute(data, extras) {
const fcpResult = extras?.fcpResult;
if (!fcpResult) {
throw new Error('FCP is required to calculate the SpeedIndex metric');
}

const metricResult = await super.compute(data, extras);
metricResult.timing = Math.max(metricResult.timing, fcpResult.timing);
return metricResult;
}

/**
* Approximate speed index using layout events from the simulated node timings.
* The layout-based speed index is the weighted average of the endTime of CPU nodes that contained
* a 'Layout' task. log(duration) is used as the weight to stand for "significance" to the page.
*
* If no layout events can be found or the endTime of a CPU task is too early, FCP is used instead.
*
* This approach was determined after evaluating the accuracy/complexity tradeoff of many
* different methods. Read more in the evaluation doc.
*
* @see https://docs.google.com/document/d/1qJWXwxoyVLVadezIp_Tgdk867G3tDNkkVRvUJSH3K1E/edit#
* @param {LH.Gatherer.Simulation.Result['nodeTimings']} nodeTimings
* @param {number} fcpTimeInMs
* @return {number}
*/
static computeLayoutBasedSpeedIndex(nodeTimings, fcpTimeInMs) {
/** @type {Array<{time: number, weight: number}>} */
const layoutWeights = [];
for (const [node, timing] of nodeTimings.entries()) {
if (node.type !== BaseNode.TYPES.CPU) continue;

if (node.childEvents.some(x => x.name === 'Layout')) {
const timingWeight = Math.max(Math.log2(timing.endTime - timing.startTime), 0);
layoutWeights.push({time: timing.endTime, weight: timingWeight});
}
}

const totalWeightedTime = layoutWeights
.map(evt => evt.weight * Math.max(evt.time, fcpTimeInMs))
.reduce((a, b) => a + b, 0);
const totalWeight = layoutWeights.map(evt => evt.weight).reduce((a, b) => a + b, 0);

if (!totalWeight) return fcpTimeInMs;
return totalWeightedTime / totalWeight;
}
}

export {SpeedIndex};
4 changes: 2 additions & 2 deletions core/test/lib/lantern/metrics/first-contentful-paint-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const devtoolsLog = readJson('../../../fixtures/traces/progressive-app-m60.devto

describe('Metrics: Lantern FCP', () => {
it('should compute predicted value', async () => {
const data = await getComputationDataFromFixture(trace, devtoolsLog);
const data = await getComputationDataFromFixture({trace, devtoolsLog});
const result = await FirstContentfulPaint.compute(data);

expect({
Expand Down Expand Up @@ -58,7 +58,7 @@ describe('Metrics: Lantern FCP', () => {
mainDocumentUrl: 'https://example.com/',
finalDisplayedUrl: 'https://example.com/',
};
const data = await getComputationDataFromFixture(trace, devtoolsLog, URL);
const data = await getComputationDataFromFixture({trace, devtoolsLog, URL});
const result = await FirstContentfulPaint.compute(data);

const optimisticNodes = [];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const devtoolsLog = readJson('../../../fixtures/traces/progressive-app-m60.devto

describe('Metrics: Lantern FMP', () => {
it('should compute predicted value', async () => {
const data = await getComputationDataFromFixture(trace, devtoolsLog);
const data = await getComputationDataFromFixture({trace, devtoolsLog});
const result = await FirstMeaningfulPaint.compute(data);

expect({
Expand Down
7 changes: 5 additions & 2 deletions core/test/lib/lantern/metrics/interactive-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ const iframeDevtoolsLog = readJson('../../../fixtures/traces/iframe-m79.devtools

describe('Metrics: Lantern TTI', () => {
it('should compute predicted value', async () => {
const data = await getComputationDataFromFixture(trace, devtoolsLog);
const data = await getComputationDataFromFixture({trace, devtoolsLog});
const result = await Interactive.compute(data, {
fmpResult: await FirstMeaningfulPaint.compute(data),
});
Expand All @@ -35,7 +35,10 @@ describe('Metrics: Lantern TTI', () => {
});

it('should compute predicted value on iframes with substantial layout', async () => {
const data = await getComputationDataFromFixture(iframeTrace, iframeDevtoolsLog);
const data = await getComputationDataFromFixture({
trace: iframeTrace,
devtoolsLog: iframeDevtoolsLog,
});
const result = await Interactive.compute(data, {
fmpResult: await FirstMeaningfulPaint.compute(data),
});
Expand Down
10 changes: 4 additions & 6 deletions core/test/lib/lantern/metrics/metric-test-utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,13 @@ import {getURLArtifactFromDevtoolsLog} from '../../../test-utils.js';
// TODO(15841): remove usage of Lighthouse code to create test data

/**
* @param {LH.Trace} trace
* @param {LH.DevtoolsLog} devtoolsLog
* @param {LH.Artifacts.URL=} URL
* @param {{trace: LH.Trace, devtoolsLog: LH.DevtoolsLog, settings?: LH.Audit.Context['settings'], URL?: LH.Artifacts.URL}} opts
*/
function getComputationDataFromFixture(trace, devtoolsLog, URL = null) {
function getComputationDataFromFixture({trace, devtoolsLog, settings, URL}) {
settings = settings || {};
URL = URL || getURLArtifactFromDevtoolsLog(devtoolsLog);
const gatherContext = {gatherMode: 'navigation'};
const settings = {};
const context = {settings, computedCache: new Map()};
URL = URL || getURLArtifactFromDevtoolsLog(devtoolsLog);
return getComputationDataParams({trace, devtoolsLog, gatherContext, settings, URL}, context);
}

Expand Down
Loading

0 comments on commit f0db7c3

Please sign in to comment.