From 048ae847ae73fc75c0fb812afbdd5fd9a6ca104b Mon Sep 17 00:00:00 2001 From: Kenzie Davisson <43759233+kenzieschmoll@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:48:34 -0700 Subject: [PATCH] Add flutter frame timings to benchmark metrics (#7759) --- packages/web_benchmarks/CHANGELOG.md | 6 ++ packages/web_benchmarks/lib/metrics.dart | 5 ++ packages/web_benchmarks/lib/src/metrics.dart | 68 +++++++++++++++++++ packages/web_benchmarks/lib/src/recorder.dart | 64 +++++++++++------ packages/web_benchmarks/lib/src/runner.dart | 5 +- packages/web_benchmarks/pubspec.yaml | 2 +- .../test/src/analysis_test.dart | 4 -- .../benchmark/web_benchmarks_test.dart | 13 ++-- 8 files changed, 131 insertions(+), 36 deletions(-) create mode 100644 packages/web_benchmarks/lib/metrics.dart create mode 100644 packages/web_benchmarks/lib/src/metrics.dart diff --git a/packages/web_benchmarks/CHANGELOG.md b/packages/web_benchmarks/CHANGELOG.md index e399fd367ee0..7a5b8d5366da 100644 --- a/packages/web_benchmarks/CHANGELOG.md +++ b/packages/web_benchmarks/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.1.0-wip + +* Add `flutter_frame.total_time`, `flutter_frame.build_time`, and `flutter_frame.raster_time` +metrics to benchmark results. These values are derived from the Flutter `FrameTiming` API. +* Expose a new library `metrics.dart` that contains definitions for the benchmark metrics. + ## 3.0.0 * **Breaking change:** removed the `initialPage` parameter from the `serveWebBenchmark` diff --git a/packages/web_benchmarks/lib/metrics.dart b/packages/web_benchmarks/lib/metrics.dart new file mode 100644 index 000000000000..cfc5d9fcc294 --- /dev/null +++ b/packages/web_benchmarks/lib/metrics.dart @@ -0,0 +1,5 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/metrics.dart'; diff --git a/packages/web_benchmarks/lib/src/metrics.dart b/packages/web_benchmarks/lib/src/metrics.dart new file mode 100644 index 000000000000..e80692ee58b2 --- /dev/null +++ b/packages/web_benchmarks/lib/src/metrics.dart @@ -0,0 +1,68 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// The names for the metrics collected by the benchmark recorder. +enum BenchmarkMetric { + /// The name for the benchmark metric that includes frame-related computations + /// prior to submitting layer and picture operations to the underlying + /// renderer, such as HTML and CanvasKit. + /// + /// During this phase we compute transforms, clips, and other information + /// needed for rendering. + prerollFrame('preroll_frame'), + + /// The name for the benchmark metric that includes submitting layer and + /// picture information to the renderer. + applyFrame('apply_frame'), + + /// The name for the benchmark metric that measures the time spent in + /// [PlatformDispatcher]'s onDrawFrame callback. + drawFrame('draw_frame'), + + /// The name for the benchmark metric that tracks the timespan between vsync + /// start and raster finish for a Flutter frame. + /// + /// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine. + flutterFrameTotalTime('flutter_frame.total_time'), + + /// The name for the benchmark metric that tracks the duration to build the + /// Flutter frame on the Dart UI thread. + /// + /// This value corresponds to [FrameTiming.buildDuration] from the Flutter + /// Engine. + flutterFrameBuildTime('flutter_frame.build_time'), + + /// The name for the benchmark metric that tracks the duration to rasterize + /// the Flutter frame on the Dart raster thread. + /// + /// This value corresponds to [FrameTiming.rasterDuration] from the Flutter + /// Engine. + flutterFrameRasterTime('flutter_frame.raster_time'); + + const BenchmarkMetric(this.label); + + /// The metric name used in the recorded benchmark data. + final String label; +} + +/// The name for the benchmark metric that records the 'averageTotalUIFrameTime' +/// from the Blink trace summary. +const String totalUiFrameAverage = 'totalUiFrame.average'; + +/// The list of expected benchmark metrics for the current compilation mode, as +/// determined by the value of [useWasm]. +List expectedBenchmarkMetrics({required bool useWasm}) { + return [ + // The skwasm renderer doesn't have preroll or apply frame steps in its + // rendering. + if (!useWasm) ...[ + BenchmarkMetric.prerollFrame, + BenchmarkMetric.applyFrame, + ], + BenchmarkMetric.drawFrame, + BenchmarkMetric.flutterFrameTotalTime, + BenchmarkMetric.flutterFrameBuildTime, + BenchmarkMetric.flutterFrameRasterTime, + ]; +} diff --git a/packages/web_benchmarks/lib/src/recorder.dart b/packages/web_benchmarks/lib/src/recorder.dart index 85d841f1f8cd..82a490c942a2 100644 --- a/packages/web_benchmarks/lib/src/recorder.dart +++ b/packages/web_benchmarks/lib/src/recorder.dart @@ -18,6 +18,7 @@ import 'package:meta/meta.dart'; import 'package:web/web.dart' as html; import 'common.dart'; +import 'metrics.dart'; /// The number of samples from warm-up iterations. /// @@ -27,16 +28,6 @@ const int _kWarmUpSampleCount = 200; /// The total number of samples collected by a benchmark. const int kTotalSampleCount = _kWarmUpSampleCount + kMeasuredSampleCount; -/// A benchmark metric that includes frame-related computations prior to -/// submitting layer and picture operations to the underlying renderer, such as -/// HTML and CanvasKit. During this phase we compute transforms, clips, and -/// other information needed for rendering. -const String kProfilePrerollFrame = 'preroll_frame'; - -/// A benchmark metric that includes submitting layer and picture information -/// to the renderer. -const String kProfileApplyFrame = 'apply_frame'; - /// Measures the amount of time [action] takes. Duration timeAction(VoidCallback action) { final Stopwatch stopwatch = Stopwatch()..start(); @@ -250,7 +241,7 @@ abstract class SceneBuilderRecorder extends Recorder { PlatformDispatcher.instance.onDrawFrame = () { final FlutterView? view = PlatformDispatcher.instance.implicitView; try { - _profile.record('drawFrameDuration', () { + _profile.record(BenchmarkMetric.drawFrame.label, () { final SceneBuilder sceneBuilder = SceneBuilder(); onDrawFrame(sceneBuilder); _profile.record('sceneBuildDuration', () { @@ -390,8 +381,11 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { @mustCallSuper void frameDidDraw() { endMeasureFrame(); - profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, - reported: true); + profile.addDataPoint( + BenchmarkMetric.drawFrame.label, + _drawFrameStopwatch.elapsed, + reported: true, + ); if (shouldContinue()) { PlatformDispatcher.instance.scheduleFrame(); @@ -417,29 +411,54 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { _RecordingWidgetsBinding.ensureInitialized(); final Widget widget = createWidget(); - registerEngineBenchmarkValueListener(kProfilePrerollFrame, (num value) { + registerEngineBenchmarkValueListener(BenchmarkMetric.prerollFrame.label, + (num value) { localProfile.addDataPoint( - kProfilePrerollFrame, + BenchmarkMetric.prerollFrame.label, Duration(microseconds: value.toInt()), reported: false, ); }); - registerEngineBenchmarkValueListener(kProfileApplyFrame, (num value) { + registerEngineBenchmarkValueListener(BenchmarkMetric.applyFrame.label, + (num value) { localProfile.addDataPoint( - kProfileApplyFrame, + BenchmarkMetric.applyFrame.label, Duration(microseconds: value.toInt()), reported: false, ); }); + late void Function(List frameTimings) frameTimingsCallback; + binding.addTimingsCallback( + frameTimingsCallback = (List frameTimings) { + for (final FrameTiming frameTiming in frameTimings) { + localProfile.addDataPoint( + BenchmarkMetric.flutterFrameTotalTime.label, + frameTiming.totalSpan, + reported: false, + ); + localProfile.addDataPoint( + BenchmarkMetric.flutterFrameBuildTime.label, + frameTiming.buildDuration, + reported: false, + ); + localProfile.addDataPoint( + BenchmarkMetric.flutterFrameRasterTime.label, + frameTiming.rasterDuration, + reported: false, + ); + } + }); + binding._beginRecording(this, widget); try { await _runCompleter.future; return localProfile; } finally { - stopListeningToEngineBenchmarkValues(kProfilePrerollFrame); - stopListeningToEngineBenchmarkValues(kProfileApplyFrame); + stopListeningToEngineBenchmarkValues(BenchmarkMetric.prerollFrame.label); + stopListeningToEngineBenchmarkValues(BenchmarkMetric.applyFrame.label); + binding.removeTimingsCallback(frameTimingsCallback); } } } @@ -508,8 +527,11 @@ abstract class WidgetBuildRecorder extends Recorder implements FrameRecorder { // Only record frames that show the widget. if (showWidget) { endMeasureFrame(); - profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, - reported: true); + profile.addDataPoint( + BenchmarkMetric.drawFrame.label, + _drawFrameStopwatch.elapsed, + reported: true, + ); } if (shouldContinue()) { diff --git a/packages/web_benchmarks/lib/src/runner.dart b/packages/web_benchmarks/lib/src/runner.dart index a78ef20bbc49..4e52a88f61ab 100644 --- a/packages/web_benchmarks/lib/src/runner.dart +++ b/packages/web_benchmarks/lib/src/runner.dart @@ -19,6 +19,7 @@ import 'benchmark_result.dart'; import 'browser.dart'; import 'common.dart'; import 'compilation_options.dart'; +import 'metrics.dart'; /// The default port number used by the local benchmark server. const int defaultBenchmarkServerPort = 9999; @@ -224,11 +225,11 @@ class BenchmarkServer { if (latestPerformanceTrace != null) { final BlinkTraceSummary? traceSummary = BlinkTraceSummary.fromJson(latestPerformanceTrace!); - profile['totalUiFrame.average'] = + profile[totalUiFrameAverage] = traceSummary?.averageTotalUIFrameTime.inMicroseconds; profile['scoreKeys'] ??= []; // using dynamic for consistency with JSON - (profile['scoreKeys'] as List).add('totalUiFrame.average'); + (profile['scoreKeys'] as List).add(totalUiFrameAverage); latestPerformanceTrace = null; } collectedProfiles.add(profile); diff --git a/packages/web_benchmarks/pubspec.yaml b/packages/web_benchmarks/pubspec.yaml index 94964f30230c..54ca82807e41 100644 --- a/packages/web_benchmarks/pubspec.yaml +++ b/packages/web_benchmarks/pubspec.yaml @@ -2,7 +2,7 @@ name: web_benchmarks description: A benchmark harness for performance-testing Flutter apps in Chrome. repository: https://github.com/flutter/packages/tree/main/packages/web_benchmarks issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+web_benchmarks%22 -version: 3.0.0 +version: 3.1.0-wip environment: sdk: ^3.3.0 diff --git a/packages/web_benchmarks/test/src/analysis_test.dart b/packages/web_benchmarks/test/src/analysis_test.dart index 852cb4c64083..c584f58085c8 100644 --- a/packages/web_benchmarks/test/src/analysis_test.dart +++ b/packages/web_benchmarks/test/src/analysis_test.dart @@ -2,10 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - import 'package:flutter_test/flutter_test.dart'; import 'package:web_benchmarks/analysis.dart'; diff --git a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart index 63b33dc8a14e..a6900a126d31 100644 --- a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart +++ b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart @@ -7,6 +7,7 @@ import 'dart:io'; import 'package:test/test.dart'; +import 'package:web_benchmarks/metrics.dart'; import 'package:web_benchmarks/server.dart'; import 'package:web_benchmarks/src/common.dart'; @@ -90,14 +91,10 @@ Future _runBenchmarks({ compilationOptions: compilationOptions, ); - // The skwasm renderer doesn't have preroll or apply frame steps in its rendering. - final List expectedMetrics = compilationOptions.useWasm - ? ['drawFrameDuration'] - : [ - 'preroll_frame', - 'apply_frame', - 'drawFrameDuration', - ]; + final List expectedMetrics = + expectedBenchmarkMetrics(useWasm: compilationOptions.useWasm) + .map((BenchmarkMetric metric) => metric.label) + .toList(); for (final String benchmarkName in benchmarkNames) { for (final String metricName in expectedMetrics) {