Add flutter frame timings to benchmark metrics (#7759)

packages/web_benchmarks/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.1.0-wip
+
+* Add `flutter_frame.total_time`, `flutter_frame.build_time`, and `flutter_frame.raster_time`
+metrics to benchmark results. These values are derived from the Flutter `FrameTiming` API.
+* Expose a new library `metrics.dart` that contains definitions for the benchmark metrics.
+
 ## 3.0.0
 
 * **Breaking change:** removed the `initialPage` parameter from the `serveWebBenchmark`

packages/web_benchmarks/lib/metrics.dart

@@ -0,0 +1,5 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+export 'src/metrics.dart';

packages/web_benchmarks/lib/src/metrics.dart

@@ -0,0 +1,68 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// The names for the metrics collected by the benchmark recorder.
+enum BenchmarkMetric {
+  /// The name for the benchmark metric that includes frame-related computations
+  /// prior to submitting layer and picture operations to the underlying
+  /// renderer, such as HTML and CanvasKit.
+  ///
+  /// During this phase we compute transforms, clips, and other information
+  /// needed for rendering.
+  prerollFrame('preroll_frame'),
+
+  /// The name for the benchmark metric that includes submitting layer and
+  /// picture information to the renderer.
+  applyFrame('apply_frame'),
+
+  /// The name for the benchmark metric that measures the time spent in
+  /// [PlatformDispatcher]'s onDrawFrame callback.
+  drawFrame('draw_frame'),
+
+  /// The name for the benchmark metric that tracks the timespan between vsync
+  /// start and raster finish for a Flutter frame.
+  ///
+  /// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine.
+  flutterFrameTotalTime('flutter_frame.total_time'),
+
+  /// The name for the benchmark metric that tracks the duration to build the
+  /// Flutter frame on the Dart UI thread.
+  ///
+  /// This value corresponds to [FrameTiming.buildDuration] from the Flutter
+  /// Engine.
+  flutterFrameBuildTime('flutter_frame.build_time'),
+
+  /// The name for the benchmark metric that tracks the duration to rasterize
+  /// the Flutter frame on the Dart raster thread.
+  ///
+  /// This value corresponds to [FrameTiming.rasterDuration] from the Flutter
+  /// Engine.
+  flutterFrameRasterTime('flutter_frame.raster_time');
+
+  const BenchmarkMetric(this.label);
+
+  /// The metric name used in the recorded benchmark data.
+  final String label;
+}
+
+/// The name for the benchmark metric that records the 'averageTotalUIFrameTime'
+/// from the Blink trace summary.
+const String totalUiFrameAverage = 'totalUiFrame.average';
+
+/// The list of expected benchmark metrics for the current compilation mode, as
+/// determined by the value of [useWasm].
+List<BenchmarkMetric> expectedBenchmarkMetrics({required bool useWasm}) {
+  return <BenchmarkMetric>[
+    // The skwasm renderer doesn't have preroll or apply frame steps in its
+    // rendering.
+    if (!useWasm) ...<BenchmarkMetric>[
+      BenchmarkMetric.prerollFrame,
+      BenchmarkMetric.applyFrame,
+    ],
+    BenchmarkMetric.drawFrame,
+    BenchmarkMetric.flutterFrameTotalTime,
+    BenchmarkMetric.flutterFrameBuildTime,
+    BenchmarkMetric.flutterFrameRasterTime,
+  ];
+}

packages/web_benchmarks/lib/src/recorder.dart

@@ -18,6 +18,7 @@ import 'package:meta/meta.dart';
 import 'package:web/web.dart' as html;
 
 import 'common.dart';
+import 'metrics.dart';
 
 /// The number of samples from warm-up iterations.
 ///
@@ -27,16 +28,6 @@ const int _kWarmUpSampleCount = 200;
 /// The total number of samples collected by a benchmark.
 const int kTotalSampleCount = _kWarmUpSampleCount + kMeasuredSampleCount;
 
-/// A benchmark metric that includes frame-related computations prior to
-/// submitting layer and picture operations to the underlying renderer, such as
-/// HTML and CanvasKit. During this phase we compute transforms, clips, and
-/// other information needed for rendering.
-const String kProfilePrerollFrame = 'preroll_frame';
-
-/// A benchmark metric that includes submitting layer and picture information
-/// to the renderer.
-const String kProfileApplyFrame = 'apply_frame';
-
 /// Measures the amount of time [action] takes.
 Duration timeAction(VoidCallback action) {
   final Stopwatch stopwatch = Stopwatch()..start();
@@ -250,7 +241,7 @@ abstract class SceneBuilderRecorder extends Recorder {
     PlatformDispatcher.instance.onDrawFrame = () {
       final FlutterView? view = PlatformDispatcher.instance.implicitView;
       try {
-        _profile.record('drawFrameDuration', () {
+        _profile.record(BenchmarkMetric.drawFrame.label, () {
           final SceneBuilder sceneBuilder = SceneBuilder();
           onDrawFrame(sceneBuilder);
           _profile.record('sceneBuildDuration', () {
@@ -390,8 +381,11 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder {
   @mustCallSuper
   void frameDidDraw() {
     endMeasureFrame();
-    profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed,
-        reported: true);
+    profile.addDataPoint(
+      BenchmarkMetric.drawFrame.label,
+      _drawFrameStopwatch.elapsed,
+      reported: true,
+    );
 
     if (shouldContinue()) {
       PlatformDispatcher.instance.scheduleFrame();
@@ -417,29 +411,54 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder {
         _RecordingWidgetsBinding.ensureInitialized();
     final Widget widget = createWidget();
 
-    registerEngineBenchmarkValueListener(kProfilePrerollFrame, (num value) {
+    registerEngineBenchmarkValueListener(BenchmarkMetric.prerollFrame.label,
+        (num value) {
       localProfile.addDataPoint(
-        kProfilePrerollFrame,
+        BenchmarkMetric.prerollFrame.label,
         Duration(microseconds: value.toInt()),
         reported: false,
       );
     });
-    registerEngineBenchmarkValueListener(kProfileApplyFrame, (num value) {
+    registerEngineBenchmarkValueListener(BenchmarkMetric.applyFrame.label,
+        (num value) {
       localProfile.addDataPoint(
-        kProfileApplyFrame,
+        BenchmarkMetric.applyFrame.label,
         Duration(microseconds: value.toInt()),
         reported: false,
       );
     });
 
+    late void Function(List<FrameTiming> frameTimings) frameTimingsCallback;
+    binding.addTimingsCallback(
+        frameTimingsCallback = (List<FrameTiming> frameTimings) {
+      for (final FrameTiming frameTiming in frameTimings) {
+        localProfile.addDataPoint(
+          BenchmarkMetric.flutterFrameTotalTime.label,
+          frameTiming.totalSpan,
+          reported: false,
+        );
+        localProfile.addDataPoint(
+          BenchmarkMetric.flutterFrameBuildTime.label,
+          frameTiming.buildDuration,
+          reported: false,
+        );
+        localProfile.addDataPoint(
+          BenchmarkMetric.flutterFrameRasterTime.label,
+          frameTiming.rasterDuration,
+          reported: false,
+        );
+      }
+    });
+
     binding._beginRecording(this, widget);
 
     try {
       await _runCompleter.future;
       return localProfile;
     } finally {
-      stopListeningToEngineBenchmarkValues(kProfilePrerollFrame);
-      stopListeningToEngineBenchmarkValues(kProfileApplyFrame);
+      stopListeningToEngineBenchmarkValues(BenchmarkMetric.prerollFrame.label);
+      stopListeningToEngineBenchmarkValues(BenchmarkMetric.applyFrame.label);
+      binding.removeTimingsCallback(frameTimingsCallback);
     }
   }
 }
@@ -508,8 +527,11 @@ abstract class WidgetBuildRecorder extends Recorder implements FrameRecorder {
     // Only record frames that show the widget.
     if (showWidget) {
       endMeasureFrame();
-      profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed,
-          reported: true);
+      profile.addDataPoint(
+        BenchmarkMetric.drawFrame.label,
+        _drawFrameStopwatch.elapsed,
+        reported: true,
+      );
     }
 
     if (shouldContinue()) {

packages/web_benchmarks/lib/src/runner.dart

@@ -19,6 +19,7 @@ import 'benchmark_result.dart';
 import 'browser.dart';
 import 'common.dart';
 import 'compilation_options.dart';
+import 'metrics.dart';
 
 /// The default port number used by the local benchmark server.
 const int defaultBenchmarkServerPort = 9999;
@@ -224,11 +225,11 @@ class BenchmarkServer {
           if (latestPerformanceTrace != null) {
             final BlinkTraceSummary? traceSummary =
                 BlinkTraceSummary.fromJson(latestPerformanceTrace!);
-            profile['totalUiFrame.average'] =
+            profile[totalUiFrameAverage] =
                 traceSummary?.averageTotalUIFrameTime.inMicroseconds;
             profile['scoreKeys'] ??=
                 <dynamic>[]; // using dynamic for consistency with JSON
-            (profile['scoreKeys'] as List<dynamic>).add('totalUiFrame.average');
+            (profile['scoreKeys'] as List<dynamic>).add(totalUiFrameAverage);
             latestPerformanceTrace = null;
           }
           collectedProfiles.add(profile);

packages/web_benchmarks/pubspec.yaml

@@ -2,7 +2,7 @@ name: web_benchmarks
 description: A benchmark harness for performance-testing Flutter apps in Chrome.
 repository: https://github.com/flutter/packages/tree/main/packages/web_benchmarks
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+web_benchmarks%22
-version: 3.0.0
+version: 3.1.0-wip
 
 environment:
   sdk: ^3.3.0

packages/web_benchmarks/test/src/analysis_test.dart

@@ -2,10 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Copyright 2013 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
 import 'package:flutter_test/flutter_test.dart';
 import 'package:web_benchmarks/analysis.dart';
 

packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart

@@ -7,6 +7,7 @@ import 'dart:io';
 
 import 'package:test/test.dart';
 
+import 'package:web_benchmarks/metrics.dart';
 import 'package:web_benchmarks/server.dart';
 import 'package:web_benchmarks/src/common.dart';
 
@@ -90,14 +91,10 @@ Future<BenchmarkResults> _runBenchmarks({
     compilationOptions: compilationOptions,
   );
 
-  // The skwasm renderer doesn't have preroll or apply frame steps in its rendering.
-  final List<String> expectedMetrics = compilationOptions.useWasm
-      ? <String>['drawFrameDuration']
-      : <String>[
-          'preroll_frame',
-          'apply_frame',
-          'drawFrameDuration',
-        ];
+  final List<String> expectedMetrics =
+      expectedBenchmarkMetrics(useWasm: compilationOptions.useWasm)
+          .map((BenchmarkMetric metric) => metric.label)
+          .toList();
 
   for (final String benchmarkName in benchmarkNames) {
     for (final String metricName in expectedMetrics) {