Add option to expose package metrics via the BEP.

(Automated rollback of commit 0ebf7bc.)

*** Reason for rollback ***

Rolling forward with changes.

PiperOrigin-RevId: 529017479
Change-Id: I37bc2211815e846133c76871e607232407ffaa29

scripts/bootstrap/compile.sh

@@ -25,7 +25,7 @@ fi
 
 # Parse third_party/googleapis/BUILD.bazel to find the proto files we need to compile from googleapis
 GOOGLE_API_PROTOS="$(grep -o '".*\.proto"' third_party/googleapis/BUILD.bazel | sed 's/"//g' | sed 's|^|third_party/googleapis/|g')"
-PROTO_FILES=$(find third_party/remoteapis ${GOOGLE_API_PROTOS} third_party/pprof src/main/protobuf src/main/java/com/google/devtools/build/lib/buildeventstream/proto src/main/java/com/google/devtools/build/skyframe src/main/java/com/google/devtools/build/lib/skyframe/proto src/main/java/com/google/devtools/build/lib/bazel/debug src/main/java/com/google/devtools/build/lib/starlarkdebug/proto src/main/java/com/google/devtools/build/lib/packages/metrics/package_metrics.proto -name "*.proto")
+PROTO_FILES=$(find third_party/remoteapis ${GOOGLE_API_PROTOS} third_party/pprof src/main/protobuf src/main/java/com/google/devtools/build/lib/buildeventstream/proto src/main/java/com/google/devtools/build/skyframe src/main/java/com/google/devtools/build/lib/skyframe/proto src/main/java/com/google/devtools/build/lib/bazel/debug src/main/java/com/google/devtools/build/lib/starlarkdebug/proto src/main/java/com/google/devtools/build/lib/packages/metrics/package_load_metrics.proto -name "*.proto")
 # For protobuf jars, derived/jars/com_google_protobuf/java/core/libcore.jar must be in front of derived/jars/com_google_protobuf/java/core/liblite.jar, so we sort jars here
 LIBRARY_JARS=$(find $ADDITIONAL_JARS -name '*.jar' | sort | grep -Fv JavaBuilder | tr "\n" " ")
 MAVEN_JARS=$(find maven -name '*.jar' | grep -Fv netty-tcnative | tr "\n" " ")

src/main/java/com/google/devtools/build/lib/buildeventstream/proto/BUILD

@@ -26,6 +26,7 @@ proto_library(
     name = "build_event_stream_proto",
     srcs = ["build_event_stream.proto"],
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/packages/metrics:package_load_metrics_proto",
         "//src/main/protobuf:command_line_proto",
         "//src/main/protobuf:failure_details_proto",
         "//src/main/protobuf:invocation_policy_proto",

src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto

@@ -18,6 +18,7 @@ package build_event_stream;
 
 import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
+import "src/main/java/com/google/devtools/build/lib/packages/metrics/package_load_metrics.proto";
 import "src/main/protobuf/command_line.proto";
 import "src/main/protobuf/failure_details.proto";
 import "src/main/protobuf/invocation_policy.proto";
@@ -944,6 +945,10 @@ message BuildMetrics {
     // After Feb 2022, this value is the number of packages successfully
     // loaded.
     int64 packages_loaded = 1;
+
+    // Loading time metrics per package.
+    repeated devtools.build.lib.packages.metrics.PackageLoadMetrics
+        package_load_metrics = 2;
   }
   PackageMetrics package_metrics = 4;
 

src/main/java/com/google/devtools/build/lib/metrics/BUILD

@@ -44,6 +44,8 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/bugreport",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/clock",
+        "//src/main/java/com/google/devtools/build/lib/packages/metrics",
+        "//src/main/java/com/google/devtools/build/lib/packages/metrics:package_load_metrics_java_proto",
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/profiler:network_metrics_collector",
         "//src/main/java/com/google/devtools/build/lib/skyframe:execution_finished_event",

src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java

@@ -45,6 +45,10 @@
 import com.google.devtools.build.lib.clock.BlazeClock.NanosToMillisSinceEpochConverter;
 import com.google.devtools.build.lib.metrics.MetricsModule.Options;
 import com.google.devtools.build.lib.metrics.PostGCMemoryUseRecorder.PeakHeap;
+import com.google.devtools.build.lib.packages.metrics.ExtremaPackageMetricsRecorder;
+import com.google.devtools.build.lib.packages.metrics.PackageLoadMetrics;
+import com.google.devtools.build.lib.packages.metrics.PackageMetricsPackageLoadingListener;
+import com.google.devtools.build.lib.packages.metrics.PackageMetricsRecorder;
 import com.google.devtools.build.lib.profiler.MemoryProfiler;
 import com.google.devtools.build.lib.profiler.NetworkMetricsCollector;
 import com.google.devtools.build.lib.profiler.Profiler;
@@ -129,8 +133,26 @@ public void onAnalysisPhaseComplete(AnalysisPhaseCompleteEvent event) {
     targetMetrics
         .setTargetsConfigured(targetsConfigured.total())
         .setTargetsConfiguredNotIncludingAspects(targetsConfigured.configuredTargetsOnly());
-    packageMetrics.setPackagesLoaded(event.getPkgManagerStats().getPackagesSuccessfullyLoaded());
     timingMetrics.setAnalysisPhaseTimeInMs(event.getTimeInMs());
+
+    packageMetrics.setPackagesLoaded(event.getPkgManagerStats().getPackagesSuccessfullyLoaded());
+
+    if (PackageMetricsPackageLoadingListener.getInstance().getPublishPackageMetricsInBep()) {
+      PackageMetricsRecorder recorder =
+          PackageMetricsPackageLoadingListener.getInstance().getPackageMetricsRecorder();
+      if (recorder != null) {
+        Stream<PackageLoadMetrics> metrics = recorder.getPackageLoadMetrics().stream();
+
+        if (recorder.getRecorderType() == PackageMetricsRecorder.Type.ONLY_EXTREMES) {
+          ExtremaPackageMetricsRecorder extremaPackageMetricsRecorder =
+              (ExtremaPackageMetricsRecorder) recorder;
+          // Safeguard: we have 5 metrics, so print at most 5 times the number of packages as being
+          // tracked per metric.
+          metrics = metrics.limit(5L * extremaPackageMetricsRecorder.getNumPackagesToTrack());
+        }
+        metrics.forEach(packageMetrics::addPackageLoadMetrics);
+      }
+    }
   }
 
   @SuppressWarnings("unused")

src/main/java/com/google/devtools/build/lib/packages/metrics/BUILD

@@ -3,7 +3,7 @@ load("//tools/build_rules:utilities.bzl", "java_library_srcs")
 
 package(
     default_applicable_licenses = ["//:license"],
-    default_visibility = ["//src:__subpackages__"],
+    default_visibility = ["//visibility:public"],
 )
 
 filegroup(
@@ -16,6 +16,7 @@ java_library(
     name = "metrics",
     srcs = glob(["*.java"]),
     deps = [
+        ":package_load_metrics_java_proto",
         ":package_metrics_java_proto",
         "//src/main/java/com/google/devtools/build/lib:runtime",
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
@@ -54,7 +55,30 @@ java_library_srcs(
     deps = [":package_metrics_java_proto"],
 )
 
+proto_library(
+    name = "package_load_metrics_proto",
+    srcs = ["package_load_metrics.proto"],
+    deps = [
+        "@com_google_protobuf//:duration_proto",
+    ],
+)
+
+java_proto_library(
+    name = "package_load_metrics_java_proto",
+    deps = [
+        ":package_load_metrics_proto",
+    ],
+)
+
+java_library_srcs(
+    name = "package_load_metrics_java_proto_srcs",
+    deps = [":package_load_metrics_java_proto"],
+)
+
 filegroup(
     name = "dist_jars",
-    srcs = [":package_metrics_java_proto_srcs"],
+    srcs = [
+        ":package_load_metrics_java_proto_srcs",
+        ":package_metrics_java_proto_srcs",
+    ],
 )

src/main/java/com/google/devtools/build/lib/packages/metrics/CompletePackageMetricsRecorder.java

@@ -27,40 +27,40 @@
 final class CompletePackageMetricsRecorder implements PackageMetricsRecorder {
 
   @GuardedBy("this")
-  private final HashMap<PackageIdentifier, PackageMetrics> metrics = new HashMap<>();
+  private final HashMap<PackageIdentifier, PackageLoadMetrics> metrics = new HashMap<>();
 
   CompletePackageMetricsRecorder() {}
 
   @Override
-  public synchronized void recordMetrics(PackageIdentifier pkgId, PackageMetrics metrics) {
+  public synchronized void recordMetrics(PackageIdentifier pkgId, PackageLoadMetrics metrics) {
     this.metrics.put(pkgId, metrics);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Duration> getLoadTimes() {
-    return Maps.transformValues(metrics, PackageMetrics::getLoadDuration);
+    return Maps.transformValues(metrics, PackageLoadMetrics::getLoadDuration);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getComputationSteps() {
-    return Maps.transformValues(metrics, PackageMetrics::getComputationSteps);
+    return Maps.transformValues(metrics, PackageLoadMetrics::getComputationSteps);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getNumTargets() {
-    return Maps.transformValues(metrics, PackageMetrics::getNumTargets);
+    return Maps.transformValues(metrics, PackageLoadMetrics::getNumTargets);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getNumTransitiveLoads() {
-    return Maps.transformValues(metrics, PackageMetrics::getNumTransitiveLoads);
+    return Maps.transformValues(metrics, PackageLoadMetrics::getNumTransitiveLoads);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getPackageOverhead() {
     return Maps.transformValues(
-        Maps.filterValues(metrics, PackageMetrics::hasPackageOverhead),
-        PackageMetrics::getPackageOverhead);
+        Maps.filterValues(metrics, PackageLoadMetrics::hasPackageOverhead),
+        PackageLoadMetrics::getPackageOverhead);
   }
 
   @Override
@@ -79,7 +79,7 @@ public Type getRecorderType() {
   }
 
   @Override
-  public synchronized ImmutableCollection<PackageMetrics> getPackageMetrics() {
+  public synchronized ImmutableCollection<PackageLoadMetrics> getPackageLoadMetrics() {
     // lazily set the pkgName when requested.
     return metrics.entrySet().stream()
         .map(e -> e.getValue().toBuilder().setName(e.getKey().toString()).build())

src/main/java/com/google/devtools/build/lib/packages/metrics/ExtremaPackageMetricsRecorder.java

@@ -38,42 +38,42 @@ public class ExtremaPackageMetricsRecorder implements PackageMetricsRecorder {
   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
 
   @GuardedBy("this")
-  private final Extrema<PackageMetricsContainer> slowestPackagesToLoad;
+  private final Extrema<PackageLoadMetricsContainer> slowestPackagesToLoad;
 
   @GuardedBy("this")
-  private final Extrema<PackageMetricsContainer> largestPackages;
+  private final Extrema<PackageLoadMetricsContainer> largestPackages;
 
   @GuardedBy("this")
-  private final Extrema<PackageMetricsContainer> packagesWithMostTransitiveLoads;
+  private final Extrema<PackageLoadMetricsContainer> packagesWithMostTransitiveLoads;
 
   @GuardedBy("this")
-  private final Extrema<PackageMetricsContainer> packagesWithMostComputationSteps;
+  private final Extrema<PackageLoadMetricsContainer> packagesWithMostComputationSteps;
 
   @GuardedBy("this")
-  private final Extrema<PackageMetricsContainer> packagesWithMostOverhead;
+  private final Extrema<PackageLoadMetricsContainer> packagesWithMostOverhead;
 
   ExtremaPackageMetricsRecorder(int currentNumPackagesToTrack) {
     Preconditions.checkArgument(currentNumPackagesToTrack >= 0, "num packages must be >= 0");
     this.currentNumPackagesToTrack = currentNumPackagesToTrack;
     this.slowestPackagesToLoad =
-        Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.LOAD_TIMES_COMP);
+        Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.LOAD_TIMES_COMP);
     this.largestPackages =
-        Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.NUM_TARGETS_COMP);
+        Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.NUM_TARGETS_COMP);
     this.packagesWithMostTransitiveLoads =
-        Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.TRANSITIVE_LOADS_COMP);
+        Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.TRANSITIVE_LOADS_COMP);
     this.packagesWithMostComputationSteps =
-        Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.COMPUTATION_STEPS_COMP);
+        Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.COMPUTATION_STEPS_COMP);
     this.packagesWithMostOverhead =
-        Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.OVERHEAD_COMP);
+        Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.OVERHEAD_COMP);
   }
 
   public int getNumPackagesToTrack() {
     return currentNumPackagesToTrack;
   }
 
   @Override
-  public synchronized void recordMetrics(PackageIdentifier pkgId, PackageMetrics metrics) {
-    PackageMetricsContainer cont = PackageMetricsContainer.create(pkgId, metrics);
+  public synchronized void recordMetrics(PackageIdentifier pkgId, PackageLoadMetrics metrics) {
+    PackageLoadMetricsContainer cont = PackageLoadMetricsContainer.create(pkgId, metrics);
     slowestPackagesToLoad.aggregate(cont);
     packagesWithMostComputationSteps.aggregate(cont);
     largestPackages.aggregate(cont);
@@ -88,40 +88,40 @@ public synchronized Map<PackageIdentifier, Duration> getLoadTimes() {
     return slowestPackagesToLoad.getExtremeElements().stream()
         .collect(
             Collectors.toMap(
-                PackageMetricsContainer::getPackageIdentifier,
-                v -> v.getPackageMetricsInternal().getLoadDuration(),
+                PackageLoadMetricsContainer::getPackageIdentifier,
+                v -> v.getPackageLoadMetricsInternal().getLoadDuration(),
                 (k, v) -> v,
                 LinkedHashMap::new)); // use a LinkedHashMap to ensure iteration order is maintained
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getComputationSteps() {
-    return toMap(packagesWithMostComputationSteps, PackageMetrics::getComputationSteps);
+    return toMap(packagesWithMostComputationSteps, PackageLoadMetrics::getComputationSteps);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getNumTargets() {
-    return toMap(largestPackages, PackageMetrics::getNumTargets);
+    return toMap(largestPackages, PackageLoadMetrics::getNumTargets);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getNumTransitiveLoads() {
-    return toMap(packagesWithMostTransitiveLoads, PackageMetrics::getNumTransitiveLoads);
+    return toMap(packagesWithMostTransitiveLoads, PackageLoadMetrics::getNumTransitiveLoads);
   }
 
   @Override
   public synchronized Map<PackageIdentifier, Long> getPackageOverhead() {
-    return toMap(packagesWithMostOverhead, PackageMetrics::getPackageOverhead);
+    return toMap(packagesWithMostOverhead, PackageLoadMetrics::getPackageOverhead);
   }
 
   private synchronized Map<PackageIdentifier, Long> toMap(
-      Extrema<PackageMetricsContainer> ext, Function<PackageMetrics, Long> fn) {
+      Extrema<PackageLoadMetricsContainer> ext, Function<PackageLoadMetrics, Long> fn) {
 
     return ext.getExtremeElements().stream()
         .collect(
             Collectors.toMap(
-                PackageMetricsContainer::getPackageIdentifier,
-                v -> fn.apply(v.getPackageMetricsInternal()),
+                PackageLoadMetricsContainer::getPackageIdentifier,
+                v -> fn.apply(v.getPackageLoadMetricsInternal()),
                 (k, v) -> v,
                 LinkedHashMap::new)); // use a LinkedHashMap to ensure iteration order is maintained
   }
@@ -140,23 +140,23 @@ public synchronized void loadingFinished() {
     logIfNonEmpty(
         "Slowest packages (ms)",
         slowestPackagesToLoad.getExtremeElements(),
-        c -> Durations.toMillis(c.getPackageMetricsInternal().getLoadDuration()));
+        c -> Durations.toMillis(c.getPackageLoadMetricsInternal().getLoadDuration()));
     logIfNonEmpty(
         "Largest packages (num targets)",
         largestPackages.getExtremeElements(),
-        c -> c.getPackageMetricsInternal().getNumTargets());
+        c -> c.getPackageLoadMetricsInternal().getNumTargets());
     logIfNonEmpty(
         "Packages with most computation steps",
         packagesWithMostComputationSteps.getExtremeElements(),
-        c -> c.getPackageMetricsInternal().getComputationSteps());
+        c -> c.getPackageLoadMetricsInternal().getComputationSteps());
     logIfNonEmpty(
         "Packages with most transitive loads (num bzl files)",
         packagesWithMostTransitiveLoads.getExtremeElements(),
-        c -> c.getPackageMetricsInternal().getNumTransitiveLoads());
+        c -> c.getPackageLoadMetricsInternal().getNumTransitiveLoads());
     logIfNonEmpty(
         "Packages with most overhead",
         packagesWithMostOverhead.getExtremeElements(),
-        c -> c.getPackageMetricsInternal().getPackageOverhead());
+        c -> c.getPackageLoadMetricsInternal().getPackageOverhead());
     clear();
   }
 
@@ -166,21 +166,21 @@ public Type getRecorderType() {
   }
 
   @Override
-  public synchronized Collection<PackageMetrics> getPackageMetrics() {
+  public synchronized Collection<PackageLoadMetrics> getPackageLoadMetrics() {
     return Streams.concat(
             slowestPackagesToLoad.getExtremeElements().stream(),
             packagesWithMostComputationSteps.getExtremeElements().stream(),
             largestPackages.getExtremeElements().stream(),
             packagesWithMostTransitiveLoads.getExtremeElements().stream(),
             packagesWithMostOverhead.getExtremeElements().stream())
-        .map(PackageMetricsContainer::getPackageMetrics)
+        .map(PackageLoadMetricsContainer::getPackageLoadMetrics)
         .collect(toImmutableSet());
   }
 
   private static void logIfNonEmpty(
       String logLinePrefix,
-      List<PackageMetricsContainer> extremeElements,
-      Function<PackageMetricsContainer, Long> valueMapper) {
+      List<PackageLoadMetricsContainer> extremeElements,
+      Function<PackageLoadMetricsContainer, Long> valueMapper) {
     List<String> logString =
         extremeElements.stream()
             .map(v -> String.format("%s (%d)", v.getPackageIdentifier(), valueMapper.apply(v)))