diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh index 82bb2f949d12b1..56dcf8753fd97c 100755 --- a/scripts/bootstrap/compile.sh +++ b/scripts/bootstrap/compile.sh @@ -25,7 +25,7 @@ fi # Parse third_party/googleapis/BUILD.bazel to find the proto files we need to compile from googleapis GOOGLE_API_PROTOS="$(grep -o '".*\.proto"' third_party/googleapis/BUILD.bazel | sed 's/"//g' | sed 's|^|third_party/googleapis/|g')" -PROTO_FILES=$(find third_party/remoteapis ${GOOGLE_API_PROTOS} third_party/pprof src/main/protobuf src/main/java/com/google/devtools/build/lib/buildeventstream/proto src/main/java/com/google/devtools/build/skyframe src/main/java/com/google/devtools/build/lib/skyframe/proto src/main/java/com/google/devtools/build/lib/bazel/debug src/main/java/com/google/devtools/build/lib/starlarkdebug/proto src/main/java/com/google/devtools/build/lib/packages/metrics/package_metrics.proto -name "*.proto") +PROTO_FILES=$(find third_party/remoteapis ${GOOGLE_API_PROTOS} third_party/pprof src/main/protobuf src/main/java/com/google/devtools/build/lib/buildeventstream/proto src/main/java/com/google/devtools/build/skyframe src/main/java/com/google/devtools/build/lib/skyframe/proto src/main/java/com/google/devtools/build/lib/bazel/debug src/main/java/com/google/devtools/build/lib/starlarkdebug/proto src/main/java/com/google/devtools/build/lib/packages/metrics/package_load_metrics.proto -name "*.proto") # For protobuf jars, derived/jars/com_google_protobuf/java/core/libcore.jar must be in front of derived/jars/com_google_protobuf/java/core/liblite.jar, so we sort jars here LIBRARY_JARS=$(find $ADDITIONAL_JARS -name '*.jar' | sort | grep -Fv JavaBuilder | tr "\n" " ") MAVEN_JARS=$(find maven -name '*.jar' | grep -Fv netty-tcnative | tr "\n" " ") diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/BUILD b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/BUILD index 313dccb6e1af12..d30768bb507356 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/BUILD +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/BUILD @@ -26,6 +26,7 @@ proto_library( name = "build_event_stream_proto", srcs = ["build_event_stream.proto"], deps = [ + "//src/main/java/com/google/devtools/build/lib/packages/metrics:package_load_metrics_proto", "//src/main/protobuf:command_line_proto", "//src/main/protobuf:failure_details_proto", "//src/main/protobuf:invocation_policy_proto", diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto index aacdeabea2cae5..fcb8dc139b1710 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto @@ -18,6 +18,7 @@ package build_event_stream; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; +import "src/main/java/com/google/devtools/build/lib/packages/metrics/package_load_metrics.proto"; import "src/main/protobuf/command_line.proto"; import "src/main/protobuf/failure_details.proto"; import "src/main/protobuf/invocation_policy.proto"; @@ -944,6 +945,10 @@ message BuildMetrics { // After Feb 2022, this value is the number of packages successfully // loaded. int64 packages_loaded = 1; + + // Loading time metrics per package. + repeated devtools.build.lib.packages.metrics.PackageLoadMetrics + package_load_metrics = 2; } PackageMetrics package_metrics = 4; diff --git a/src/main/java/com/google/devtools/build/lib/metrics/BUILD b/src/main/java/com/google/devtools/build/lib/metrics/BUILD index f5236811422ca9..fd915effd339e9 100644 --- a/src/main/java/com/google/devtools/build/lib/metrics/BUILD +++ b/src/main/java/com/google/devtools/build/lib/metrics/BUILD @@ -44,6 +44,8 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bugreport", "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/clock", + "//src/main/java/com/google/devtools/build/lib/packages/metrics", + "//src/main/java/com/google/devtools/build/lib/packages/metrics:package_load_metrics_java_proto", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/profiler:network_metrics_collector", "//src/main/java/com/google/devtools/build/lib/skyframe:execution_finished_event", diff --git a/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java b/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java index 4a5fa67e79e364..78fbd860fed668 100644 --- a/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java +++ b/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java @@ -45,6 +45,10 @@ import com.google.devtools.build.lib.clock.BlazeClock.NanosToMillisSinceEpochConverter; import com.google.devtools.build.lib.metrics.MetricsModule.Options; import com.google.devtools.build.lib.metrics.PostGCMemoryUseRecorder.PeakHeap; +import com.google.devtools.build.lib.packages.metrics.ExtremaPackageMetricsRecorder; +import com.google.devtools.build.lib.packages.metrics.PackageLoadMetrics; +import com.google.devtools.build.lib.packages.metrics.PackageMetricsPackageLoadingListener; +import com.google.devtools.build.lib.packages.metrics.PackageMetricsRecorder; import com.google.devtools.build.lib.profiler.MemoryProfiler; import com.google.devtools.build.lib.profiler.NetworkMetricsCollector; import com.google.devtools.build.lib.profiler.Profiler; @@ -129,8 +133,26 @@ public void onAnalysisPhaseComplete(AnalysisPhaseCompleteEvent event) { targetMetrics .setTargetsConfigured(targetsConfigured.total()) .setTargetsConfiguredNotIncludingAspects(targetsConfigured.configuredTargetsOnly()); - packageMetrics.setPackagesLoaded(event.getPkgManagerStats().getPackagesSuccessfullyLoaded()); timingMetrics.setAnalysisPhaseTimeInMs(event.getTimeInMs()); + + packageMetrics.setPackagesLoaded(event.getPkgManagerStats().getPackagesSuccessfullyLoaded()); + + if (PackageMetricsPackageLoadingListener.getInstance().getPublishPackageMetricsInBep()) { + PackageMetricsRecorder recorder = + PackageMetricsPackageLoadingListener.getInstance().getPackageMetricsRecorder(); + if (recorder != null) { + Stream metrics = recorder.getPackageLoadMetrics().stream(); + + if (recorder.getRecorderType() == PackageMetricsRecorder.Type.ONLY_EXTREMES) { + ExtremaPackageMetricsRecorder extremaPackageMetricsRecorder = + (ExtremaPackageMetricsRecorder) recorder; + // Safeguard: we have 5 metrics, so print at most 5 times the number of packages as being + // tracked per metric. + metrics = metrics.limit(5L * extremaPackageMetricsRecorder.getNumPackagesToTrack()); + } + metrics.forEach(packageMetrics::addPackageLoadMetrics); + } + } } @SuppressWarnings("unused") diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/BUILD b/src/main/java/com/google/devtools/build/lib/packages/metrics/BUILD index 4bbc6f10d08fb3..23d2309d8fc12a 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/BUILD +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/BUILD @@ -3,7 +3,7 @@ load("//tools/build_rules:utilities.bzl", "java_library_srcs") package( default_applicable_licenses = ["//:license"], - default_visibility = ["//src:__subpackages__"], + default_visibility = ["//visibility:public"], ) filegroup( @@ -16,6 +16,7 @@ java_library( name = "metrics", srcs = glob(["*.java"]), deps = [ + ":package_load_metrics_java_proto", ":package_metrics_java_proto", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", @@ -54,7 +55,30 @@ java_library_srcs( deps = [":package_metrics_java_proto"], ) +proto_library( + name = "package_load_metrics_proto", + srcs = ["package_load_metrics.proto"], + deps = [ + "@com_google_protobuf//:duration_proto", + ], +) + +java_proto_library( + name = "package_load_metrics_java_proto", + deps = [ + ":package_load_metrics_proto", + ], +) + +java_library_srcs( + name = "package_load_metrics_java_proto_srcs", + deps = [":package_load_metrics_java_proto"], +) + filegroup( name = "dist_jars", - srcs = [":package_metrics_java_proto_srcs"], + srcs = [ + ":package_load_metrics_java_proto_srcs", + ":package_metrics_java_proto_srcs", + ], ) diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/CompletePackageMetricsRecorder.java b/src/main/java/com/google/devtools/build/lib/packages/metrics/CompletePackageMetricsRecorder.java index 9a2222dea77b4c..2b29d4eea5e13a 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/CompletePackageMetricsRecorder.java +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/CompletePackageMetricsRecorder.java @@ -27,40 +27,40 @@ final class CompletePackageMetricsRecorder implements PackageMetricsRecorder { @GuardedBy("this") - private final HashMap metrics = new HashMap<>(); + private final HashMap metrics = new HashMap<>(); CompletePackageMetricsRecorder() {} @Override - public synchronized void recordMetrics(PackageIdentifier pkgId, PackageMetrics metrics) { + public synchronized void recordMetrics(PackageIdentifier pkgId, PackageLoadMetrics metrics) { this.metrics.put(pkgId, metrics); } @Override public synchronized Map getLoadTimes() { - return Maps.transformValues(metrics, PackageMetrics::getLoadDuration); + return Maps.transformValues(metrics, PackageLoadMetrics::getLoadDuration); } @Override public synchronized Map getComputationSteps() { - return Maps.transformValues(metrics, PackageMetrics::getComputationSteps); + return Maps.transformValues(metrics, PackageLoadMetrics::getComputationSteps); } @Override public synchronized Map getNumTargets() { - return Maps.transformValues(metrics, PackageMetrics::getNumTargets); + return Maps.transformValues(metrics, PackageLoadMetrics::getNumTargets); } @Override public synchronized Map getNumTransitiveLoads() { - return Maps.transformValues(metrics, PackageMetrics::getNumTransitiveLoads); + return Maps.transformValues(metrics, PackageLoadMetrics::getNumTransitiveLoads); } @Override public synchronized Map getPackageOverhead() { return Maps.transformValues( - Maps.filterValues(metrics, PackageMetrics::hasPackageOverhead), - PackageMetrics::getPackageOverhead); + Maps.filterValues(metrics, PackageLoadMetrics::hasPackageOverhead), + PackageLoadMetrics::getPackageOverhead); } @Override @@ -79,7 +79,7 @@ public Type getRecorderType() { } @Override - public synchronized ImmutableCollection getPackageMetrics() { + public synchronized ImmutableCollection getPackageLoadMetrics() { // lazily set the pkgName when requested. return metrics.entrySet().stream() .map(e -> e.getValue().toBuilder().setName(e.getKey().toString()).build()) diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/ExtremaPackageMetricsRecorder.java b/src/main/java/com/google/devtools/build/lib/packages/metrics/ExtremaPackageMetricsRecorder.java index b27fde8e93440f..f40e8000cf7313 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/ExtremaPackageMetricsRecorder.java +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/ExtremaPackageMetricsRecorder.java @@ -38,33 +38,33 @@ public class ExtremaPackageMetricsRecorder implements PackageMetricsRecorder { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); @GuardedBy("this") - private final Extrema slowestPackagesToLoad; + private final Extrema slowestPackagesToLoad; @GuardedBy("this") - private final Extrema largestPackages; + private final Extrema largestPackages; @GuardedBy("this") - private final Extrema packagesWithMostTransitiveLoads; + private final Extrema packagesWithMostTransitiveLoads; @GuardedBy("this") - private final Extrema packagesWithMostComputationSteps; + private final Extrema packagesWithMostComputationSteps; @GuardedBy("this") - private final Extrema packagesWithMostOverhead; + private final Extrema packagesWithMostOverhead; ExtremaPackageMetricsRecorder(int currentNumPackagesToTrack) { Preconditions.checkArgument(currentNumPackagesToTrack >= 0, "num packages must be >= 0"); this.currentNumPackagesToTrack = currentNumPackagesToTrack; this.slowestPackagesToLoad = - Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.LOAD_TIMES_COMP); + Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.LOAD_TIMES_COMP); this.largestPackages = - Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.NUM_TARGETS_COMP); + Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.NUM_TARGETS_COMP); this.packagesWithMostTransitiveLoads = - Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.TRANSITIVE_LOADS_COMP); + Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.TRANSITIVE_LOADS_COMP); this.packagesWithMostComputationSteps = - Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.COMPUTATION_STEPS_COMP); + Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.COMPUTATION_STEPS_COMP); this.packagesWithMostOverhead = - Extrema.max(currentNumPackagesToTrack, PackageMetricsContainer.OVERHEAD_COMP); + Extrema.max(currentNumPackagesToTrack, PackageLoadMetricsContainer.OVERHEAD_COMP); } public int getNumPackagesToTrack() { @@ -72,8 +72,8 @@ public int getNumPackagesToTrack() { } @Override - public synchronized void recordMetrics(PackageIdentifier pkgId, PackageMetrics metrics) { - PackageMetricsContainer cont = PackageMetricsContainer.create(pkgId, metrics); + public synchronized void recordMetrics(PackageIdentifier pkgId, PackageLoadMetrics metrics) { + PackageLoadMetricsContainer cont = PackageLoadMetricsContainer.create(pkgId, metrics); slowestPackagesToLoad.aggregate(cont); packagesWithMostComputationSteps.aggregate(cont); largestPackages.aggregate(cont); @@ -88,40 +88,40 @@ public synchronized Map getLoadTimes() { return slowestPackagesToLoad.getExtremeElements().stream() .collect( Collectors.toMap( - PackageMetricsContainer::getPackageIdentifier, - v -> v.getPackageMetricsInternal().getLoadDuration(), + PackageLoadMetricsContainer::getPackageIdentifier, + v -> v.getPackageLoadMetricsInternal().getLoadDuration(), (k, v) -> v, LinkedHashMap::new)); // use a LinkedHashMap to ensure iteration order is maintained } @Override public synchronized Map getComputationSteps() { - return toMap(packagesWithMostComputationSteps, PackageMetrics::getComputationSteps); + return toMap(packagesWithMostComputationSteps, PackageLoadMetrics::getComputationSteps); } @Override public synchronized Map getNumTargets() { - return toMap(largestPackages, PackageMetrics::getNumTargets); + return toMap(largestPackages, PackageLoadMetrics::getNumTargets); } @Override public synchronized Map getNumTransitiveLoads() { - return toMap(packagesWithMostTransitiveLoads, PackageMetrics::getNumTransitiveLoads); + return toMap(packagesWithMostTransitiveLoads, PackageLoadMetrics::getNumTransitiveLoads); } @Override public synchronized Map getPackageOverhead() { - return toMap(packagesWithMostOverhead, PackageMetrics::getPackageOverhead); + return toMap(packagesWithMostOverhead, PackageLoadMetrics::getPackageOverhead); } private synchronized Map toMap( - Extrema ext, Function fn) { + Extrema ext, Function fn) { return ext.getExtremeElements().stream() .collect( Collectors.toMap( - PackageMetricsContainer::getPackageIdentifier, - v -> fn.apply(v.getPackageMetricsInternal()), + PackageLoadMetricsContainer::getPackageIdentifier, + v -> fn.apply(v.getPackageLoadMetricsInternal()), (k, v) -> v, LinkedHashMap::new)); // use a LinkedHashMap to ensure iteration order is maintained } @@ -140,23 +140,23 @@ public synchronized void loadingFinished() { logIfNonEmpty( "Slowest packages (ms)", slowestPackagesToLoad.getExtremeElements(), - c -> Durations.toMillis(c.getPackageMetricsInternal().getLoadDuration())); + c -> Durations.toMillis(c.getPackageLoadMetricsInternal().getLoadDuration())); logIfNonEmpty( "Largest packages (num targets)", largestPackages.getExtremeElements(), - c -> c.getPackageMetricsInternal().getNumTargets()); + c -> c.getPackageLoadMetricsInternal().getNumTargets()); logIfNonEmpty( "Packages with most computation steps", packagesWithMostComputationSteps.getExtremeElements(), - c -> c.getPackageMetricsInternal().getComputationSteps()); + c -> c.getPackageLoadMetricsInternal().getComputationSteps()); logIfNonEmpty( "Packages with most transitive loads (num bzl files)", packagesWithMostTransitiveLoads.getExtremeElements(), - c -> c.getPackageMetricsInternal().getNumTransitiveLoads()); + c -> c.getPackageLoadMetricsInternal().getNumTransitiveLoads()); logIfNonEmpty( "Packages with most overhead", packagesWithMostOverhead.getExtremeElements(), - c -> c.getPackageMetricsInternal().getPackageOverhead()); + c -> c.getPackageLoadMetricsInternal().getPackageOverhead()); clear(); } @@ -166,21 +166,21 @@ public Type getRecorderType() { } @Override - public synchronized Collection getPackageMetrics() { + public synchronized Collection getPackageLoadMetrics() { return Streams.concat( slowestPackagesToLoad.getExtremeElements().stream(), packagesWithMostComputationSteps.getExtremeElements().stream(), largestPackages.getExtremeElements().stream(), packagesWithMostTransitiveLoads.getExtremeElements().stream(), packagesWithMostOverhead.getExtremeElements().stream()) - .map(PackageMetricsContainer::getPackageMetrics) + .map(PackageLoadMetricsContainer::getPackageLoadMetrics) .collect(toImmutableSet()); } private static void logIfNonEmpty( String logLinePrefix, - List extremeElements, - Function valueMapper) { + List extremeElements, + Function valueMapper) { List logString = extremeElements.stream() .map(v -> String.format("%s (%d)", v.getPackageIdentifier(), valueMapper.apply(v))) diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsContainer.java b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageLoadMetricsContainer.java similarity index 52% rename from src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsContainer.java rename to src/main/java/com/google/devtools/build/lib/packages/metrics/PackageLoadMetricsContainer.java index 4cab05ac223b7c..6ff079975de05d 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsContainer.java +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageLoadMetricsContainer.java @@ -20,36 +20,37 @@ /** Container class holding a PackageIdentifier and PackageMetrics proto. */ @AutoValue -public abstract class PackageMetricsContainer { +public abstract class PackageLoadMetricsContainer { /** Sorts by LoadTime Duration. */ - public static final Comparator LOAD_TIMES_COMP = + public static final Comparator LOAD_TIMES_COMP = Comparator.comparing( - c -> c.getPackageMetricsInternal().getLoadDuration(), Durations.comparator()); + c -> c.getPackageLoadMetricsInternal().getLoadDuration(), Durations.comparator()); /** Sorts by Num Target count . */ - public static final Comparator NUM_TARGETS_COMP = - Comparator.comparingLong(c -> c.getPackageMetricsInternal().getNumTargets()); + public static final Comparator NUM_TARGETS_COMP = + Comparator.comparingLong(c -> c.getPackageLoadMetricsInternal().getNumTargets()); /** Sorts by Comutation Steps count. */ - public static final Comparator COMPUTATION_STEPS_COMP = - Comparator.comparingLong(c -> c.getPackageMetricsInternal().getComputationSteps()); + public static final Comparator COMPUTATION_STEPS_COMP = + Comparator.comparingLong(c -> c.getPackageLoadMetricsInternal().getComputationSteps()); /** Sorts by Transitive Load Count. */ - public static final Comparator TRANSITIVE_LOADS_COMP = - Comparator.comparingLong(c -> c.getPackageMetricsInternal().getNumTransitiveLoads()); + public static final Comparator TRANSITIVE_LOADS_COMP = + Comparator.comparingLong(c -> c.getPackageLoadMetricsInternal().getNumTransitiveLoads()); /** Sorts by Package Overhead. */ - public static final Comparator OVERHEAD_COMP = - Comparator.comparingLong(c -> c.getPackageMetricsInternal().getPackageOverhead()); + public static final Comparator OVERHEAD_COMP = + Comparator.comparingLong(c -> c.getPackageLoadMetricsInternal().getPackageOverhead()); - public static PackageMetricsContainer create(PackageIdentifier pkgId, PackageMetrics metrics) { - return new AutoValue_PackageMetricsContainer(pkgId, metrics); + public static PackageLoadMetricsContainer create( + PackageIdentifier pkgId, PackageLoadMetrics metrics) { + return new AutoValue_PackageLoadMetricsContainer(pkgId, metrics); } public abstract PackageIdentifier getPackageIdentifier(); - abstract PackageMetrics getPackageMetricsInternal(); + abstract PackageLoadMetrics getPackageLoadMetricsInternal(); /** Construct a full PackageMetrics object with the name set lazily from the PackageIdentifier. */ - public PackageMetrics getPackageMetrics() { - return getPackageMetricsInternal().toBuilder() + public PackageLoadMetrics getPackageLoadMetrics() { + return getPackageLoadMetricsInternal().toBuilder() .setName(getPackageIdentifier().toString()) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsModule.java b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsModule.java index 19b532377349f2..f96d10d9ce297b 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsModule.java +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsModule.java @@ -48,6 +48,14 @@ public static class Options extends OptionsBase { "Configures PackageMetrics to record all metrics for all packages. Disables Top-n INFO" + " logging.") public boolean enableAllMetrics; + + @Option( + name = "experimental_publish_package_metrics_in_bep", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.BAZEL_MONITORING}, + help = "Whether to publish package metrics in the BEP.") + public boolean publishPackageMetricsInBep; } private final PackageMetricsPackageLoadingListener packageLoadingListener; @@ -83,6 +91,7 @@ public void beforeCommand(CommandEnvironment commandEnvironment) { ? new CompletePackageMetricsRecorder() : new ExtremaPackageMetricsRecorder(Math.max(options.numberOfPackagesToTrack, 0)); packageLoadingListener.setPackageMetricsRecorder(recorder); + packageLoadingListener.setPublishPackageMetricsInBep(options.publishPackageMetricsInBep); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListener.java b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListener.java index 053acc4512abff..274efc2b66772c 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListener.java +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListener.java @@ -26,6 +26,8 @@ public class PackageMetricsPackageLoadingListener implements PackageLoadingListe @GuardedBy("this") private PackageMetricsRecorder recorder; + private boolean publishPackageMetricsInBep = false; + @GuardedBy("PackageMetricsPackageLoadingListener.class") private static PackageMetricsPackageLoadingListener instance = null; @@ -51,8 +53,8 @@ public synchronized void onLoadingCompleteAndSuccessful( return; } - PackageMetrics.Builder builder = - PackageMetrics.newBuilder() + PackageLoadMetrics.Builder builder = + PackageLoadMetrics.newBuilder() .setLoadDuration(Durations.fromNanos(loadTimeNanos)) .setComputationSteps(pkg.getComputationSteps()) .setNumTargets(pkg.getTargets().size()) @@ -70,6 +72,14 @@ public synchronized void setPackageMetricsRecorder(PackageMetricsRecorder record this.recorder = recorder; } + public void setPublishPackageMetricsInBep(boolean publishPackageMetricsInBep) { + this.publishPackageMetricsInBep = publishPackageMetricsInBep; + } + + public boolean getPublishPackageMetricsInBep() { + return publishPackageMetricsInBep; + } + /** Returns the PackageMetricsRecorder, if any, for the PackageLoadingListener. */ public synchronized PackageMetricsRecorder getPackageMetricsRecorder() { return recorder; diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsRecorder.java b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsRecorder.java index 79ea6c6641d750..ed2efd23f7796a 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsRecorder.java +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsRecorder.java @@ -13,6 +13,9 @@ // limitations under the License. package com.google.devtools.build.lib.packages.metrics; +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.collect.ImmutableCollection; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.protobuf.Duration; import java.util.Collection; @@ -28,7 +31,7 @@ enum Type { } /** Records the metrics for a given package. */ - void recordMetrics(PackageIdentifier pkgId, PackageMetrics metrics); + void recordMetrics(PackageIdentifier pkgId, PackageLoadMetrics metrics); /** * Returns a {@code Map} of recorded load durations. This may contain @@ -70,5 +73,22 @@ enum Type { Type getRecorderType(); /** If Type is ALL returns metrics for all Packages loaded. */ - Collection getPackageMetrics(); + Collection getPackageLoadMetrics(); + + /* TODO(twerth): Remove method after migration is complete. */ + default ImmutableCollection getPackageMetrics() { + Collection packageLoadMetrics = getPackageLoadMetrics(); + return packageLoadMetrics.stream() + .map( + plm -> + PackageMetrics.newBuilder() + .setName(plm.getName()) + .setPackageOverhead(plm.getPackageOverhead()) + .setComputationSteps(plm.getComputationSteps()) + .setLoadDuration(plm.getLoadDuration()) + .setNumTargets(plm.getNumTargets()) + .setNumTransitiveLoads(plm.getNumTransitiveLoads()) + .build()) + .collect(toImmutableList()); + } } diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/package_load_metrics.proto b/src/main/java/com/google/devtools/build/lib/packages/metrics/package_load_metrics.proto new file mode 100644 index 00000000000000..e27ecceaf4456b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/package_load_metrics.proto @@ -0,0 +1,44 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto2"; + +package devtools.build.lib.packages.metrics; + +import "google/protobuf/duration.proto"; + +option java_package = "com.google.devtools.build.lib.packages.metrics"; +option java_multiple_files = true; + +// Message used to concisely report all package metrics. +message PackageLoadMetrics { + // Name of the package. + optional string name = 1; + + // Wall-time duration it took to construct the package. + optional google.protobuf.Duration load_duration = 2; + + // Number of targets created in the package. + optional uint64 num_targets = 3; + + // Number of Starlark computation steps required to create the package. + optional uint64 computation_steps = 4; + + // Number of transitive Starlark load()s required to create the package. + optional uint64 num_transitive_loads = 5; + + // Numeric value given to the memory and general accounting costs associated + // with a loaded package. Values are an approximate but reasonable proxy for + // the real storage costs of a package. + optional uint64 package_overhead = 6; +} diff --git a/src/main/java/com/google/devtools/build/lib/packages/metrics/package_metrics.proto b/src/main/java/com/google/devtools/build/lib/packages/metrics/package_metrics.proto index c643fcd7e5bb11..3782122606c58d 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/metrics/package_metrics.proto +++ b/src/main/java/com/google/devtools/build/lib/packages/metrics/package_metrics.proto @@ -19,6 +19,7 @@ import "google/protobuf/duration.proto"; option java_package = "com.google.devtools.build.lib.packages.metrics"; option java_multiple_files = true; +option deprecated = true; // Use PackageLoadMetrics instead. // Message used to consisely report all package metrics. message PackageMetrics { diff --git a/src/test/java/com/google/devtools/build/lib/packages/metrics/BUILD b/src/test/java/com/google/devtools/build/lib/packages/metrics/BUILD index db6852fff51ba0..366c3ca6eb55f0 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/metrics/BUILD +++ b/src/test/java/com/google/devtools/build/lib/packages/metrics/BUILD @@ -21,7 +21,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/packages/metrics", - "//src/main/java/com/google/devtools/build/lib/packages/metrics:package_metrics_java_proto", + "//src/main/java/com/google/devtools/build/lib/packages/metrics:package_load_metrics_java_proto", "//src/main/java/net/starlark/java/eval", "//third_party:guava", "//third_party:junit4", diff --git a/src/test/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListenerTest.java b/src/test/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListenerTest.java index 1991882d60564c..9df24da06cc0ff 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListenerTest.java +++ b/src/test/java/com/google/devtools/build/lib/packages/metrics/PackageMetricsPackageLoadingListenerTest.java @@ -410,8 +410,8 @@ public void metricMap_extrema() { recordEverything(); - PackageMetrics pkg1 = - PackageMetrics.newBuilder() + PackageLoadMetrics pkg1 = + PackageLoadMetrics.newBuilder() .setName("my/pkg1") .setLoadDuration(Durations.fromMillis(42)) .setComputationSteps(1000) @@ -420,8 +420,8 @@ public void metricMap_extrema() { .setPackageOverhead(100_000) .build(); - PackageMetrics pkg2 = - PackageMetrics.newBuilder() + PackageLoadMetrics pkg2 = + PackageLoadMetrics.newBuilder() .setName("my/pkg2") .setLoadDuration(Durations.fromMillis(43)) .setComputationSteps(100) @@ -430,8 +430,8 @@ public void metricMap_extrema() { .setPackageOverhead(200_000) .build(); - PackageMetrics pkg3 = - PackageMetrics.newBuilder() + PackageLoadMetrics pkg3 = + PackageLoadMetrics.newBuilder() .setName("my/pkg3") .setLoadDuration(Durations.fromMillis(44)) .setComputationSteps(10) @@ -440,7 +440,7 @@ public void metricMap_extrema() { .setPackageOverhead(300_000) .build(); - assertThat(underTest.getPackageMetricsRecorder().getPackageMetrics()) + assertThat(underTest.getPackageMetricsRecorder().getPackageLoadMetrics()) .containsExactly(pkg1, pkg2, pkg3); recorder.loadingFinished(); assertAllMapsEmpty(recorder); @@ -453,8 +453,8 @@ public void metricMap_complete() { recordEverything(); - PackageMetrics pkg1 = - PackageMetrics.newBuilder() + PackageLoadMetrics pkg1 = + PackageLoadMetrics.newBuilder() .setName("my/pkg1") .setLoadDuration(Durations.fromMillis(42)) .setComputationSteps(1000) @@ -463,8 +463,8 @@ public void metricMap_complete() { .setPackageOverhead(100_000) .build(); - PackageMetrics pkg2 = - PackageMetrics.newBuilder() + PackageLoadMetrics pkg2 = + PackageLoadMetrics.newBuilder() .setName("my/pkg2") .setLoadDuration(Durations.fromMillis(43)) .setComputationSteps(100) @@ -473,8 +473,8 @@ public void metricMap_complete() { .setPackageOverhead(200_000) .build(); - PackageMetrics pkg3 = - PackageMetrics.newBuilder() + PackageLoadMetrics pkg3 = + PackageLoadMetrics.newBuilder() .setName("my/pkg3") .setLoadDuration(Durations.fromMillis(44)) .setComputationSteps(10) @@ -483,7 +483,7 @@ public void metricMap_complete() { .setPackageOverhead(300_000) .build(); - assertThat(underTest.getPackageMetricsRecorder().getPackageMetrics()) + assertThat(underTest.getPackageMetricsRecorder().getPackageLoadMetrics()) .containsExactly(pkg1, pkg2, pkg3); recorder.loadingFinished(); assertAllMapsEmpty(recorder);