From 74aadb265c1cae614ae636146f7a1e9e195b7e97 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Mon, 21 Aug 2023 17:06:36 -0700 Subject: [PATCH] Intern repository mapping entries All repositories generated by a module extension have the same repository mapping entries. Without interning, if a module extension generates N repositories, each such repository would have its own copy with (more than) N entries, resulting in overall memory usage quadratic in N. We intern the entries, not the `RepositoryMapping` itself, since the latter also includes the owner repo, which differs between extension repos. Output of `bazel info used-heap-size-after-gc` after running a build with a synthetic module extension generating N + 1 repos and requesting all of them: ``` before N=100: 32MB N=1000: 77MB N=3000: 371MB N=5000: 961MB N=10000: 3614MB after N=100: 32MB N=1000: 44MB N=3000: 71MB N=5000: 91MB N=10000: 158MB ``` Closes #19269. PiperOrigin-RevId: 558940840 Change-Id: I07402f203b5f11bf448a1ae9e9ee4637ad4c536d --- .../build/lib/cmdline/RepositoryMapping.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java index 0246c930171776..d1e5f35cc7c53f 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java @@ -17,6 +17,8 @@ import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Interner; +import com.google.devtools.build.lib.concurrent.BlazeInterners; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -34,6 +36,15 @@ @AutoValue public abstract class RepositoryMapping { + // All repositories generated by a module extension have the same repository mapping + // entries. Without interning, if a module extension generates N repositories, each such + // repository would have its own copy with (more than) N entries, resulting in memory usage that + // is quadratic in N. + // Note: We intern the entries, not the RepositoryMapping itself, because the latter also includes + // the owner repo, which differs between extension repos. + private static final Interner> ENTRIES_INTERNER = + BlazeInterners.newWeakInterner(); + // Always fallback to the requested name public static final RepositoryMapping ALWAYS_FALLBACK = createAllowingFallback(ImmutableMap.of()); @@ -60,7 +71,8 @@ public static RepositoryMapping createAllowingFallback(Map entries, RepositoryName ownerRepo) { - return new AutoValue_RepositoryMapping(ImmutableMap.copyOf(entries), ownerRepo); + return new AutoValue_RepositoryMapping( + ENTRIES_INTERNER.intern(ImmutableMap.copyOf(entries)), ownerRepo); } /**