From 3348e42d0d9d1a98850097e82ab43e2ce6e71c5f Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Sat, 13 Jul 2024 17:41:41 +0200 Subject: [PATCH 1/3] [MRESOLVER-587] Memory usage improvements Do not use Artifact instance as keys, and intern the List on artifact descriptors as well. --- https://issues.apache.org/jira/browse/MRESOLVER-587 --- .../internal/impl/collect/DataPool.java | 83 +++++++++++++++++-- .../collect/bf/BfDependencyCollector.java | 2 +- .../collect/df/DfDependencyCollector.java | 2 +- .../internal/impl/collect/DataPoolTest.java | 2 +- 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java index c86f3527d..87194b6fa 100644 --- a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java +++ b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java @@ -58,12 +58,17 @@ public final class DataPool { private static final String CONFIG_PROP_COLLECTOR_POOL_DESCRIPTOR = "aether.dependencyCollector.pool.descriptor"; + private static final String CONFIG_PROP_COLLECTOR_POOL_DEPENDENCY_LISTS = + "aether.dependencyCollector.pool.dependencyLists"; + private static final String ARTIFACT_POOL = DataPool.class.getName() + "$Artifact"; private static final String DEPENDENCY_POOL = DataPool.class.getName() + "$Dependency"; private static final String DESCRIPTORS = DataPool.class.getName() + "$Descriptors"; + private static final String DEPENDENCY_LISTS_POOL = DataPool.class.getName() + "$DependencyLists"; + public static final ArtifactDescriptorResult NO_DESCRIPTOR = new ArtifactDescriptorResult(new ArtifactDescriptorRequest()); @@ -80,7 +85,12 @@ public final class DataPool { /** * Descriptor interning pool, lives across session (if session carries non-null {@link RepositoryCache}). */ - private final InternPool descriptors; + private final InternPool descriptors; + + /** + * {@link Dependency} list interning pool, lives across session (if session carries non-null {@link RepositoryCache}). + */ + private final InternPool, List> dependencyLists; /** * Constraint cache, lives during single collection invocation (same as this DataPool instance). @@ -98,11 +108,14 @@ public DataPool(RepositorySystemSession session) { InternPool artifactsPool = null; InternPool dependenciesPool = null; - InternPool descriptorsPool = null; + InternPool descriptorsPool = null; + InternPool, List> dependencyListsPool = null; if (cache != null) { artifactsPool = (InternPool) cache.get(session, ARTIFACT_POOL); dependenciesPool = (InternPool) cache.get(session, DEPENDENCY_POOL); - descriptorsPool = (InternPool) cache.get(session, DESCRIPTORS); + descriptorsPool = (InternPool) cache.get(session, DESCRIPTORS); + dependencyListsPool = + (InternPool, List>) cache.get(session, DEPENDENCY_LISTS_POOL); } if (artifactsPool == null) { @@ -132,9 +145,20 @@ public DataPool(RepositorySystemSession session) { } } + if (dependencyListsPool == null) { + String dependencyListsPoolType = + ConfigUtils.getString(session, HARD, CONFIG_PROP_COLLECTOR_POOL_DEPENDENCY_LISTS); + + dependencyListsPool = createPool(dependencyListsPoolType); + if (cache != null) { + cache.put(session, DEPENDENCY_LISTS_POOL, dependencyListsPool); + } + } + this.artifacts = artifactsPool; this.dependencies = dependenciesPool; this.descriptors = descriptorsPool; + this.dependencyLists = dependencyListsPool; this.constraints = new ConcurrentHashMap<>(256); this.nodes = new ConcurrentHashMap<>(256); @@ -148,11 +172,11 @@ public Dependency intern(Dependency dependency) { return dependencies.intern(dependency, dependency); } - public Object toKey(ArtifactDescriptorRequest request) { - return request.getArtifact(); + public DescriptorKey toKey(ArtifactDescriptorRequest request) { + return new DescriptorKey(request.getArtifact()); } - public ArtifactDescriptorResult getDescriptor(Object key, ArtifactDescriptorRequest request) { + public ArtifactDescriptorResult getDescriptor(DescriptorKey key, ArtifactDescriptorRequest request) { Descriptor descriptor = descriptors.get(key); if (descriptor != null) { return descriptor.toResult(request); @@ -160,14 +184,20 @@ public ArtifactDescriptorResult getDescriptor(Object key, ArtifactDescriptorRequ return null; } - public void putDescriptor(Object key, ArtifactDescriptorResult result) { + public void putDescriptor(DescriptorKey key, ArtifactDescriptorResult result) { + result.setDependencies(intern(result.getDependencies())); + result.setManagedDependencies(intern(result.getManagedDependencies())); descriptors.intern(key, new GoodDescriptor(result)); } - public void putDescriptor(Object key, ArtifactDescriptorException e) { + public void putDescriptor(DescriptorKey key, ArtifactDescriptorException e) { descriptors.intern(key, BadDescriptor.INSTANCE); } + private List intern(List dependencies) { + return dependencyLists.intern(dependencies, dependencies); + } + public Object toKey(VersionRangeRequest request) { return new ConstraintKey(request); } @@ -202,8 +232,43 @@ public void putChildren(Object key, List children) { nodes.put(key, children); } - abstract static class Descriptor { + public static final class DescriptorKey { + private final Artifact artifact; + private final int hashCode; + private DescriptorKey(Artifact artifact) { + this.artifact = artifact; + this.hashCode = buildHashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DescriptorKey that = (DescriptorKey) o; + return Objects.equals(artifact, that.artifact); + } + + @Override + public int hashCode() { + return hashCode; + } + + private int buildHashCode() { + return Objects.hashCode(artifact); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{" + "artifact='" + artifact + '\'' + '}'; + } + } + + abstract static class Descriptor { public abstract ArtifactDescriptorResult toResult(ArtifactDescriptorRequest request); } diff --git a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java index 85d267e9c..84f49822f 100644 --- a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java +++ b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/bf/BfDependencyCollector.java @@ -448,7 +448,7 @@ private ArtifactDescriptorResult resolveCachedArtifactDescriptor( RepositorySystemSession session, DependencyProcessingContext context, Results results) { - Object key = pool.toKey(descriptorRequest); + DataPool.DescriptorKey key = pool.toKey(descriptorRequest); ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest); if (descriptorResult == null) { try { diff --git a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/df/DfDependencyCollector.java b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/df/DfDependencyCollector.java index 5948d4fd8..bc248cb0c 100644 --- a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/df/DfDependencyCollector.java +++ b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/df/DfDependencyCollector.java @@ -375,7 +375,7 @@ private ArtifactDescriptorResult resolveCachedArtifactDescriptor( Dependency d, Results results, Args args) { - Object key = pool.toKey(descriptorRequest); + DataPool.DescriptorKey key = pool.toKey(descriptorRequest); ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest); if (descriptorResult == null) { try { diff --git a/maven-resolver-impl/src/test/java/org/eclipse/aether/internal/impl/collect/DataPoolTest.java b/maven-resolver-impl/src/test/java/org/eclipse/aether/internal/impl/collect/DataPoolTest.java index f4295f073..428645c1e 100644 --- a/maven-resolver-impl/src/test/java/org/eclipse/aether/internal/impl/collect/DataPoolTest.java +++ b/maven-resolver-impl/src/test/java/org/eclipse/aether/internal/impl/collect/DataPoolTest.java @@ -51,7 +51,7 @@ public void testArtifactDescriptorCaching() { result.addAlias(new DefaultArtifact("gid:alias:4")); DataPool pool = newDataPool(); - Object key = pool.toKey(request); + DataPool.DescriptorKey key = pool.toKey(request); pool.putDescriptor(key, result); ArtifactDescriptorResult cached = pool.getDescriptor(key, request); assertNotNull(cached); From 779a630bb93be3b40ef446391eef9b304ac99200 Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Fri, 2 Aug 2024 12:24:59 +0200 Subject: [PATCH 2/3] Make newly added interning configurable (default: false to both, as before) Adds two new config properties that controls interning of ArtifactDescriptor dependencies and managedDependencies. Interning both radically lower memory consumption but at same time increases runtime, --- .../internal/impl/collect/DataPool.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java index 87194b6fa..7ee696fda 100644 --- a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java +++ b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java @@ -61,6 +61,12 @@ public final class DataPool { private static final String CONFIG_PROP_COLLECTOR_POOL_DEPENDENCY_LISTS = "aether.dependencyCollector.pool.dependencyLists"; + private static final String CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_DEPENDENCIES = + "aether.dependencyCollector.pool.internArtifactDescriptorDependencies"; + + private static final String CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES = + "aether.dependencyCollector.pool.internArtifactDescriptorManagedDependencies"; + private static final String ARTIFACT_POOL = DataPool.class.getName() + "$Artifact"; private static final String DEPENDENCY_POOL = DataPool.class.getName() + "$Dependency"; @@ -102,10 +108,19 @@ public final class DataPool { */ private final ConcurrentHashMap> nodes; + private final boolean internArtifactDescriptorDependencies; + + private final boolean internArtifactDescriptorManagedDependencies; + @SuppressWarnings("unchecked") public DataPool(RepositorySystemSession session) { final RepositoryCache cache = session.getCache(); + internArtifactDescriptorDependencies = ConfigUtils.getBoolean( + session, false, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_DEPENDENCIES); + internArtifactDescriptorManagedDependencies = ConfigUtils.getBoolean( + session, false, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES); + InternPool artifactsPool = null; InternPool dependenciesPool = null; InternPool descriptorsPool = null; @@ -185,8 +200,12 @@ public ArtifactDescriptorResult getDescriptor(DescriptorKey key, ArtifactDescrip } public void putDescriptor(DescriptorKey key, ArtifactDescriptorResult result) { - result.setDependencies(intern(result.getDependencies())); - result.setManagedDependencies(intern(result.getManagedDependencies())); + if (internArtifactDescriptorDependencies) { + result.setDependencies(intern(result.getDependencies())); + } + if (internArtifactDescriptorManagedDependencies) { + result.setManagedDependencies(intern(result.getManagedDependencies())); + } descriptors.intern(key, new GoodDescriptor(result)); } From b4ebda06965a29e9cb045809fb0e3284062a7f8d Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Fri, 2 Aug 2024 13:04:38 +0200 Subject: [PATCH 3/3] Make default false/true Intern only managed deps. Also simplify a bit. --- .../eclipse/aether/internal/impl/collect/DataPool.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java index 7ee696fda..40461f42f 100644 --- a/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java +++ b/maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/DataPool.java @@ -119,7 +119,7 @@ public DataPool(RepositorySystemSession session) { internArtifactDescriptorDependencies = ConfigUtils.getBoolean( session, false, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_DEPENDENCIES); internArtifactDescriptorManagedDependencies = ConfigUtils.getBoolean( - session, false, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES); + session, true, CONFIG_PROP_COLLECTOR_POOL_INTERN_ARTIFACT_DESCRIPTOR_MANAGED_DEPENDENCIES); InternPool artifactsPool = null; InternPool dependenciesPool = null; @@ -257,7 +257,7 @@ public static final class DescriptorKey { private DescriptorKey(Artifact artifact) { this.artifact = artifact; - this.hashCode = buildHashCode(); + this.hashCode = Objects.hashCode(artifact); } @Override @@ -277,10 +277,6 @@ public int hashCode() { return hashCode; } - private int buildHashCode() { - return Objects.hashCode(artifact); - } - @Override public String toString() { return getClass().getSimpleName() + "{" + "artifact='" + artifact + '\'' + '}';