diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java index fda60443dca2..848ee08019d2 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -37,6 +36,8 @@ import com.oracle.svm.core.heap.ObjectReferenceVisitor; import com.oracle.svm.core.util.DuplicatedInNativeCode; +import jdk.graal.compiler.word.Word; + /** * References from the runtime compiled code to the Java heap must be considered either strong or * weak references, depending on whether the code is currently on the execution stack. Otherwise, @@ -75,7 +76,7 @@ public boolean visitCode(CodeInfo codeInfo) { Object tether = UntetheredCodeInfoAccess.getTetherUnsafe(codeInfo); if (tether != null && !isReachable(tether)) { int state = CodeInfoAccess.getState(codeInfo); - if (state == CodeInfo.STATE_PARTIALLY_FREED) { + if (state == CodeInfo.STATE_INVALIDATED) { /* * The tether object is not reachable and the CodeInfo was already invalidated, so * we only need to visit references that will be accessed before the unmanaged diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeInfoGCSupportImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeInfoGCSupportImpl.java index a0f7c43b3d59..b3d37bc1d829 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeInfoGCSupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeInfoGCSupportImpl.java @@ -59,16 +59,4 @@ public void registerFrameMetadata(CodeInfo codeInfo) { public void registerDeoptMetadata(CodeInfo codeInfo) { // nothing to do (see above) } - - @Override - @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true) - public void unregisterCodeConstants(CodeInfo codeInfo) { - // nothing to do (see above) - } - - @Override - @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true) - public void unregisterRuntimeCodeInfo(CodeInfo codeInfo) { - // nothing to do (see above) - } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java index 0440b8703d2b..598ae00f97e0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java @@ -28,7 +28,6 @@ import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -48,6 +47,8 @@ import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.api.replacements.Fold; + /** * Parses a small subset of the runtime arguments before the image heap is mapped and before the * isolate is fully started. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfo.java index c7b4adb9f7ef..4094497c4893 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfo.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfo.java @@ -75,23 +75,22 @@ public interface CodeInfo extends UntetheredCodeInfo { int STATE_READY_FOR_INVALIDATION = STATE_NON_ENTRANT + 1; /** - * Indicates that this {@link CodeInfo} object was invalidated and parts of its data (including - * the code memory) were freed. The remaining data will be freed by the GC once the tether - * object becomes unreachable. Until then, the GC must continue visiting all heap references - * (except for the code constants as the code is no longer installed). + * Indicates that this {@link CodeInfo} object was invalidated. The data will be freed by the GC + * once the tether object becomes unreachable. Until then, the GC must continue visiting all + * heap references, including code constants that are directly embedded into the machine code. */ @DuplicatedInNativeCode // - int STATE_PARTIALLY_FREED = STATE_READY_FOR_INVALIDATION + 1; + int STATE_INVALIDATED = STATE_READY_FOR_INVALIDATION + 1; /** * This state is only a temporary state when the VM is at a safepoint. It indicates that a - * previously already partially freed {@link CodeInfo} object is no longer reachable from the GC - * point of view. The GC will free the {@link CodeInfo} object during the current safepoint. It - * is crucial that the GC still visits all heap references that may be accessed while freeing - * the {@link CodeInfo} object (i.e., all object fields). + * previously invalidated {@link CodeInfo} object is no longer reachable from the GC point of + * view. The GC will free the {@link CodeInfo} object during the current safepoint. It is + * crucial that the GC still visits all heap references that may be accessed while freeing the + * {@link CodeInfo} object (i.e., all object fields). */ @DuplicatedInNativeCode // - int STATE_UNREACHABLE = STATE_PARTIALLY_FREED + 1; + int STATE_UNREACHABLE = STATE_INVALIDATED + 1; /** * Indicates that the {@link CodeInfo} object was already freed. This state should never be diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java index 0e391a5c34e9..bfb2d6194adb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java @@ -157,8 +157,8 @@ public static String stateToString(int codeInfoState) { return "non-entrant"; case CodeInfo.STATE_READY_FOR_INVALIDATION: return "ready for invalidation"; - case CodeInfo.STATE_PARTIALLY_FREED: - return "partially freed"; + case CodeInfo.STATE_INVALIDATED: + return "invalidated"; case CodeInfo.STATE_UNREACHABLE: return "unreachable"; case CodeInfo.STATE_FREED: diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoImpl.java index d3abe8b94d8d..67172d1f53ab 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoImpl.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.code; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawFieldOffset; @@ -39,6 +38,7 @@ import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.word.Word; import jdk.vm.ci.code.InstalledCode; /** @@ -49,9 +49,12 @@ * As {@link CodeInfo} objects have a complicated life-cycle that also involves the GC, it is * crucial that all places that access or use this data observe the following rules: * diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java index 87b821c755d9..4328ae0ace38 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java @@ -217,7 +217,7 @@ private static void invalidateInstalledCodeAtSafepoint(SubstrateInstalledCode in if (CodeInfoAccess.isAlive(info)) { invalidateCodeAtSafepoint0(info); } - assert CodeInfoAccess.getState(info) == CodeInfo.STATE_PARTIALLY_FREED; + assert CodeInfoAccess.getState(info) == CodeInfo.STATE_INVALIDATED; } finally { CodeInfoAccess.releaseTether(untetheredInfo, tether); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java index ab304e8edca8..20bfca0c158a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java @@ -28,9 +28,6 @@ import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer; import org.graalvm.collections.EconomicMap; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionKey; -import jdk.graal.compiler.options.OptionType; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; @@ -57,6 +54,10 @@ import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.Counter; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionType; + public class RuntimeCodeCache { public static class Options { @@ -210,14 +211,14 @@ protected void invalidateMethod(CodeInfo info) { */ Deoptimizer.deoptimizeInRange(CodeInfoAccess.getCodeStart(info), CodeInfoAccess.getCodeEnd(info), false); - finishInvalidation(info, true); + finishInvalidation(info); } protected void invalidateNonStackMethod(CodeInfo info) { assert VMOperation.isGCInProgress() : "may only be called by the GC"; prepareInvalidation(info); assert codeNotOnStackVerifier.verify(info); - finishInvalidation(info, false); + finishInvalidation(info); } private void prepareInvalidation(CodeInfo info) { @@ -236,14 +237,14 @@ private void prepareInvalidation(CodeInfo info) { } } - private void finishInvalidation(CodeInfo info, boolean notifyGC) { + private void finishInvalidation(CodeInfo info) { InstalledCodeObserverSupport.removeObservers(RuntimeCodeInfoAccess.getCodeObserverHandles(info)); - finishInvalidation0(info, notifyGC); + finishInvalidation0(info); RuntimeCodeInfoHistory.singleton().logInvalidate(info); } @Uninterruptible(reason = "Modifying code tables that are used by the GC") - private void finishInvalidation0(CodeInfo info, boolean notifyGC) { + private void finishInvalidation0(CodeInfo info) { /* * Now it is guaranteed that the InstalledCode is not on the stack and cannot be invoked * anymore, so we can free the code and all metadata. @@ -256,7 +257,7 @@ private void finishInvalidation0(CodeInfo info, boolean notifyGC) { numCodeInfos--; NonmovableArrays.setWord(codeInfos, numCodeInfos, WordFactory.nullPointer()); - RuntimeCodeInfoAccess.freePartially(info, notifyGC); + RuntimeCodeInfoAccess.markAsInvalidated(info); assert verifyTable(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java index 762942332d12..e27a627fe3f6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java @@ -204,23 +204,11 @@ public static CodeInfo allocateMethodInfo(NonmovableObjectArray objectDa } @Uninterruptible(reason = "Prevent the GC from running - otherwise, it could accidentally visit the freed memory.") - static void freePartially(CodeInfo info, boolean notifyGC) { + static void markAsInvalidated(CodeInfo info) { CodeInfoImpl impl = cast(info); assert CodeInfoAccess.isAliveState(impl.getState()) || impl.getState() == CodeInfo.STATE_READY_FOR_INVALIDATION : "unexpected state (probably already released)"; - if (notifyGC) { - // Notify the GC as long as the object data is still valid. - Heap.getHeap().getRuntimeCodeInfoGCSupport().unregisterCodeConstants(info); - } - - NonmovableArrays.releaseUnmanagedArray(impl.getCodeObserverHandles()); - impl.setCodeObserverHandles(NonmovableArrays.nullArray()); - - releaseCodeMemory(impl.getCodeStart(), impl.getCodeAndDataMemorySize()); - /* - * Note that we must not null-out any CodeInfo metadata as it can be accessed in a stack - * walk even when the CodeInfo data is already partially freed. - */ - CodeInfoAccess.setState(info, CodeInfo.STATE_PARTIALLY_FREED); + /* We can't free any data because only the GC is allowed to free CodeInfo data. */ + CodeInfoAccess.setState(info, CodeInfo.STATE_INVALIDATED); } public static CodePointer allocateCodeMemory(UnsignedWord size) { @@ -245,7 +233,7 @@ static void releaseMethodInfoOnTearDown(CodeInfo info) { InstalledCodeObserverSupport.removeObserversOnTearDown(getCodeObserverHandles(info)); assert ((CodeInfoTether) UntetheredCodeInfoAccess.getTetherUnsafe(info)).getCount() == 1 : "CodeInfo tether must not be referenced by non-teardown code."; - free(info, true); + free(info); } public interface NonmovableArrayAction { @@ -262,16 +250,14 @@ public void apply(NonmovableArray array) { }; @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void free(CodeInfo info, boolean notifyGC) { + public static void free(CodeInfo info) { CodeInfoImpl impl = cast(info); - if (CodeInfoAccess.isAliveState(impl.getState()) || impl.getState() == CodeInfo.STATE_READY_FOR_INVALIDATION) { - freePartially(info, notifyGC); - } - if (notifyGC) { - // Notify the GC as long as the object data is still valid. - Heap.getHeap().getRuntimeCodeInfoGCSupport().unregisterRuntimeCodeInfo(info); - } + /* Free the code observers handles unconditionally (they are never in the image heap). */ + NonmovableArrays.releaseUnmanagedArray(impl.getCodeObserverHandles()); + impl.setCodeObserverHandles(NonmovableArrays.nullArray()); + + releaseCodeMemory(impl.getCodeStart(), impl.getCodeAndDataMemorySize()); if (!impl.getAllObjectsAreInImageHeap()) { forEachArray(info, RELEASE_ACTION); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index 192394d06ab4..677a26df2e17 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -26,7 +26,6 @@ import java.util.concurrent.locks.ReentrantLock; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -46,6 +45,8 @@ import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.api.replacements.Fold; + /** * Keeps track of {@link CodeInfo} structures of runtime-compiled methods (including invalidated and * not yet freed ones) and releases their memory on tear-down. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java index 384438ec486c..d81daf130498 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java @@ -72,7 +72,7 @@ public boolean visitCode(CodeInfo codeInfo) { } else if (state == CodeInfo.STATE_READY_FOR_INVALIDATION) { // All objects that are accessed during invalidation must still be reachable. CodeInfoTable.invalidateNonStackCodeAtSafepoint(codeInfo); - assert CodeInfoAccess.getState(codeInfo) == CodeInfo.STATE_PARTIALLY_FREED; + assert CodeInfoAccess.getState(codeInfo) == CodeInfo.STATE_INVALIDATED; freeMemory(codeInfo); } return true; @@ -82,6 +82,6 @@ private static void freeMemory(CodeInfo codeInfo) { boolean removed = RuntimeCodeInfoMemory.singleton().removeDuringGC(codeInfo); assert removed : "must have been present"; RuntimeCodeInfoHistory.singleton().logFree(codeInfo); - RuntimeCodeInfoAccess.free(codeInfo, false); + RuntimeCodeInfoAccess.free(codeInfo); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeInfoGCSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeInfoGCSupport.java index 5f2198e1e5c7..e02a237a08fa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeInfoGCSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeInfoGCSupport.java @@ -53,20 +53,4 @@ public abstract class RuntimeCodeInfoGCSupport { */ @Uninterruptible(reason = "Called when installing code.", callerMustBe = true) public abstract void registerDeoptMetadata(CodeInfo codeInfo); - - /** - * Notify the GC that the application is going to invalidate run-time compiled code that has - * embedded references to Java heap objects. This notification must not be triggered if the GC - * itself frees a code metadata object. - */ - @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true) - public abstract void unregisterCodeConstants(CodeInfo codeInfo); - - /** - * Notify the GC that the application is going to free a code metadata object that references - * Java heap objects from native-memory. This notification must not be triggered if the GC - * itself frees a code metadata object. - */ - @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true) - public abstract void unregisterRuntimeCodeInfo(CodeInfo codeInfo); }