[GR-48811] Change lifecycle of JIT-compiled code.

PullRequest: graal/16377

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -875,14 +875,11 @@ private void walkStack(JavaStackWalk walk) {
 
             if (RuntimeCompilation.isEnabled() && codeInfo != CodeInfoTable.getImageCodeInfo()) {
                 /*
-                 * For runtime-compiled code that is currently on the stack, we need to treat all
-                 * the references to Java heap objects as strong references. It is important that we
-                 * really walk *all* those references here. Otherwise, RuntimeCodeCacheWalker might
-                 * decide to invalidate too much code, depending on the order in which the CodeInfo
-                 * objects are visited.
+                 * Runtime-compiled code that is currently on the stack must be kept alive. So, we
+                 * mark the tether as strongly reachable. The RuntimeCodeCacheWalker will handle all
+                 * other object references later on.
                  */
-                RuntimeCodeInfoAccess.walkStrongReferences(codeInfo, greyToBlackObjRefVisitor);
-                RuntimeCodeInfoAccess.walkWeakReferences(codeInfo, greyToBlackObjRefVisitor);
+                RuntimeCodeInfoAccess.walkTether(codeInfo, greyToBlackObjRefVisitor);
             }
 
             if (!JavaStackWalker.continueWalk(walk, queryResult, deoptFrame)) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeCacheWalker.java

@@ -24,7 +24,6 @@
  */
 package com.oracle.svm.core.genscavenge;
 
-import jdk.graal.compiler.word.Word;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
@@ -37,6 +36,8 @@
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.util.DuplicatedInNativeCode;
 
+import jdk.graal.compiler.word.Word;
+
 /**
  * References from the runtime compiled code to the Java heap must be considered either strong or
  * weak references, depending on whether the code is currently on the execution stack. Otherwise,
@@ -75,7 +76,7 @@ public boolean visitCode(CodeInfo codeInfo) {
         Object tether = UntetheredCodeInfoAccess.getTetherUnsafe(codeInfo);
         if (tether != null && !isReachable(tether)) {
             int state = CodeInfoAccess.getState(codeInfo);
-            if (state == CodeInfo.STATE_PARTIALLY_FREED) {
+            if (state == CodeInfo.STATE_INVALIDATED) {
                 /*
                  * The tether object is not reachable and the CodeInfo was already invalidated, so
                  * we only need to visit references that will be accessed before the unmanaged

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RuntimeCodeInfoGCSupportImpl.java

@@ -59,16 +59,4 @@ public void registerFrameMetadata(CodeInfo codeInfo) {
     public void registerDeoptMetadata(CodeInfo codeInfo) {
         // nothing to do (see above)
     }
-
-    @Override
-    @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true)
-    public void unregisterCodeConstants(CodeInfo codeInfo) {
-        // nothing to do (see above)
-    }
-
-    @Override
-    @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true)
-    public void unregisterRuntimeCodeInfo(CodeInfo codeInfo) {
-        // nothing to do (see above)
-    }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java

@@ -28,7 +28,6 @@
 import java.nio.ByteOrder;
 import java.nio.charset.StandardCharsets;
 
-import jdk.graal.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -48,6 +47,8 @@
 import com.oracle.svm.core.option.RuntimeOptionKey;
 import com.oracle.svm.core.util.VMError;
 
+import jdk.graal.compiler.api.replacements.Fold;
+
 /**
  * Parses a small subset of the runtime arguments before the image heap is mapped and before the
  * isolate is fully started.

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfo.java

@@ -75,23 +75,22 @@ public interface CodeInfo extends UntetheredCodeInfo {
     int STATE_READY_FOR_INVALIDATION = STATE_NON_ENTRANT + 1;
 
     /**
-     * Indicates that this {@link CodeInfo} object was invalidated and parts of its data (including
-     * the code memory) were freed. The remaining data will be freed by the GC once the tether
-     * object becomes unreachable. Until then, the GC must continue visiting all heap references
-     * (except for the code constants as the code is no longer installed).
+     * Indicates that this {@link CodeInfo} object was invalidated. The data will be freed by the GC
+     * once the tether object becomes unreachable. Until then, the GC must continue visiting all
+     * heap references, including code constants that are directly embedded into the machine code.
      */
     @DuplicatedInNativeCode //
-    int STATE_PARTIALLY_FREED = STATE_READY_FOR_INVALIDATION + 1;
+    int STATE_INVALIDATED = STATE_READY_FOR_INVALIDATION + 1;
 
     /**
      * This state is only a temporary state when the VM is at a safepoint. It indicates that a
-     * previously already partially freed {@link CodeInfo} object is no longer reachable from the GC
-     * point of view. The GC will free the {@link CodeInfo} object during the current safepoint. It
-     * is crucial that the GC still visits all heap references that may be accessed while freeing
-     * the {@link CodeInfo} object (i.e., all object fields).
+     * previously invalidated {@link CodeInfo} object is no longer reachable from the GC point of
+     * view. The GC will free the {@link CodeInfo} object during the current safepoint. It is
+     * crucial that the GC still visits all heap references that may be accessed while freeing the
+     * {@link CodeInfo} object (i.e., all object fields).
      */
     @DuplicatedInNativeCode //
-    int STATE_UNREACHABLE = STATE_PARTIALLY_FREED + 1;
+    int STATE_UNREACHABLE = STATE_INVALIDATED + 1;
 
     /**
      * Indicates that the {@link CodeInfo} object was already freed. This state should never be

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoAccess.java

@@ -157,8 +157,8 @@ public static String stateToString(int codeInfoState) {
                 return "non-entrant";
             case CodeInfo.STATE_READY_FOR_INVALIDATION:
                 return "ready for invalidation";
-            case CodeInfo.STATE_PARTIALLY_FREED:
-                return "partially freed";
+            case CodeInfo.STATE_INVALIDATED:
+                return "invalidated";
             case CodeInfo.STATE_UNREACHABLE:
                 return "unreachable";
             case CodeInfo.STATE_FREED:

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoImpl.java

@@ -24,7 +24,6 @@
  */
 package com.oracle.svm.core.code;
 
-import jdk.graal.compiler.word.Word;
 import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.nativeimage.c.struct.RawField;
 import org.graalvm.nativeimage.c.struct.RawFieldOffset;
@@ -39,6 +38,7 @@
 import com.oracle.svm.core.util.DuplicatedInNativeCode;
 import com.oracle.svm.core.util.VMError;
 
+import jdk.graal.compiler.word.Word;
 import jdk.vm.ci.code.InstalledCode;
 
 /**
@@ -49,9 +49,12 @@
  * As {@link CodeInfo} objects have a complicated life-cycle that also involves the GC, it is
  * crucial that all places that access or use this data observe the following rules:
  * <ul>
- * <li>When heap objects are stored into native-memory that is referenced by a {@link CodeInfo}
- * object, then it is necessary to notify the GC about that (see
- * {@link RuntimeCodeInfoGCSupport}).</li>
+ * <li>{@link CodeInfo} objects use native memory and store references to Java heap objects in that
+ * native-memory. The VM must notify the GC about all Java heap references (see
+ * {@link RuntimeCodeInfoGCSupport}) so that the GC can update the references when it moves objects.
+ * Once the VM notified the GC about a Java heap reference in native memory, the VM must no longer
+ * overwrite, null-out, or free that memory (only the GC is allowed to do that during code unloading
+ * at a safepoint).</li>
  * <li><b>NEVER</b> do a direct cast from {@link UntetheredCodeInfo} or {@link CodeInfo} to
  * {@link CodeInfoImpl}. For more details, refer to the {@link CodeInfoAccess} documentation.</li>
  * </ul>

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java

@@ -217,7 +217,7 @@ private static void invalidateInstalledCodeAtSafepoint(SubstrateInstalledCode in
             if (CodeInfoAccess.isAlive(info)) {
                 invalidateCodeAtSafepoint0(info);
             }
-            assert CodeInfoAccess.getState(info) == CodeInfo.STATE_PARTIALLY_FREED;
+            assert CodeInfoAccess.getState(info) == CodeInfo.STATE_INVALIDATED;
         } finally {
             CodeInfoAccess.releaseTether(untetheredInfo, tether);
         }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java

@@ -28,9 +28,6 @@
 import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer;
 
 import org.graalvm.collections.EconomicMap;
-import jdk.graal.compiler.options.Option;
-import jdk.graal.compiler.options.OptionKey;
-import jdk.graal.compiler.options.OptionType;
 import org.graalvm.nativeimage.CurrentIsolate;
 import org.graalvm.nativeimage.IsolateThread;
 import org.graalvm.nativeimage.Platform;
@@ -57,6 +54,10 @@
 import com.oracle.svm.core.thread.VMThreads;
 import com.oracle.svm.core.util.Counter;
 
+import jdk.graal.compiler.options.Option;
+import jdk.graal.compiler.options.OptionKey;
+import jdk.graal.compiler.options.OptionType;
+
 public class RuntimeCodeCache {
 
     public static class Options {
@@ -210,14 +211,14 @@ protected void invalidateMethod(CodeInfo info) {
          */
         Deoptimizer.deoptimizeInRange(CodeInfoAccess.getCodeStart(info), CodeInfoAccess.getCodeEnd(info), false);
 
-        finishInvalidation(info, true);
+        finishInvalidation(info);
     }
 
     protected void invalidateNonStackMethod(CodeInfo info) {
         assert VMOperation.isGCInProgress() : "may only be called by the GC";
         prepareInvalidation(info);
         assert codeNotOnStackVerifier.verify(info);
-        finishInvalidation(info, false);
+        finishInvalidation(info);
     }
 
     private void prepareInvalidation(CodeInfo info) {
@@ -236,14 +237,14 @@ private void prepareInvalidation(CodeInfo info) {
         }
     }
 
-    private void finishInvalidation(CodeInfo info, boolean notifyGC) {
+    private void finishInvalidation(CodeInfo info) {
         InstalledCodeObserverSupport.removeObservers(RuntimeCodeInfoAccess.getCodeObserverHandles(info));
-        finishInvalidation0(info, notifyGC);
+        finishInvalidation0(info);
         RuntimeCodeInfoHistory.singleton().logInvalidate(info);
     }
 
     @Uninterruptible(reason = "Modifying code tables that are used by the GC")
-    private void finishInvalidation0(CodeInfo info, boolean notifyGC) {
+    private void finishInvalidation0(CodeInfo info) {
         /*
          * Now it is guaranteed that the InstalledCode is not on the stack and cannot be invoked
          * anymore, so we can free the code and all metadata.
@@ -256,7 +257,7 @@ private void finishInvalidation0(CodeInfo info, boolean notifyGC) {
         numCodeInfos--;
         NonmovableArrays.setWord(codeInfos, numCodeInfos, WordFactory.nullPointer());
 
-        RuntimeCodeInfoAccess.freePartially(info, notifyGC);
+        RuntimeCodeInfoAccess.markAsInvalidated(info);
         assert verifyTable();
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java

@@ -179,6 +179,21 @@ public static boolean walkWeakReferences(CodeInfo info, ObjectReferenceVisitor v
         return continueVisiting;
     }
 
+    /**
+     * This method only walks the tether. You typically want to use {@link #walkStrongReferences}
+     * and/or {@link #walkWeakReferences} instead.
+     */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean walkTether(CodeInfo info, ObjectReferenceVisitor visitor) {
+        Pointer address = NonmovableArrays.addressOf(cast(info).getObjectFields(), CodeInfoImpl.TETHER_OBJFIELD);
+        return callVisitor(visitor, address);
+    }
+
+    @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false)
+    private static boolean callVisitor(ObjectReferenceVisitor visitor, Pointer address) {
+        return visitor.visitObjectReference(address, true, null);
+    }
+
     /**
      * This method only visits a very specific subset of all the references, so you typically want
      * to use {@link #walkStrongReferences} and/or {@link #walkWeakReferences} instead.
@@ -204,23 +219,11 @@ public static CodeInfo allocateMethodInfo(NonmovableObjectArray<Object> objectDa
     }
 
     @Uninterruptible(reason = "Prevent the GC from running - otherwise, it could accidentally visit the freed memory.")
-    static void freePartially(CodeInfo info, boolean notifyGC) {
+    static void markAsInvalidated(CodeInfo info) {
         CodeInfoImpl impl = cast(info);
         assert CodeInfoAccess.isAliveState(impl.getState()) || impl.getState() == CodeInfo.STATE_READY_FOR_INVALIDATION : "unexpected state (probably already released)";
-        if (notifyGC) {
-            // Notify the GC as long as the object data is still valid.
-            Heap.getHeap().getRuntimeCodeInfoGCSupport().unregisterCodeConstants(info);
-        }
-
-        NonmovableArrays.releaseUnmanagedArray(impl.getCodeObserverHandles());
-        impl.setCodeObserverHandles(NonmovableArrays.nullArray());
-
-        releaseCodeMemory(impl.getCodeStart(), impl.getCodeAndDataMemorySize());
-        /*
-         * Note that we must not null-out any CodeInfo metadata as it can be accessed in a stack
-         * walk even when the CodeInfo data is already partially freed.
-         */
-        CodeInfoAccess.setState(info, CodeInfo.STATE_PARTIALLY_FREED);
+        /* We can't free any data because only the GC is allowed to free CodeInfo data. */
+        CodeInfoAccess.setState(info, CodeInfo.STATE_INVALIDATED);
     }
 
     public static CodePointer allocateCodeMemory(UnsignedWord size) {
@@ -245,7 +248,7 @@ static void releaseMethodInfoOnTearDown(CodeInfo info) {
         InstalledCodeObserverSupport.removeObserversOnTearDown(getCodeObserverHandles(info));
 
         assert ((CodeInfoTether) UntetheredCodeInfoAccess.getTetherUnsafe(info)).getCount() == 1 : "CodeInfo tether must not be referenced by non-teardown code.";
-        free(info, true);
+        free(info);
     }
 
     public interface NonmovableArrayAction {
@@ -262,16 +265,14 @@ public void apply(NonmovableArray<?> array) {
     };
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static void free(CodeInfo info, boolean notifyGC) {
+    public static void free(CodeInfo info) {
         CodeInfoImpl impl = cast(info);
-        if (CodeInfoAccess.isAliveState(impl.getState()) || impl.getState() == CodeInfo.STATE_READY_FOR_INVALIDATION) {
-            freePartially(info, notifyGC);
-        }
 
-        if (notifyGC) {
-            // Notify the GC as long as the object data is still valid.
-            Heap.getHeap().getRuntimeCodeInfoGCSupport().unregisterRuntimeCodeInfo(info);
-        }
+        /* Free the code observers handles unconditionally (they are never in the image heap). */
+        NonmovableArrays.releaseUnmanagedArray(impl.getCodeObserverHandles());
+        impl.setCodeObserverHandles(NonmovableArrays.nullArray());
+
+        releaseCodeMemory(impl.getCodeStart(), impl.getCodeAndDataMemorySize());
 
         if (!impl.getAllObjectsAreInImageHeap()) {
             forEachArray(info, RELEASE_ACTION);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java

@@ -26,7 +26,6 @@
 
 import java.util.concurrent.locks.ReentrantLock;
 
-import jdk.graal.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -46,6 +45,8 @@
 import com.oracle.svm.core.thread.VMOperation;
 import com.oracle.svm.core.util.VMError;
 
+import jdk.graal.compiler.api.replacements.Fold;
+
 /**
  * Keeps track of {@link CodeInfo} structures of runtime-compiled methods (including invalidated and
  * not yet freed ones) and releases their memory on tear-down.

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/SubstrateSpeculationLog.java

@@ -65,11 +65,11 @@ private LogEntry(SpeculationReason reason, LogEntry next) {
 
     public void addFailedSpeculation(SpeculationReason speculation) {
         /*
-         * This method is called from inside the VMOperation that performs deoptimization, and
-         * thefore must not be synchronization free. Note that this even precludes using a
+         * This method may be called from inside a VMOperation that performs deoptimization, and
+         * therefore must be synchronization free. Note that this even precludes using a
          * ConcurrentHashMap, because it also has some code paths that require synchronization.
          *
-         * Therefore we use our own very simple atomic linked list.
+         * Therefore, we use our own very simple atomic linked list.
          */
         while (true) {
             LogEntry oldHead = addedFailedSpeculationsHead;

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeCacheCleaner.java

@@ -50,7 +50,7 @@ public final class RuntimeCodeCacheCleaner implements CodeInfoVisitor {
      * <ul>
      * <li>{@link SpeculationReason} objects are embedded in the code and only needed when a
      * deoptimization is triggered.</li>
-     * <li>{@link SharedRuntimeMethod} objects are sometimes used as artifical methods (e.g., for
+     * <li>{@link SharedRuntimeMethod} objects are sometimes used as artificial methods (e.g., for
      * adapter code) and are located in the frame info object constants.</li>
      * </ul>
      */
@@ -72,7 +72,7 @@ public boolean visitCode(CodeInfo codeInfo) {
         } else if (state == CodeInfo.STATE_READY_FOR_INVALIDATION) {
             // All objects that are accessed during invalidation must still be reachable.
             CodeInfoTable.invalidateNonStackCodeAtSafepoint(codeInfo);
-            assert CodeInfoAccess.getState(codeInfo) == CodeInfo.STATE_PARTIALLY_FREED;
+            assert CodeInfoAccess.getState(codeInfo) == CodeInfo.STATE_INVALIDATED;
             freeMemory(codeInfo);
         }
         return true;
@@ -82,6 +82,6 @@ private static void freeMemory(CodeInfo codeInfo) {
         boolean removed = RuntimeCodeInfoMemory.singleton().removeDuringGC(codeInfo);
         assert removed : "must have been present";
         RuntimeCodeInfoHistory.singleton().logFree(codeInfo);
-        RuntimeCodeInfoAccess.free(codeInfo, false);
+        RuntimeCodeInfoAccess.free(codeInfo);
     }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/RuntimeCodeInfoGCSupport.java

@@ -53,20 +53,4 @@ public abstract class RuntimeCodeInfoGCSupport {
      */
     @Uninterruptible(reason = "Called when installing code.", callerMustBe = true)
     public abstract void registerDeoptMetadata(CodeInfo codeInfo);
-
-    /**
-     * Notify the GC that the application is going to invalidate run-time compiled code that has
-     * embedded references to Java heap objects. This notification must not be triggered if the GC
-     * itself frees a code metadata object.
-     */
-    @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true)
-    public abstract void unregisterCodeConstants(CodeInfo codeInfo);
-
-    /**
-     * Notify the GC that the application is going to free a code metadata object that references
-     * Java heap objects from native-memory. This notification must not be triggered if the GC
-     * itself frees a code metadata object.
-     */
-    @Uninterruptible(reason = "Called when freeing code.", callerMustBe = true)
-    public abstract void unregisterRuntimeCodeInfo(CodeInfo codeInfo);
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/stack/JavaStackWalker.java

@@ -251,7 +251,7 @@ public static boolean continueWalk(JavaStackWalk walk, SimpleCodeInfoQueryResult
     @Uninterruptible(reason = "Not really uninterruptible, but we are about to fatally fail.", calleeMustBe = false)
     public static RuntimeException reportUnknownFrameEncountered(Pointer sp, CodePointer ip, DeoptimizedFrame deoptFrame) {
         Log log = Log.log().string("Stack walk must walk only frames of known code:");
-        log.string("  sp=").hex(sp).string("  ip=").hex(ip);
+        log.string("  sp=").zhex(sp).string("  ip=").zhex(ip);
         if (DeoptimizationSupport.enabled()) {
             log.string("  deoptFrame=").object(deoptFrame);
         }