Implement Truffle compiler control based on HotSpot's CompileBroker compilation activity

truffle/src/com.oracle.truffle.compiler/src/com/oracle/truffle/compiler/TruffleCompilerRuntime.java

@@ -359,4 +359,31 @@ default ResolvedJavaType resolveType(MetaAccessProvider metaAccess, String class
      */
     boolean isSuppressedFailure(TruffleCompilable compilable, Supplier<String> serializedException);
 
+    /**
+     * Represents HotSpot's compilation activity mode which is one of:
+     * {@code stop_compilation = 0}, {@code run_compilation = 1} or {@code shutdown_compilation = 2}
+     * Should be in sync with the {@code CompilerActivity} enum in {@code hotspot/share/compiler/compileBroker.hpp}
+     */
+    enum CompilationActivityMode {
+        STOP_COMPILATION,
+        RUN_COMPILATION,
+        SHUTDOWN_COMPILATION;
+
+        static public CompilationActivityMode fromInteger(int i) {
+            return switch (i) {
+                case 0 -> STOP_COMPILATION;
+                case 1 -> RUN_COMPILATION;
+                case 2 -> SHUTDOWN_COMPILATION;
+                default -> throw new RuntimeException("Invalid CompilationActivityMode " + i);
+            };
+        }
+    }
+
+    /**
+     * Returns the current host compilation activity mode which is one of:
+     * {@code STOP_COMPILATION}, {@code RUN_COMPILATION} or {@code SHUTDOWN_COMPILATION}
+     */
+     default CompilationActivityMode getCompilationActivityMode() {
+         return CompilationActivityMode.RUN_COMPILATION;
+     }
 }

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedRuntimeOptions.java

@@ -158,6 +158,14 @@ public ExceptionAction apply(String s) {
     // TODO: GR-29949
     public static final OptionKey<Long> CompilerIdleDelay = new OptionKey<>(10000L);
 
+    @Option(help = "Before the Truffle runtime submits an OptimizedCallTarget for compilation, it checks for the " +
+                   "compilation activity mode in the host VM. If the activity mode is 'STOP_COMPILATION' because " +
+                   "of a full code cache, no new compilation requests are submitted and the compilation queue is flushed. " +
+                   "After 'StoppedCompilationRetryDelay' milliseconds new compilations will be submitted again " +
+                   "(which might trigger a sweep of the code cache and a reset of the compilation activity mode in the host JVM).",
+                   usageSyntax = "<ms>", category = OptionCategory.EXPERT)
+    public static final OptionKey<Long> StoppedCompilationRetryDelay = new OptionKey<>(1000L);
+
     @Option(help = "Manually set the number of compiler threads. By default, the number of compiler threads is scaled with the number of available cores on the CPU.", usageSyntax = "[1, inf)", category = OptionCategory.EXPERT, //
                     stability = OptionStability.STABLE, sandbox = SandboxPolicy.UNTRUSTED) //
     public static final OptionKey<Integer> CompilerThreads = new OptionKey<>(-1);

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedTruffleRuntime.java

@@ -41,6 +41,7 @@
 package com.oracle.truffle.runtime;
 
 import static com.oracle.truffle.runtime.OptimizedRuntimeOptions.CompilerIdleDelay;
+import static com.oracle.truffle.runtime.OptimizedRuntimeOptions.StoppedCompilationRetryDelay;
 
 import java.io.CharArrayWriter;
 import java.io.PrintWriter;
@@ -912,10 +913,45 @@ private void notifyCompilationFailure(OptimizedCallTarget callTarget, Throwable
     protected void onEngineCreated(EngineData engine) {
     }
 
+    private long stoppedCompilationTime = 0;
+    private boolean logShutdownCompilations = true;
+
     @SuppressWarnings("try")
     public CompilationTask submitForCompilation(OptimizedCallTarget optimizedCallTarget, boolean lastTierCompilation) {
-        Priority priority = new Priority(optimizedCallTarget.getCallAndLoopCount(), lastTierCompilation ? Priority.Tier.LAST : Priority.Tier.FIRST);
-        return getCompileQueue().submitCompilation(priority, optimizedCallTarget);
+        BackgroundCompileQueue queue = getCompileQueue();
+        CompilationActivityMode compilationActivityMode = getCompilationActivityMode();
+        if (compilationActivityMode == CompilationActivityMode.RUN_COMPILATION ||
+                (stoppedCompilationTime != 0 && System.currentTimeMillis() - stoppedCompilationTime > optimizedCallTarget.getOptionValue(StoppedCompilationRetryDelay))) {
+            stoppedCompilationTime = 0;
+            Priority priority = new Priority(optimizedCallTarget.getCallAndLoopCount(), lastTierCompilation ? Priority.Tier.LAST : Priority.Tier.FIRST);
+            return queue.submitCompilation(priority, optimizedCallTarget);
+        } else if (compilationActivityMode == CompilationActivityMode.STOP_COMPILATION) {
+            if (stoppedCompilationTime == 0) {
+                stoppedCompilationTime = System.currentTimeMillis();
+            }
+            // Flush the compilations queue. There's still a chance that compilation will be re-enabled
+            // eventually, if the hosts code cache can be cleaned up.
+            for (OptimizedCallTarget target : queue.getQueuedTargets(optimizedCallTarget.engine)) {
+                target.cancelCompilation("Compilation temporary disabled due to full code cache.");
+            }
+        } else {
+            // Compilation was shut down permanently because the hosts code cache ran full and
+            // the host was configured without support for code cache sweeping.
+            assert compilationActivityMode == CompilationActivityMode.SHUTDOWN_COMPILATION;
+            TruffleLogger logger = optimizedCallTarget.engine.getLogger("engine");
+            // The logger can be null if the engine is closed.
+            if (logger != null && logShutdownCompilations) {
+                logShutdownCompilations = false;
+                logger.log(Level.WARNING, "Truffle host compilations permanently disabled because of full code cache. "  +
+                    "Increase the code cache size using '-XX:ReservedCodeCacheSize=' and/or run with '-XX:+UseCodeCacheFlushing -XX:+MethodFlushing'.");
+            }
+            try {
+                queue.shutdownAndAwaitTermination(100 /* milliseconds */);
+            } catch (RuntimeException re) {
+                // Best effort, ignore failure
+            }
+        }
+        return null;
     }
 
     @SuppressWarnings("all")

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/hotspot/HotSpotTruffleRuntime.java

@@ -40,6 +40,9 @@
  */
 package com.oracle.truffle.runtime.hotspot;
 
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 import java.lang.ref.Reference;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
@@ -692,4 +695,30 @@ public boolean isLibGraalCompilationEnabled() {
         return compilationSupport instanceof LibGraalTruffleCompilationSupport;
     }
 
+    static MethodHandle getCompilationActivityMode;
+    static {
+        try {
+            MethodType mt = MethodType.methodType(int.class);
+            getCompilationActivityMode = MethodHandles.lookup().findVirtual(HotSpotJVMCIRuntime.class, "getCompilationActivityMode", mt);
+        } catch (NoSuchMethodException | IllegalAccessException e) {
+            // Older JVMCI runtimes might not support `getCompilationActivityMode()`
+        }
+    }
+
+    /**
+     * Returns the current host compilation activity mode which is one of:
+     * {@code stop_compilation = 0}, {@code run_compilation = 1} or {@code shutdown_compilation = 2}
+     */
+    @Override
+    public CompilationActivityMode getCompilationActivityMode() {
+        int activityMode = 1; // Default is to run compilations
+        if (getCompilationActivityMode != null) {
+            try {
+                activityMode = (int) getCompilationActivityMode.invokeExact(HotSpotJVMCIRuntime.runtime());
+            } catch (Throwable t) {
+                // Ignore and go with the default mode
+            }
+        }
+        return CompilationActivityMode.fromInteger(activityMode);
+    }
 }