Add support for different thresholds per GraalVM version

`ImageMetricsITCase` tests are skipped for versions with undefined thresholds
quarkusio · Dec 14, 2023 · 612d816 · 612d816
1 parent d4d2392
commit 612d816
Show file tree

Hide file tree

Showing 12 changed files with 147 additions and 35 deletions.
diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/GraalVM.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/GraalVM.java
@@ -297,6 +297,13 @@ public String getVersionAsString() {
             return version;
         }
 
+        public String getMajorMinorAsString() {
+            if (versions.length >= 2) {
+                return versions[0] + "." + versions[1];
+            }
+            return versions[0] + ".0";
+        }
+
         @Override
         public String toString() {
             return "Version{" +

diff --git a/...jpa-postgresql-withxml/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java b/...jpa-postgresql-withxml/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java
@@ -1,15 +1,17 @@
 package io.quarkus.it.jpa.postgresql;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 
 import io.quarkus.test.junit.QuarkusIntegrationTest;
-import io.quarkus.test.junit.nativeimage.BuildOutput;
+import io.quarkus.test.junit.nativeimage.BuildOutputExtension;
 
+@ExtendWith(BuildOutputExtension.class)
 @QuarkusIntegrationTest
 public class ImageMetricsITCase {
     @Test
     public void verifyImageMetrics() {
-        BuildOutput buildOutput = new BuildOutput();
+        BuildOutputExtension buildOutput = new BuildOutputExtension();
         buildOutput.verifyImageMetrics();
     }
 }
diff --git a/...t/resources/image-metrics-test.properties → ...age-metrics/23.0/image-metrics.properties b/...t/resources/image-metrics-test.properties → ...age-metrics/23.0/image-metrics.properties
@@ -1,16 +1,17 @@
-image_details.total_bytes=81172328
+# Properties file used by ImageMetricsITCase
+image_details.total_bytes=86417288
 image_details.total_bytes.tolerance=3
-analysis_results.types.reachable=18854
+analysis_results.types.reachable=20067
 analysis_results.types.reachable.tolerance=3
-analysis_results.methods.reachable=93385
+analysis_results.methods.reachable=99493
 analysis_results.methods.reachable.tolerance=3
-analysis_results.fields.reachable=28156
+analysis_results.fields.reachable=29845
 analysis_results.fields.reachable.tolerance=3
-analysis_results.types.reflection=5978
+analysis_results.types.reflection=6394
 analysis_results.types.reflection.tolerance=3
-analysis_results.methods.reflection=4107
+analysis_results.methods.reflection=4580
 analysis_results.methods.reflection.tolerance=3
-analysis_results.fields.reflection=127
+analysis_results.fields.reflection=143
 analysis_results.fields.reflection.tolerance=3
 analysis_results.types.jni=63
 analysis_results.types.jni.tolerance=1

diff --git a/...sts/jpa-postgresql-withxml/src/test/resources/image-metrics/23.1/image-metrics.properties b/...sts/jpa-postgresql-withxml/src/test/resources/image-metrics/23.1/image-metrics.properties
@@ -0,0 +1,21 @@
+# Properties file used by ImageMetricsITCase
+image_details.total_bytes=91650760
+image_details.total_bytes.tolerance=3
+analysis_results.types.reachable=20387
+analysis_results.types.reachable.tolerance=3
+analysis_results.methods.reachable=100956
+analysis_results.methods.reachable.tolerance=3
+analysis_results.fields.reachable=29789
+analysis_results.fields.reachable.tolerance=3
+analysis_results.types.reflection=6522
+analysis_results.types.reflection.tolerance=3
+analysis_results.methods.reflection=4682
+analysis_results.methods.reflection.tolerance=3
+analysis_results.fields.reflection=163
+analysis_results.fields.reflection.tolerance=3
+analysis_results.types.jni=61
+analysis_results.types.jni.tolerance=1
+analysis_results.methods.jni=55
+analysis_results.methods.jni.tolerance=1
+analysis_results.fields.jni=59
+analysis_results.fields.jni.tolerance=1
diff --git a/...n-tests/jpa-postgresql/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java b/...n-tests/jpa-postgresql/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java
@@ -1,15 +1,18 @@
 package io.quarkus.it.jpa.postgresql;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 
 import io.quarkus.test.junit.QuarkusIntegrationTest;
-import io.quarkus.test.junit.nativeimage.BuildOutput;
+import io.quarkus.test.junit.nativeimage.BuildOutputExtension;
 
+@ExtendWith(BuildOutputExtension.class)
 @QuarkusIntegrationTest
 public class ImageMetricsITCase {
+
     @Test
     public void verifyImageMetrics() {
-        BuildOutput buildOutput = new BuildOutput();
+        BuildOutputExtension buildOutput = new BuildOutputExtension();
         buildOutput.verifyImageMetrics();
     }
 }
diff --git a/...t/resources/image-metrics-test.properties → ...age-metrics/23.0/image-metrics.properties b/...t/resources/image-metrics-test.properties → ...age-metrics/23.0/image-metrics.properties
@@ -1,16 +1,17 @@
-image_details.total_bytes=72185704
+# Properties file used by ImageMetricsITCase
+image_details.total_bytes=78626464
 image_details.total_bytes.tolerance=3
-analysis_results.types.reachable=17905
+analysis_results.types.reachable=19172
 analysis_results.types.reachable.tolerance=3
-analysis_results.methods.reachable=87856
+analysis_results.methods.reachable=95204
 analysis_results.methods.reachable.tolerance=3
-analysis_results.fields.reachable=25370
+analysis_results.fields.reachable=27139
 analysis_results.fields.reachable.tolerance=3
-analysis_results.types.reflection=5659
+analysis_results.types.reflection=5939
 analysis_results.types.reflection.tolerance=3
-analysis_results.methods.reflection=4200
+analysis_results.methods.reflection=4401
 analysis_results.methods.reflection.tolerance=3
-analysis_results.fields.reflection=154
+analysis_results.fields.reflection=170
 analysis_results.fields.reflection.tolerance=3
 analysis_results.types.jni=63
 analysis_results.types.jni.tolerance=1

diff --git a/...ation-tests/jpa-postgresql/src/test/resources/image-metrics/23.1/image-metrics.properties b/...ation-tests/jpa-postgresql/src/test/resources/image-metrics/23.1/image-metrics.properties
@@ -0,0 +1,21 @@
+# Properties file used by ImageMetricsITCase
+image_details.total_bytes=83036632
+image_details.total_bytes.tolerance=3
+analysis_results.types.reachable=19394
+analysis_results.types.reachable.tolerance=3
+analysis_results.methods.reachable=96465
+analysis_results.methods.reachable.tolerance=3
+analysis_results.fields.reachable=27025
+analysis_results.fields.reachable.tolerance=3
+analysis_results.types.reflection=6048
+analysis_results.types.reflection.tolerance=3
+analysis_results.methods.reflection=4495
+analysis_results.methods.reflection.tolerance=3
+analysis_results.fields.reflection=192
+analysis_results.fields.reflection.tolerance=3
+analysis_results.types.jni=61
+analysis_results.types.jni.tolerance=1
+analysis_results.methods.jni=55
+analysis_results.methods.jni.tolerance=1
+analysis_results.fields.jni=59
+analysis_results.fields.jni.tolerance=1
diff --git a/integration-tests/main/src/test/java/io/quarkus/it/main/ImageMetricsITCase.java b/integration-tests/main/src/test/java/io/quarkus/it/main/ImageMetricsITCase.java
@@ -1,15 +1,17 @@
 package io.quarkus.it.main;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 
 import io.quarkus.test.junit.QuarkusIntegrationTest;
-import io.quarkus.test.junit.nativeimage.BuildOutput;
+import io.quarkus.test.junit.nativeimage.BuildOutputExtension;
 
+@ExtendWith(BuildOutputExtension.class)
 @QuarkusIntegrationTest
 public class ImageMetricsITCase {
     @Test
     public void verifyImageMetrics() {
-        BuildOutput buildOutput = new BuildOutput();
+        BuildOutputExtension buildOutput = new BuildOutputExtension();
         buildOutput.verifyImageMetrics();
     }
 }
diff --git a/integration-tests/main/src/test/resources/image-metrics/23.0/image-metrics.properties b/integration-tests/main/src/test/resources/image-metrics/23.0/image-metrics.properties
@@ -0,0 +1,21 @@
+# Properties file used by ImageMetricsITCase
+image_details.total_bytes=139447360
+image_details.total_bytes.tolerance=3
+analysis_results.types.reachable=30117
+analysis_results.types.reachable.tolerance=3
+analysis_results.methods.reachable=150060
+analysis_results.methods.reachable.tolerance=3
+analysis_results.fields.reachable=44502
+analysis_results.fields.reachable.tolerance=3
+analysis_results.types.reflection=8989
+analysis_results.types.reflection.tolerance=3
+analysis_results.methods.reflection=7393
+analysis_results.methods.reflection.tolerance=3
+analysis_results.fields.reflection=438
+analysis_results.fields.reflection.tolerance=3
+analysis_results.types.jni=64
+analysis_results.types.jni.tolerance=1
+analysis_results.methods.jni=55
+analysis_results.methods.jni.tolerance=1
+analysis_results.fields.jni=70
+analysis_results.fields.jni.tolerance=1
diff --git a/...t/resources/image-metrics-test.properties → ...age-metrics/23.1/image-metrics.properties b/...t/resources/image-metrics-test.properties → ...age-metrics/23.1/image-metrics.properties
@@ -1,23 +1,24 @@
-image_details.total_bytes=138778008
+# Properties file used by ImageMetricsITCase
+image_details.total_bytes=147268552
 image_details.total_bytes.tolerance=3
 # TODO: Switch to using analysis_results.types.reachable key once we drop support for GraalVM 22.3.0
-analysis_results.classes.reachable=30005
+analysis_results.classes.reachable=30415
 analysis_results.classes.reachable.tolerance=3
-analysis_results.methods.reachable=149440
+analysis_results.methods.reachable=151296
 analysis_results.methods.reachable.tolerance=3
-analysis_results.fields.reachable=44161
+analysis_results.fields.reachable=44325
 analysis_results.fields.reachable.tolerance=3
 # TODO: Switch to using analysis_results.types.reflection key once we drop support for GraalVM 22.3.0
-analysis_results.classes.reflection=8966
+analysis_results.classes.reflection=9118
 analysis_results.classes.reflection.tolerance=3
-analysis_results.methods.reflection=7346
+analysis_results.methods.reflection=7741
 analysis_results.methods.reflection.tolerance=3
-analysis_results.fields.reflection=438
+analysis_results.fields.reflection=480
 analysis_results.fields.reflection.tolerance=3
 # TODO: Switch to using analysis_results.types.jni key once we drop support for GraalVM 22.3.0
-analysis_results.classes.jni=64
+analysis_results.classes.jni=62
 analysis_results.classes.jni.tolerance=1
 analysis_results.methods.jni=55
 analysis_results.methods.jni.tolerance=1
-analysis_results.fields.jni=70
+analysis_results.fields.jni=61
 analysis_results.fields.jni.tolerance=1
diff --git a/test-framework/junit5/src/main/java/io/quarkus/test/junit/IntegrationTestUtil.java b/test-framework/junit5/src/main/java/io/quarkus/test/junit/IntegrationTestUtil.java
@@ -422,7 +422,7 @@ public void close() {
         }
     }
 
-    static Properties readQuarkusArtifactProperties(ExtensionContext context) {
+    public static Properties readQuarkusArtifactProperties(ExtensionContext context) {
         Path buildOutputDirectory = determineBuildOutputDirectory(context);
         Path artifactProperties = buildOutputDirectory.resolve("quarkus-artifact.properties");
         if (!Files.exists(artifactProperties)) {

diff --git a/...s/test/junit/nativeimage/BuildOutput.java → ...nit/nativeimage/BuildOutputExtension.java b/...s/test/junit/nativeimage/BuildOutput.java → ...nit/nativeimage/BuildOutputExtension.java
@@ -1,5 +1,7 @@
 package io.quarkus.test.junit.nativeimage;
 
+import static io.quarkus.test.junit.IntegrationTestUtil.readQuarkusArtifactProperties;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -13,6 +15,11 @@
 import jakarta.json.JsonObject;
 
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+import io.quarkus.deployment.pkg.steps.GraalVM;
 
 /**
  * This is a general utility to assert via
@@ -22,17 +29,21 @@
  * <a href="https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/BuildOutput.md">the upstream GraalVM
  * documentation</a>.
  */
-public class BuildOutput {
+public class BuildOutputExtension implements BeforeAllCallback {
 
-    private static final String IMAGE_METRICS_TEST_PROPERTIES = "image-metrics-test.properties";
+    private static final String IMAGE_METRICS_TEST_PROPERTIES = "image-metrics.properties";
+    private static final String IMAGE_METRICS_DIR = "image-metrics";
     private final JsonObject buildOutput;
+    private static GraalVM.Version mandrelVersion;
 
-    public BuildOutput() {
+    public BuildOutputExtension() {
         this.buildOutput = getBuildOutput();
     }
 
     public void verifyImageMetrics() {
-        verifyImageMetrics(IMAGE_METRICS_TEST_PROPERTIES);
+        String version = mandrelVersion.getMajorMinorAsString();
+        String propertiesFileName = IMAGE_METRICS_DIR + "/" + version + "/" + IMAGE_METRICS_TEST_PROPERTIES;
+        verifyImageMetrics(propertiesFileName);
     }
 
     public void verifyImageMetrics(String propertiesFileName) {
@@ -52,7 +63,10 @@ public void verifyImageMetrics(String propertiesFileName) {
     private Properties getProperties(String propertiesFileName) {
         Properties properties = new Properties();
         try {
-            properties.load(getClass().getClassLoader().getResourceAsStream(propertiesFileName));
+            InputStream resourceAsStream = getClass().getClassLoader().getResourceAsStream(propertiesFileName);
+            Assumptions.assumeTrue(resourceAsStream != null,
+                    "Could not find properties file matching the Mandrel version being used: " + propertiesFileName);
+            properties.load(resourceAsStream);
         } catch (IOException e) {
             Assertions.fail("Could not load properties from " + propertiesFileName, e);
         }
@@ -103,4 +117,22 @@ private static Path locateNativeImageBuildDirectory() {
         Assertions.assertEquals(1, files.length, "Could not identify the native image build directory");
         return files[0].toPath();
     }
+
+    @Override
+    public void beforeAll(ExtensionContext extensionContext) throws Exception {
+        mandrelVersion = getMandrelVersion(extensionContext);
+    }
+
+    private GraalVM.Version getMandrelVersion(ExtensionContext context) {
+        Properties quarkusArtifactProperties = readQuarkusArtifactProperties(context);
+        String fullVersion = quarkusArtifactProperties.getProperty("metadata.graalvm.version.full");
+        try {
+            return GraalVM.Version.of(fullVersion.lines());
+        } catch (NumberFormatException e) {
+            System.out.println(
+                    "WARNING: Unable to determine the GraalVM version with which the native binary was built. metadata.graalvm.version.full = "
+                            + fullVersion);
+            return GraalVM.Version.CURRENT;
+        }
+    }
 }