Add RegressionTest error message for when possible non-deterministic …

…behavior is detected (#883)
NationalSecurityAgency · Aug 14, 2024 · 6b49867 · 6b49867
1 parent dcae255
commit 6b49867
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/src/test/java/emissary/test/core/junit5/RegressionTest.java b/src/test/java/emissary/test/core/junit5/RegressionTest.java
@@ -337,6 +337,7 @@ protected void setupPayload(final IBaseDataObject payload, final Document answer
     @Override
     protected void checkAnswers(final Document answers, final IBaseDataObject payload,
             final List<IBaseDataObject> attachments, final String tname) {
-        RegressionTestUtil.checkAnswers(answers, payload, actualSimplifiedLogEvents, attachments, place.getClass().getName(), getDecoders());
+        RegressionTestUtil.checkAnswers(answers, payload, actualSimplifiedLogEvents, attachments, place.getClass().getName(), getDecoders(),
+                generateAnswers());
     }
 }
diff --git a/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java b/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java
@@ -70,6 +70,9 @@ public final class RegressionTestUtil {
      */
     public static final String THROWABLE_MESSAGE_NAME = "throwableMessage";
 
+    public static final String NON_DETERMINISTIC_ERROR_MESSAGE =
+            "\nNOTE: Since 'generateAnswers' is true, these differences could indicate non-deterministic processing in the tested code path\n";
+
     /**
      * XML builder to read XML answer file in
      */
@@ -119,7 +122,7 @@ public static Path getTestResx() {
     }
 
     public static void checkAnswers(final Document answers, final IBaseDataObject payload, final List<SimplifiedLogEvent> actualSimplifiedLogEvents,
-            final List<IBaseDataObject> attachments, final String placeName, final ElementDecoders decoders) {
+            final List<IBaseDataObject> attachments, final String placeName, final ElementDecoders decoders, final boolean generateAnswers) {
         final Element root = answers.getRootElement();
         final Element parent = root.getChild(ANSWERS);
 
@@ -130,7 +133,7 @@ public static void checkAnswers(final Document answers, final IBaseDataObject pa
         final String differences = PlaceComparisonHelper.checkDifferences(expectedIbdo, payload, expectedAttachments,
                 attachments, placeName, DIFF_CHECK);
 
-        assertNull(differences, differences);
+        assertNull(differences, generateAnswers ? differences + NON_DETERMINISTIC_ERROR_MESSAGE : differences);
 
         final List<SimplifiedLogEvent> expectedSimplifiedLogEvents = getSimplifiedLogEvents(parent);