From 793768e308de2514576070d94b6c89a656a8b98d Mon Sep 17 00:00:00 2001 From: James Cover jdcove2 Date: Thu, 8 Aug 2024 19:51:32 +0000 Subject: [PATCH 1/2] Add RegressionTest error message for when possible non-deterministic behaviour is detected. --- .../java/emissary/test/core/junit5/RegressionTest.java | 3 ++- .../java/emissary/test/core/junit5/RegressionTestUtil.java | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/java/emissary/test/core/junit5/RegressionTest.java b/src/test/java/emissary/test/core/junit5/RegressionTest.java index 5dd04d3129..5f6c1462bb 100644 --- a/src/test/java/emissary/test/core/junit5/RegressionTest.java +++ b/src/test/java/emissary/test/core/junit5/RegressionTest.java @@ -337,6 +337,7 @@ protected void setupPayload(final IBaseDataObject payload, final Document answer @Override protected void checkAnswers(final Document answers, final IBaseDataObject payload, final List attachments, final String tname) { - RegressionTestUtil.checkAnswers(answers, payload, actualSimplifiedLogEvents, attachments, place.getClass().getName(), getDecoders()); + RegressionTestUtil.checkAnswers(answers, payload, actualSimplifiedLogEvents, attachments, place.getClass().getName(), getDecoders(), + generateAnswers()); } } diff --git a/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java b/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java index c9dea7054e..46af83a15a 100644 --- a/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java +++ b/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java @@ -70,6 +70,9 @@ public final class RegressionTestUtil { */ public static final String THROWABLE_MESSAGE_NAME = "throwableMessage"; + public static final String NON_DETERMINISTIC_ERROR_MESSAGE = + "\nNOTE: Since 'generateAnswers' is true, these differences could be because of non-deterministic state in the IBDO\n"; + /** * XML builder to read XML answer file in */ @@ -119,7 +122,7 @@ public static Path getTestResx() { } public static void checkAnswers(final Document answers, final IBaseDataObject payload, final List actualSimplifiedLogEvents, - final List attachments, final String placeName, final ElementDecoders decoders) { + final List attachments, final String placeName, final ElementDecoders decoders, final boolean generateAnswers) { final Element root = answers.getRootElement(); final Element parent = root.getChild(ANSWERS); @@ -130,7 +133,7 @@ public static void checkAnswers(final Document answers, final IBaseDataObject pa final String differences = PlaceComparisonHelper.checkDifferences(expectedIbdo, payload, expectedAttachments, attachments, placeName, DIFF_CHECK); - assertNull(differences, differences); + assertNull(differences, generateAnswers ? differences + NON_DETERMINISTIC_ERROR_MESSAGE : differences); final List expectedSimplifiedLogEvents = getSimplifiedLogEvents(parent); From ec194171fb69c0dec4418a1b169e2c75b600394d Mon Sep 17 00:00:00 2001 From: James Cover jdcove2 Date: Tue, 13 Aug 2024 17:53:55 +0000 Subject: [PATCH 2/2] Changes from PR feedback. --- src/test/java/emissary/test/core/junit5/RegressionTestUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java b/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java index 46af83a15a..88c6b61705 100644 --- a/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java +++ b/src/test/java/emissary/test/core/junit5/RegressionTestUtil.java @@ -71,7 +71,7 @@ public final class RegressionTestUtil { public static final String THROWABLE_MESSAGE_NAME = "throwableMessage"; public static final String NON_DETERMINISTIC_ERROR_MESSAGE = - "\nNOTE: Since 'generateAnswers' is true, these differences could be because of non-deterministic state in the IBDO\n"; + "\nNOTE: Since 'generateAnswers' is true, these differences could indicate non-deterministic processing in the tested code path\n"; /** * XML builder to read XML answer file in