From b394ef08ad2493494032ef9179cbd050e21a614c Mon Sep 17 00:00:00 2001 From: Marc Philipp Date: Sat, 30 Jan 2021 21:32:29 +0100 Subject: [PATCH] Use jOOX to test XML reports Instead of asserting string subsequences, we select elements and attributes in the DOM via jOOX which allows for much more expressive assertions. --- dependencies/dependencies.gradle.kts | 1 + gradle.properties | 1 + platform-tests/platform-tests.gradle.kts | 1 + ...egacyXmlReportGeneratingListenerTests.java | 312 ++++++++---------- .../legacy/xml/XmlReportAssertions.java | 11 +- .../legacy/xml/XmlReportWriterTests.java | 159 ++++----- 6 files changed, 207 insertions(+), 278 deletions(-) diff --git a/dependencies/dependencies.gradle.kts b/dependencies/dependencies.gradle.kts index 090ab495abd5..ea15d7116f4e 100644 --- a/dependencies/dependencies.gradle.kts +++ b/dependencies/dependencies.gradle.kts @@ -33,5 +33,6 @@ dependencies { api("org.mockito:mockito-junit-jupiter:${versions["mockito"]}") api("biz.aQute.bnd:biz.aQute.bndlib:${versions["bnd"]}") api("org.spockframework:spock-core:${versions["spock"]}") + api("org.jooq:joox:${versions["joox"]}") } } diff --git a/gradle.properties b/gradle.properties index f0e981630fe6..401df118b762 100644 --- a/gradle.properties +++ b/gradle.properties @@ -36,6 +36,7 @@ log4j.version=2.13.3 mockito.version=3.5.0 slf4j.version=1.7.30 spock.version=1.3-groovy-2.5 +joox.version=1.6.2 # Tools checkstyle.version=8.31 diff --git a/platform-tests/platform-tests.gradle.kts b/platform-tests/platform-tests.gradle.kts index 89662f6ee73d..9ec70ae67346 100644 --- a/platform-tests/platform-tests.gradle.kts +++ b/platform-tests/platform-tests.gradle.kts @@ -21,6 +21,7 @@ dependencies { testImplementation(testFixtures(project(":junit-platform-launcher"))) testImplementation(project(":junit-jupiter-engine")) testImplementation("org.apiguardian:apiguardian-api") + testImplementation("org.jooq:joox") // --- Test run-time dependencies --------------------------------------------- testRuntimeOnly(project(":junit-vintage-engine")) diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java index bf834c0ed939..76a1276649d6 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java @@ -11,6 +11,7 @@ package org.junit.platform.reporting.legacy.xml; import static org.assertj.core.api.Assertions.assertThat; +import static org.joox.JOOX.$; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeFalse; @@ -37,8 +38,8 @@ import java.util.Map; import java.util.Set; +import org.joox.Match; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestReporter; import org.junit.jupiter.api.io.TempDir; import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.UniqueId; @@ -65,29 +66,26 @@ void writesFileForSingleSucceedingTest(@TempDir Path tempDirectory) throws Excep executeTests(engine, tempDirectory); - var content = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); - - // @formatter:off - assertThat(content) - .containsSubsequence( - "", - "unique-id: [engine:dummy]/[test:succeedingTest]", - "display-name: display<-->Name 😎", - "", - "", - - "", - "unique-id: [engine:dummy]", - "display-name: dummy", - "", - "") - .doesNotContain("Name 😎"); + assertThat(testcase.attr("classname")).isEqualTo("dummy"); + assertThat(testcase.child("system-out").text()) // + .containsSubsequence("unique-id: [engine:dummy]/[test:succeedingTest]", + "display-name: display<-->Name 😎"); + + assertThat(testsuite.find("skipped")).isEmpty(); + assertThat(testsuite.find("failure")).isEmpty(); + assertThat(testsuite.find("error")).isEmpty(); } @Test @@ -97,22 +95,23 @@ void writesFileForSingleFailingTest(@TempDir Path tempDirectory) throws Exceptio executeTests(engine, tempDirectory); - var content = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); - - // @formatter:off - assertThat(content) - .containsSubsequence( - "", - "AssertionFailedError: expected to fail", - "\tat", - "", - "", - "") - .doesNotContain("fail"); + assertThat(failure.attr("type")).isEqualTo(AssertionFailedError.class.getName()); + assertThat(failure.text()).containsSubsequence("AssertionFailedError: expected to fail", "\tat"); + + assertThat(testsuite.find("skipped")).isEmpty(); + assertThat(testsuite.find("error")).isEmpty(); } @Test @@ -124,22 +123,23 @@ void writesFileForSingleErroneousTest(@TempDir Path tempDirectory) throws Except executeTests(engine, tempDirectory); - var content = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); - - // @formatter:off - assertThat(content) - .containsSubsequence( - "", - "RuntimeException: error occurred", - "\tat ", - "", - "", - "") - .doesNotContain("", - "should be skipped", - "", - "", - "") - .doesNotContain("", - "TestAbortedException: ", - "deliberately aborted", - "at ", - "", - "", - "") - .doesNotContain("", - "parent was skipped: should be skipped", - "", - "", - ""); - // @formatter:on + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + + assertThat(testsuite.attr("tests", int.class)).isEqualTo(1); + assertThat(testsuite.attr("skipped", int.class)).isEqualTo(1); + + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("test"); + assertThat(testcase.child("skipped").text()).isEqualTo("parent was skipped: should be skipped"); } @Test @@ -268,20 +254,18 @@ void writesFileForFailingContainer(@TempDir Path tempDirectory) throws Exception executeTests(engine, tempDirectory); - var content = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); - - // @formatter:off - assertThat(content) - .containsSubsequence( - "", - "AssertionFailedError: failure before all tests", - "\tat", - "", - "", - ""); - // @formatter:on + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + + assertThat(testsuite.attr("tests", int.class)).isEqualTo(1); + assertThat(testsuite.attr("failures", int.class)).isEqualTo(1); + + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("test"); + + var failure = testcase.child("failure"); + assertThat(failure.attr("message")).isEqualTo("failure before all tests"); + assertThat(failure.attr("type")).isEqualTo(AssertionFailedError.class.getName()); + assertThat(failure.text()).containsSubsequence("AssertionFailedError: failure before all tests", "\tat"); } @Test @@ -292,19 +276,10 @@ void writesSystemProperties(@TempDir Path tempDirectory) throws Exception { executeTests(engine, tempDirectory); - var content = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); - - // @formatter:off - assertThat(content) - .containsSubsequence( - "", - "", - "", - "", - ""); - // @formatter:on + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + var properties = testsuite.child("properties").children("property"); + assertThat(properties.matchAttr("name", "file\\.separator").attr("value")).isEqualTo(File.separator); + assertThat(properties.matchAttr("name", "path\\.separator").attr("value")).isEqualTo(File.pathSeparator); } @Test @@ -318,17 +293,9 @@ void writesHostNameAndTimestamp(@TempDir Path tempDirectory) throws Exception { executeTests(engine, tempDirectory, Clock.fixed(ZonedDateTime.of(now, zone).toInstant(), zone)); - var content = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); - - // @formatter:off - assertThat(content) - .containsSubsequence( - ""); - // @formatter:on + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + assertThat(testsuite.attr("hostname")).isEqualTo(InetAddress.getLocalHost().getHostName()); + assertThat(testsuite.attr("timestamp")).isEqualTo("2016-01-28T14:02:59"); } @Test @@ -362,8 +329,7 @@ void printsExceptionWhenReportCouldNotBeWritten(@TempDir Path tempDirectory) thr } @Test - void writesReportEntriesToSystemOutElement(@TempDir Path tempDirectory, TestReporter testReporter) - throws Exception { + void writesReportEntriesToSystemOutElement(@TempDir Path tempDirectory) throws Exception { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); @@ -382,24 +348,12 @@ void writesReportEntriesToSystemOutElement(@TempDir Path tempDirectory, TestRepo listener.executionFinished(testIdentifier, successful()); listener.executionFinished(testPlan.getTestIdentifier("[engine:engine]"), successful()); - var content = readValidXmlFile(tempDirectory.resolve("TEST-engine.xml")); - //testReporter.publishEntry("xml", content); - - // @formatter:off - assertThat(content) - .containsSubsequence( - "", - "Report Entry #1 (timestamp: " + Year.now(), - "- foo: bar\n", - "Report Entry #2 (timestamp: " + Year.now(), - "- bar: baz\n", - "- qux: foo\n", - "", - "", - ""); - // @formatter:on + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-engine.xml")); + + assertThat(String.join("\n", testsuite.child("testcase").children("system-out").texts())) // + .containsSubsequence( // + "Report Entry #1 (timestamp: " + Year.now(), "- foo: bar\n", + "Report Entry #2 (timestamp: " + Year.now(), "- bar: baz\n", "- qux: foo\n"); } private void executeTests(TestEngine engine, Path tempDirectory) { @@ -414,11 +368,13 @@ private void executeTests(TestEngine engine, Path tempDirectory, Clock clock) { launcher.execute(request().selectors(selectUniqueId(UniqueId.forEngine(engine.getId()))).build()); } - private String readValidXmlFile(Path xmlFile) throws Exception { + private Match readValidXmlFile(Path xmlFile) throws Exception { assertTrue(Files.exists(xmlFile), () -> "File does not exist: " + xmlFile); - var content = Files.readString(xmlFile); - assertValidAccordingToJenkinsSchema(content); - return content; + try (var reader = Files.newBufferedReader(xmlFile)) { + var xml = $(reader); + assertValidAccordingToJenkinsSchema(xml.document()); + return xml; + } } } diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportAssertions.java b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportAssertions.java index f9428edf52ac..0973bdb86a40 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportAssertions.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportAssertions.java @@ -12,14 +12,13 @@ import static org.junit.jupiter.api.Assertions.fail; -import java.io.StringReader; - import javax.xml.XMLConstants; -import javax.xml.transform.stream.StreamSource; +import javax.xml.transform.dom.DOMSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; import javax.xml.validation.Validator; +import org.w3c.dom.Document; import org.xml.sax.SAXException; /** @@ -27,14 +26,14 @@ */ class XmlReportAssertions { - static void assertValidAccordingToJenkinsSchema(String content) throws Exception { + static void assertValidAccordingToJenkinsSchema(Document document) throws Exception { try { // Schema is thread-safe, Validator is not var validator = CachedSchema.JENKINS.newValidator(); - validator.validate(new StreamSource(new StringReader(content))); + validator.validate(new DOMSource(document)); } catch (SAXException e) { - fail("Invalid XML document: " + content, e); + fail("Invalid XML document: " + document, e); } } diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportWriterTests.java b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportWriterTests.java index c47dae5d1239..b0079bb6f253 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportWriterTests.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportWriterTests.java @@ -11,6 +11,7 @@ package org.junit.platform.reporting.legacy.xml; import static org.assertj.core.api.Assertions.assertThat; +import static org.joox.JOOX.$; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.junit.platform.commons.util.CollectionUtils.getOnlyElement; @@ -20,6 +21,7 @@ import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY; import static org.junit.platform.reporting.legacy.xml.XmlReportAssertions.assertValidAccordingToJenkinsSchema; +import java.io.StringReader; import java.io.StringWriter; import java.io.Writer; import java.time.Clock; @@ -27,8 +29,8 @@ import java.util.Set; import java.util.stream.Stream; +import org.joox.Match; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -52,16 +54,13 @@ void writesTestsuiteElementsWithoutTestcaseElementsWithoutAnyTests() throws Exce var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); - var content = writeXmlReport(testPlan, reportData); + var testsuite = writeXmlReport(testPlan, reportData); - assertValidAccordingToJenkinsSchema(content); - //@formatter:off - assertThat(content) - .containsSubsequence( - "") - .doesNotContain("", - "Report Entry #1 (timestamp: ", - "- myKey: myValue", - ""); - //@formatter:on + var testsuite = writeXmlReport(testPlan, reportData); + + assertValidAccordingToJenkinsSchema(testsuite.document()); + assertThat(String.join("\n", testsuite.find("system-out").texts())) // + .containsSubsequence("Report Entry #1 (timestamp: ", "- myKey: myValue"); } @Test @@ -104,30 +97,18 @@ void writesCapturedOutput() throws Exception { reportData.addReportEntry(TestIdentifier.from(testDescriptor), ReportEntry.from(Map.of("baz", "qux"))); reportData.markFinished(testPlan.getTestIdentifier(uniqueId.toString()), successful()); - var content = writeXmlReport(testPlan, reportData); - - assertValidAccordingToJenkinsSchema(content); - //@formatter:off - assertThat(content) - .containsSubsequence( - "", - "unique-id: ", "test:test", - "display-name: successfulTest", - "", - "", - "Report Entry #1 (timestamp: ", - "- foo: bar", - "Report Entry #2 (timestamp: ", - "- baz: qux", - "", - "", - "normal output", - "", - "", - "error output", - "") - .doesNotContain(STDOUT_REPORT_ENTRY_KEY, STDERR_REPORT_ENTRY_KEY); - //@formatter:on + var testsuite = writeXmlReport(testPlan, reportData); + + assertValidAccordingToJenkinsSchema(testsuite.document()); + assertThat(testsuite.find("system-out").text(0)) // + .containsSubsequence("unique-id: ", "test:test", "display-name: successfulTest"); + assertThat(testsuite.find("system-out").text(1)) // + .containsSubsequence("Report Entry #1 (timestamp: ", "- foo: bar", "Report Entry #2 (timestamp: ", + "- baz: qux"); + assertThat(testsuite.find("system-out").text(2).trim()) // + .isEqualTo("normal output"); + assertThat(testsuite.find("system-err").text().trim()) // + .isEqualTo("error output"); } @Test @@ -139,16 +120,14 @@ void writesEmptySkippedElementForSkippedTestWithoutReason() throws Exception { var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); reportData.markSkipped(testPlan.getTestIdentifier(uniqueId.toString()), null); - var content = writeXmlReport(testPlan, reportData); + var testsuite = writeXmlReport(testPlan, reportData); - assertValidAccordingToJenkinsSchema(content); - //@formatter:off - assertThat(content) - .containsSubsequence( - "", - ""); - //@formatter:on + assertValidAccordingToJenkinsSchema(testsuite.document()); + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("skippedTest"); + var skipped = testcase.child("skipped"); + assertThat(skipped.size()).isEqualTo(1); + assertThat(skipped.children()).isEmpty(); } @Test @@ -171,16 +150,15 @@ public String getLegacyReportingName() { var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); reportData.markFinished(testPlan.getTestIdentifier(uniqueId.toString()), failed(null)); - var content = writeXmlReport(testPlan, reportData); + var testsuite = writeXmlReport(testPlan, reportData); - assertValidAccordingToJenkinsSchema(content); - //@formatter:off - assertThat(content) - .containsSubsequence( - "", - ""); - //@formatter:on + assertValidAccordingToJenkinsSchema(testsuite.document()); + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("failedTest"); + assertThat(testcase.attr("classname")).isEqualTo("myEngine"); + var error = testcase.child("error"); + assertThat(error.size()).isEqualTo(1); + assertThat(error.children()).isEmpty(); } @Test @@ -192,16 +170,12 @@ void omitsMessageAttributeForFailedTestWithThrowableWithoutMessage() throws Exce var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); reportData.markFinished(testPlan.getTestIdentifier(uniqueId.toString()), failed(new NullPointerException())); - var content = writeXmlReport(testPlan, reportData); + var testsuite = writeXmlReport(testPlan, reportData); - assertValidAccordingToJenkinsSchema(content); - //@formatter:off - assertThat(content) - .containsSubsequence( - "", - ""); - //@formatter:on + assertValidAccordingToJenkinsSchema(testsuite.document()); + var error = testsuite.find("error"); + assertThat(error.attr("type")).isEqualTo("java.lang.NullPointerException"); + assertThat(error.attr("message")).isNull(); } @Test @@ -214,21 +188,14 @@ void writesValidXmlEvenIfExceptionMessageContainsCData() throws Exception { var assertionError = new AssertionError(""); reportData.markFinished(testPlan.getTestIdentifier(uniqueId.toString()), failed(assertionError)); - var content = writeXmlReport(testPlan, reportData); - - assertValidAccordingToJenkinsSchema(content); - //@formatter:off - assertThat(content) - .containsSubsequence( - "", - "]]>") - .doesNotContain(assertionError.getMessage()); - //@formatter:on + var testsuite = writeXmlReport(testPlan, reportData); + + assertValidAccordingToJenkinsSchema(testsuite.document()); + assertThat(testsuite.find("failure").attr("message")).isEqualTo(""); } @Test - void escapesInvalidCharactersInSystemPropertiesAndExceptionMessages(TestInfo testInfo) throws Exception { + void escapesInvalidCharactersInSystemPropertiesAndExceptionMessages() throws Exception { var uniqueId = engineDescriptor.getUniqueId().append("test", "test"); engineDescriptor.addChild(new TestDescriptorStub(uniqueId, "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); @@ -238,18 +205,21 @@ void escapesInvalidCharactersInSystemPropertiesAndExceptionMessages(TestInfo tes reportData.markFinished(testPlan.getTestIdentifier(uniqueId.toString()), failed(assertionError)); System.setProperty("foo.bar", "\1"); - String content; + Match testsuite; try { - content = writeXmlReport(testPlan, reportData); + testsuite = writeXmlReport(testPlan, reportData); } finally { System.getProperties().remove("foo.bar"); } - assertValidAccordingToJenkinsSchema(content); - assertThat(content) // - .contains("") // - .contains("failure message=\"expected: <A> but was: <B&#0;>\"") // + assertValidAccordingToJenkinsSchema(testsuite.document()); + assertThat(testsuite.find("property").matchAttr("name", "foo\\.bar").attr("value")) // + .isEqualTo(""); + var failure = testsuite.find("failure"); + assertThat(failure.attr("message")) // + .isEqualTo("expected: but was: "); + assertThat(failure.text()) // .contains("AssertionError: expected: but was: "); } @@ -264,9 +234,10 @@ void doesNotReopenCDataWithinCDataContent() throws Exception { reportData.markFinished(testPlan.getTestIdentifier(uniqueId.toString()), failed(assertionError)); Writer assertingWriter = new StringWriter() { + @SuppressWarnings("NullableProblems") @Override - public void write(char[] cbuf, int off, int len) { - assertThat(new String(cbuf, off, len)).doesNotContain("]]> stringPairs() { ); } - private String writeXmlReport(TestPlan testPlan, XmlReportData reportData) throws Exception { + private Match writeXmlReport(TestPlan testPlan, XmlReportData reportData) throws Exception { var out = new StringWriter(); writeXmlReport(testPlan, reportData, out); - return out.toString(); + return $(new StringReader(out.toString())); } private void writeXmlReport(TestPlan testPlan, XmlReportData reportData, Writer out) throws Exception {