From a4c3408f5c7061b68559a8d50af9f0a437e5ae90 Mon Sep 17 00:00:00 2001 From: Juliette de Rancourt Date: Tue, 30 Jul 2024 12:22:52 +0200 Subject: [PATCH] Introduce BeforeSuite and AfterSuite annotations for Suite classes These new lifecycle callback method can be used to run code before the first and after the last test in the suite. Resolves #456. Co-authored-by: Marc Philipp --- .../release-notes-5.11.0-RC1.adoc | 1 + .../junit-platform-suite-engine.adoc | 11 + .../java/example/BeforeAndAfterSuiteDemo.java | 34 +++ .../junit/platform/suite/api/AfterSuite.java | 75 ++++++ .../junit/platform/suite/api/BeforeSuite.java | 75 ++++++ .../suite/engine/LifecycleMethodUtils.java | 90 +++++++ .../suite/engine/SuiteTestDescriptor.java | 54 +++- .../engine/BeforeAndAfterSuiteTests.java | 229 ++++++++++++++++ .../engine/testcases/StatefulTestCase.java | 46 ++++ .../testsuites/LifecycleMethodsSuites.java | 245 ++++++++++++++++++ 10 files changed, 857 insertions(+), 3 deletions(-) create mode 100644 documentation/src/test/java/example/BeforeAndAfterSuiteDemo.java create mode 100644 junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/AfterSuite.java create mode 100644 junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/BeforeSuite.java create mode 100644 junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java create mode 100644 platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java create mode 100644 platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/StatefulTestCase.java create mode 100644 platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc index 1a90b20a5407..8ede33a6efa0 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.11.0-RC1.adoc @@ -33,6 +33,7 @@ repository on GitHub. exception's _root_ cause matches all supplied conditions, for use with the `EngineTestKit`. * `ReflectionSupport` now supports scanning for classpath resources. +* Introduce `@BeforeSuite` and `@AfterSuite` annotations. [[release-notes-5.11.0-RC1-junit-jupiter]] diff --git a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc index d38a312d799f..ebb1d6495b04 100644 --- a/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc +++ b/documentation/src/docs/asciidoc/user-guide/advanced-topics/junit-platform-suite-engine.adoc @@ -48,3 +48,14 @@ include::{testDir}/example/SuiteDemo.java[tags=user_guide] NOTE: There are numerous configuration options for discovering and filtering tests in a test suite. Please consult the Javadoc of the `{suite-api-package}` package for a full list of supported annotations and further details. + +==== @BeforeSuite and @AfterSuite + +`@BeforeSuite` and `@AfterSuite` annotations can be used on methods inside a +`@Suite`-annotated class. They will be executed respectively before and after +all tests of the test suite. + +[source,java,indent=0] +---- +include::{testDir}/example/BeforeAndAfterSuiteDemo.java[tags=user_guide] +---- diff --git a/documentation/src/test/java/example/BeforeAndAfterSuiteDemo.java b/documentation/src/test/java/example/BeforeAndAfterSuiteDemo.java new file mode 100644 index 000000000000..c461417361bc --- /dev/null +++ b/documentation/src/test/java/example/BeforeAndAfterSuiteDemo.java @@ -0,0 +1,34 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package example; + +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; +import org.junit.platform.suite.api.SelectPackages; +import org.junit.platform.suite.api.Suite; + +//tag::user_guide[] +@Suite +@SelectPackages("example") +class BeforeAndAfterSuiteDemo { + + @BeforeSuite + static void beforeSuite() { + // executes before the test suite + } + + @AfterSuite + static void afterSuite() { + // executes after the test suite + } + +} +//end::user_guide[] diff --git a/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/AfterSuite.java b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/AfterSuite.java new file mode 100644 index 000000000000..c2b5e33a3467 --- /dev/null +++ b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/AfterSuite.java @@ -0,0 +1,75 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.api; + +import static org.apiguardian.api.API.Status.EXPERIMENTAL; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apiguardian.api.API; + +/** + * {@code @AfterSuite} is used to signal that the annotated method should be + * executed after all tests in the current test suite. + * + *

Method Signatures

+ * + *

{@code @AfterSuite} methods must have a {@code void} return type, must + * be {@code static} and must not be {@code private}. + * + *

Inheritance and Execution Order

+ * + *

{@code @AfterSuite} methods are inherited from superclasses as long as they + * are not overridden according to the visibility rules of the Java + * language. Furthermore, {@code @AfterSuite} methods from superclasses will be + * executed after {@code @AfterSuite} methods in subclasses. + * + *

The JUnit Platform Suite Engine does not guarantee the execution order of + * multiple {@code @AfterSuite} methods that are declared within a single test + * class or test interface. While it may at times appear that these methods are + * invoked in alphabetical order, they are in fact sorted using an algorithm + * that is deterministic but intentionally non-obvious. + * + *

In addition, {@code @AfterSuite} methods are in no way linked to + * {@code @BeforeSuite} methods. Consequently, there are no guarantees with regard + * to their wrapping behavior. For example, given two + * {@code @BeforeSuite} methods {@code createA()} and {@code createB()} as well as + * two {@code @AfterSuite} methods {@code destroyA()} and {@code destroyB()}, the + * order in which the {@code @BeforeSuite} methods are executed (e.g. + * {@code createA()} before {@code createB()}) does not imply any order for the + * seemingly corresponding {@code @AfterSuite} methods. In other words, + * {@code destroyA()} might be called before or after + * {@code destroyB()}. The JUnit Team therefore recommends that developers + * declare at most one {@code @BeforeSuite} method and at most one + * {@code @AfterSuite} method per test class or test interface unless there are no + * dependencies between the {@code @BeforeSuite} methods or between the + * {@code @AfterSuite} methods. + * + *

Composition

+ * + *

{@code @AfterSuite} may be used as a meta-annotation in order to create + * a custom composed annotation that inherits the semantics of + * {@code @AfterSuite}. + * + * @since 1.11 + * @see BeforeSuite + * @see Suite + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@API(status = EXPERIMENTAL, since = "1.11") +public @interface AfterSuite { +} diff --git a/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/BeforeSuite.java b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/BeforeSuite.java new file mode 100644 index 000000000000..5b4c0d3c93f0 --- /dev/null +++ b/junit-platform-suite-api/src/main/java/org/junit/platform/suite/api/BeforeSuite.java @@ -0,0 +1,75 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.api; + +import static org.apiguardian.api.API.Status.EXPERIMENTAL; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apiguardian.api.API; + +/** + * {@code @BeforeSuite} is used to signal that the annotated method should be + * executed before all tests in the current test suite. + * + *

Method Signatures

+ * + *

{@code @BeforeSuite} methods must have a {@code void} return type, must + * be {@code static} and must not be {@code private}. + * + *

Inheritance and Execution Order

+ * + *

{@code @BeforeSuite} methods are inherited from superclasses as long as they + * are not overridden according to the visibility rules of the Java + * language. Furthermore, {@code @BeforeSuite} methods from superclasses will be + * executed before {@code @BeforeSuite} methods in subclasses. + * + *

The JUnit Platform Suite Engine does not guarantee the execution order of + * multiple {@code @BeforeSuite} methods that are declared within a single test + * class or test interface. While it may at times appear that these methods are + * invoked in alphabetical order, they are in fact sorted using an algorithm + * that is deterministic but intentionally non-obvious. + * + *

In addition, {@code @BeforeSuite} methods are in no way linked to + * {@code @AfterSuite} methods. Consequently, there are no guarantees with regard + * to their wrapping behavior. For example, given two + * {@code @BeforeSuite} methods {@code createA()} and {@code createB()} as well as + * two {@code @AfterSuite} methods {@code destroyA()} and {@code destroyB()}, the + * order in which the {@code @BeforeSuite} methods are executed (e.g. + * {@code createA()} before {@code createB()}) does not imply any order for the + * seemingly corresponding {@code @AfterSuite} methods. In other words, + * {@code destroyA()} might be called before or after + * {@code destroyB()}. The JUnit Team therefore recommends that developers + * declare at most one {@code @BeforeSuite} method and at most one + * {@code @AfterSuite} method per test class or test interface unless there are no + * dependencies between the {@code @BeforeSuite} methods or between the + * {@code @AfterSuite} methods. + * + *

Composition

+ * + *

{@code @BeforeSuite} may be used as a meta-annotation in order to create + * a custom composed annotation that inherits the semantics of + * {@code @BeforeSuite}. + * + * @since 1.11 + * @see AfterSuite + * @see Suite + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@API(status = EXPERIMENTAL, since = "1.11") +public @interface BeforeSuite { +} diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java new file mode 100644 index 000000000000..b00bca64a131 --- /dev/null +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/LifecycleMethodUtils.java @@ -0,0 +1,90 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine; + +import static org.junit.platform.commons.util.AnnotationUtils.findAnnotatedMethods; +import static org.junit.platform.commons.util.ReflectionUtils.returnsPrimitiveVoid; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.util.List; + +import org.junit.platform.commons.JUnitException; +import org.junit.platform.commons.util.ReflectionUtils; +import org.junit.platform.commons.util.ReflectionUtils.HierarchyTraversalMode; +import org.junit.platform.engine.support.hierarchical.ThrowableCollector; +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; + +/** + * Collection of utilities for working with test lifecycle methods. + * + * @since 1.11 + */ +final class LifecycleMethodUtils { + + private LifecycleMethodUtils() { + /* no-op */ + } + + static List findBeforeSuiteMethods(Class testClass, ThrowableCollector throwableCollector) { + return findMethodsAndAssertStaticAndNonPrivate(testClass, BeforeSuite.class, HierarchyTraversalMode.TOP_DOWN, + throwableCollector); + } + + static List findAfterSuiteMethods(Class testClass, ThrowableCollector throwableCollector) { + return findMethodsAndAssertStaticAndNonPrivate(testClass, AfterSuite.class, HierarchyTraversalMode.BOTTOM_UP, + throwableCollector); + } + + private static List findMethodsAndAssertStaticAndNonPrivate(Class testClass, + Class annotationType, HierarchyTraversalMode traversalMode, + ThrowableCollector throwableCollector) { + + List methods = findAnnotatedMethods(testClass, annotationType, traversalMode); + throwableCollector.execute(() -> methods.forEach(method -> { + assertVoid(annotationType, method); + assertStatic(annotationType, method); + assertNonPrivate(annotationType, method); + assertNoParameters(annotationType, method); + })); + return methods; + } + + private static void assertStatic(Class annotationType, Method method) { + if (ReflectionUtils.isNotStatic(method)) { + throw new JUnitException(String.format("@%s method '%s' must be static.", annotationType.getSimpleName(), + method.toGenericString())); + } + } + + private static void assertNonPrivate(Class annotationType, Method method) { + if (ReflectionUtils.isPrivate(method)) { + throw new JUnitException(String.format("@%s method '%s' must not be private.", + annotationType.getSimpleName(), method.toGenericString())); + } + } + + private static void assertVoid(Class annotationType, Method method) { + if (!returnsPrimitiveVoid(method)) { + throw new JUnitException(String.format("@%s method '%s' must not return a value.", + annotationType.getSimpleName(), method.toGenericString())); + } + } + + private static void assertNoParameters(Class annotationType, Method method) { + if (method.getParameterCount() > 0) { + throw new JUnitException(String.format("@%s method '%s' must not accept parameters.", + annotationType.getSimpleName(), method.toGenericString())); + } + } + +} diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java index 9fa107cb5f2c..ccf40405f7bd 100644 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java @@ -13,8 +13,12 @@ import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation; import static org.junit.platform.suite.commons.SuiteLauncherDiscoveryRequestBuilder.request; +import java.lang.reflect.Method; +import java.util.List; + import org.junit.platform.commons.JUnitException; import org.junit.platform.commons.util.Preconditions; +import org.junit.platform.commons.util.ReflectionUtils; import org.junit.platform.commons.util.StringUtils; import org.junit.platform.engine.ConfigurationParameters; import org.junit.platform.engine.EngineExecutionListener; @@ -24,6 +28,8 @@ import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector; +import org.junit.platform.engine.support.hierarchical.ThrowableCollector; import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.core.LauncherDiscoveryResult; import org.junit.platform.launcher.listeners.TestExecutionSummary; @@ -121,16 +127,58 @@ private static String getSuiteDisplayName(Class testClass) { void execute(EngineExecutionListener parentEngineExecutionListener) { parentEngineExecutionListener.executionStarted(this); + ThrowableCollector throwableCollector = new OpenTest4JAwareThrowableCollector(); + + List beforeSuiteMethods = LifecycleMethodUtils.findBeforeSuiteMethods(suiteClass, throwableCollector); + List afterSuiteMethods = LifecycleMethodUtils.findAfterSuiteMethods(suiteClass, throwableCollector); + + executeBeforeSuiteMethods(beforeSuiteMethods, throwableCollector); + + TestExecutionSummary summary = executeTests(parentEngineExecutionListener, throwableCollector); + + executeAfterSuiteMethods(afterSuiteMethods, throwableCollector); + + TestExecutionResult testExecutionResult = computeTestExecutionResult(summary, throwableCollector); + parentEngineExecutionListener.executionFinished(this, testExecutionResult); + } + + private void executeBeforeSuiteMethods(List beforeSuiteMethods, ThrowableCollector throwableCollector) { + if (throwableCollector.isNotEmpty()) { + return; + } + for (Method beforeSuiteMethod : beforeSuiteMethods) { + throwableCollector.execute(() -> ReflectionUtils.invokeMethod(beforeSuiteMethod, null)); + if (throwableCollector.isNotEmpty()) { + return; + } + } + } + + private TestExecutionSummary executeTests(EngineExecutionListener parentEngineExecutionListener, + ThrowableCollector throwableCollector) { + if (throwableCollector.isNotEmpty()) { + return null; + } + // #2838: The discovery result from a suite may have been filtered by // post discovery filters from the launcher. The discovery result should // be pruned accordingly. LauncherDiscoveryResult discoveryResult = this.launcherDiscoveryResult.withRetainedEngines( getChildren()::contains); - TestExecutionSummary summary = launcher.execute(discoveryResult, parentEngineExecutionListener); - parentEngineExecutionListener.executionFinished(this, computeTestExecutionResult(summary)); + return launcher.execute(discoveryResult, parentEngineExecutionListener); } - private TestExecutionResult computeTestExecutionResult(TestExecutionSummary summary) { + private void executeAfterSuiteMethods(List afterSuiteMethods, ThrowableCollector throwableCollector) { + for (Method afterSuiteMethod : afterSuiteMethods) { + throwableCollector.execute(() -> ReflectionUtils.invokeMethod(afterSuiteMethod, null)); + } + } + + private TestExecutionResult computeTestExecutionResult(TestExecutionSummary summary, + ThrowableCollector throwableCollector) { + if (throwableCollector.isNotEmpty()) { + return TestExecutionResult.failed(throwableCollector.getThrowable()); + } if (failIfNoTests && summary.getTestsFoundCount() == 0) { return TestExecutionResult.failed(new NoTestsDiscoveredException(suiteClass)); } diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java new file mode 100644 index 000000000000..c892eee146bb --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/BeforeAndAfterSuiteTests.java @@ -0,0 +1,229 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Named.named; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.suite.engine.SuiteEngineDescriptor.ENGINE_ID; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.FailingAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.FailingBeforeAndAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.FailingBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonStaticAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonStaticBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonVoidAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.NonVoidBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.ParameterAcceptingAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.ParameterAcceptingBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.PrivateAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.PrivateBeforeSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.SeveralFailingBeforeAndAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.SubclassWithBeforeAndAfterSuite; +import static org.junit.platform.suite.engine.testsuites.LifecycleMethodsSuites.SuccessfulBeforeAndAfterSuite; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.suppressed; + +import java.util.ArrayList; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.platform.commons.JUnitException; +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; +import org.junit.platform.suite.engine.testcases.StatefulTestCase; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; + +/** + * Integration tests that verify support for {@link BeforeSuite} and {@link AfterSuite}, + * in the {@link SuiteTestEngine}. + * + * @since 1.11 + */ +public class BeforeAndAfterSuiteTests { + + @BeforeEach + void setUp() { + StatefulTestCase.callSequence = new ArrayList<>(); + } + + @Test + void successfulBeforeAndAfterSuite() { + // @formatter:off + executeSuite(SuccessfulBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(7).finished(7).succeeded(6).failed(1)) + .assertThatEvents() + .haveExactly(1, event(test(StatefulTestCase.Test1.class.getName()), finishedSuccessfully())) + .haveExactly(1, event(test(StatefulTestCase.Test2.class.getName()), finishedWithFailure())); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "test1", + "test2", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void beforeAndAfterSuiteInheritance() { + // @formatter:off + executeSuite(SubclassWithBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(7).finished(7).succeeded(6).failed(1)); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "superclassBeforeSuiteMethod", + "subclassBeforeSuiteMethod", + "test1", + "test2", + "subclassAfterSuiteMethod", + "superclassAfterSuiteMethod" + ); + // @formatter:on + } + + @Test + void failingBeforeSuite() { + // @formatter:off + executeSuite(FailingBeforeSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(2).finished(2).succeeded(1).failed(1)) + .assertThatEvents() + .haveExactly(1, event( + container(FailingBeforeSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @BeforeSuite method")))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void failingAfterSuite() { + // @formatter:off + executeSuite(FailingAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(7).finished(7).succeeded(5).failed(2)) + .assertThatEvents() + .haveExactly(1, event( + container(FailingAfterSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method")))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "test1", + "test2", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void failingBeforeAndAfterSuite() { + // @formatter:off + executeSuite(FailingBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(2).finished(2).succeeded(1).failed(1)) + .assertThatEvents() + .haveExactly(1, event( + container(FailingBeforeAndAfterSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @BeforeSuite method"), + suppressed(0, instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method"))))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "afterSuiteMethod" + ); + // @formatter:on + } + + @Test + void severalFailingBeforeAndAfterSuite() { + // @formatter:off + executeSuite(SeveralFailingBeforeAndAfterSuite.class) + .allEvents() + .assertStatistics(stats -> stats.started(2).finished(2).succeeded(1).failed(1)) + .assertThatEvents() + .haveExactly(1, event( + container(SeveralFailingBeforeAndAfterSuite.class), + finishedWithFailure(instanceOf(RuntimeException.class), + message("Exception thrown by @BeforeSuite method"), + suppressed(0, instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method")), + suppressed(1, instanceOf(RuntimeException.class), + message("Exception thrown by @AfterSuite method"))))); + + assertThat(StatefulTestCase.callSequence).containsExactly( + "beforeSuiteMethod", + "afterSuiteMethod", + "afterSuiteMethod" + ); + // @formatter:on + } + + @ParameterizedTest(name = "{0}") + @MethodSource + void invalidBeforeOrAfterSuiteMethod(Class testSuiteClass, Predicate failureMessagePredicate) { + // @formatter:off + executeSuite(testSuiteClass) + .allEvents() + .assertThatEvents() + .haveExactly(1, event( + container(testSuiteClass), + finishedWithFailure(instanceOf(JUnitException.class), message(failureMessagePredicate)))); + // @formatter:on + } + + private static Stream invalidBeforeOrAfterSuiteMethod() { + return Stream.of( + invalidBeforeOrAfterSuiteCase(NonVoidBeforeSuite.class, "@BeforeSuite method", "must not return a value."), + invalidBeforeOrAfterSuiteCase(ParameterAcceptingBeforeSuite.class, "@BeforeSuite method", + "must not accept parameters."), + invalidBeforeOrAfterSuiteCase(NonStaticBeforeSuite.class, "@BeforeSuite method", "must be static."), + invalidBeforeOrAfterSuiteCase(PrivateBeforeSuite.class, "@BeforeSuite method", "must not be private."), + invalidBeforeOrAfterSuiteCase(NonVoidAfterSuite.class, "@AfterSuite method", "must not return a value."), + invalidBeforeOrAfterSuiteCase(ParameterAcceptingAfterSuite.class, "@AfterSuite method", + "must not accept parameters."), + invalidBeforeOrAfterSuiteCase(NonStaticAfterSuite.class, "@AfterSuite method", "must be static."), + invalidBeforeOrAfterSuiteCase(PrivateAfterSuite.class, "@AfterSuite method", "must not be private.")); + } + + private static Arguments invalidBeforeOrAfterSuiteCase(Class suiteClass, String failureMessageStart, + String failureMessageEnd) { + return arguments(named(suiteClass.getSimpleName(), suiteClass), + (Predicate) s -> s.startsWith(failureMessageStart) && s.endsWith(failureMessageEnd)); + } + + private static EngineExecutionResults executeSuite(Class suiteClass) { + return EngineTestKit.engine(ENGINE_ID).selectors(selectClass(suiteClass)).execute(); + } + +} diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/StatefulTestCase.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/StatefulTestCase.java new file mode 100644 index 000000000000..3a2a32230c27 --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/StatefulTestCase.java @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine.testcases; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * @since 1.11 + */ +public class StatefulTestCase { + + public static List callSequence = new ArrayList<>(); + + public static class Test1 { + + @Test + void statefulTest() { + callSequence.add("test1"); + } + + } + + public static class Test2 { + + @Test + void statefulTest() { + callSequence.add("test2"); + fail("This is a failing test"); + } + + } + +} diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java new file mode 100644 index 000000000000..8a5af3587d38 --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/LifecycleMethodsSuites.java @@ -0,0 +1,245 @@ +/* + * Copyright 2015-2024 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine.testsuites; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.junit.platform.suite.api.AfterSuite; +import org.junit.platform.suite.api.BeforeSuite; +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; +import org.junit.platform.suite.engine.BeforeAndAfterSuiteTests; +import org.junit.platform.suite.engine.testcases.StatefulTestCase; + +/** + * Test suites used in {@link BeforeAndAfterSuiteTests}. + * + * @since 1.11 + */ +public class LifecycleMethodsSuites { + + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + @Suite + @SelectClasses({ StatefulTestCase.Test1.class, StatefulTestCase.Test2.class }) + private @interface TestSuite { + } + + @TestSuite + public static class SuccessfulBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + } + + } + + @TestSuite + public static class FailingBeforeSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + } + + } + + @TestSuite + public static class FailingAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + } + + @TestSuite + public static class FailingBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + } + + @TestSuite + public static class SeveralFailingBeforeAndAfterSuite { + + @BeforeSuite + static void setUp1() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @BeforeSuite + static void setUp2() { + StatefulTestCase.callSequence.add("beforeSuiteMethod"); + throw new RuntimeException("Exception thrown by @BeforeSuite method"); + } + + @AfterSuite + static void tearDown1() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + @AfterSuite + static void tearDown2() { + StatefulTestCase.callSequence.add("afterSuiteMethod"); + throw new RuntimeException("Exception thrown by @AfterSuite method"); + } + + } + + @TestSuite + public static class SuperclassWithBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("superclassBeforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("superclassAfterSuiteMethod"); + } + + } + + public static class SubclassWithBeforeAndAfterSuite extends SuperclassWithBeforeAndAfterSuite { + + @BeforeSuite + static void setUp() { + StatefulTestCase.callSequence.add("subclassBeforeSuiteMethod"); + } + + @AfterSuite + static void tearDown() { + StatefulTestCase.callSequence.add("subclassAfterSuiteMethod"); + } + + } + + @TestSuite + public static class NonVoidBeforeSuite { + + @BeforeSuite + static String nonVoidBeforeSuite() { + fail("Should not be called"); + return ""; + } + + } + + @TestSuite + public static class ParameterAcceptingBeforeSuite { + + @BeforeSuite + static void parameterAcceptingBeforeSuite(String param) { + fail("Should not be called"); + } + + } + + @TestSuite + public static class NonStaticBeforeSuite { + + @BeforeSuite + void nonStaticBeforeSuite() { + fail("Should not be called"); + } + + } + + @TestSuite + public static class PrivateBeforeSuite { + + @BeforeSuite + private static void privateBeforeSuite() { + fail("Should not be called"); + } + + } + + @TestSuite + public static class NonVoidAfterSuite { + + @AfterSuite + static String nonVoidAfterSuite() { + fail("Should not be called"); + return ""; + } + + } + + @TestSuite + public static class ParameterAcceptingAfterSuite { + + @AfterSuite + static void parameterAcceptingAfterSuite(String param) { + fail("Should not be called"); + } + + } + + @TestSuite + public static class NonStaticAfterSuite { + + @AfterSuite + void nonStaticAfterSuite() { + fail("Should not be called"); + } + + } + + @TestSuite + public static class PrivateAfterSuite { + + @AfterSuite + private static void privateAfterSuite() { + fail("Should not be called"); + } + + } + +}