Skip to content

Commit

Permalink
[SUREFIRE-2065] Fix parameterized JUnit4 test reporting (#608)
Browse files Browse the repository at this point in the history
* [SUREFIRE-2065] Fix parameterized JUnit4 test reporting

---------

Co-authored-by: Christian Marquez Grabia <4968250+chalmagr@users.noreply.github.com>
  • Loading branch information
andpab and chalmagr authored Feb 6, 2023
1 parent b8887b4 commit 11d0541
Show file tree
Hide file tree
Showing 14 changed files with 579 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,25 @@ public TestFile assertContainsText( String text )
return assertContainsText( containsString( text ) );
}

public TestFile assertNotContainsText( Matcher<String> matcher )
{
final List<String> list = surefireVerifier.loadFile( file, encoding );
for ( String line : list )
{
if ( matcher.matches( line ) )
{
Assert.fail( "Found unexpected message in log" );
return null;
}
}
return this;
}

public TestFile assertNotContainsText( String text )
{
return assertNotContainsText( containsString( text ) );
}

public URI toURI()
{
return file.toURI();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package org.apache.maven.surefire.its.jiras;

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import static java.nio.charset.StandardCharsets.UTF_8;

import org.apache.maven.it.VerificationException;
import org.apache.maven.surefire.its.fixture.OutputValidator;
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
import org.hamcrest.Matchers;
import org.junit.Test;

/**
* Integration Tests for SUREFIRE-2065
*/
@SuppressWarnings( "checkstyle:magicnumber" )
public class Surefire2065IT extends SurefireJUnit4IntegrationTestCase
{
@Test
public void shouldNotDetectFlakyTestsWhenCombiningJunit4And5Tests() throws VerificationException
{
OutputValidator validator = unpack( "surefire-2065-common" )
.mavenTestFailureIgnore( true )
.executeTest()
.assertTestSuiteResults( 8, 0, 4, 0, 0 );

assertJunit4( validator );
assertJunit5( validator );
}

@Test
public void shouldNotDetectFlakyTestsWhenRunningOnlyJunit4() throws VerificationException
{
OutputValidator validator = unpack( "surefire-2065-junit4" )
.mavenTestFailureIgnore( true )
.executeTest()
.assertTestSuiteResults( 4, 0, 2, 0, 0 );

assertJunit4( validator );
}

@Test
public void shouldNotDetectFlakyTestsWhenRunningOnlyJunit5() throws VerificationException
{
OutputValidator validator = unpack( "surefire-2065-junit5" )
.mavenTestFailureIgnore( true )
.executeTest()
.assertTestSuiteResults( 4, 0, 2, 0, 0 );

assertJunit5( validator );
}

private static void assertJunit4( OutputValidator validator )
{
validator.getSurefireReportsFile( "TEST-pkg.junit4.ParameterizedTest.xml", UTF_8 )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[0]\".*/>$" ) )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[1]\".*[^/]>$" ) )
.assertContainsText( "<failure message=" )
.assertContainsText( "<rerunFailure message=" )
.assertNotContainsText( "<flakyFailure message=" );

validator.getSurefireReportsFile( "TEST-pkg.junit4.ParameterizedWithDisplayNameTest.xml", UTF_8 )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[value=0]\".*/>$" ) )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\[value=1]\".*[^/]>$" ) )
.assertContainsText( "<failure message=" )
.assertContainsText( "<rerunFailure message=" )
.assertNotContainsText( "<flakyFailure message=" );
}

private static void assertJunit5( OutputValidator validator )
{
validator.getSurefireReportsFile( "TEST-pkg.junit5.ParameterizedTest.xml", UTF_8 )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[1]\".*/>$" ) )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[2]\".*[^/]>$" ) )
.assertContainsText( "<failure message=" )
.assertContainsText( "<rerunFailure message=" )
.assertNotContainsText( "<flakyFailure message=" );

validator.getSurefireReportsFile( "TEST-pkg.junit5.ParameterizedWithDisplayNameTest.xml", UTF_8 )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[1]\".*/>$" ) )
.assertContainsText( Matchers.matchesPattern( "^ *<testcase name=\"notFlaky\\(int\\)\\[2]\".*[^/]>$" ) )
.assertContainsText( "<failure message=" )
.assertContainsText( "<rerunFailure message=" )
.assertNotContainsText( "<flakyFailure message=" );
}
}
72 changes: 72 additions & 0 deletions surefire-its/src/test/resources/surefire-2065-common/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.example</groupId>
<artifactId>surefire-2065-common</artifactId>
<version>1.0-SNAPSHOT</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
<junit4.version>4.12</junit4.version>
<junit5.version>5.3.2</junit5.version>
</properties>

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit4.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit5.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<version>${junit5.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.version}</version>
<configuration>
<rerunFailingTestsCount>1</rerunFailingTestsCount>
</configuration>
</plugin>
</plugins>
</build>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package pkg.junit4;

import static org.junit.Assert.assertEquals;

import java.util.Arrays;
import java.util.List;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

@RunWith(Parameterized.class)
public class ParameterizedTest
{
@Parameterized.Parameters
public static List<Integer> parameters() throws Exception
{
return Arrays.asList( 0, 1 );
}

@Parameterized.Parameter(0)
public int expected;

@Test
public void notFlaky()
{
assertEquals( expected, 0 );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package pkg.junit4;

import static org.junit.Assert.assertEquals;

import java.util.Arrays;
import java.util.List;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

@RunWith(Parameterized.class)
public class ParameterizedWithDisplayNameTest
{
private static int count = 0;

@Parameterized.Parameters(name = "value={0}")
public static List<Integer> parameters() throws Exception
{
return Arrays.asList( 0, 1 );
}

@Parameterized.Parameter(0)
public int expected;

@Test
public void notFlaky()
{
assertEquals( expected, 0 );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package pkg.junit5;

import static org.junit.jupiter.api.Assertions.assertEquals;

import java.util.Arrays;
import java.util.stream.Stream;

import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.ArgumentsProvider;
import org.junit.jupiter.params.provider.ArgumentsSource;

class ParameterizedTest
{
static class ParameterSource implements ArgumentsProvider
{
@Override
public Stream<? extends Arguments> provideArguments( ExtensionContext context ) throws Exception
{
return Arrays.asList(0, 1).stream().map( Arguments::of );
}
}

@org.junit.jupiter.params.ParameterizedTest
@ArgumentsSource(value = ParameterSource.class)
public void notFlaky(int expected)
{
assertEquals( expected, 0 );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package pkg.junit5;

import static org.junit.jupiter.api.Assertions.assertEquals;

import java.util.Arrays;
import java.util.stream.Stream;

import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.ArgumentsProvider;
import org.junit.jupiter.params.provider.ArgumentsSource;

class ParameterizedWithDisplayNameTest
{
static class ParameterSource implements ArgumentsProvider
{
@Override
public Stream<? extends Arguments> provideArguments( ExtensionContext context ) throws Exception
{
return Arrays.asList(0, 1).stream().map( Arguments::of );
}
}

@org.junit.jupiter.params.ParameterizedTest(name = "value={0}")
@ArgumentsSource(value = ParameterSource.class)
public void notFlaky(int expected)
{
assertEquals( expected, 0 );
}
}
59 changes: 59 additions & 0 deletions surefire-its/src/test/resources/surefire-2065-junit4/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.example</groupId>
<artifactId>surefire-2065-junit4</artifactId>
<version>1.0-SNAPSHOT</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
<junit4.version>4.12</junit4.version>
</properties>

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit4.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.version}</version>
<configuration>
<rerunFailingTestsCount>1</rerunFailingTestsCount>
</configuration>
</plugin>
</plugins>
</build>

</project>
Loading

0 comments on commit 11d0541

Please sign in to comment.