Mutation testing

.azure/build-pipeline.yaml

@@ -46,6 +46,13 @@ jobs:
         parameters:
           JDK_VERSION: $(JDK_VERSION)
 
+       # Run mutation testing before ITs tests
+      - bash: mvn clean test-compile org.pitest:pitest-maven:mutationCoverage
+        displayName: 'Run Mutation Testing'
+        env:
+          # Test container optimization
+          TESTCONTAINERS_RYUK_DISABLED: TRUE
+          TESTCONTAINERS_CHECKS_DISABLE: TRUE
       - bash: mvn -Dfailsafe.rerunFailingTestsCount=3 clean verify
         env:
           # Test container optimization

README.md

@@ -235,6 +235,26 @@ StrimziKafkaCluster kafkaCluster = new StrimziKafkaCluster.StrimziKafkaClusterBu
 kafkaCluster.start();
 ```
 
+### Running Mutation Testing
+
+To run mutation testing and assess your code’s robustness against small changes, use the following Maven command:
+```bash
+mvn clean test-compile org.pitest:pitest-maven:mutationCoverage
+```
+This command will execute mutation tests based on the pitest-maven plugin and display results directly in the console.
+
+#### Viewing Mutation Testing Results
+
+After running the command, results will be available in an HTML report located in target/pit-reports/<timestamp>. 
+Open the index.html file in a browser to view detailed information about the mutation coverage.
+
+#### Using `@DoNotMutate` Annotation
+
+For parts of the code primarily covered by integration tests, we can use the `@DoNotMutate` annotation. 
+Applying this annotation to code ensures that mutation testing will ignore it. 
+This is particularly useful for code components that are challenging to test at a unit level but well-covered in integration tests. 
+Using `@DoNotMutate` helps keep mutation coverage metrics meaningful by excluding areas where mutation detection would not add value.
+
 ### Additional tips
 
 1. In case you are using `Azure pipelines` Ryuk needs to be turned off, since Azure does not allow starting privileged containers.

pom.xml

@@ -105,6 +105,7 @@
         <fasterxml.jackson-core.version>2.18.0</fasterxml.jackson-core.version>
         <fasterxml.jackson-databind.version>2.18.0</fasterxml.jackson-databind.version>
         <toxiproxy.java.version>2.1.7</toxiproxy.java.version>
+        <pitest-annotations.version>1.3.3</pitest-annotations.version>
 
         <!-- DEPENDENCY TEST VERSIONS      -->
         <jupiter.version>5.11.0</jupiter.version>
@@ -124,6 +125,8 @@
         <maven.checkstyle.version>3.5.0</maven.checkstyle.version>
         <maven.gpg.version>1.6</maven.gpg.version>
         <sonatype.nexus.staging>1.7.0</sonatype.nexus.staging>
+        <pit-junit-plugin.version>1.2.1</pit-junit-plugin.version>
+        <pit-plugin.version>1.17.0</pit-plugin.version>
 
         <!-- FIX VULNERABILITY VERSIONS  -->
         <commons-compress.version>1.26.1</commons-compress.version>
@@ -190,6 +193,12 @@
             <artifactId>toxiproxy-java</artifactId>
             <version>${toxiproxy.java.version}</version>
         </dependency>
+        <!-- for mutation testing       -->
+        <dependency>
+            <groupId>com.arcmutate</groupId>
+            <artifactId>pitest-annotations</artifactId>
+            <version>${pitest-annotations.version}</version>
+        </dependency>
 
         <!-- TEST DEPENDENCY -->
         <dependency>
@@ -241,6 +250,13 @@
             <version>${mockito.version}</version>
             <scope>test</scope>
         </dependency>
+        <!--  needed for PIT (mutation testing)   -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
 
         <!-- overriding version of commons-compress for Test container - Vulnerability -->
         <dependency>
@@ -398,6 +414,37 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.pitest</groupId>
+                <artifactId>pitest-maven</artifactId>
+                <version>${pit-plugin.version}</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.pitest</groupId>
+                        <artifactId>pitest-junit5-plugin</artifactId>
+                        <version>${pit-junit-plugin.version}</version>
+                    </dependency>
+                </dependencies>
+                <configuration>
+                    <targetClasses>
+                        <!-- we not want to include also WaitException, Utils classes etc. -->
+                        <param>io.strimzi.test.container.Strimzi*</param>
+                    </targetClasses>
+                    <targetTests>
+                        <!-- include just UTs within mutation and prevent image pulling -->
+                        <param>io.strimzi.test.container.Strimzi*Test</param>
+                    </targetTests>
+                    <avoidCallsTo>
+                        <avoidCallsTo>java.util.logging</avoidCallsTo>
+                        <avoidCallsTo>org.apache.log4j</avoidCallsTo>
+                        <avoidCallsTo>org.slf4j</avoidCallsTo>
+                        <avoidCallsTo>org.apache.commons.logging</avoidCallsTo>
+                    </avoidCallsTo>
+                    <mutationThreshold>80</mutationThreshold>
+                    <coverageThreshold>71</coverageThreshold>
+                    <verbose>true</verbose>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

src/main/java/io/strimzi/test/container/StrimziKafkaCluster.java

@@ -4,6 +4,7 @@
  */
 package io.strimzi.test.container;
 
+import com.groupcdg.pitest.annotations.DoNotMutate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.Container;
@@ -263,12 +264,14 @@ public Collection<KafkaContainer> getBrokers() {
     }
 
     @Override
+    @DoNotMutate
     public boolean hasKraftOrExternalZooKeeperConfigured() {
         KafkaContainer broker0 = brokers.iterator().next();
         return broker0.hasKraftOrExternalZooKeeperConfigured() ? true : false;
     }
 
     @Override
+    @DoNotMutate
     public String getInternalZooKeeperConnect() {
         if (hasKraftOrExternalZooKeeperConfigured()) {
             throw new IllegalStateException("Connect string is not available when using KRaft or external ZooKeeper");
@@ -322,15 +325,25 @@ private void configureQuorumVoters(final Map<String, String> additionalKafkaConf
         additionalKafkaConfiguration.put("controller.quorum.voters", quorumVoters);
     }
 
-    @SuppressWarnings({"CyclomaticComplexity"})
+    @SuppressWarnings({"CyclomaticComplexity", "NPathComplexity"})
     @Override
+    @DoNotMutate
     public void start() {
         Stream<KafkaContainer> startables = this.brokers.stream();
         try {
             Startables.deepStart(startables).get(60, TimeUnit.SECONDS);
-        } catch (InterruptedException | ExecutionException | TimeoutException e) {
+        } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
-            throw new RuntimeException("Failed to start Kafka containers", e);
+            throw new RuntimeException("Interrupted while starting Kafka containers", e);
+        } catch (ExecutionException | UnsupportedKraftKafkaVersionException e) {
+            Throwable cause = e.getCause();
+            if (cause instanceof UnsupportedKraftKafkaVersionException) {
+                throw (UnsupportedKraftKafkaVersionException) cause;
+            } else {
+                throw new RuntimeException("Failed to start Kafka containers", e);
+            }
+        } catch (TimeoutException e) {
+            throw new RuntimeException("Timed out while starting Kafka containers", e);
         }
 
         if (this.isZooKeeperBasedKafkaCluster()) {
@@ -400,6 +413,7 @@ public void start() {
     }
 
     @Override
+    @DoNotMutate
     public void stop() {
         if (this.isZooKeeperBasedKafkaCluster()) {
             // firstly we shut-down zookeeper -> reason: 'On the command line if I kill ZK first it sometimes prevents a broker from shutting down quickly.'
@@ -419,4 +433,8 @@ public void stop() {
     public StrimziZookeeperContainer getZookeeper() {
         return zookeeper;
     }
+
+    protected Network getNetwork() {
+        return network;
+    }
 }

src/main/java/io/strimzi/test/container/StrimziKafkaContainer.java

@@ -6,6 +6,7 @@
 
 import com.github.dockerjava.api.command.InspectContainerResponse;
 import com.github.dockerjava.api.model.ContainerNetwork;
+import com.groupcdg.pitest.annotations.DoNotMutate;
 import eu.rekawek.toxiproxy.Proxy;
 import eu.rekawek.toxiproxy.ToxiproxyClient;
 import org.apache.logging.log4j.Level;
@@ -144,6 +145,7 @@ private StrimziKafkaContainer(CompletableFuture<String> imageName) {
 
     @Override
     @SuppressWarnings({"NPathComplexity", "CyclomaticComplexity"})
+    @DoNotMutate
     protected void doStart() {
         if (this.proxyContainer != null && !this.proxyContainer.isRunning()) {
             this.proxyContainer.start();
@@ -188,6 +190,7 @@ protected void doStart() {
     }
 
     @Override
+    @DoNotMutate
     public void stop() {
         if (proxyContainer != null && proxyContainer.isRunning()) {
             proxyContainer.stop();
@@ -213,6 +216,7 @@ protected String runStarterScript() {
      *
      * @return StrimziKafkaContainer instance
      */
+    @DoNotMutate
     public StrimziKafkaContainer waitForRunning() {
         if (this.useKraft) {
             super.waitingFor(Wait.forLogMessage(".*Transitioning from RECOVERY to RUNNING.*", 1));
@@ -223,6 +227,7 @@ public StrimziKafkaContainer waitForRunning() {
     }
 
     @Override
+    @DoNotMutate
     protected void containerIsStarting(final InspectContainerResponse containerInfo, final boolean reused) {
         super.containerIsStarting(containerInfo, reused);
 
@@ -286,6 +291,7 @@ protected void containerIsStarting(final InspectContainerResponse containerInfo,
     }
 
     @Override
+    @DoNotMutate
     public boolean hasKraftOrExternalZooKeeperConfigured() {
         return this.useKraft || this.externalZookeeperConnect != null;
     }
@@ -371,6 +377,7 @@ protected String[] buildListenersConfig(final InspectContainerResponse container
      * In order to avoid any compile dependency on kafka-clients' <code>Uuid</code> specific class,
      * we implement our own uuid generator by replicating the Kafka's base64 encoded uuid generation logic.
      */
+    @DoNotMutate
     private String randomUuid() {
         final UUID metadataTopicIdInternal = new UUID(0L, 1L);
         final UUID zeroIdImpactInternal = new UUID(0L, 0L);
@@ -570,6 +577,7 @@ public String getInternalZooKeeperConnect() {
      * @return the bootstrap servers URL
      */
     @Override
+    @DoNotMutate
     public String getBootstrapServers() {
         if (proxyContainer != null) {
             // returning the proxy host and port for indirect connection

src/main/java/io/strimzi/test/container/StrimziZookeeperContainer.java

@@ -5,6 +5,7 @@
 package io.strimzi.test.container;
 
 import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.groupcdg.pitest.annotations.DoNotMutate;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,6 +61,7 @@ public StrimziZookeeperContainer(String dockerImageName) {
     /**
      * Image name is lazily set in {@link #doStart()} method
      */
+    @DoNotMutate
     private StrimziZookeeperContainer(CompletableFuture<String> imageName) {
         super(imageName);
         this.imageNameProvider = imageName;
@@ -76,6 +78,7 @@ private StrimziZookeeperContainer(CompletableFuture<String> imageName) {
     }
 
     @Override
+    @DoNotMutate
     protected void doStart() {
         if (!imageNameProvider.isDone()) {
             imageNameProvider.complete(KafkaVersionService.strimziTestContainerImageName(kafkaVersion));
@@ -86,6 +89,7 @@ protected void doStart() {
     }
 
     @Override
+    @DoNotMutate
     protected void containerIsStarting(InspectContainerResponse containerInfo, boolean reused) {
         super.containerIsStarting(containerInfo, reused);
 
@@ -111,6 +115,7 @@ protected void containerIsStarting(InspectContainerResponse containerInfo, boole
      * @param zooKeeperPropertiesFile the mountable config file
      * @return StrimziZookeeperContainer instance
      */
+    @DoNotMutate
     public StrimziZookeeperContainer withZooKeeperPropertiesFile(final MountableFile zooKeeperPropertiesFile) {
         Utils.asTransferableBytes(zooKeeperPropertiesFile)
             .ifPresent(properties -> withCopyToContainer(properties, "/opt/kafka/config/zookeeper.properties"));
@@ -123,6 +128,7 @@ public StrimziZookeeperContainer withZooKeeperPropertiesFile(final MountableFile
      * @param kafkaVersion kafka version
      * @return StrimziKafkaContainer instance
      */
+    @DoNotMutate
     public StrimziZookeeperContainer withKafkaVersion(final String kafkaVersion) {
         this.kafkaVersion = kafkaVersion;
         return this;
@@ -133,6 +139,7 @@ public StrimziZookeeperContainer withKafkaVersion(final String kafkaVersion) {
      *
      * @return zookeeper connect string `host:port`
      */
+    @DoNotMutate
     public String getConnectString() {
         return this.getHost() + ":" + this.getMappedPort(ZOOKEEPER_PORT);
     }