Updates to support assumption violations (#108)

* Addung support for skipping tests when assumption violations are thrown --------- Co-authored-by: noconnor <niall.oconnor@yahooinc.com>
noconnor · Jun 29, 2023 · 1dfc33b · 1dfc33b
1 parent 5548b26
commit 1dfc33b
Show file tree

Hide file tree

Showing 25 changed files with 1,383 additions and 183 deletions.
diff --git a/...t4-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java b/...t4-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java
@@ -11,6 +11,7 @@
 import com.github.noconnor.junitperf.JUnitPerfTest;
 
 import static com.github.noconnor.junitperf.examples.utils.ReportingUtils.newHtmlReporter;
+import static org.junit.Assume.assumeFalse;
 
 public class ExampleSuccessTests {
 
@@ -34,4 +35,15 @@ public void whenNoRequirementsArePresent_thenTestShouldAlwaysPass() throws IOExc
       socket.connect(new InetSocketAddress("www.google.com", 80), 1000);
     }
   }
+
+  @Test
+  @JUnitPerfTest(threads = 10, durationMs = 10_000, warmUpMs = 1_000, rampUpPeriodMs = 2_000, totalExecutions = 100)
+  public void whenAssumptionFails_thenTestShouldBeSkipped() throws IOException {
+    //noinspection DataFlowIssue
+    assumeFalse(true); // dummy test to illustrate skipped tests
+    try (Socket socket = new Socket()) {
+      socket.connect(new InetSocketAddress("www.google.com", 80), 1000);
+    }
+  }
+
 }
diff --git a/...t5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java b/...t5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java
@@ -1,9 +1,9 @@
 package com.github.noconnor.junitperf.examples;
 
 import com.github.noconnor.junitperf.JUnitPerfInterceptor;
+import com.github.noconnor.junitperf.JUnitPerfReportingConfig;
 import com.github.noconnor.junitperf.JUnitPerfTest;
 import com.github.noconnor.junitperf.JUnitPerfTestActiveConfig;
-import com.github.noconnor.junitperf.JUnitPerfReportingConfig;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -14,6 +14,7 @@
 import java.net.Socket;
 
 import static com.github.noconnor.junitperf.examples.utils.ReportingUtils.newHtmlReporter;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
 
 @ExtendWith(JUnitPerfInterceptor.class)
 public class ExampleSuccessTests {
@@ -49,4 +50,15 @@ public void whenTotalNumberOfExecutionsIsSet_thenTotalExecutionsShouldOverrideDu
             socket.connect(new InetSocketAddress("www.google.com", 80), 1000);
         }
     }
+
+    @Test
+    @JUnitPerfTest(threads = 10, durationMs = 10_000, warmUpMs = 1_000, rampUpPeriodMs = 2_000, maxExecutionsPerSecond = 100)
+    public void whenAssumptionFails_thenTestWillBeSkipped() throws IOException {
+        //noinspection DataFlowIssue
+        assumeFalse(true); // dummy test to illustrate skipped tests
+
+        try (Socket socket = new Socket()) {
+            socket.connect(new InetSocketAddress("www.google.com", 80), 1000);
+        }
+    }
 }
diff --git a/...-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassOne.java b/...-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassOne.java
@@ -2,6 +2,8 @@
 
 import org.junit.jupiter.api.Test;
 
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
 public class TestClassOne {
     @Test
     public void sample_test1_class1() throws InterruptedException {
@@ -14,4 +16,10 @@ public void sample_test2_class1() throws InterruptedException {
         Thread.sleep(1);
     }
 
+    @Test
+    public void sample_test3_class1() throws InterruptedException {
+        //noinspection DataFlowIssue
+        assumeFalse(true); // dummy test to illustrate skipped tests
+    }
+
 }
diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java
@@ -52,6 +52,9 @@ public class EvaluationContext {
   private long finishTimeNs;
   @Getter
   private final boolean isAsyncEvaluation;
+  @Setter
+  @Getter
+  private Throwable abortedException;
 
   @Getter
   private Map<Integer, Float> requiredPercentiles = emptyMap();
@@ -118,6 +121,10 @@ public EvaluationContext(String testName, long startTimeNs, boolean isAsyncEvalu
     this.isAsyncEvaluation = isAsyncEvaluation;
   }
 
+  public boolean isAborted() {
+    return nonNull(abortedException);
+  }
+
   @SuppressWarnings("WeakerAccess")
   public long getThroughputQps() {
     return (long)((evaluationCount/ ((float)configuredDuration - configuredWarmUp)) * 1000);

diff --git a/...c/main/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGenerator.java b/...c/main/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGenerator.java
@@ -28,44 +28,49 @@ public void generateReport(LinkedHashSet<EvaluationContext> testContexts) {
   }
 
   public void updateReport(EvaluationContext context) {
-    String throughputStatus = context.isThroughputAchieved() ? PASSED : FAILED;
-    String errorRateStatus = context.isErrorThresholdAchieved() ? PASSED : FAILED;
+    if (context.isAborted()) {
+      log.info("Test {} was SKIPPED", context.getTestName());
+    } else {
 
-    log.info("Test Name:    {}", context.getTestName());
-    log.info("Started at:   {}", context.getStartTime());
-    log.info("Invocations:  {}", context.getEvaluationCount());
-    log.info("  - Success:  {}", context.getEvaluationCount() - context.getErrorCount());
-    log.info("  - Errors:   {}", context.getErrorCount());
-    log.info("  - Errors:   {}% - {}", context.getErrorPercentage(), errorRateStatus);
-    log.info("");
-    log.info("Thread Count: {}", context.getConfiguredThreads());
-    log.info("Warm up:      {} ms", context.getConfiguredWarmUp());
-    log.info("Ramp up:      {} ms", context.getConfiguredRampUpPeriodMs());
-    log.info("");
-    log.info("Execution time: {}", context.getTestDurationFormatted());
-    log.info("Throughput:     {}/s (Required: {}/s) - {}",
-      context.getThroughputQps(),
-      context.getRequiredThroughput(),
-      throughputStatus);
-    log.info("Min. latency:   {} ms (Required: {}ms) - {}",
-      context.getMinLatencyMs(),
-      format(context.getRequiredMinLatency()));
-    log.info("Max. latency:    {} ms (Required: {}ms) - {}",
-      context.getMaxLatencyMs(),
-      format(context.getRequiredMaxLatency()));
-    log.info("Ave. latency:    {} ms (Required: {}ms) - {}",
-      context.getMeanLatencyMs(),
-      format(context.getRequiredMeanLatency()));
-    context.getRequiredPercentiles().forEach((percentile, threshold) -> {
-      String percentileStatus = context.getPercentileResults().get(percentile) ? PASSED : FAILED;
-      log.info("{}:    {}ms (Required: {} ms) - {}",
-        percentile,
-        context.getLatencyPercentileMs(percentile),
-        format(threshold),
-        percentileStatus);
-    });
-    log.info("");
-    log.info("");
+      String throughputStatus = context.isThroughputAchieved() ? PASSED : FAILED;
+      String errorRateStatus = context.isErrorThresholdAchieved() ? PASSED : FAILED;
+
+      log.info("Test Name:    {}", context.getTestName());
+      log.info("Started at:   {}", context.getStartTime());
+      log.info("Invocations:  {}", context.getEvaluationCount());
+      log.info("  - Success:  {}", context.getEvaluationCount() - context.getErrorCount());
+      log.info("  - Errors:   {}", context.getErrorCount());
+      log.info("  - Errors:   {}% - {}", context.getErrorPercentage(), errorRateStatus);
+      log.info("");
+      log.info("Thread Count: {}", context.getConfiguredThreads());
+      log.info("Warm up:      {} ms", context.getConfiguredWarmUp());
+      log.info("Ramp up:      {} ms", context.getConfiguredRampUpPeriodMs());
+      log.info("");
+      log.info("Execution time: {}", context.getTestDurationFormatted());
+      log.info("Throughput:     {}/s (Required: {}/s) - {}",
+              context.getThroughputQps(),
+              context.getRequiredThroughput(),
+              throughputStatus);
+      log.info("Min. latency:   {} ms (Required: {}ms) - {}",
+              context.getMinLatencyMs(),
+              format(context.getRequiredMinLatency()));
+      log.info("Max. latency:    {} ms (Required: {}ms) - {}",
+              context.getMaxLatencyMs(),
+              format(context.getRequiredMaxLatency()));
+      log.info("Ave. latency:    {} ms (Required: {}ms) - {}",
+              context.getMeanLatencyMs(),
+              format(context.getRequiredMeanLatency()));
+      context.getRequiredPercentiles().forEach((percentile, threshold) -> {
+        String percentileStatus = context.getPercentileResults().get(percentile) ? PASSED : FAILED;
+        log.info("{}:    {}ms (Required: {} ms) - {}",
+                percentile,
+                context.getLatencyPercentileMs(percentile),
+                format(threshold),
+                percentileStatus);
+      });
+      log.info("");
+      log.info("");
+    }
   }
 
   @Override

diff --git a/...e/src/main/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGenerator.java b/...e/src/main/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGenerator.java
@@ -46,9 +46,11 @@ public void generateReport(LinkedHashSet<EvaluationContext> testContexts) {
       writer.newLine();
       history.forEach(context -> {
 
-        String record = String.format("%s,%d,%d,%d,%.4f,%.4f,%.4f,%s",
-          context.getTestName(),
-          context.getConfiguredDuration(),
+        String name = context.isAborted() ? context.getTestName() + " (skipped)" : context.getTestName();
+        int duration = context.isAborted() ? 0 : context.getConfiguredDuration();
+        String record = String.format("%s,%s,%d,%d,%.4f,%.4f,%.4f,%s", 
+          name,
+          duration,
           context.getConfiguredThreads(),
           context.getThroughputQps(),
           context.getMinLatencyMs(),

diff --git a/.../src/main/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGenerator.java b/.../src/main/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGenerator.java
@@ -24,6 +24,7 @@
 
 import static java.lang.System.getProperty;
 import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
 
 @Slf4j
 public class HtmlReportGenerator implements ReportGenerator {
@@ -85,17 +86,21 @@ private void renderTemplate() {
                 ViewData c = new ViewData(context);
 
                 String overview = ViewProcessor.populateTemplate(c, "context", blocks.get(OVERVIEW_MARKER));
-                String detail = ViewProcessor.populateTemplate(c, "context", blocks.get(DETAILS_MARKER));
-                String percentileData = ViewProcessor.populateTemplate(
-                        c.getRequiredPercentiles(),
-                        "context.percentiles",
-                        blocks.get(PERCENTILE_TARGETS_MARKER)
-                );
-
-                detail = detail.replaceAll(asRegex(PERCENTILE_TARGETS_MARKER), percentileData);
-
+
+                if (context.isAborted()) {
+                    overview = overview.replaceAll("href=", "nolink=");
+                } else {
+                    String detail = ViewProcessor.populateTemplate(c, "context", blocks.get(DETAILS_MARKER));
+                    String percentileData = ViewProcessor.populateTemplate(
+                            c.getRequiredPercentiles(),
+                            "context.percentiles",
+                            blocks.get(PERCENTILE_TARGETS_MARKER)
+                    );
+
+                    detail = detail.replaceAll(asRegex(PERCENTILE_TARGETS_MARKER), percentileData);
+                    details.append(detail).append("\n");
+                }
                 overviews.append(overview).append("\n");
-                details.append(detail).append("\n");
             }
 
             root = root.replaceAll(asRegex(OVERVIEW_MARKER), overviews.toString());

diff --git a/...-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewData.java b/...-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewData.java
@@ -6,17 +6,20 @@
 import lombok.Setter;
 import lombok.ToString;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static java.util.Objects.isNull;
 import static java.util.Objects.nonNull;
 
 @Getter
 public class ViewData {
 
     static final String SUCCESS_COLOUR = "#2b67a4";
     static final String FAILED_COLOUR = "#d9534f";
+    static final String SKIPPED_COLOUR = "#dcdcdc";
 
     @Getter
     @Setter
@@ -59,7 +62,7 @@ public static final class RequiredPercentilesData {
 
     public ViewData(EvaluationContext context) {
         this.testName = buildTestName(context);
-        this.testNameColour = context.isSuccessful() ? SUCCESS_COLOUR : FAILED_COLOUR;
+        this.testNameColour = context.isAborted() ? SKIPPED_COLOUR : context.isSuccessful() ? SUCCESS_COLOUR : FAILED_COLOUR;
         this.chartData = buildChartData(context);
         this.csvData = buildCsvData(context);
         this.startTime = context.getStartTime();
@@ -88,10 +91,17 @@ public ViewData(EvaluationContext context) {
     }
 
     private static String buildTestName(EvaluationContext context) {
-        return nonNull(context.getGroupName()) ? context.getGroupName() + " : " + context.getTestName() : context.getTestName();
+        String baseName = nonNull(context.getGroupName()) ? context.getGroupName() + " : " + context.getTestName() : context.getTestName();
+        if (context.isAborted()){
+            baseName = baseName + (" (skipped)");
+        }
+        return baseName;
     }
 
     private List<RequiredPercentilesData> buildRequiredPercentileData(EvaluationContext context) {
+        if (isNull(context.getPercentileResults())) {
+            return Collections.emptyList();
+        }
         return context.getRequiredPercentiles().entrySet()
                 .stream()
                 .map(entry -> {