From 1dfc33be52cd69a3ab9bffc8f9dfd39751c7ccae Mon Sep 17 00:00:00 2001 From: noconnor Date: Thu, 29 Jun 2023 16:19:50 +0100 Subject: [PATCH] Updates to support assumption violations (#108) * Addung support for skipping tests when assumption violations are thrown --------- Co-authored-by: noconnor --- .../examples/ExampleSuccessTests.java | 12 + .../examples/ExampleSuccessTests.java | 14 +- .../examples/existing/TestClassOne.java | 8 + .../junitperf/data/EvaluationContext.java | 7 + .../providers/ConsoleReportGenerator.java | 79 +- .../providers/CsvReportGenerator.java | 8 +- .../providers/HtmlReportGenerator.java | 25 +- .../reporting/providers/utils/ViewData.java | 14 +- .../junitperf/statements/EvaluationTask.java | 238 +++--- .../statements/ExceptionsRegistry.java | 51 ++ .../PerformanceEvaluationStatement.java | 29 +- .../junitperf/data/EvaluationContextTest.java | 10 + .../reporting/BaseReportGeneratorTest.java | 34 + .../providers/ConsoleReportGeneratorTest.java | 7 + .../providers/CsvReportGeneratorTest.java | 8 + .../providers/HtmlReportGeneratorTest.java | 7 + .../statements/EvaluationTaskTest.java | 66 +- .../statements/ExceptionsRegistryTest.java | 66 ++ .../PerformanceEvaluationStatementTest.java | 36 + .../test/resources/csv/fail_abort_succeed.csv | 4 + .../html/example_aborted_failed_success.html | 778 ++++++++++++++++++ .../noconnor/junitperf/JUnitPerfRule.java | 8 + .../noconnor/junitperf/JUnitPerfRuleTest.java | 38 +- .../junitperf/JUnitPerfInterceptor.java | 7 + .../junitperf/JUnitPerfInterceptorTest.java | 12 + 25 files changed, 1383 insertions(+), 183 deletions(-) create mode 100644 junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/ExceptionsRegistry.java create mode 100644 junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/ExceptionsRegistryTest.java create mode 100644 junitperf-core/src/test/resources/csv/fail_abort_succeed.csv create mode 100644 junitperf-core/src/test/resources/html/example_aborted_failed_success.html diff --git a/junit4-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java b/junit4-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java index 34ba27a..e141702 100644 --- a/junit4-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java +++ b/junit4-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java @@ -11,6 +11,7 @@ import com.github.noconnor.junitperf.JUnitPerfTest; import static com.github.noconnor.junitperf.examples.utils.ReportingUtils.newHtmlReporter; +import static org.junit.Assume.assumeFalse; public class ExampleSuccessTests { @@ -34,4 +35,15 @@ public void whenNoRequirementsArePresent_thenTestShouldAlwaysPass() throws IOExc socket.connect(new InetSocketAddress("www.google.com", 80), 1000); } } + + @Test + @JUnitPerfTest(threads = 10, durationMs = 10_000, warmUpMs = 1_000, rampUpPeriodMs = 2_000, totalExecutions = 100) + public void whenAssumptionFails_thenTestShouldBeSkipped() throws IOException { + //noinspection DataFlowIssue + assumeFalse(true); // dummy test to illustrate skipped tests + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress("www.google.com", 80), 1000); + } + } + } diff --git a/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java b/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java index 8345cb6..8cd9523 100644 --- a/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java +++ b/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/ExampleSuccessTests.java @@ -1,9 +1,9 @@ package com.github.noconnor.junitperf.examples; import com.github.noconnor.junitperf.JUnitPerfInterceptor; +import com.github.noconnor.junitperf.JUnitPerfReportingConfig; import com.github.noconnor.junitperf.JUnitPerfTest; import com.github.noconnor.junitperf.JUnitPerfTestActiveConfig; -import com.github.noconnor.junitperf.JUnitPerfReportingConfig; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -14,6 +14,7 @@ import java.net.Socket; import static com.github.noconnor.junitperf.examples.utils.ReportingUtils.newHtmlReporter; +import static org.junit.jupiter.api.Assumptions.assumeFalse; @ExtendWith(JUnitPerfInterceptor.class) public class ExampleSuccessTests { @@ -49,4 +50,15 @@ public void whenTotalNumberOfExecutionsIsSet_thenTotalExecutionsShouldOverrideDu socket.connect(new InetSocketAddress("www.google.com", 80), 1000); } } + + @Test + @JUnitPerfTest(threads = 10, durationMs = 10_000, warmUpMs = 1_000, rampUpPeriodMs = 2_000, maxExecutionsPerSecond = 100) + public void whenAssumptionFails_thenTestWillBeSkipped() throws IOException { + //noinspection DataFlowIssue + assumeFalse(true); // dummy test to illustrate skipped tests + + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress("www.google.com", 80), 1000); + } + } } diff --git a/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassOne.java b/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassOne.java index 097491e..0e3b362 100644 --- a/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassOne.java +++ b/junit5-examples/src/test/java/com/github/noconnor/junitperf/examples/existing/TestClassOne.java @@ -2,6 +2,8 @@ import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + public class TestClassOne { @Test public void sample_test1_class1() throws InterruptedException { @@ -14,4 +16,10 @@ public void sample_test2_class1() throws InterruptedException { Thread.sleep(1); } + @Test + public void sample_test3_class1() throws InterruptedException { + //noinspection DataFlowIssue + assumeFalse(true); // dummy test to illustrate skipped tests + } + } diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java index 61c8647..56d9ff0 100644 --- a/junitperf-core/src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java @@ -52,6 +52,9 @@ public class EvaluationContext { private long finishTimeNs; @Getter private final boolean isAsyncEvaluation; + @Setter + @Getter + private Throwable abortedException; @Getter private Map requiredPercentiles = emptyMap(); @@ -118,6 +121,10 @@ public EvaluationContext(String testName, long startTimeNs, boolean isAsyncEvalu this.isAsyncEvaluation = isAsyncEvaluation; } + public boolean isAborted() { + return nonNull(abortedException); + } + @SuppressWarnings("WeakerAccess") public long getThroughputQps() { return (long)((evaluationCount/ ((float)configuredDuration - configuredWarmUp)) * 1000); diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGenerator.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGenerator.java index d0d346b..b57b293 100644 --- a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGenerator.java +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGenerator.java @@ -28,44 +28,49 @@ public void generateReport(LinkedHashSet testContexts) { } public void updateReport(EvaluationContext context) { - String throughputStatus = context.isThroughputAchieved() ? PASSED : FAILED; - String errorRateStatus = context.isErrorThresholdAchieved() ? PASSED : FAILED; + if (context.isAborted()) { + log.info("Test {} was SKIPPED", context.getTestName()); + } else { - log.info("Test Name: {}", context.getTestName()); - log.info("Started at: {}", context.getStartTime()); - log.info("Invocations: {}", context.getEvaluationCount()); - log.info(" - Success: {}", context.getEvaluationCount() - context.getErrorCount()); - log.info(" - Errors: {}", context.getErrorCount()); - log.info(" - Errors: {}% - {}", context.getErrorPercentage(), errorRateStatus); - log.info(""); - log.info("Thread Count: {}", context.getConfiguredThreads()); - log.info("Warm up: {} ms", context.getConfiguredWarmUp()); - log.info("Ramp up: {} ms", context.getConfiguredRampUpPeriodMs()); - log.info(""); - log.info("Execution time: {}", context.getTestDurationFormatted()); - log.info("Throughput: {}/s (Required: {}/s) - {}", - context.getThroughputQps(), - context.getRequiredThroughput(), - throughputStatus); - log.info("Min. latency: {} ms (Required: {}ms) - {}", - context.getMinLatencyMs(), - format(context.getRequiredMinLatency())); - log.info("Max. latency: {} ms (Required: {}ms) - {}", - context.getMaxLatencyMs(), - format(context.getRequiredMaxLatency())); - log.info("Ave. latency: {} ms (Required: {}ms) - {}", - context.getMeanLatencyMs(), - format(context.getRequiredMeanLatency())); - context.getRequiredPercentiles().forEach((percentile, threshold) -> { - String percentileStatus = context.getPercentileResults().get(percentile) ? PASSED : FAILED; - log.info("{}: {}ms (Required: {} ms) - {}", - percentile, - context.getLatencyPercentileMs(percentile), - format(threshold), - percentileStatus); - }); - log.info(""); - log.info(""); + String throughputStatus = context.isThroughputAchieved() ? PASSED : FAILED; + String errorRateStatus = context.isErrorThresholdAchieved() ? PASSED : FAILED; + + log.info("Test Name: {}", context.getTestName()); + log.info("Started at: {}", context.getStartTime()); + log.info("Invocations: {}", context.getEvaluationCount()); + log.info(" - Success: {}", context.getEvaluationCount() - context.getErrorCount()); + log.info(" - Errors: {}", context.getErrorCount()); + log.info(" - Errors: {}% - {}", context.getErrorPercentage(), errorRateStatus); + log.info(""); + log.info("Thread Count: {}", context.getConfiguredThreads()); + log.info("Warm up: {} ms", context.getConfiguredWarmUp()); + log.info("Ramp up: {} ms", context.getConfiguredRampUpPeriodMs()); + log.info(""); + log.info("Execution time: {}", context.getTestDurationFormatted()); + log.info("Throughput: {}/s (Required: {}/s) - {}", + context.getThroughputQps(), + context.getRequiredThroughput(), + throughputStatus); + log.info("Min. latency: {} ms (Required: {}ms) - {}", + context.getMinLatencyMs(), + format(context.getRequiredMinLatency())); + log.info("Max. latency: {} ms (Required: {}ms) - {}", + context.getMaxLatencyMs(), + format(context.getRequiredMaxLatency())); + log.info("Ave. latency: {} ms (Required: {}ms) - {}", + context.getMeanLatencyMs(), + format(context.getRequiredMeanLatency())); + context.getRequiredPercentiles().forEach((percentile, threshold) -> { + String percentileStatus = context.getPercentileResults().get(percentile) ? PASSED : FAILED; + log.info("{}: {}ms (Required: {} ms) - {}", + percentile, + context.getLatencyPercentileMs(percentile), + format(threshold), + percentileStatus); + }); + log.info(""); + log.info(""); + } } @Override diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGenerator.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGenerator.java index b1c446a..366219a 100644 --- a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGenerator.java +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGenerator.java @@ -46,9 +46,11 @@ public void generateReport(LinkedHashSet testContexts) { writer.newLine(); history.forEach(context -> { - String record = String.format("%s,%d,%d,%d,%.4f,%.4f,%.4f,%s", - context.getTestName(), - context.getConfiguredDuration(), + String name = context.isAborted() ? context.getTestName() + " (skipped)" : context.getTestName(); + int duration = context.isAborted() ? 0 : context.getConfiguredDuration(); + String record = String.format("%s,%s,%d,%d,%.4f,%.4f,%.4f,%s", + name, + duration, context.getConfiguredThreads(), context.getThroughputQps(), context.getMinLatencyMs(), diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGenerator.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGenerator.java index 897d340..fd41521 100644 --- a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGenerator.java +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGenerator.java @@ -24,6 +24,7 @@ import static java.lang.System.getProperty; import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; @Slf4j public class HtmlReportGenerator implements ReportGenerator { @@ -85,17 +86,21 @@ private void renderTemplate() { ViewData c = new ViewData(context); String overview = ViewProcessor.populateTemplate(c, "context", blocks.get(OVERVIEW_MARKER)); - String detail = ViewProcessor.populateTemplate(c, "context", blocks.get(DETAILS_MARKER)); - String percentileData = ViewProcessor.populateTemplate( - c.getRequiredPercentiles(), - "context.percentiles", - blocks.get(PERCENTILE_TARGETS_MARKER) - ); - - detail = detail.replaceAll(asRegex(PERCENTILE_TARGETS_MARKER), percentileData); - + + if (context.isAborted()) { + overview = overview.replaceAll("href=", "nolink="); + } else { + String detail = ViewProcessor.populateTemplate(c, "context", blocks.get(DETAILS_MARKER)); + String percentileData = ViewProcessor.populateTemplate( + c.getRequiredPercentiles(), + "context.percentiles", + blocks.get(PERCENTILE_TARGETS_MARKER) + ); + + detail = detail.replaceAll(asRegex(PERCENTILE_TARGETS_MARKER), percentileData); + details.append(detail).append("\n"); + } overviews.append(overview).append("\n"); - details.append(detail).append("\n"); } root = root.replaceAll(asRegex(OVERVIEW_MARKER), overviews.toString()); diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewData.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewData.java index 5413701..8c10ea7 100644 --- a/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewData.java +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/reporting/providers/utils/ViewData.java @@ -6,10 +6,12 @@ import lombok.Setter; import lombok.ToString; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Objects.isNull; import static java.util.Objects.nonNull; @Getter @@ -17,6 +19,7 @@ public class ViewData { static final String SUCCESS_COLOUR = "#2b67a4"; static final String FAILED_COLOUR = "#d9534f"; + static final String SKIPPED_COLOUR = "#dcdcdc"; @Getter @Setter @@ -59,7 +62,7 @@ public static final class RequiredPercentilesData { public ViewData(EvaluationContext context) { this.testName = buildTestName(context); - this.testNameColour = context.isSuccessful() ? SUCCESS_COLOUR : FAILED_COLOUR; + this.testNameColour = context.isAborted() ? SKIPPED_COLOUR : context.isSuccessful() ? SUCCESS_COLOUR : FAILED_COLOUR; this.chartData = buildChartData(context); this.csvData = buildCsvData(context); this.startTime = context.getStartTime(); @@ -88,10 +91,17 @@ public ViewData(EvaluationContext context) { } private static String buildTestName(EvaluationContext context) { - return nonNull(context.getGroupName()) ? context.getGroupName() + " : " + context.getTestName() : context.getTestName(); + String baseName = nonNull(context.getGroupName()) ? context.getGroupName() + " : " + context.getTestName() : context.getTestName(); + if (context.isAborted()){ + baseName = baseName + (" (skipped)"); + } + return baseName; } private List buildRequiredPercentileData(EvaluationContext context) { + if (isNull(context.getPercentileResults())) { + return Collections.emptyList(); + } return context.getRequiredPercentiles().entrySet() .stream() .map(entry -> { diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/EvaluationTask.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/EvaluationTask.java index 0f0c1ed..05b4ff2 100644 --- a/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/EvaluationTask.java +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/EvaluationTask.java @@ -3,11 +3,14 @@ import com.github.noconnor.junitperf.statistics.StatisticsCalculator; import com.google.common.util.concurrent.RateLimiter; import lombok.Builder; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import java.util.function.Supplier; +import static com.github.noconnor.junitperf.statements.ExceptionsRegistry.reThrowIfAbort; import static java.lang.System.nanoTime; +import static java.util.Objects.isNull; import static java.util.Objects.nonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -15,118 +18,143 @@ @Slf4j final class EvaluationTask implements Runnable { - private final TestStatement statement; - private final RateLimiter rateLimiter; - private final Supplier terminator; - private final StatisticsCalculator stats; - private final long warmUpPeriodNs; - private final long executionTarget; - - @Builder - EvaluationTask(TestStatement statement, - RateLimiter rateLimiter, - StatisticsCalculator stats, - Supplier terminator, - int warmUpPeriodMs, - int executionTarget) { - this(statement, rateLimiter, terminator, stats, warmUpPeriodMs, executionTarget); - } - - // Test only - EvaluationTask(TestStatement statement, - RateLimiter rateLimiter, - Supplier terminator, - StatisticsCalculator stats, - int warmUpPeriodMs, - int executionTarget) { - this.statement = statement; - this.rateLimiter = rateLimiter; - this.terminator = terminator; - this.stats = stats; - this.warmUpPeriodNs = NANOSECONDS.convert(Math.max(warmUpPeriodMs, 0), MILLISECONDS); - this.executionTarget = executionTarget; - } - - @Override - public void run() { - long startTimeNs = nanoTime(); - long startMeasurements = startTimeNs + warmUpPeriodNs; - while (terminationFlagNotSet() && threadNotInterrupted() && executionTargetNotMet()) { - waitForPermit(); - evaluateStatement(startMeasurements); + private final TestStatement statement; + private final RateLimiter rateLimiter; + private final Supplier terminator; + private final StatisticsCalculator stats; + private final long warmUpPeriodNs; + private final long executionTarget; + + @Builder + EvaluationTask(TestStatement statement, + RateLimiter rateLimiter, + StatisticsCalculator stats, + Supplier terminator, + int warmUpPeriodMs, + int executionTarget) { + this(statement, rateLimiter, terminator, stats, warmUpPeriodMs, executionTarget); } - } - - private boolean terminationFlagNotSet() { - return !terminator.get(); - } - - private static boolean threadNotInterrupted() { - return !Thread.currentThread().isInterrupted(); - } - - private boolean executionTargetNotMet() { - return executionTarget <= 0 || stats.getEvaluationCount() < executionTarget; - } - - private void evaluateStatement(long startMeasurements) { - if (nanoTime() < startMeasurements) { - try { - statement.runBefores(); - statement.evaluate(); - statement.runAfters(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Throwable throwable) { - log.trace("Warmup error", throwable); - } - } else { - - try { - statement.runBefores(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Throwable throwable) { - log.trace("Setup error", throwable); - if (!(throwable.getCause() instanceof InterruptedException)) { - throw new IllegalStateException("Before method failed", throwable); + + // Test only + EvaluationTask(TestStatement statement, + RateLimiter rateLimiter, + Supplier terminator, + StatisticsCalculator stats, + int warmUpPeriodMs, + int executionTarget) { + this.statement = statement; + this.rateLimiter = rateLimiter; + this.terminator = terminator; + this.stats = stats; + this.warmUpPeriodNs = NANOSECONDS.convert(Math.max(warmUpPeriodMs, 0), MILLISECONDS); + this.executionTarget = executionTarget; + } + + @SneakyThrows + @Override + public void run() { + long startTimeNs = nanoTime(); + long startMeasurements = startTimeNs + warmUpPeriodNs; + while (terminationFlagNotSet() && threadNotInterrupted() && executionTargetNotMet()) { + waitForPermit(); + evaluateStatement(startMeasurements); } - } - - long startTimeNs = nanoTime(); - try { - statement.evaluate(); - stats.addLatencyMeasurement(nanoTime() - startTimeNs); - stats.incrementEvaluationCount(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Throwable throwable) { - if (!(throwable.getCause() instanceof InterruptedException)) { - stats.incrementEvaluationCount(); - stats.incrementErrorCount(); - stats.addLatencyMeasurement(nanoTime() - startTimeNs); + } + + private boolean terminationFlagNotSet() { + return !terminator.get(); + } + + private static boolean threadNotInterrupted() { + return !Thread.currentThread().isInterrupted(); + } + + private boolean executionTargetNotMet() { + return executionTarget <= 0 || stats.getEvaluationCount() < executionTarget; + } + + private void evaluateStatement(long startMeasurements) throws Throwable { + if (nanoTime() < startMeasurements) { + try { + statement.runBefores(); + statement.evaluate(); + statement.runAfters(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable throwable) { + log.trace("Warmup error", throwable); + } + } else { + + try { + statement.runBefores(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable throwable) { + log.trace("Setup error", throwable); + reThrowIfAbort(throwable); + if (isTerminalException(throwable)) { + throw new IllegalStateException("Before method failed", throwable); + } + } + + long startTimeNs = nanoTime(); + try { + statement.evaluate(); + stats.addLatencyMeasurement(nanoTime() - startTimeNs); + stats.incrementEvaluationCount(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable throwable) { + log.trace("Execution error", throwable); + reThrowIfAbort(throwable); + checkForIgnorable(throwable); + stats.addLatencyMeasurement(nanoTime() - startTimeNs); + } + + try { + statement.runAfters(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable throwable) { + log.trace("Teardown error", throwable); + reThrowIfAbort(throwable); + if (isTerminalException(throwable)) { + throw new IllegalStateException("After method failed", throwable); + } + } + } - log.trace("Execution error", throwable); - } - - try { - statement.runAfters(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Throwable throwable) { - log.trace("Teardown error", throwable); - if (!(throwable.getCause() instanceof InterruptedException)) { - throw new IllegalStateException("After method failed", throwable); + } + + private void checkForIgnorable(Throwable throwable) { + if (isIgnorableException(throwable)) { + stats.incrementEvaluationCount(); + } else { + stats.incrementEvaluationCount(); + stats.incrementErrorCount(); } - } + } + + private boolean isTerminalException(Throwable throwable) { + return !isIgnorableException(throwable); + } + private boolean isIgnorableException(Throwable throwable) { + return isIgnorable(throwable); } - } - private void waitForPermit() { - if (nonNull(rateLimiter)) { - rateLimiter.acquire(); + private boolean isIgnorable(Throwable throwable) { + if (isNull(throwable)) { + return false; + } + return ExceptionsRegistry.isIgnorable(throwable) || isIgnorable(throwable.getCause()); + } + + private void waitForPermit() { + if (nonNull(rateLimiter)) { + rateLimiter.acquire(); + } } - } } diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/ExceptionsRegistry.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/ExceptionsRegistry.java new file mode 100644 index 0000000..08b2195 --- /dev/null +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/ExceptionsRegistry.java @@ -0,0 +1,51 @@ +package com.github.noconnor.junitperf.statements; + +import lombok.experimental.UtilityClass; + +import java.lang.reflect.InvocationTargetException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +@UtilityClass +public class ExceptionsRegistry { + + private static final Set> IGNORABLE_EXCEPTIONS_REGISTRY = new HashSet<>(); + private static final Set> ABORT_EXCEPTIONS_REGISTRY = new HashSet<>(); + + public static void registerIgnorable(Class exception) { + IGNORABLE_EXCEPTIONS_REGISTRY.add(exception); + } + + public static void registerAbort(Class exception) { + ABORT_EXCEPTIONS_REGISTRY.add(exception); + } + + public static boolean isIgnorable(Throwable throwable) { + return IGNORABLE_EXCEPTIONS_REGISTRY.contains(throwable.getClass()); + } + + public static void reThrowIfAbort(Throwable throwable) throws Throwable { + Throwable targetException = throwable; + if (throwable instanceof InvocationTargetException) { + targetException = ((InvocationTargetException) throwable).getTargetException(); + } + if (ABORT_EXCEPTIONS_REGISTRY.contains(targetException.getClass())) { + // re-throw abortable exceptions + throw targetException; + } + } + + public static Set> ignorables() { + return Collections.unmodifiableSet(IGNORABLE_EXCEPTIONS_REGISTRY); + } + + public static Set> abortables() { + return Collections.unmodifiableSet(ABORT_EXCEPTIONS_REGISTRY); + } + + static void clearRegistry() { + IGNORABLE_EXCEPTIONS_REGISTRY.clear(); + ABORT_EXCEPTIONS_REGISTRY.clear(); + } +} diff --git a/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatement.java b/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatement.java index 23e3698..2f4c1ef 100644 --- a/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatement.java +++ b/junitperf-core/src/main/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatement.java @@ -68,6 +68,10 @@ public void runParallelEvaluation() throws Throwable { stopSignal.set(true); threads.forEach(Thread::interrupt); } + if (context.isAborted()) { + listener.accept(null); + throw context.getAbortedException(); + } context.setFinishTimeNs(nanoTime()); context.setStatistics(statistics); context.runValidation(); @@ -78,16 +82,21 @@ public void runParallelEvaluation() throws Throwable { private Runnable createTask(AtomicBoolean stopSignal, CountDownLatch latch) { StatisticsCalculator stats = context.isAsyncEvaluation() ? NoOpStatisticsCollector.INSTANCE : statistics; return () -> { - EvaluationTask.builder() - .statement(baseStatement) - .rateLimiter(rateLimiter) - .stats(stats) - .terminator(stopSignal::get) - .warmUpPeriodMs(context.getConfiguredWarmUp()) - .executionTarget(context.getConfiguredExecutionTarget()) - .build() - .run(); - latch.countDown(); + try { + EvaluationTask.builder() + .statement(baseStatement) + .rateLimiter(rateLimiter) + .stats(stats) + .terminator(stopSignal::get) + .warmUpPeriodMs(context.getConfiguredWarmUp()) + .executionTarget(context.getConfiguredExecutionTarget()) + .build() + .run(); + } catch (Throwable t) { + context.setAbortedException(t); + } finally { + latch.countDown(); + } }; } diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/data/EvaluationContextTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/data/EvaluationContextTest.java index 5ad1924..e36df09 100644 --- a/junitperf-core/src/test/java/com/github/noconnor/junitperf/data/EvaluationContextTest.java +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/data/EvaluationContextTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.junit.After; +import org.junit.AssumptionViolatedException; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; @@ -422,6 +423,15 @@ public void whenDurationIsValid_thenDurationShouldBeFormatted() { assertEquals("300ms", context.getTestDurationFormatted()); } + @Test + public void whenContextContainsAbortedException_thenIsAbortedShouldBeTrue() { + Exception abort = new AssumptionViolatedException("unittest"); + assertFalse(context.isAborted()); + context.setAbortedException(abort); + assertTrue(context.isAborted()); + assertEquals(abort, context.getAbortedException()); + } + private void initialiseContext() { context.loadConfiguration(perfTestAnnotation); context.loadRequirements(perfTestRequirement); diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/BaseReportGeneratorTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/BaseReportGeneratorTest.java index af9241e..3fc3d82 100644 --- a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/BaseReportGeneratorTest.java +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/BaseReportGeneratorTest.java @@ -9,6 +9,8 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.util.LinkedHashSet; + +import org.junit.AssumptionViolatedException; import org.junit.Rule; import org.junit.rules.TemporaryFolder; import org.mockito.Mock; @@ -25,6 +27,8 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -85,6 +89,15 @@ protected EvaluationContext createdSuccessfulEvaluationContext(String name) { return context; } + protected EvaluationContext createdAbortedEvaluationContext(String name) { + EvaluationContext context = new EvaluationContext(name, nanoTime()); + context.loadConfiguration(perfTestAnnotationMock); + context.loadRequirements(perfTestRequirementAnnotationMock); + context.setStatistics(createAbortedMock()); + context.setAbortedException(new AssumptionViolatedException("unittest")); + return context; + } + protected EvaluationContext createdSomeFailuresEvaluationContext(String name) { EvaluationContext context = new EvaluationContext(name, nanoTime()); context.setFinishTimeNs(nanoTime() + SECONDS.toNanos(10)); @@ -137,6 +150,18 @@ private StatisticsCalculator createAllFailureMock() { return statisticsMock; } + private StatisticsCalculator createAbortedMock() { + StatisticsCalculator statisticsMock = mock(StatisticsCalculator.class); + when(statisticsMock.getErrorPercentage()).thenReturn(0.0F); + when(statisticsMock.getLatencyPercentile(anyInt(), eq(MILLISECONDS))).thenReturn(0F); + when(statisticsMock.getEvaluationCount()).thenReturn(0L); + when(statisticsMock.getErrorCount()).thenReturn(0L); + when(statisticsMock.getMaxLatency(MILLISECONDS)).thenReturn(0F); + when(statisticsMock.getMinLatency(MILLISECONDS)).thenReturn(0F); + when(statisticsMock.getMeanLatency(MILLISECONDS)).thenReturn(0F); + return statisticsMock; + } + protected void initialisePerfTestAnnotationMock() { when(perfTestAnnotationMock.durationMs()).thenReturn(10_000); when(perfTestAnnotationMock.warmUpMs()).thenReturn(100); @@ -186,6 +211,15 @@ protected LinkedHashSet generateMixedOrderedContexts() { return newLinkedHashSet(newArrayList(context1, context2)); } + protected LinkedHashSet generateAbortedFailedAndSuccessContexts() { + EvaluationContext context1 = createdFailedEvaluationContext("unittest1"); + EvaluationContext context2 = createdAbortedEvaluationContext("unittest2"); + EvaluationContext context3 = createdSuccessfulEvaluationContext("unittest3"); + verifyAllValidationFailed(context1); + verifyAllValidationPassed(context3); + return newLinkedHashSet(newArrayList(context1, context2, context3)); + } + protected LinkedHashSet generateSomeFailuresContext() { EvaluationContext context = createdSomeFailuresEvaluationContext("unittest1"); return newLinkedHashSet(newArrayList(context)); diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGeneratorTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGeneratorTest.java index bd59dff..e40bcd1 100644 --- a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGeneratorTest.java +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGeneratorTest.java @@ -39,6 +39,13 @@ public void whenGeneratingAReport_andTestsContainsSomeFailures_thenAppropriateRe reportGenerator.generateReport(generateSomeFailuresContext()); } + @Test + public void whenGeneratingAReport_andTestsContainsSomeAborts_thenAppropriateReportShouldBeGenerated() { + LinkedHashSet contexts = new LinkedHashSet<>(); + contexts.add(createdAbortedEvaluationContext("unittest1")); + reportGenerator.generateReport(contexts); + } + @Test public void whenGeneratingAReport_andGenerateIsCalledMultipleTimes_thenOnlyNewResultsShouldBePrinted() { LinkedHashSet contexts = generateSomeFailuresContext(); diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGeneratorTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGeneratorTest.java index 567cb95..e29fc61 100644 --- a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGeneratorTest.java +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/CsvReportGeneratorTest.java @@ -8,6 +8,7 @@ import static java.lang.System.getProperty; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; public class CsvReportGeneratorTest extends BaseReportGeneratorTest { @@ -63,6 +64,13 @@ public void whenGeneratingAReport_andTestsContainsSomeFailures_thenAppropriateRe assertThat(readFileContents(reportFile), is(readFileContents(expectedContents))); } + @Test + public void whenGeneratingAReport_andTestsContainsSomeAbortsAndFailures_thenAppropriateReportShouldBeGenerated() throws IOException { + reportGenerator.generateReport(generateAbortedFailedAndSuccessContexts()); + File expectedContents = getResourceFile("csv/fail_abort_succeed.csv"); + assertEquals(readFileContents(expectedContents), readFileContents(reportFile)); + } + @Test public void whenCallingGetReportPath_andCustomPathHasBeenSpecified_thenCorrectPathShouldBeReturned() { assertThat(reportGenerator.getReportPath(), is(reportFile.getPath())); diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGeneratorTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGeneratorTest.java index ad3578c..f5e348d 100644 --- a/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGeneratorTest.java +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/reporting/providers/HtmlReportGeneratorTest.java @@ -67,6 +67,13 @@ public void whenGeneratingAReport_andTestsContainsSomeFailures_thenAppropriateRe assertEquals(readFileContents(expectedContents), readFileContents(reportFile)); } + @Test + public void whenGeneratingAReport_andTestsContainsSomeAbortsAndFailures_thenAppropriateReportShouldBeGenerated() throws IOException { + reportGenerator.generateReport(generateAbortedFailedAndSuccessContexts()); + File expectedContents = getResourceFile("html/example_aborted_failed_success.html"); + assertEquals(readFileContents(expectedContents), readFileContents(reportFile)); + } + @Test public void whenCallingGetReportPath_andCustomPathHasBeenSpecified_thenCorrectPathShouldBeReturned() { assertThat(reportGenerator.getReportPath(), is(reportFile.getPath())); diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/EvaluationTaskTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/EvaluationTaskTest.java index 2e46a7a..cb557e7 100644 --- a/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/EvaluationTaskTest.java +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/EvaluationTaskTest.java @@ -42,6 +42,8 @@ public class EvaluationTaskTest extends BaseTest { @Before public void setup() { + ExceptionsRegistry.registerIgnorable(InterruptedException.class); + ExceptionsRegistry.registerAbort(AbortTestException.class); initialiseRateLimiterMock(); task = new EvaluationTask(statementMock, rateLimiterMock, terminatorMock, statsMock, 0, 0); } @@ -49,6 +51,7 @@ public void setup() { @After public void teardown() { // clear thread interrupt + ExceptionsRegistry.clearRegistry(); Thread.interrupted(); } @@ -75,6 +78,23 @@ public void whenRunning_andStatementEvaluationThrowsAnException_thenStatsErrorCo verify(statsMock, times(5)).incrementErrorCount(); } + @Test + public void whenRunning_andStatementEvaluationThrowsATestAbortedException_thenStatsExecutionCounterShouldBeIncremented() throws Throwable { + setExecutionCount(10); + mockAssumptionFailures(1); + + try { + task.run(); + fail("Expected exception"); + } catch (AbortTestException e){ + // expected + } finally { + verify(statsMock, never()).incrementEvaluationCount(); + verify(statsMock, never()).addLatencyMeasurement(anyLong()); + verify(statsMock, never()).incrementErrorCount(); + } + } + @Test public void whenRunning_andStatementEvaluationThrowsAnException_thenLatencyMeasurementShouldBeTaken() throws Throwable { setExecutionCount(10); @@ -88,8 +108,8 @@ public void whenRunning_andStatementEvaluationThrowsAnInterruptException_thenNoM setExecutionCount(10); mockNestedInterruptAfter(9); task.run(); - verify(statsMock, times(9)).addLatencyMeasurement(anyLong()); - verify(statsMock, times(9)).incrementEvaluationCount(); + verify(statsMock, times(10)).addLatencyMeasurement(anyLong()); + verify(statsMock, times(10)).incrementEvaluationCount(); verify(statsMock, never()).incrementErrorCount(); } @@ -171,6 +191,20 @@ public void whenRunning_andRunBeforesThrowsAnException_thenExceptionShouldBeThro } } + @Test + public void whenRunning_andRunBeforesAnAbortException_thenNoExceptionShouldBeThrown() throws Throwable { + setExecutionCount(1); + doThrow(new AbortTestException()).when(statementMock).runBefores(); + try { + task.run(); + fail("Expected exception"); + } catch (AbortTestException e) { + // expected + } finally { + verify(statementMock, never()).evaluate(); + } + } + @Test public void whenRunning_andRunBeforesThrowsAnInterruptedException_thenNoExceptionShouldBeThrown() throws Throwable { setExecutionCount(1); @@ -187,6 +221,20 @@ public void whenRunning_andRunAftersThrowsAnInterruptedException_thenNoException verify(statementMock).evaluate(); } + @Test + public void whenRunning_andRunAftersThrowsATestAbortedException_thenNoExceptionShouldBeThrown() throws Throwable { + setExecutionCount(1); + doThrow(new AbortTestException()).when(statementMock).runAfters(); + try { + task.run(); + fail("Expected exception"); + } catch (AbortTestException e) { + // expected + } finally { + verify(statementMock).evaluate(); + } + } + @Test public void whenRunning_andRunAftersThrowsAnException_thenExceptionShouldBeThrown() throws Throwable { setExecutionCount(1); @@ -202,6 +250,16 @@ public void whenRunning_andRunAftersThrowsAnException_thenExceptionShouldBeThrow } } + private void mockAssumptionFailures(int desiredFailureCount) throws Throwable { + AtomicInteger executions = new AtomicInteger(); + doAnswer(invocation -> { + if (executions.getAndIncrement() < desiredFailureCount) { + throw new AbortTestException(); + } + return null; + }).when(statementMock).evaluate(); + } + private void mockEvaluationFailures(int desiredFailureCount) throws Throwable { AtomicInteger executions = new AtomicInteger(); doAnswer(invocation -> { @@ -255,4 +313,8 @@ private void setExecutionCount(int loops) { stub.thenReturn(true); } + private static class AbortTestException extends RuntimeException { + + } + } diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/ExceptionsRegistryTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/ExceptionsRegistryTest.java new file mode 100644 index 0000000..9fe8a1d --- /dev/null +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/ExceptionsRegistryTest.java @@ -0,0 +1,66 @@ +package com.github.noconnor.junitperf.statements; + +import org.junit.After; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ExceptionsRegistryTest { + + @Before + public void setup() { + ExceptionsRegistry.registerIgnorable(InterruptedException.class); + ExceptionsRegistry.registerAbort(AssumptionViolatedException.class); + } + + @After + public void teardown() { + ExceptionsRegistry.clearRegistry(); + } + + @Test + public void testRegistry() { + assertEquals(1, ExceptionsRegistry.ignorables().size()); + assertEquals(1, ExceptionsRegistry.abortables().size()); + assertTrue(ExceptionsRegistry.ignorables().contains(InterruptedException.class)); + assertTrue(ExceptionsRegistry.abortables().contains(AssumptionViolatedException.class)); + } + + @Test + public void ifIgnoreExceptionIsRegistered_thenTestingForIgnoreExceptionShouldReturnTrue() { + assertTrue(ExceptionsRegistry.isIgnorable(new InterruptedException())); + } + + @Test + public void ifIgnoreExceptionIsNotRegistered_thenTestingForIgnoreExceptionShouldReturnFalse() { + assertFalse(ExceptionsRegistry.isIgnorable(new IllegalStateException())); + } + + @Test + public void ifAbortExceptionIsRegistered_thenTestingForAbortExceptionShouldRethrowException() { + AssumptionViolatedException abort = new AssumptionViolatedException("unittest"); + try { + ExceptionsRegistry.reThrowIfAbort(abort); + fail("Expected exception to be re-thrown"); + } catch (AssumptionViolatedException e) { + // expected + } catch (Throwable t) { + fail("Unexpected exception thrown"); + } + } + + @Test + public void ifAbortExceptionIsNotRegistered_thenTestingForAbortExceptionShouldNotRethrowException() { + IllegalStateException exception = new IllegalStateException("unittest"); + try { + ExceptionsRegistry.reThrowIfAbort(exception); + } catch (Throwable t) { + fail("Unexpected exception thrown"); + } + } +} \ No newline at end of file diff --git a/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatementTest.java b/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatementTest.java index b25a129..7a27a4a 100644 --- a/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatementTest.java +++ b/junitperf-core/src/test/java/com/github/noconnor/junitperf/statements/PerformanceEvaluationStatementTest.java @@ -4,10 +4,12 @@ import com.github.noconnor.junitperf.data.EvaluationContext; import com.github.noconnor.junitperf.statistics.StatisticsCalculator; import com.google.common.collect.ImmutableMap; +import org.junit.AssumptionViolatedException; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import java.util.concurrent.ThreadFactory; @@ -23,6 +25,9 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -54,6 +59,7 @@ public class PerformanceEvaluationStatementTest extends BaseTest { @Before public void setup() { + ExceptionsRegistry.clearRegistry(); initialiseThreadFactoryMock(); initialiseContext(); statement = PerformanceEvaluationStatement.builder() @@ -206,6 +212,36 @@ public void whenRunningEvaluation_thenStatisticsShouldBeReset() throws Throwable verify(statisticsCalculatorMock, times(3)).reset(); } + @Test + public void whenBaseStatementThrowsAnAbortException_thenExceptionShouldBeReThrown() throws Throwable { + + ExceptionsRegistry.registerAbort(AssumptionViolatedException.class); + + ArgumentCaptor captor = ArgumentCaptor.forClass(Runnable.class); + when(threadFactoryMock.newThread(captor.capture())).thenReturn(threadMock); + doAnswer((invocation) -> { + captor.getValue().run(); + return null; + }).when(threadMock).start(); + + AssumptionViolatedException abort = new AssumptionViolatedException("unittest"); + doThrow(abort).when(baseStatementMock).evaluate(); + when(contextMock.isAborted()).thenReturn(true); + when(contextMock.getAbortedException()).thenReturn(abort); + + try { + statement.runParallelEvaluation(); + } catch (AssumptionViolatedException e) { + // expected + } catch (Throwable t) { + fail("Unexpected exception"); + } finally { + verify(contextMock).setAbortedException(abort); + verify(listenerMock).accept(null); + verify(contextMock, never()).runValidation(); + } + } + private void initialiseThreadFactoryMock() { when(threadFactoryMock.newThread(any(Runnable.class))).thenReturn(threadMock); } diff --git a/junitperf-core/src/test/resources/csv/fail_abort_succeed.csv b/junitperf-core/src/test/resources/csv/fail_abort_succeed.csv new file mode 100644 index 0000000..10c578a --- /dev/null +++ b/junitperf-core/src/test/resources/csv/fail_abort_succeed.csv @@ -0,0 +1,4 @@ +testName,duration,threadCount,throughput,minLatencyNs,maxLatencyNs,meanLatencyNs,1st,2nd,3rd,4th,5th,6th,7th,8th,9th,10th,11th,12th,13th,14th,15th,16th,17th,18th,19th,20th,21st,22nd,23rd,24th,25th,26th,27th,28th,29th,30th,31st,32nd,33rd,34th,35th,36th,37th,38th,39th,40th,41st,42nd,43rd,44th,45th,46th,47th,48th,49th,50th,51st,52nd,53rd,54th,55th,56th,57th,58th,59th,60th,61st,62nd,63rd,64th,65th,66th,67th,68th,69th,70th,71st,72nd,73rd,74th,75th,76th,77th,78th,79th,80th,81st,82nd,83rd,84th,85th,86th,87th,88th,89th,90th,91st,92nd,93rd,94th,95th,96th,97th,98th,99th,100th +unittest1,10000,50,101,12.7000,234.6800,61.7000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,4.6364,48.3435,234.6800 +unittest2 (skipped),0,50,0,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +unittest3,10000,50,13131,1.6364,38.5485,17.5400,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,1.6364,28.3435,38.5485 diff --git a/junitperf-core/src/test/resources/html/example_aborted_failed_success.html b/junitperf-core/src/test/resources/html/example_aborted_failed_success.html new file mode 100644 index 0000000..f21ee48 --- /dev/null +++ b/junitperf-core/src/test/resources/html/example_aborted_failed_success.html @@ -0,0 +1,778 @@ + + + + + + JUnit Performance Report + + + +
+

JUnit Performance Report

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
   Tests
 unittest1
 unittest2 (skipped)
 unittest3
+
+
+ + + +

unittest1

 + + + + + +
+ + +
+ Download as csv + 		+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Started at:unittest o'clock
Invocations:1,000
- Success:600
- Errors:40040.00%
   
Thread Count:50
Warm up:100 ms
Ramp up:50 ms
   
 Measured
(system)		Required
Execution time:10s
Throughput:101 / s13,000 / s
Min latency:12.70 ms10.00 ms
Average latency:61.70 ms55.10 ms
98:4.64 ms3.30 ms
99:48.34 ms32.60 ms
100:234.68 ms47.00 ms
Max latency:234.68 ms200.66 ms
+
+
+


+ + +

unittest3

 + + + + + +
+ + +
+ Download as csv + 		+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Started at:unittest o'clock
Invocations:130,000
- Success:104,000
- Errors:26,00020.00%
   
Thread Count:50
Warm up:100 ms
Ramp up:50 ms
   
 Measured
(system)		Required
Execution time:10s
Throughput:13,131 / s13,000 / s
Min latency:1.64 ms10.00 ms
Average latency:17.54 ms55.10 ms
98:1.64 ms3.30 ms
99:28.34 ms32.60 ms
100:38.55 ms47.00 ms
Max latency:38.55 ms200.66 ms
+
+
+


+ + + +
+
Report created by JunitPerf
+
+ + diff --git a/junitperf-junit4/src/main/java/com/github/noconnor/junitperf/JUnitPerfRule.java b/junitperf-junit4/src/main/java/com/github/noconnor/junitperf/JUnitPerfRule.java index 2ffd46a..ced85b1 100644 --- a/junitperf-junit4/src/main/java/com/github/noconnor/junitperf/JUnitPerfRule.java +++ b/junitperf-junit4/src/main/java/com/github/noconnor/junitperf/JUnitPerfRule.java @@ -8,6 +8,7 @@ import com.github.noconnor.junitperf.reporting.ReportGenerator; import com.github.noconnor.junitperf.reporting.providers.HtmlReportGenerator; import com.github.noconnor.junitperf.statements.DefaultStatement; +import com.github.noconnor.junitperf.statements.ExceptionsRegistry; import com.github.noconnor.junitperf.statements.MeasurableStatement; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement.PerformanceEvaluationStatementBuilder; @@ -19,6 +20,8 @@ import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; + +import org.junit.AssumptionViolatedException; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; @@ -27,6 +30,11 @@ public class JUnitPerfRule implements TestRule { static final Map, LinkedHashSet> ACTIVE_CONTEXTS = new HashMap<>(); + + static { + ExceptionsRegistry.registerIgnorable(InterruptedException.class); + ExceptionsRegistry.registerAbort(AssumptionViolatedException.class); + } private final Set reporters; diff --git a/junitperf-junit4/src/test/java/com/github/noconnor/junitperf/JUnitPerfRuleTest.java b/junitperf-junit4/src/test/java/com/github/noconnor/junitperf/JUnitPerfRuleTest.java index 12a5b3b..e9d4d2e 100644 --- a/junitperf-junit4/src/test/java/com/github/noconnor/junitperf/JUnitPerfRuleTest.java +++ b/junitperf-junit4/src/test/java/com/github/noconnor/junitperf/JUnitPerfRuleTest.java @@ -1,27 +1,16 @@ package com.github.noconnor.junitperf; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertTrue; -import static org.mockito.Answers.RETURNS_SELF; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - import com.github.noconnor.junitperf.data.EvaluationContext; import com.github.noconnor.junitperf.reporting.ReportGenerator; import com.github.noconnor.junitperf.statements.DefaultStatement; +import com.github.noconnor.junitperf.statements.ExceptionsRegistry; import com.github.noconnor.junitperf.statements.MeasurableStatement; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement.PerformanceEvaluationStatementBuilder; import com.github.noconnor.junitperf.statements.TestStatement; import com.github.noconnor.junitperf.statistics.StatisticsCalculator; -import java.util.LinkedHashSet; -import java.util.function.Consumer; import org.junit.After; +import org.junit.AssumptionViolatedException; import org.junit.Before; import org.junit.Test; import org.junit.runner.Description; @@ -31,6 +20,21 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import java.util.LinkedHashSet; +import java.util.function.Consumer; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Answers.RETURNS_SELF; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + public class JUnitPerfRuleTest { @@ -174,6 +178,14 @@ public void whenCallingCreateEvaluationContext_thenContextShouldHaveAsyncFlagSet assertThat(context.isAsyncEvaluation(), is(false)); } + @Test + public void verifyCorrectExceptionsAreRegistered() { + assertEquals(1, ExceptionsRegistry.ignorables().size()); + assertEquals(1, ExceptionsRegistry.abortables().size()); + assertTrue(ExceptionsRegistry.ignorables().contains(InterruptedException.class)); + assertTrue(ExceptionsRegistry.abortables().contains(AssumptionViolatedException.class)); + } + private void triggerReportGeneration() { Consumer listener = captureListener(); listener.accept(null); diff --git a/junitperf-junit5/src/main/java/com/github/noconnor/junitperf/JUnitPerfInterceptor.java b/junitperf-junit5/src/main/java/com/github/noconnor/junitperf/JUnitPerfInterceptor.java index 12378bd..de81726 100644 --- a/junitperf-junit5/src/main/java/com/github/noconnor/junitperf/JUnitPerfInterceptor.java +++ b/junitperf-junit5/src/main/java/com/github/noconnor/junitperf/JUnitPerfInterceptor.java @@ -4,6 +4,7 @@ import com.github.noconnor.junitperf.reporting.ReportGenerator; import com.github.noconnor.junitperf.reporting.providers.ConsoleReportGenerator; import com.github.noconnor.junitperf.statements.FullStatement; +import com.github.noconnor.junitperf.statements.ExceptionsRegistry; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement.PerformanceEvaluationStatementBuilder; import com.github.noconnor.junitperf.statistics.StatisticsCalculator; @@ -19,6 +20,7 @@ import org.junit.jupiter.api.extension.ParameterResolver; import org.junit.jupiter.api.extension.ReflectiveInvocationContext; import org.junit.jupiter.api.extension.TestInstancePostProcessor; +import org.opentest4j.TestAbortedException; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -42,6 +44,11 @@ public class JUnitPerfInterceptor implements InvocationInterceptor, TestInstance protected static final Map testContexts = new ConcurrentHashMap<>(); protected static final Map sharedContexts = new ConcurrentHashMap<>(); + static { + ExceptionsRegistry.registerIgnorable(InterruptedException.class); + ExceptionsRegistry.registerAbort(TestAbortedException.class); + } + @Data protected static class TestDetails { private Class testClass; diff --git a/junitperf-junit5/src/test/java/com/github/noconnor/junitperf/JUnitPerfInterceptorTest.java b/junitperf-junit5/src/test/java/com/github/noconnor/junitperf/JUnitPerfInterceptorTest.java index 5359ae0..3bc9a2a 100644 --- a/junitperf-junit5/src/test/java/com/github/noconnor/junitperf/JUnitPerfInterceptorTest.java +++ b/junitperf-junit5/src/test/java/com/github/noconnor/junitperf/JUnitPerfInterceptorTest.java @@ -6,12 +6,15 @@ import com.github.noconnor.junitperf.reporting.ReportGenerator; import com.github.noconnor.junitperf.reporting.providers.ConsoleReportGenerator; import com.github.noconnor.junitperf.reporting.providers.HtmlReportGenerator; +import com.github.noconnor.junitperf.statements.ExceptionsRegistry; import com.github.noconnor.junitperf.statements.FullStatement; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement; import com.github.noconnor.junitperf.statements.PerformanceEvaluationStatement.PerformanceEvaluationStatementBuilder; import com.github.noconnor.junitperf.statistics.StatisticsCalculator; import com.github.noconnor.junitperf.statistics.providers.DescriptiveStatisticsCalculator; import com.github.noconnor.junitperf.suite.SuiteRegistry; +import org.junit.Assert; +import org.junit.AssumptionViolatedException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -26,6 +29,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import org.opentest4j.TestAbortedException; import java.lang.reflect.Method; import java.lang.reflect.Parameter; @@ -427,6 +431,14 @@ void whenInterceptorResolveParameterIsCalled_thenTestContextSupplierShouldBeRetu interceptor.postProcessTestInstance(this, getParent(extensionContextMock)); assertTrue(interceptor.resolveParameter(null, extensionContextMock) instanceof TestContextSupplier); } + + @Test + void verifyCorrectExceptionsAreRegistered() { + assertEquals(1, ExceptionsRegistry.ignorables().size()); + assertEquals(1, ExceptionsRegistry.abortables().size()); + assertTrue(ExceptionsRegistry.ignorables().contains(InterruptedException.class)); + assertTrue(ExceptionsRegistry.abortables().contains(TestAbortedException.class)); + } private static void mockActiveSuite(ExtensionContext testMethodContext, Class suiteClass) { ExtensionContext suiteRoot = mock(ExtensionContext.class);