diff --git a/core/src/main/java/cucumber/formatter/FormatterConverter.java b/core/src/main/java/cucumber/formatter/FormatterConverter.java index f36791eae5..3f187273ba 100644 --- a/core/src/main/java/cucumber/formatter/FormatterConverter.java +++ b/core/src/main/java/cucumber/formatter/FormatterConverter.java @@ -27,6 +27,7 @@ public class FormatterConverter { put("progress", ProgressFormatter.class); put("json", JSONFormatter.class); put("json-pretty", JSONPrettyFormatter.class); + put("usage", UsageFormatter.class); }}; private static final Pattern FORMATTER_WITH_FILE_PATTERN = Pattern.compile("([^:]+):(.*)"); private Appendable defaultOut = System.out; diff --git a/core/src/main/java/cucumber/formatter/UsageFormatter.java b/core/src/main/java/cucumber/formatter/UsageFormatter.java new file mode 100644 index 0000000000..4f30ff92fb --- /dev/null +++ b/core/src/main/java/cucumber/formatter/UsageFormatter.java @@ -0,0 +1,312 @@ +package cucumber.formatter; + +import java.io.InputStream; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import cucumber.formatter.usage.AverageUsageStatisticStrategy; +import cucumber.formatter.usage.MedianUsageStatisticStrategy; +import cucumber.formatter.usage.UsageStatisticStrategy; +import cucumber.runtime.StepDefinitionMatch; +import gherkin.deps.com.google.gson.Gson; +import gherkin.deps.com.google.gson.GsonBuilder; +import gherkin.formatter.Formatter; +import gherkin.formatter.NiceAppendable; +import gherkin.formatter.Reporter; +import gherkin.formatter.model.Background; +import gherkin.formatter.model.Examples; +import gherkin.formatter.model.Feature; +import gherkin.formatter.model.Match; +import gherkin.formatter.model.Result; +import gherkin.formatter.model.Scenario; +import gherkin.formatter.model.ScenarioOutline; +import gherkin.formatter.model.Step; + +/** + * Formatter to measure performance of steps. Aggregated results for all steps can be computed + * by adding {@link UsageStatisticStrategy} to the usageFormatter + */ +public class UsageFormatter implements Formatter, Reporter +{ + final Map> usageMap = new HashMap>(); + final Map statisticStrategies = new HashMap(); + + private final NiceAppendable out; + + private Match match; + + /** + * Constructor + * @param out {@link Appendable} to print the result + */ + public UsageFormatter(Appendable out) + { + this.out = new NiceAppendable(out); + + statisticStrategies.put("median", new MedianUsageStatisticStrategy()); + statisticStrategies.put("average", new AverageUsageStatisticStrategy()); + } + + @Override + public void uri(String uri) + { + } + + @Override + public void feature(Feature feature) + { + } + + @Override + public void background(Background background) + { + } + + @Override + public void scenario(Scenario scenario) + { + } + + @Override + public void scenarioOutline(ScenarioOutline scenarioOutline) + { + } + + @Override + public void examples(Examples examples) + { + } + + @Override + public void embedding(String mimeType, InputStream data) + { + } + + @Override + public void write(String text) + { + } + + @Override + public void step(Step step) + { + } + + @Override + public void eof() + { + } + + @Override + public void syntaxError(String state, String event, List legalEvents, String uri, int line) + { + } + + @Override + public void done() + { + List stepDefContainers = new ArrayList(); + for (Map.Entry> usageEntry : usageMap.entrySet()) + { + StepDefContainer stepDefContainer = new StepDefContainer(); + stepDefContainers.add(stepDefContainer); + + stepDefContainer.source = usageEntry.getKey(); + stepDefContainer.steps = createStepContainer(usageEntry.getValue()); + } + + out.append(gson().toJson(stepDefContainers)); + } + + private List createStepContainer(List stepContainers) + { + for (StepContainer stepContainer : stepContainers) + { + stepContainer.aggregatedResults = createAggregatedResults(stepContainer); + formatDurationAsSeconds(stepContainer.durations); + } + return stepContainers; + } + + private void formatDurationAsSeconds(List durations) + { + for (StepDuration duration : durations) + { + duration.duration = toSeconds(duration.duration.longValue()); + } + } + + private Map createAggregatedResults(StepContainer stepContainer) + { + Map aggregatedResults = new HashMap(); + for (Map.Entry calculatorEntry : statisticStrategies.entrySet()) + { + UsageStatisticStrategy statisticStrategy = calculatorEntry.getValue(); + List rawDurations = getRawDurations(stepContainer.durations); + Long calculationResult = statisticStrategy.calculate(rawDurations); + + String strategy = calculatorEntry.getKey(); + aggregatedResults.put(strategy, toSeconds(calculationResult)); + } + return aggregatedResults; + } + + private BigDecimal toSeconds(Long nanoSeconds) + { + return BigDecimal.valueOf(nanoSeconds).divide(BigDecimal.valueOf(1000000000)); + } + + private List getRawDurations(List stepDurations) + { + List rawDurations = new ArrayList(); + + for(StepDuration stepDuration : stepDurations) + { + rawDurations.add(stepDuration.duration.longValue()); + } + return rawDurations; + } + + private Gson gson() { + return new GsonBuilder().setPrettyPrinting().create(); + } + + @Override + public void close() + { + out.close(); + } + + @Override + public void result(Result result) + { + String stepDefinition = getStepDefinition(); + String stepName = getStepName(); + if (result.getStatus().equals(Result.PASSED)) + { + addUsageEntry(result, stepDefinition, stepName); + } + } + + private String getStepName() + { + if (match instanceof StepDefinitionMatch) + { + return ((StepDefinitionMatch) match).getStepName(); + } + return null; + } + + private String getStepDefinition() + { + if (match instanceof StepDefinitionMatch) + { + return ((StepDefinitionMatch) match).getPattern(); + } + return null; + } + + private void addUsageEntry(Result result, String stepDefinition, String stepNameWithArgs) + { + List stepContainers = usageMap.get(stepDefinition); + if (stepContainers == null) + { + stepContainers = new ArrayList(); + usageMap.put(stepDefinition, stepContainers); + } + StepContainer stepContainer = findOrCreateStepContainer(stepNameWithArgs, stepContainers); + + String stepLocation = getStepLocation(); + Long duration = result.getDuration(); + StepDuration stepDuration = createStepDuration(duration, stepLocation); + stepContainer.durations.add(stepDuration); + } + + private String getStepLocation() + { + if(match instanceof StepDefinitionMatch) + { + StackTraceElement stepLocation = ((StepDefinitionMatch) match).getStepLocation(); + return stepLocation.getFileName() + ":" + stepLocation.getLineNumber(); + } + return null; + } + + private StepDuration createStepDuration(Long duration, String location) + { + StepDuration stepDuration = new StepDuration(); + if (duration == null) + { + stepDuration.duration = BigDecimal.ZERO; + } else + { + stepDuration.duration = BigDecimal.valueOf(duration); + } + stepDuration.location = location; + return stepDuration; + } + + private StepContainer findOrCreateStepContainer(String stepNameWithArgs, List stepContainers) + { + for (StepContainer container : stepContainers) + { + if (stepNameWithArgs.equals(container.name)) + { + return container; + } + } + StepContainer stepContainer = new StepContainer(); + stepContainer.name = stepNameWithArgs; + stepContainers.add(stepContainer); + return stepContainer; + } + + @Override + public void match(Match match) + { + this.match = match; + } + + /** + * Add a {@link UsageStatisticStrategy} to the formatter + * @param key the key, will be displayed in the output + * @param strategy the strategy + */ + public void addUsageStatisticStrategy(String key, UsageStatisticStrategy strategy) + { + statisticStrategies.put(key, strategy); + } + + /** + * Container of Step Definitions (patterns) + */ + static class StepDefContainer + { + /** + * The StepDefinition (pattern) + */ + public String source; + + /** + * A list of Steps + */ + public List steps; + } + + /** + * Contains for usage-entries of steps + */ + static class StepContainer { + public String name; + public Map aggregatedResults = new HashMap(); + public List durations = new ArrayList(); + } + + static class StepDuration { + public BigDecimal duration; + public String location; + } +} diff --git a/core/src/main/java/cucumber/formatter/usage/AverageUsageStatisticStrategy.java b/core/src/main/java/cucumber/formatter/usage/AverageUsageStatisticStrategy.java new file mode 100644 index 0000000000..b0b032db09 --- /dev/null +++ b/core/src/main/java/cucumber/formatter/usage/AverageUsageStatisticStrategy.java @@ -0,0 +1,30 @@ +package cucumber.formatter.usage; + +import java.util.List; + +/** + * Calculate the average of a list of duration entries + */ +public class AverageUsageStatisticStrategy implements UsageStatisticStrategy +{ + @Override + public Long calculate(List durationEntries) + { + if (verifyNoNulls(durationEntries)) + { + return 0L; + } + + long sum = 0; + for (Long duration : durationEntries) + { + sum += duration; + } + return sum / durationEntries.size(); + } + + private boolean verifyNoNulls(List durationEntries) + { + return durationEntries == null || durationEntries.isEmpty() || durationEntries.contains(null); + } +} diff --git a/core/src/main/java/cucumber/formatter/usage/MedianUsageStatisticStrategy.java b/core/src/main/java/cucumber/formatter/usage/MedianUsageStatisticStrategy.java new file mode 100644 index 0000000000..98573f3353 --- /dev/null +++ b/core/src/main/java/cucumber/formatter/usage/MedianUsageStatisticStrategy.java @@ -0,0 +1,34 @@ +package cucumber.formatter.usage; + +import java.util.Collections; +import java.util.List; + +/** + * Calculate the median of a list of duration entries + */ +public class MedianUsageStatisticStrategy implements UsageStatisticStrategy +{ + @Override + public Long calculate(List durationEntries) + { + if (verifyNoNulls(durationEntries)) + { + return 0L; + } + Collections.sort(durationEntries); + int middle = durationEntries.size() / 2; + if (durationEntries.size() % 2 == 1) + { + return durationEntries.get(middle); + } + else + { + return (durationEntries.get(middle - 1) + durationEntries.get(middle)) / 2; + } + } + + private boolean verifyNoNulls(List durationEntries) + { + return durationEntries == null || durationEntries.isEmpty() || durationEntries.contains(null); + } +} diff --git a/core/src/main/java/cucumber/formatter/usage/UsageStatisticStrategy.java b/core/src/main/java/cucumber/formatter/usage/UsageStatisticStrategy.java new file mode 100644 index 0000000000..c4a612ed14 --- /dev/null +++ b/core/src/main/java/cucumber/formatter/usage/UsageStatisticStrategy.java @@ -0,0 +1,15 @@ +package cucumber.formatter.usage; + +import java.util.List; + +/** + * Calculate a statistical value to be displayed in the usage-file + */ +public interface UsageStatisticStrategy +{ + /** + * @param durationEntries list of execution times of steps as nanoseconds + * @return a statistical value (e.g. median, average, ..) + */ + Long calculate(List durationEntries); +} diff --git a/core/src/main/java/cucumber/runtime/StepDefinitionMatch.java b/core/src/main/java/cucumber/runtime/StepDefinitionMatch.java index d70abaaf10..3493eb8dd4 100644 --- a/core/src/main/java/cucumber/runtime/StepDefinitionMatch.java +++ b/core/src/main/java/cucumber/runtime/StepDefinitionMatch.java @@ -180,4 +180,9 @@ public String getPattern() { public StackTraceElement getStepLocation() { return step.getStackTraceElement(uri); } + + public String getStepName() + { + return step.getName(); + } } diff --git a/core/src/test/java/cucumber/formatter/FormatterConverterTest.java b/core/src/test/java/cucumber/formatter/FormatterConverterTest.java index 44d635e9b2..2c0875a06d 100644 --- a/core/src/test/java/cucumber/formatter/FormatterConverterTest.java +++ b/core/src/test/java/cucumber/formatter/FormatterConverterTest.java @@ -35,4 +35,16 @@ public void instantiates_pretty_formatter_without_file_arg() { Formatter formatter = fc.convert("pretty"); assertEquals(CucumberPrettyFormatter.class, formatter.getClass()); } + + @Test + public void instantiates_usage_formatter_without_file_arg() { + Formatter formatter = fc.convert("usage"); + assertEquals(UsageFormatter.class, formatter.getClass()); + } + + @Test + public void instantiates_usage_formatter_with_file_arg() throws IOException { + Formatter formatter = fc.convert("usage:" + createTempFile().getAbsolutePath()); + assertEquals(UsageFormatter.class, formatter.getClass()); + } } diff --git a/core/src/test/java/cucumber/formatter/UsageFormatterTest.java b/core/src/test/java/cucumber/formatter/UsageFormatterTest.java new file mode 100644 index 0000000000..f271f9932d --- /dev/null +++ b/core/src/test/java/cucumber/formatter/UsageFormatterTest.java @@ -0,0 +1,171 @@ +package cucumber.formatter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import java.io.Closeable; +import java.io.IOException; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.junit.Test; +import org.mockito.Mockito; + +import cucumber.formatter.usage.UsageStatisticStrategy; +import cucumber.runtime.StepDefinitionMatch; +import gherkin.formatter.model.Result; + +public class UsageFormatterTest +{ + @Test + public void close() throws IOException + { + Appendable out = mock(Appendable.class, withSettings().extraInterfaces(Closeable.class)); + UsageFormatter usageFormatter = new UsageFormatter(out); + usageFormatter.close(); + verify((Closeable) out).close(); + } + + @Test + public void resultWithoutSkippedSteps() + { + Appendable out = mock(Appendable.class); + UsageFormatter usageFormatter = new UsageFormatter(out); + Result result = mock(Result.class); + when(result.getStatus()).thenReturn(Result.SKIPPED.getStatus()); + + usageFormatter.result(result); + verifyZeroInteractions(out); + } + + @Test + public void resultWithStep() + { + Appendable out = mock(Appendable.class); + UsageFormatter usageFormatter = new UsageFormatter(out); + + StepDefinitionMatch match = mockStepDefinitionMatch(); + usageFormatter.match(match); + + Result result = mock(Result.class); + when(result.getDuration()).thenReturn(12345L); + when(result.getStatus()).thenReturn(Result.PASSED); + + usageFormatter.result(result); + + Map> usageMap = usageFormatter.usageMap; + assertEquals(usageMap.size(), 1); + List durationEntries = usageMap.get("stepDef"); + assertEquals(durationEntries.size(), 1); + assertEquals(durationEntries.get(0).name, "step"); + assertEquals(durationEntries.get(0).durations.size(), 1); + assertEquals(durationEntries.get(0).durations.get(0).duration, BigDecimal.valueOf(12345)); + } + + private StepDefinitionMatch mockStepDefinitionMatch() + { + StepDefinitionMatch match = mock(StepDefinitionMatch.class, Mockito.RETURNS_MOCKS); + when(match.getPattern()).thenReturn("stepDef"); + when(match.getStepLocation()).thenReturn(new StackTraceElement("x", "y", "z", 3)); + when(match.getStepName()).thenReturn("step"); + return match; + } + + @Test + public void resultWithZeroDuration() + { + Appendable out = mock(Appendable.class); + UsageFormatter usageFormatter = new UsageFormatter(out); + + StepDefinitionMatch match = mockStepDefinitionMatch(); + usageFormatter.match(match); + + Result result = mock(Result.class); + when(result.getDuration()).thenReturn(0L); + when(result.getStatus()).thenReturn(Result.PASSED); + + usageFormatter.result(result); + + Map> usageMap = usageFormatter.usageMap; + assertEquals(usageMap.size(), 1); + List durationEntries = usageMap.get("stepDef"); + assertEquals(durationEntries.size(), 1); + assertEquals(durationEntries.get(0).name, "step"); + assertEquals(durationEntries.get(0).durations.size(), 1); + assertEquals(durationEntries.get(0).durations.get(0).duration, BigDecimal.ZERO); + } + + @Test + public void resultWithNullDuration() + { + Appendable out = mock(Appendable.class); + UsageFormatter usageFormatter = new UsageFormatter(out); + + StepDefinitionMatch match = mockStepDefinitionMatch(); + usageFormatter.match(match); + + Result result = mock(Result.class); + when(result.getDuration()).thenReturn(null); + when(result.getStatus()).thenReturn(Result.PASSED); + + usageFormatter.result(result); + + Map> usageMap = usageFormatter.usageMap; + assertEquals(usageMap.size(), 1); + List durationEntries = usageMap.get("stepDef"); + assertEquals(durationEntries.size(), 1); + assertEquals(durationEntries.get(0).name, "step"); + assertEquals(durationEntries.get(0).durations.size(), 1); + assertEquals(durationEntries.get(0).durations.get(0).duration, BigDecimal.ZERO); + } + + @Test + public void doneWithoutUsageStatisticStrategies() throws IOException + { + StringBuffer out = new StringBuffer(); + UsageFormatter usageFormatter = new UsageFormatter(out); + + UsageFormatter.StepContainer stepContainer = new UsageFormatter.StepContainer(); + UsageFormatter.StepDuration stepDuration = new UsageFormatter.StepDuration(); + stepDuration.duration = BigDecimal.valueOf(12345678L); + stepDuration.location = "location.feature"; + stepContainer.durations = Arrays.asList(stepDuration); + + usageFormatter.usageMap.put("aStep", Arrays.asList(stepContainer)); + + usageFormatter.done(); + + assertTrue(out.toString().contains("0.012345678")); + } + + @Test + public void doneWithUsageStatisticStrategies() throws IOException + { + StringBuffer out = new StringBuffer(); + UsageFormatter usageFormatter = new UsageFormatter(out); + + UsageFormatter.StepContainer stepContainer = new UsageFormatter.StepContainer(); + UsageFormatter.StepDuration stepDuration = new UsageFormatter.StepDuration(); + stepDuration.duration = BigDecimal.valueOf(12345678L); + stepDuration.location = "location.feature"; + stepContainer.durations = Arrays.asList(stepDuration); + + usageFormatter.usageMap.put("aStep", Arrays.asList(stepContainer)); + + UsageStatisticStrategy usageStatisticStrategy = mock(UsageStatisticStrategy.class); + when(usageStatisticStrategy.calculate(Arrays.asList(12345678L))).thenReturn(23456L); + usageFormatter.addUsageStatisticStrategy("average", usageStatisticStrategy); + + usageFormatter.done(); + + assertTrue(out.toString().contains("0.000023456")); + assertTrue(out.toString().contains("0.012345678")); + } +} diff --git a/core/src/test/java/cucumber/formatter/usage/AverageUsageStatisticStrategyTest.java b/core/src/test/java/cucumber/formatter/usage/AverageUsageStatisticStrategyTest.java new file mode 100644 index 0000000000..03fec9d05f --- /dev/null +++ b/core/src/test/java/cucumber/formatter/usage/AverageUsageStatisticStrategyTest.java @@ -0,0 +1,43 @@ +package cucumber.formatter.usage; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.Collections; + +import org.junit.Test; + +public class AverageUsageStatisticStrategyTest +{ + @Test + public void calculate() throws Exception + { + AverageUsageStatisticStrategy averageUsageStatisticStrategy = new AverageUsageStatisticStrategy(); + Long result = averageUsageStatisticStrategy.calculate(Arrays.asList(1L, 2L, 3L)); + assertEquals(result, Long.valueOf(2)); + } + + @Test + public void calculateNull() throws Exception + { + AverageUsageStatisticStrategy averageUsageStatisticStrategy = new AverageUsageStatisticStrategy(); + Long result = averageUsageStatisticStrategy.calculate(null); + assertEquals(result, Long.valueOf(0)); + } + + @Test + public void calculateEmptylist() throws Exception + { + AverageUsageStatisticStrategy averageUsageStatisticStrategy = new AverageUsageStatisticStrategy(); + Long result = averageUsageStatisticStrategy.calculate(Collections.emptyList()); + assertEquals(result, Long.valueOf(0)); + } + + @Test + public void calculateListWithNulls() throws Exception + { + AverageUsageStatisticStrategy averageUsageStatisticStrategy = new AverageUsageStatisticStrategy(); + Long result = averageUsageStatisticStrategy.calculate(Arrays.asList(3L, null)); + assertEquals(result, Long.valueOf(0)); + } +} diff --git a/core/src/test/java/cucumber/formatter/usage/MedianUsageStatisticStrategyTest.java b/core/src/test/java/cucumber/formatter/usage/MedianUsageStatisticStrategyTest.java new file mode 100644 index 0000000000..110813ae69 --- /dev/null +++ b/core/src/test/java/cucumber/formatter/usage/MedianUsageStatisticStrategyTest.java @@ -0,0 +1,51 @@ +package cucumber.formatter.usage; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.Collections; + +import org.junit.Test; + +public class MedianUsageStatisticStrategyTest +{ + @Test + public void calculateOddEntries() throws Exception + { + MedianUsageStatisticStrategy medianUsageStatisticStrategy = new MedianUsageStatisticStrategy(); + Long result = medianUsageStatisticStrategy.calculate(Arrays.asList(1L, 2L, 3L)); + assertEquals(result, Long.valueOf(2)); + } + + @Test + public void calculateEvenEntries() throws Exception + { + MedianUsageStatisticStrategy medianUsageStatisticStrategy = new MedianUsageStatisticStrategy(); + Long result = medianUsageStatisticStrategy.calculate(Arrays.asList(1L, 3L, 10L, 5L)); + assertEquals(result, Long.valueOf(4)); + } + + @Test + public void calculateNull() throws Exception + { + MedianUsageStatisticStrategy medianUsageStatisticStrategy = new MedianUsageStatisticStrategy(); + Long result = medianUsageStatisticStrategy.calculate(null); + assertEquals(result, Long.valueOf(0)); + } + + @Test + public void calculateEmptylist() throws Exception + { + MedianUsageStatisticStrategy medianUsageStatisticStrategy = new MedianUsageStatisticStrategy(); + Long result = medianUsageStatisticStrategy.calculate(Collections.emptyList()); + assertEquals(result, Long.valueOf(0)); + } + + @Test + public void calculateListWithNulls() throws Exception + { + MedianUsageStatisticStrategy medianUsageStatisticStrategy = new MedianUsageStatisticStrategy(); + Long result = medianUsageStatisticStrategy.calculate(Arrays.asList(1L, null, 3L)); + assertEquals(result, Long.valueOf(0)); + } +} diff --git a/core/src/test/java/cucumber/runtime/StepDefinitionMatchTest.java b/core/src/test/java/cucumber/runtime/StepDefinitionMatchTest.java index 3be49ef9b1..44dfd9dac3 100644 --- a/core/src/test/java/cucumber/runtime/StepDefinitionMatchTest.java +++ b/core/src/test/java/cucumber/runtime/StepDefinitionMatchTest.java @@ -13,6 +13,7 @@ import java.util.List; import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -70,4 +71,18 @@ public void can_have_doc_string_as_last_argument_among_many() throws Throwable { stepDefinitionMatch.runStep(ENGLISH); verify(stepDefinition).execute(ENGLISH, new Object[]{5, "HELLO"}); } + + @Test + public void retrieve_step_name() + { + String theName = "name"; + + Step step = mock(Step.class); + when(step.getName()).thenReturn(theName); + + StepDefinitionMatch stepDefinitionMatch = new StepDefinitionMatch(null, mock(StepDefinition.class), null, step, null); + + String stepName = stepDefinitionMatch.getStepName(); + assertEquals(theName, stepName); + } }