From 60d00df1e4a4ffb8234e979e317e63c1c0511114 Mon Sep 17 00:00:00 2001 From: draker94 <58571912+draker94@users.noreply.github.com> Date: Thu, 30 Mar 2023 22:56:16 +0300 Subject: [PATCH] Restore attachments when scenario-level fail-fast is enabled (#3778) Co-authored-by: draker94 --- .../org/vividus/steps/ui/BarcodeSteps.java | 4 +- .../vividus/steps/ui/BarcodeStepsTests.java | 16 +-- .../org/vividus/steps/db/DatabaseSteps.java | 14 ++- .../vividus/steps/db/DatabaseStepsTests.java | 29 +++-- .../org/vividus/softassert/ISoftAssert.java | 30 ++++- .../org/vividus/softassert/SoftAssert.java | 34 +++++- .../vividus/softassert/SoftAssertTests.java | 25 +++- .../java/org/vividus/steps/ArchiveSteps.java | 50 +++++--- .../org/vividus/steps/VariablesSteps.java | 16 ++- .../org/vividus/steps/ArchiveStepsTests.java | 108 ++++++++++++------ .../vividus/steps/VariablesStepsTests.java | 28 ++++- 11 files changed, 267 insertions(+), 87 deletions(-) diff --git a/vividus-extension-selenium/src/main/java/org/vividus/steps/ui/BarcodeSteps.java b/vividus-extension-selenium/src/main/java/org/vividus/steps/ui/BarcodeSteps.java index fc297c142e..a10a9353c6 100644 --- a/vividus-extension-selenium/src/main/java/org/vividus/steps/ui/BarcodeSteps.java +++ b/vividus-extension-selenium/src/main/java/org/vividus/steps/ui/BarcodeSteps.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -138,9 +138,9 @@ private void scanScreenshot(BufferedImage screenshot, Set scopes, } catch (NotFoundException e) { - softAssert.recordFailedAssertion(notFoundMessage, e); Attachment attachment = new Attachment(ImageTool.toByteArray(screenshot), "Screenshot", "image/png"); eventBus.post(new AttachmentPublishEvent(attachment)); + softAssert.recordFailedAssertion(notFoundMessage, e); } } } diff --git a/vividus-extension-selenium/src/test/java/org/vividus/steps/ui/BarcodeStepsTests.java b/vividus-extension-selenium/src/test/java/org/vividus/steps/ui/BarcodeStepsTests.java index a4914d11ee..36de41e5be 100644 --- a/vividus-extension-selenium/src/test/java/org/vividus/steps/ui/BarcodeStepsTests.java +++ b/vividus-extension-selenium/src/test/java/org/vividus/steps/ui/BarcodeStepsTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -91,8 +92,7 @@ void whenIScanBarcodeAndBarcodeIsAbsent() throws IOException, NotFoundException barCodeSteps.scanBarcode(VARIABLE_SCOPE, VARIABLE_NAME); - verify(softAssert).recordFailedAssertion("There is no barcode on the screen", exception); - verifyScreenshotAttachment(); + verifyScreenshotNotFoundBehavior("There is no barcode on the screen", exception); } @Test @@ -119,9 +119,7 @@ void whenIScanBarcodeFromSearchContextAndBarcodeIsAbsent() throws IOException, N var exception = NotFoundException.getNotFoundInstance(); when(barcodeActions.scanBarcode(QR_CODE_IMAGE)).thenThrow(exception); barCodeSteps.scanBarcodeFromContext(VARIABLE_SCOPE, VARIABLE_NAME); - verify(softAssert).recordFailedAssertion("There is no barcode on the selected context, page or screen", - exception); - verifyScreenshotAttachment(); + verifyScreenshotNotFoundBehavior("There is no barcode on the selected context, page or screen", exception); } private void mockScreenshotFromSearchContext() @@ -133,16 +131,18 @@ private void mockScreenshotFromSearchContext() when(screenshot.getImage()).thenReturn(QR_CODE_IMAGE); } - private void verifyScreenshotAttachment() throws IOException + private void verifyScreenshotNotFoundBehavior(String assertionMessage, Exception exception) throws IOException { + var ordered = inOrder(eventBus, softAssert); var eventCaptor = ArgumentCaptor.forClass(Object.class); - verify(eventBus).post(eventCaptor.capture()); + ordered.verify(eventBus).post(eventCaptor.capture()); Object event = eventCaptor.getValue(); assertThat(event, instanceOf(AttachmentPublishEvent.class)); Attachment attachment = ((AttachmentPublishEvent) event).getAttachment(); assertEquals("Screenshot", attachment.getTitle()); assertEquals("image/png", attachment.getContentType()); assertArrayEquals(ImageTool.toByteArray(QR_CODE_IMAGE), attachment.getContent()); + ordered.verify(softAssert).recordFailedAssertion(assertionMessage, exception); verifyNoInteractions(variableContext); } } diff --git a/vividus-plugin-db/src/main/java/org/vividus/steps/db/DatabaseSteps.java b/vividus-plugin-db/src/main/java/org/vividus/steps/db/DatabaseSteps.java index 69f2289323..424a2834c3 100644 --- a/vividus-plugin-db/src/main/java/org/vividus/steps/db/DatabaseSteps.java +++ b/vividus-plugin-db/src/main/java/org/vividus/steps/db/DatabaseSteps.java @@ -30,6 +30,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -318,11 +319,16 @@ private void verifyComparisonResult(DataSetComparisonRule comparisonRule, DataSo { attachmentPublisher.publishAttachment("data-sources-statistics.ftl", Map.of("statistics", dataSourceStatistics), "Data sources statistics"); - if (!softAssert.assertTrue(comparisonRule.getAssertionDescription(), result.isEmpty())) + + Consumer resultConsumer = passed -> { - attachmentPublisher.publishAttachment("/templates/maps-comparison-table.ftl", Map.of("results", result), - "Data sets comparison"); - } + if (!passed) + { + attachmentPublisher.publishAttachment("/templates/maps-comparison-table.ftl", Map.of("results", result), + "Data sets comparison"); + } + }; + softAssert.assertTrue(comparisonRule.getAssertionDescription(), result.isEmpty(), resultConsumer); } /** diff --git a/vividus-plugin-db/src/test/java/org/vividus/steps/db/DatabaseStepsTests.java b/vividus-plugin-db/src/test/java/org/vividus/steps/db/DatabaseStepsTests.java index f05c65adc6..f48f370773 100644 --- a/vividus-plugin-db/src/test/java/org/vividus/steps/db/DatabaseStepsTests.java +++ b/vividus-plugin-db/src/test/java/org/vividus/steps/db/DatabaseStepsTests.java @@ -48,6 +48,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import java.util.stream.Stream; import com.github.valfirst.slf4jtest.TestLogger; @@ -297,7 +298,7 @@ void shouldCompareQueriesResponsesAndPostDiffTable() throws InterruptedException List> result = List.of(Map.of(COL1, VAL1), Map.of(COL1, VAL1), Map.of(COL1, VAL3)); mockQueryForList(QUERY, DB_KEY, result); mockQueryForList(QUERY2, DB_KEY2, List.of(Map.of(COL1, VAL2))); - when(softAssert.assertTrue(QUERY_RESULTS_ARE_EQUAL, false)).thenReturn(false); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(false), any()); mockHashing(); configureTimeout(); databaseSteps.setDuplicateKeysStrategy(DuplicateKeysStrategy.DISTINCT); @@ -351,7 +352,7 @@ void shouldLimitDiffTable() throws InterruptedException, ExecutionException, Tim List> result = List.of(Map.of(COL1, VAL1), Map.of(COL1, VAL3)); mockQueryForList(QUERY, DB_KEY, result); mockQueryForList(QUERY, DB_KEY2, List.of(Map.of(COL1, VAL2))); - when(softAssert.assertTrue(QUERY_RESULTS_ARE_EQUAL, false)).thenReturn(false); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(false), any()); mockHashing(); configureTimeout(); databaseSteps.compareData(QUERY, DB_KEY, DataSetComparisonRule.IS_EQUAL_TO, QUERY, DB_KEY2, KEYS); @@ -453,7 +454,8 @@ static Stream notMatchingDataSets() void shouldCompareDataVsExamplesTableAndPostReportFailedChecks(DataSetComparisonRule comparisonRule, List> leftDataSet, List> rightDataSet) { - when(softAssert.assertTrue(comparisonRule.getAssertionDescription(), false)).thenReturn(false); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertTrue(eq(comparisonRule.getAssertionDescription()), + eq(false), any()); databaseSteps.setDuplicateKeysStrategy(DuplicateKeysStrategy.NOOP); mockHashing(); mockDataSource(); @@ -508,11 +510,11 @@ void testWaitUntilQueryReturnedDataEqualToTable() { databaseSteps.setDuplicateKeysStrategy(DuplicateKeysStrategy.NOOP); mockQueryForList(QUERY, DB_KEY, List.of(Map.of(COL1, VAL2))); - when(softAssert.assertTrue(QUERY_RESULTS_ARE_EQUAL, true)).thenReturn(true); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(true), any()); databaseSteps.waitForDataAppearance(TWO_SECONDS, 10, QUERY, DB_KEY, DataSetComparisonRule.IS_EQUAL_TO, TABLE); verify(attachmentPublisher).publishAttachment(eq(DATA_SOURCES_STATISTICS_FTL), any(Map.class), eq(DATA_SOURCES_STATISTICS_TITLE)); - verify(softAssert).assertTrue(QUERY_RESULTS_ARE_EQUAL, true); + verify(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(true), any()); } @Test @@ -529,11 +531,11 @@ void testWaitTwiceUntilQueryReturnedDataEqualToTable() .thenReturn(List.of(Map.of(COL1, VAL1))) .thenReturn(List.of(Map.of(COL1, VAL2))); - when(softAssert.assertTrue(QUERY_RESULTS_ARE_EQUAL, true)).thenReturn(true); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(true), any()); databaseSteps.waitForDataAppearance(TWO_SECONDS, 10, QUERY, DB_KEY, DataSetComparisonRule.IS_EQUAL_TO, TABLE); verify(attachmentPublisher).publishAttachment(eq(DATA_SOURCES_STATISTICS_FTL), any(Map.class), eq(DATA_SOURCES_STATISTICS_TITLE)); - verify(softAssert).assertTrue(QUERY_RESULTS_ARE_EQUAL, true); + verify(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(true), any()); } @Test @@ -549,6 +551,7 @@ void testWaitUntilQueryReturnedDataEqualToTableFailed() when(jdbcTemplate.queryForList(QUERY)) .thenReturn(List.of(Map.of(COL1, VAL1))) .thenReturn(List.of(Map.of(COL1, VAL3))); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(false), any()); databaseSteps.waitForDataAppearance(Duration.ofSeconds(4), 2, QUERY, DB_KEY, DataSetComparisonRule.IS_EQUAL_TO, TABLE); @@ -556,7 +559,7 @@ void testWaitUntilQueryReturnedDataEqualToTableFailed() assertThat(LOGGER.getLoggingEvents(), equalTo(List.of(info(logMessage, 1), info(logMessage, 1)))); verify(attachmentPublisher).publishAttachment(eq(DATA_SOURCES_STATISTICS_FTL), any(Map.class), eq(DATA_SOURCES_STATISTICS_TITLE)); - verify(softAssert).assertTrue(QUERY_RESULTS_ARE_EQUAL, false); + verify(softAssert).assertTrue(eq(QUERY_RESULTS_ARE_EQUAL), eq(false), any()); verify(attachmentPublisher).publishAttachment(eq(DATA_SET_COMPARISON_FTL), argThat(r -> { @SuppressWarnings("unchecked") List> results = (List>) ((Map) r) @@ -650,4 +653,14 @@ private void configureTimeout() { databaseSteps.setDbQueryTimeout(Duration.ofSeconds(20)); } + + private Answer getAssertionAnswer(boolean assertionPassed) + { + return a -> + { + Consumer consumer = a.getArgument(2); + consumer.accept(assertionPassed); + return null; + }; + } } diff --git a/vividus-soft-assert/src/main/java/org/vividus/softassert/ISoftAssert.java b/vividus-soft-assert/src/main/java/org/vividus/softassert/ISoftAssert.java index 1d5a941c16..3073dc75cc 100644 --- a/vividus-soft-assert/src/main/java/org/vividus/softassert/ISoftAssert.java +++ b/vividus-soft-assert/src/main/java/org/vividus/softassert/ISoftAssert.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package org.vividus.softassert; +import java.util.function.Consumer; import java.util.regex.Pattern; import org.hamcrest.Matcher; @@ -31,6 +32,17 @@ public interface ISoftAssert */ boolean assertTrue(String description, boolean condition); + /** + * Asserts that a condition is true. If it isn't, an assertion error is added to the collection. + * An additional operation which will always be executed is specified. + * Useful when used with fail-fast at the scenario level. + * @param description The assertion description + * @param condition Condition to be checked + * @param resultConsumer Always executed additional operation that consumes result of the assertion + * @return true if assertion is passed, otherwise false + */ + boolean assertTrue(String description, boolean condition, Consumer resultConsumer); + /** * Asserts that a condition is false. If it isn't an assertion error is added to the collection. * @param description The assertion description @@ -167,6 +179,22 @@ public interface ISoftAssert */ boolean assertThat(String description, T actual, Matcher matcher); + /** + * Asserts that actual satisfies the condition specified by matcher. + * If it is not, an assertion error is added to the collection. + * An additional operation which will always be executed is specified. + * Useful when used with fail-fast at the scenario level. + * @param description The assertion description + * @param the static type accepted by the matcher (this can flag obvious + * compile-time problems such as {@code assertThat(1, is("a"))} + * @param actual the computed value being compared + * @param matcher an expression, built of {@link Matcher}s, specifying allowed + * values + * @param resultConsumer always executed additional operation that consumes result of the assertion + * @return true if assertion is passed, otherwise false + */ + boolean assertThat(String description, T actual, Matcher matcher, Consumer resultConsumer); + /** * Logs information about passed assertion and returns true. * @param description The assertion description diff --git a/vividus-soft-assert/src/main/java/org/vividus/softassert/SoftAssert.java b/vividus-soft-assert/src/main/java/org/vividus/softassert/SoftAssert.java index 49423d52fb..5c92deadc2 100644 --- a/vividus-soft-assert/src/main/java/org/vividus/softassert/SoftAssert.java +++ b/vividus-soft-assert/src/main/java/org/vividus/softassert/SoftAssert.java @@ -17,6 +17,7 @@ package org.vividus.softassert; import java.util.List; +import java.util.function.Consumer; import java.util.regex.Pattern; import com.google.common.eventbus.EventBus; @@ -67,7 +68,13 @@ public class SoftAssert implements ISoftAssert @Override public boolean assertTrue(final String description, final boolean condition) { - return recordAssertion(condition, description, condition ? IS_TRUE : IS_FALSE); + return assertTrue(description, condition, null); + } + + @Override + public boolean assertTrue(String description, boolean condition, Consumer resultConsumer) + { + return recordAssertionWithFinally(condition, description, condition ? IS_TRUE : IS_FALSE, resultConsumer); } @Override @@ -206,10 +213,17 @@ public boolean assertNull(String description, final Object object) @Override public boolean assertThat(String description, T actual, Matcher matcher) + { + return assertThat(description, actual, matcher, null); + } + + @Override + public boolean assertThat(String description, T actual, Matcher matcher, + Consumer resultConsumer) { boolean matches = matcher.matches(actual); String matcherDescriptionString = getAssertionDescriptionString(actual, matcher); - return recordAssertion(matches, description, matcherDescriptionString); + return recordAssertionWithFinally(matches, description, matcherDescriptionString, resultConsumer); } protected String getNullAssertionDescription(final Object object) @@ -313,6 +327,22 @@ private boolean recordAssertion(boolean passed, String description, String asser return recordAssertion(passed, format(description, assertionDescription)); } + private boolean recordAssertionWithFinally(boolean passed, String description, String assertionDescription, + Consumer resultConsumer) + { + try + { + return recordAssertion(passed, format(description, assertionDescription)); + } + finally + { + if (resultConsumer != null) + { + resultConsumer.accept(passed); + } + } + } + private boolean recordAssertion(boolean passed, String description, Throwable cause) { if (passed) diff --git a/vividus-soft-assert/src/test/java/org/vividus/softassert/SoftAssertTests.java b/vividus-soft-assert/src/test/java/org/vividus/softassert/SoftAssertTests.java index f312dba332..7c6d6ab819 100644 --- a/vividus-soft-assert/src/test/java/org/vividus/softassert/SoftAssertTests.java +++ b/vividus-soft-assert/src/test/java/org/vividus/softassert/SoftAssertTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -36,6 +37,7 @@ import static uk.org.lidalia.slf4jext.Level.ERROR; import java.util.List; +import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.regex.Pattern; @@ -189,6 +191,15 @@ void testAssertTrueNot() assertFalse(softAssert.assertTrue(TRUE, TEXT.equals(EMPTY_STRING))); } + @Test + void testAssertTrueWithConsumer() + { + mockAssertionCollection(); + Consumer consumer = mock(); + assertTrue(softAssert.assertTrue(TRUE, TEXT.equals(TEXT), consumer)); + verify(consumer).accept(true); + } + @Test void testAssertNotNull() { @@ -234,6 +245,18 @@ void testAssertThatFalse() assertFalse(softAssert.assertThat("Text contains 'a'", TEXT, containsString("a"))); } + @Test + void testAssertThatWithConsumerAndException() + { + mockAssertionCollection(); + Consumer consumer = mock(); + when(failTestFastManager.isFailTestCaseFast()).thenReturn(true); + doThrow(RuntimeException.class).when(failTestFastHandler).failTestCaseFast(); + assertThrows(RuntimeException.class, + () -> softAssert.assertThat("Text contains 'b'", TEXT, containsString("b"), consumer)); + verify(consumer).accept(false); + } + @Test void testRecordFailedAssertion() { diff --git a/vividus/src/main/java/org/vividus/steps/ArchiveSteps.java b/vividus/src/main/java/org/vividus/steps/ArchiveSteps.java index 707c42d222..7d8548603a 100644 --- a/vividus/src/main/java/org/vividus/steps/ArchiveSteps.java +++ b/vividus/src/main/java/org/vividus/steps/ArchiveSteps.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,8 +24,11 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.hamcrest.Matcher; import org.jbehave.core.annotations.AsParameters; import org.jbehave.core.annotations.Then; import org.jbehave.core.annotations.When; @@ -110,28 +113,47 @@ public void saveArchiveEntriesToVariables(DataWrapper archiveData, List parameters) { Set entryNames = ZipUtils.readZipEntryNamesFromBytes(archiveData.getBytes()); - boolean verificationPassed = parameters.stream() - .map(entry -> assertArchiveEntry(entry, entryNames)) - .reduce(true, (a, b) -> a && b); - if (!verificationPassed) - { - attachmentPublisher.publishAttachment("templates/archive-entries-result-table.ftl", - Map.of("entryNames", entryNames), "Archive entries"); - } + MutableBoolean reportNotPublished = new MutableBoolean(true); + + parameters.forEach(entry -> assertArchiveEntry(entry, entryNames, reportNotPublished)); } - private boolean assertArchiveEntry(NamedEntry expectedEntry, Set entryNames) + private void assertArchiveEntry(NamedEntry expectedEntry, Set entryNames, + MutableBoolean reportNotPublished) { String expectedName = expectedEntry.getName(); StringComparisonRule comparisonRule = expectedEntry.getRule(); + + String assertionDescription; + Matcher> entryMatcher; if (comparisonRule != null) { - return softAssert.assertThat(String.format( + assertionDescription = String.format( "The archive contains entry matching the comparison " + "rule '%s' with name pattern '%s'", - comparisonRule, expectedName), entryNames, hasItem(comparisonRule.createMatcher(expectedName))); + comparisonRule, expectedName); + entryMatcher = hasItem(comparisonRule.createMatcher(expectedName)); } - return softAssert.assertThat("The archive contains entry with name " + expectedName, entryNames, - hasItem(expectedName)); + else + { + assertionDescription = "The archive contains entry with name " + expectedName; + entryMatcher = hasItem(expectedName); + } + + softAssert.assertThat(assertionDescription, entryNames, entryMatcher, + publishAttachmentOnFailure(entryNames, reportNotPublished)); + } + + private Consumer publishAttachmentOnFailure(Set entryNames, MutableBoolean reportNotPublished) + { + return passed -> + { + if (!passed && reportNotPublished.isTrue()) + { + attachmentPublisher.publishAttachment("templates/archive-entries-result-table.ftl", + Map.of("entryNames", entryNames), "Archive entries"); + reportNotPublished.setFalse(); + } + }; } @AsParameters diff --git a/vividus/src/main/java/org/vividus/steps/VariablesSteps.java b/vividus/src/main/java/org/vividus/steps/VariablesSteps.java index 01f03eb922..5207853378 100644 --- a/vividus/src/main/java/org/vividus/steps/VariablesSteps.java +++ b/vividus/src/main/java/org/vividus/steps/VariablesSteps.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.function.Consumer; import java.util.regex.Pattern; import org.jbehave.core.annotations.Given; @@ -62,12 +63,15 @@ protected > boolean compareValues(T value1, ComparisonRu { String readableCondition = EnumUtils.toHumanReadableForm(condition); String description = "Checking if \"" + value1 + "\" is " + readableCondition + " \"" + value2 + "\""; - boolean passed = softAssert.assertThat(description, value1, condition.getComparisonRule(value2)); - if (!passed) + + Consumer resultConsumer = passed -> { - diffAttachmentPublisher.publishDiff(value1, value2); - } - return passed; + if (!passed) + { + diffAttachmentPublisher.publishDiff(value1, value2); + } + }; + return softAssert.assertThat(description, value1, condition.getComparisonRule(value2), resultConsumer); } @Override diff --git a/vividus/src/test/java/org/vividus/steps/ArchiveStepsTests.java b/vividus/src/test/java/org/vividus/steps/ArchiveStepsTests.java index c78f2541da..6dc78129a7 100644 --- a/vividus/src/test/java/org/vividus/steps/ArchiveStepsTests.java +++ b/vividus/src/test/java/org/vividus/steps/ArchiveStepsTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,14 @@ package org.vividus.steps; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; import static org.vividus.steps.StringComparisonRule.CONTAINS; import static org.vividus.steps.StringComparisonRule.DOES_NOT_CONTAIN; import static org.vividus.steps.StringComparisonRule.IS_EQUAL_TO; @@ -31,12 +33,14 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.stubbing.Answer; import org.vividus.context.VariableContext; import org.vividus.reporter.event.IAttachmentPublisher; import org.vividus.softassert.ISoftAssert; @@ -98,17 +102,20 @@ void testSaveFilesContentToVariablesInvalidPath() void testVerifyArchiveContainsEntries() { var archiveEntries = createArchiveEntries(); + lenient().doAnswer(getAssertionAnswer(false)).when(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + DUMMY), + eq(archiveEntries), argThat(e -> !e.matches(archiveEntries)), any()); + archiveSteps.verifyArchiveContainsEntries(createArchiveData(), List.of( createEntry(FILE_JSON, null), createEntry(IMAGE_PNG, null), createEntry(DUMMY, null) )); verify(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + FILE_JSON), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries))); + argThat(e -> e.matches(archiveEntries)), any()); verify(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + IMAGE_PNG), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries))); + argThat(e -> e.matches(archiveEntries)), any()); verify(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + DUMMY), eq(archiveEntries), - argThat(e -> !e.matches(archiveEntries))); + argThat(e -> !e.matches(archiveEntries)), any()); verifyNoMoreInteractions(softAssert); verifyPublishAttachment(archiveEntries); verifyNoMoreInteractions(attachmentPublisher); @@ -118,38 +125,40 @@ void testVerifyArchiveContainsEntries() void testVerifyArchiveContainsEntriesIsPassed() { var archiveEntries = createArchiveEntries(); - when(softAssert.assertThat(eq(CONTAINS_ENTRY_WITH_NAME + IMAGE_PNG), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries)))).thenReturn(true); - when(softAssert.assertThat(eq(CONTAINS_ENTRY_WITH_NAME + FILE_JSON), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries)))).thenReturn(true); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + IMAGE_PNG), + eq(archiveEntries), argThat(e -> e.matches(archiveEntries)), any()); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + FILE_JSON), + eq(archiveEntries), argThat(e -> e.matches(archiveEntries)), any()); + archiveSteps.verifyArchiveContainsEntries(createArchiveData(), List.of( createEntry(FILE_JSON, null), createEntry(IMAGE_PNG, null) )); verify(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + FILE_JSON), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries))); + argThat(e -> e.matches(archiveEntries)), any()); verify(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + IMAGE_PNG), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries))); + argThat(e -> e.matches(archiveEntries)), any()); verifyNoMoreInteractions(softAssert); verifyNoInteractions(attachmentPublisher); } @Test - void testVerifyArchiveContainsEntriesIsFailled() + void testVerifyArchiveContainsEntriesIsFailed() { var archiveEntries = createArchiveEntries(); - when(softAssert.assertThat(eq(CONTAINS_ENTRY_WITH_NAME + IMAGE_PNG), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries)))).thenReturn(true); - when(softAssert.assertThat(eq(CONTAINS_ENTRY_WITH_NAME + DUMMY), eq(archiveEntries), - argThat(e -> !e.matches(archiveEntries)))).thenReturn(false); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + DUMMY), + eq(archiveEntries), argThat(e -> !e.matches(archiveEntries)), any()); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + IMAGE_PNG), + eq(archiveEntries), argThat(e -> e.matches(archiveEntries)), any()); + archiveSteps.verifyArchiveContainsEntries(createArchiveData(), List.of( createEntry(DUMMY, null), createEntry(IMAGE_PNG, null) )); verify(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + DUMMY), eq(archiveEntries), - argThat(e -> !e.matches(archiveEntries))); + argThat(e -> !e.matches(archiveEntries)), any()); verify(softAssert).assertThat(eq(CONTAINS_ENTRY_WITH_NAME + IMAGE_PNG), eq(archiveEntries), - argThat(e -> e.matches(archiveEntries))); + argThat(e -> e.matches(archiveEntries)), any()); verifyNoMoreInteractions(softAssert); verifyPublishAttachment(archiveEntries); verifyNoMoreInteractions(attachmentPublisher); @@ -159,18 +168,21 @@ void testVerifyArchiveContainsEntriesIsFailled() void testVerifyArchiveContainsEntriesWithUserRulesIsFailed() { var archiveEntries = createArchiveEntries(); - when(softAssert.assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, DUMMY)), - eq(archiveEntries), argThat(e -> !e.matches(archiveEntries)))).thenReturn(false); - when(softAssert.assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), - eq(archiveEntries), argThat(e -> e.matches(archiveEntries)))).thenReturn(true); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertThat( + eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, DUMMY)), eq(archiveEntries), + argThat(e -> !e.matches(archiveEntries)), any()); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertThat( + eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), eq(archiveEntries), + argThat(e -> e.matches(archiveEntries)), any()); + archiveSteps.verifyArchiveContainsEntries(createArchiveData(), List.of( createEntry(DUMMY, MATCHES), createEntry(MATCHES_PATTERN, MATCHES) )); verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, DUMMY)), - eq(archiveEntries), argThat(e -> !e.matches(archiveEntries))); + eq(archiveEntries), argThat(e -> !e.matches(archiveEntries)), any()); verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), - eq(archiveEntries), argThat(e -> e.matches(archiveEntries))); + eq(archiveEntries), argThat(e -> e.matches(archiveEntries)), any()); verifyNoMoreInteractions(softAssert); verifyPublishAttachment(archiveEntries); verifyNoMoreInteractions(attachmentPublisher); @@ -180,13 +192,14 @@ void testVerifyArchiveContainsEntriesWithUserRulesIsFailed() void testVerifyArchiveContainsEntriesWithUserRulesIsPassed() { var archiveEntries = createArchiveEntries(); - when(softAssert.assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), - eq(archiveEntries), argThat(e -> e.matches(archiveEntries)))).thenReturn(true); - archiveSteps.verifyArchiveContainsEntries(createArchiveData(), List.of( - createEntry(MATCHES_PATTERN, MATCHES) + doAnswer(getAssertionAnswer(true)).when(softAssert).assertThat( + eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), eq(archiveEntries), + argThat(e -> e.matches(archiveEntries)), any()); + + archiveSteps.verifyArchiveContainsEntries(createArchiveData(), List.of(createEntry(MATCHES_PATTERN, MATCHES) )); verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), - eq(archiveEntries), argThat(e -> e.matches(archiveEntries))); + eq(archiveEntries), argThat(e -> e.matches(archiveEntries)), any()); verifyNoMoreInteractions(softAssert); verifyNoInteractions(attachmentPublisher); } @@ -194,22 +207,35 @@ void testVerifyArchiveContainsEntriesWithUserRulesIsPassed() @Test void testVerifyArchiveContainsEntriesWithUserRules() { - var containsPattern = "file"; var archiveEntries = createArchiveEntries(); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertThat( + eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), eq(archiveEntries), + argThat(e -> e.matches(archiveEntries)), any()); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertThat( + eq(String.format(CONTAINS_ENTRY_WITH_RULE, CONTAINS, DUMMY)), eq(archiveEntries), + argThat(e -> !e.matches(archiveEntries)), any()); + doAnswer(getAssertionAnswer(false)).when(softAssert).assertThat( + eq(String.format(CONTAINS_ENTRY_WITH_RULE, IS_EQUAL_TO, DUMMY)), eq(archiveEntries), + argThat(e -> !e.matches(archiveEntries)), any()); + doAnswer(getAssertionAnswer(true)).when(softAssert).assertThat( + eq(String.format(CONTAINS_ENTRY_WITH_RULE, DOES_NOT_CONTAIN, DUMMY)), eq(archiveEntries), + argThat(e -> e.matches(archiveEntries)), any()); + archiveSteps.verifyArchiveContainsEntries(createArchiveData(), List.of( createEntry(MATCHES_PATTERN, MATCHES), - createEntry(containsPattern, CONTAINS), + createEntry(DUMMY, CONTAINS), createEntry(DUMMY, IS_EQUAL_TO), createEntry(DUMMY, DOES_NOT_CONTAIN) )); + verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, MATCHES, MATCHES_PATTERN)), - eq(archiveEntries), argThat(e -> e.matches(archiveEntries))); - verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, CONTAINS, containsPattern)), - eq(archiveEntries), argThat(e -> e.matches(archiveEntries))); + eq(archiveEntries), argThat(e -> e.matches(archiveEntries)), any()); + verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, CONTAINS, DUMMY)), + eq(archiveEntries), argThat(e -> !e.matches(archiveEntries)), any()); verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, IS_EQUAL_TO, DUMMY)), - eq(archiveEntries), argThat(e -> !e.matches(archiveEntries))); + eq(archiveEntries), argThat(e -> !e.matches(archiveEntries)), any()); verify(softAssert).assertThat(eq(String.format(CONTAINS_ENTRY_WITH_RULE, DOES_NOT_CONTAIN, DUMMY)), - eq(archiveEntries), argThat(e -> e.matches(archiveEntries))); + eq(archiveEntries), argThat(e -> e.matches(archiveEntries)), any()); verifyNoMoreInteractions(softAssert); verifyPublishAttachment(archiveEntries); verifyNoMoreInteractions(attachmentPublisher); @@ -243,6 +269,16 @@ private Set createArchiveEntries() return archiveEntries; } + private Answer getAssertionAnswer(boolean assertionPassed) + { + return a -> + { + Consumer consumer = a.getArgument(3); + consumer.accept(assertionPassed); + return null; + }; + } + private static ArchiveVariable createVariable(String path, String variableName, OutputFormat outputFormat) { var scopes = Set.of(VariableScope.SCENARIO); diff --git a/vividus/src/test/java/org/vividus/steps/VariablesStepsTests.java b/vividus/src/test/java/org/vividus/steps/VariablesStepsTests.java index 63524a249b..6a5c9e66e3 100644 --- a/vividus/src/test/java/org/vividus/steps/VariablesStepsTests.java +++ b/vividus/src/test/java/org/vividus/steps/VariablesStepsTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2023 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,17 +23,18 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; import java.math.BigDecimal; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Stream; import org.jbehave.core.model.ExamplesTable; @@ -47,6 +48,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.stubbing.Answer; import org.vividus.context.VariableContext; import org.vividus.publishing.DiffAttachmentPublisher; import org.vividus.reporter.event.IAttachmentPublisher; @@ -85,9 +87,15 @@ static Stream stringsAsNumbers() @MethodSource("stringsAsNumbers") void shouldCompareStringsAsNumbers(Object variable1, Object variable2) { + String assertDescription = "Checking if \"1\" is less than or equal to \"10\""; + String matcherAsString = "a value less than or equal to <10>"; + doAnswer(getAssertionAnswer(false)).when(softAssert).assertThat( + eq(assertDescription), eq(BigDecimal.ONE), + argThat(m -> matcherAsString.equals(m.toString())), any()); + variablesSteps.compareVariables(variable1, ComparisonRule.LESS_THAN_OR_EQUAL_TO, variable2); - verify(softAssert).assertThat(eq("Checking if \"1\" is less than or equal to \"10\""), - eq(BigDecimal.ONE), argThat(m -> "a value less than or equal to <10>".equals(m.toString()))); + verify(softAssert).assertThat(eq(assertDescription), + eq(BigDecimal.ONE), argThat(m -> matcherAsString.equals(m.toString())), any()); verify(diffAttachmentPublisher).publishDiff(BigDecimal.ONE, BigDecimal.TEN); } @@ -100,7 +108,7 @@ void shouldCompareStringsAsNumbers(Object variable1, Object variable2) void testCompareSimpleVariablesStrings(String variable1, String variable2, boolean passed, int published) { String description = "Checking if \"" + variable1 + "\" is equal to \"" + variable2 + "\""; - when(softAssert.assertThat(eq(description), eq(variable1), any())).thenReturn(passed); + doAnswer(getAssertionAnswer(passed)).when(softAssert).assertThat(eq(description), eq(variable1), any(), any()); assertEquals(passed, variablesSteps.compareVariables(variable1, ComparisonRule.EQUAL_TO, variable2)); verify(diffAttachmentPublisher, times(published)).publishDiff(variable1, variable2); } @@ -325,4 +333,14 @@ void testInitVariableWithGivenValues() variablesSteps.initVariableWithGivenValues(scopes, VALUE_1, listOfMaps); verify(variableContext).putVariable(scopes, VALUE_1, listOfMaps); } + + private Answer getAssertionAnswer(boolean assertionPassed) + { + return a -> + { + Consumer consumer = a.getArgument(3); + consumer.accept(assertionPassed); + return assertionPassed; + }; + } }