From 176fa1c5633ff57ea5ba2b76f6606895d724ff7d Mon Sep 17 00:00:00 2001 From: Uladzislau Arlouski Date: Fri, 8 Dec 2023 17:44:49 +0200 Subject: [PATCH] [plugin-web-app-to-rest-api] Add ability to use variable name as HTML source in from_html transformer --- .../pages/plugin-web-app-to-rest-api.adoc | 13 ++++- .../HtmlDocumentTableTransformer.java | 44 ++++++++++++--- .../HtmlDocumentTableTransformerTests.java | 55 ++++++++++++++++--- .../story/integration/TableTransformers.story | 13 ++++- 4 files changed, 107 insertions(+), 18 deletions(-) diff --git a/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc b/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc index b887f71cbd..9e15f7e9e6 100644 --- a/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc +++ b/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc @@ -250,7 +250,18 @@ Examples: |Description |`pageUrl` -|The URL of the page to build the table upon. +a|The URL of the page to build the table upon. + +WARNING: The `pageUrl` parameter is deprecated and will be removed in VIVIDUS 0.7.0, please use `variableName` instead. + +IMPORTANT: The `pageUrl` parameter can not be used together with the `variableName` parameter. + +|`variableName` +a|The name of the variable containing source HTML, only variables of scopes `global` and `next_batches` are allowed. +Exceptions are cases when the transformer using in xref:commons:vividus-steps.adoc#_initialize_variable_with_a_table[step] +which initializes a variable with a table. + +IMPORTANT: The `variableName` parameter can not be used together with the `pageUrl` parameter. |`column` |The column name in the generated table. diff --git a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformer.java b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformer.java index daed99ed32..61faf42499 100644 --- a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformer.java +++ b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformer.java @@ -26,8 +26,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Properties; import java.util.function.Function; +import java.util.function.Supplier; +import org.apache.commons.lang3.Validate; import org.jbehave.core.model.ExamplesTable.TableProperties; import org.jbehave.core.model.TableParsers; import org.jbehave.core.model.TableTransformers.TableTransformer; @@ -35,23 +38,48 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.vividus.context.VariableContext; import org.vividus.util.ExamplesTableProcessor; public class HtmlDocumentTableTransformer implements TableTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(HtmlDocumentTableTransformer.class); + private static final String SLASH = "/"; private final Optional httpConfiguration; + private final VariableContext variableContext; - public HtmlDocumentTableTransformer(Optional httpConfiguration) + public HtmlDocumentTableTransformer(Optional httpConfiguration, VariableContext variableContext) { this.httpConfiguration = httpConfiguration; + this.variableContext = variableContext; } @Override public String transform(String table, TableParsers parsers, TableProperties tableProperties) { - String pageUrl = tableProperties.getMandatoryNonBlankProperty("pageUrl", String.class); + Properties propertes = tableProperties.getProperties(); + String pageUrl = propertes.getProperty("pageUrl"); + String variableName = propertes.getProperty("variableName"); + + Validate.isTrue(pageUrl != null && variableName == null || pageUrl == null && variableName != null, + "Either 'pageUrl' or 'variableName' should be specified."); + + Supplier documentSuppler; + if (pageUrl == null) + { + documentSuppler = () -> Jsoup.parse((String) variableContext.getVariable(variableName)); + } + else + { + LOGGER.atWarn().log("The 'pageUrl' transformer parameter is deprecated and will be removed VIVIDUS 0.7.0, " + + "please use 'variableName' parameter instead."); + documentSuppler = () -> createDocument(pageUrl); + } + String column = tableProperties.getMandatoryNonBlankProperty("column", String.class); String xpathSelector = tableProperties.getMandatoryNonBlankProperty("xpathSelector", String.class); @@ -74,12 +102,12 @@ else if (lastSegment.startsWith("@")) getter = Element::outerHtml; } - return createDocument(pageUrl).selectXpath(xpathSelector) - .stream() - .map(getter) - .collect(collectingAndThen(toList(), attrs -> ExamplesTableProcessor - .buildExamplesTableFromColumns(List.of(column), List.of(attrs), - tableProperties))); + return documentSuppler.get().selectXpath(xpathSelector) + .stream() + .map(getter) + .collect(collectingAndThen(toList(), attrs -> ExamplesTableProcessor + .buildExamplesTableFromColumns(List.of(column), List.of(attrs), + tableProperties))); } private Document createDocument(String pageUrl) diff --git a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformerTests.java b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformerTests.java index 57550bfe8b..a638b119c7 100644 --- a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformerTests.java +++ b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformerTests.java @@ -17,6 +17,7 @@ package org.vividus.crawler.transformer; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; @@ -34,14 +35,21 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.vividus.context.VariableContext; import org.vividus.crawler.transformer.HtmlDocumentTableTransformer.HttpConfiguration; +@ExtendWith(MockitoExtension.class) class HtmlDocumentTableTransformerTests { private static final String PAGE_URL = "https://example.com"; - private static final Document HTML_DOC = Jsoup.parse(""" + private static final String DOC = """ @@ -50,7 +58,41 @@ class HtmlDocumentTableTransformerTests B - """); + """; + private static final Document HTML_DOC = Jsoup.parse(DOC); + private static final String RELS = """ + |col| + |/r| + |/g| + |/b|"""; + + @Mock private VariableContext variableContext; + + @ParameterizedTest + @CsvSource(value = { + "pageUrl=https://example.com, variableName=html", + "column=col" + }, delimiterString = "none") + void shouldFailOnInvalidInputParameters(String parameters) + { + TableProperties tableProperties = new TableProperties(parameters, new Keywords(), new ParameterConverters()); + HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(Optional.empty(), variableContext); + IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, + () -> transformer.transform("", null, tableProperties)); + assertEquals("Either 'pageUrl' or 'variableName' should be specified.", thrown.getMessage()); + } + + @Test + void shouldBuildTableByElementAttributeFromVariableValue() throws IOException + { + when(variableContext.getVariable("html")).thenReturn(DOC); + TableProperties tableProperties = new TableProperties("column=col, variableName=html, xpathSelector=//a/@href", + new Keywords(), new ParameterConverters()); + HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(Optional.empty(), + variableContext); + String table = transformer.transform("", null, tableProperties); + assertEquals(RELS, table); + } @Test void shouldBuildTableByElementAttribute() throws IOException @@ -58,11 +100,7 @@ void shouldBuildTableByElementAttribute() throws IOException HttpConfiguration cfg = new HttpConfiguration(); Map headers = Map.of("credit-card-pin", "1234"); cfg.setHeaders(headers); - performTest(Optional.of(cfg), "//a/@href", t -> assertEquals(""" - |col| - |/r| - |/g| - |/b|""", t), con -> verify(con).headers(headers)); + performTest(Optional.of(cfg), "//a/@href", t -> assertEquals(RELS, t), con -> verify(con).headers(headers)); } @Test @@ -98,7 +136,8 @@ private void performTest(Optional httpConfiguration, String x TableProperties tableProperties = new TableProperties( "column=col, pageUrl=https://example.com, xpathSelector=%s".formatted(xpathSelector), new Keywords(), new ParameterConverters()); - HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(httpConfiguration); + HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(httpConfiguration, + variableContext); String table = transformer.transform("", null, tableProperties); tableConsumer.accept(table); verify(connection).get(); diff --git a/vividus-tests/src/main/resources/story/integration/TableTransformers.story b/vividus-tests/src/main/resources/story/integration/TableTransformers.story index 7b59afbece..ba696505ee 100644 --- a/vividus-tests/src/main/resources/story/integration/TableTransformers.story +++ b/vividus-tests/src/main/resources/story/integration/TableTransformers.story @@ -449,7 +449,7 @@ Then `${documentTable}` is equal to table: |Link to unexistent element| |Link with tooltip | -Scenario: Verify FROM_HTML transformer with HTML +Scenario: [Deprecated] Verify FROM_HTML transformer with page HTML When I initialize scenario variable `documentTable` with values: {transformer=FROM_HTML, column=col, pageUrl=$\{vividus-test-site-url\}/links.html, xpathSelector=//a} Then `${documentTable}` is equal to table: @@ -457,3 +457,14 @@ Then `${documentTable}` is equal to table: |Link to an element | |Link to unexistent element | |Link with tooltip| + +Scenario: Verify FROM_HTML transformer with variable HTML +When I execute HTTP GET request for resource with URL `${vividus-test-site-url}/links.html` +Given I initialize story variable `pageSource` with value `${response}` +When I initialize scenario variable `documentTable` with values: +{transformer=FROM_HTML, column=col, variableName=pageSource, xpathSelector=//a} +Then `${documentTable}` is equal to table: +|col | +|Link to an element | +|Link to unexistent element | +|Link with tooltip|