[plugin-web-app-to-rest-api] Add ability to use variable name as HTML…

docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc

@@ -250,7 +250,18 @@ Examples:
 |Description
 
 |`pageUrl`
-|The URL of the page to build the table upon.
+a|The URL of the page to build the table upon.
+
+WARNING: The `pageUrl` parameter is deprecated and will be removed in VIVIDUS 0.7.0, please use `variableName` instead.
+
+IMPORTANT: The `pageUrl` parameter can not be used together with the `variableName` parameter.
+
+|`variableName`
+a|The name of the variable containing source HTML, only variables of scopes `global` and `next_batches` are allowed.
+Exceptions are cases when the transformer using in xref:commons:vividus-steps.adoc#_initialize_variable_with_a_table[step]
+which initializes a variable with a table.
+
+IMPORTANT: The `variableName` parameter can not be used together with the `pageUrl` parameter.
 
 |`column`
 |The column name in the generated table.

vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformer.java

@@ -26,32 +26,60 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Properties;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
+import org.apache.commons.lang3.Validate;
 import org.jbehave.core.model.ExamplesTable.TableProperties;
 import org.jbehave.core.model.TableParsers;
 import org.jbehave.core.model.TableTransformers.TableTransformer;
 import org.jsoup.Connection;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.vividus.context.VariableContext;
 import org.vividus.util.ExamplesTableProcessor;
 
 public class HtmlDocumentTableTransformer implements TableTransformer
 {
+    private static final Logger LOGGER = LoggerFactory.getLogger(HtmlDocumentTableTransformer.class);
+
     private static final String SLASH = "/";
 
     private final Optional<HttpConfiguration> httpConfiguration;
+    private final VariableContext variableContext;
 
-    public HtmlDocumentTableTransformer(Optional<HttpConfiguration> httpConfiguration)
+    public HtmlDocumentTableTransformer(Optional<HttpConfiguration> httpConfiguration, VariableContext variableContext)
     {
         this.httpConfiguration = httpConfiguration;
+        this.variableContext = variableContext;
     }
 
     @Override
     public String transform(String table, TableParsers parsers, TableProperties tableProperties)
     {
-        String pageUrl = tableProperties.getMandatoryNonBlankProperty("pageUrl", String.class);
+        Properties propertes = tableProperties.getProperties();
+        String pageUrl = propertes.getProperty("pageUrl");
+        String variableName = propertes.getProperty("variableName");
+
+        Validate.isTrue(pageUrl != null && variableName == null || pageUrl == null && variableName != null,
+                "Either 'pageUrl' or 'variableName' should be specified.");
+
+        Supplier<Document> documentSuppler;
+        if (pageUrl == null)
+        {
+            documentSuppler = () -> Jsoup.parse((String) variableContext.getVariable(variableName));
+        }
+        else
+        {
+            LOGGER.atWarn().log("The 'pageUrl' transformer parameter is deprecated and will be removed VIVIDUS 0.7.0, "
+                    + "please use 'variableName' parameter instead.");
+            documentSuppler = () -> createDocument(pageUrl);
+        }
+
         String column = tableProperties.getMandatoryNonBlankProperty("column", String.class);
 
         String xpathSelector = tableProperties.getMandatoryNonBlankProperty("xpathSelector", String.class);
@@ -74,12 +102,12 @@ else if (lastSegment.startsWith("@"))
             getter = Element::outerHtml;
         }
 
-        return createDocument(pageUrl).selectXpath(xpathSelector)
-                                      .stream()
-                                      .map(getter)
-                                      .collect(collectingAndThen(toList(), attrs -> ExamplesTableProcessor
-                                              .buildExamplesTableFromColumns(List.of(column), List.of(attrs),
-                                                      tableProperties)));
+        return documentSuppler.get().selectXpath(xpathSelector)
+                                    .stream()
+                                    .map(getter)
+                                    .collect(collectingAndThen(toList(), attrs -> ExamplesTableProcessor
+                                            .buildExamplesTableFromColumns(List.of(column), List.of(attrs),
+                                                    tableProperties)));
     }
 
     private Document createDocument(String pageUrl)

vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformerTests.java

@@ -17,6 +17,7 @@
 package org.vividus.crawler.transformer;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
 import static org.mockito.Mockito.verify;
@@ -34,14 +35,21 @@
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.mockito.Mock;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.vividus.context.VariableContext;
 import org.vividus.crawler.transformer.HtmlDocumentTableTransformer.HttpConfiguration;
 
+@ExtendWith(MockitoExtension.class)
 class HtmlDocumentTableTransformerTests
 {
     private static final String PAGE_URL = "https://example.com";
-    private static final Document HTML_DOC = Jsoup.parse("""
+    private static final String DOC = """
         <!DOCTYPE html>
         <html>
         <body>
@@ -50,19 +58,49 @@ class HtmlDocumentTableTransformerTests
         <a href="/b">B</a>
         </body>
         </html>
-        """);
+        """;
+    private static final Document HTML_DOC = Jsoup.parse(DOC);
+    private static final String RELS = """
+        |col|
+        |/r|
+        |/g|
+        |/b|""";
+
+    @Mock private VariableContext variableContext;
+
+    @ParameterizedTest
+    @CsvSource(value = {
+        "pageUrl=https://example.com, variableName=html",
+        "column=col"
+    }, delimiterString = "none")
+    void shouldFailOnInvalidInputParameters(String parameters)
+    {
+        TableProperties tableProperties = new TableProperties(parameters, new Keywords(), new ParameterConverters());
+        HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(Optional.empty(), variableContext);
+        IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class,
+                () -> transformer.transform("", null, tableProperties));
+        assertEquals("Either 'pageUrl' or 'variableName' should be specified.", thrown.getMessage());
+    }
+
+    @Test
+    void shouldBuildTableByElementAttributeFromVariableValue() throws IOException
+    {
+        when(variableContext.getVariable("html")).thenReturn(DOC);
+        TableProperties tableProperties = new TableProperties("column=col, variableName=html, xpathSelector=//a/@href",
+                new Keywords(), new ParameterConverters());
+        HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(Optional.empty(),
+                variableContext);
+        String table = transformer.transform("", null, tableProperties);
+        assertEquals(RELS, table);
+    }
 
     @Test
     void shouldBuildTableByElementAttribute() throws IOException
     {
         HttpConfiguration cfg = new HttpConfiguration();
         Map<String, String> headers = Map.of("credit-card-pin", "1234");
         cfg.setHeaders(headers);
-        performTest(Optional.of(cfg), "//a/@href", t -> assertEquals("""
-                |col|
-                |/r|
-                |/g|
-                |/b|""", t), con -> verify(con).headers(headers));
+        performTest(Optional.of(cfg), "//a/@href", t -> assertEquals(RELS, t), con -> verify(con).headers(headers));
     }
 
     @Test
@@ -98,7 +136,8 @@ private void performTest(Optional<HttpConfiguration> httpConfiguration, String x
             TableProperties tableProperties = new TableProperties(
                     "column=col, pageUrl=https://example.com, xpathSelector=%s".formatted(xpathSelector),
                     new Keywords(), new ParameterConverters());
-            HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(httpConfiguration);
+            HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(httpConfiguration,
+                    variableContext);
             String table = transformer.transform("", null, tableProperties);
             tableConsumer.accept(table);
             verify(connection).get();

vividus-tests/src/main/resources/story/integration/TableTransformers.story

@@ -449,11 +449,22 @@ Then `${documentTable}` is equal to table:
 |Link to unexistent element|
 |Link with tooltip         |
 
-Scenario: Verify FROM_HTML transformer with HTML
+Scenario: [Deprecated] Verify FROM_HTML transformer with page HTML
 When I initialize scenario variable `documentTable` with values:
 {transformer=FROM_HTML, column=col, pageUrl=$\{vividus-test-site-url\}/links.html, xpathSelector=//a}
 Then `${documentTable}` is equal to table:
 |col                                                                              |
 |<a href="#ElementId">Link to an element</a>                                      |
 |<a href="#notFound">Link to unexistent element</a>                               |
 |<a href="#" title="Link title" onclick="onLinkClick(event)">Link with tooltip</a>|
+
+Scenario: Verify FROM_HTML transformer with variable HTML
+When I execute HTTP GET request for resource with URL `${vividus-test-site-url}/links.html`
+Given I initialize story variable `pageSource` with value `${response}`
+When I initialize scenario variable `documentTable` with values:
+{transformer=FROM_HTML, column=col, variableName=pageSource, xpathSelector=//a}
+Then `${documentTable}` is equal to table:
+|col                                                                              |
+|<a href="#ElementId">Link to an element</a>                                      |
+|<a href="#notFound">Link to unexistent element</a>                               |
+|<a href="#" title="Link title" onclick="onLinkClick(event)">Link with tooltip</a>|