From e9b0a634767831e56fa006ed67aaf4a668eefc04 Mon Sep 17 00:00:00 2001 From: Uladzislau Arlouski Date: Thu, 11 Jan 2024 19:50:38 +0200 Subject: [PATCH] Extends headless and sitemap transformers to pass app URL through transformer parameter --- .../pages/plugin-web-app-to-rest-api.adoc | 40 ++++++++++++--- .../AbstractFetchingUrlsTableTransformer.java | 30 ++++++++++-- .../HeadlessCrawlerTableTransformer.java | 44 ++++++++++------- .../transformer/SiteMapTableTransformer.java | 4 +- .../properties/defaults/default.properties | 2 + .../main/resources/vividus-plugin/spring.xml | 2 + .../FetchingUrlsTableTransformerTests.java | 49 ++++++++++++++++++- .../HeadlessCrawlerTableTransformerTests.java | 17 +++++-- .../SiteMapTableTransformerTests.java | 11 +++-- .../story/integration/TableTransformers.story | 7 +++ 10 files changed, 165 insertions(+), 41 deletions(-) diff --git a/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc b/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc index 1eb02a6f1c..09849f2510 100644 --- a/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc +++ b/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc @@ -13,11 +13,20 @@ include::partial$plugin-installation.adoc[] `FROM_SITEMAP` transformer generates table based on the website sitemap. +IMPORTANT: The use of `web-application.main-page-url` property for setting of main page for crawling is deprecated and will be removed in VIVIDUS 0.7.0, pelase see either `mainPageUrl` transformer parameter or `transformer.from-sitemap.main-page-url` property. + [cols="1,3", options="header"] + |=== + |Parameter |Description +|`mainPageUrl` +a|main application page URL, used as initial seed URL that is fetched by the crawler to extract new URLs in it and follow them for crawling. + +IMPORTANT: The main page url value defined by this parameter overrides the value defined by the `transformer.from-sitemap.main-page-url` property. + |`siteMapRelativeUrl` |relative URL of `sitemap.xml` @@ -26,15 +35,25 @@ include::partial$plugin-installation.adoc[] |`column` |the column name in the generated table + |=== [cols="3,1,1,3", options="header"] + |=== + |Property Name |Acceptable values |Default |Description +|`transformer.from-sitemap.main-page-url` +|`URL` +| +a|main application page URL, used as initial seed URL that is fetched by the crawler to extract new URLs in it and follow them for crawling. + +IMPORTANT: The main page url value defined by this property gets overriden by the value defined in `mainPageUrl` transformer parameter. + |`transformer.from-sitemap.ignore-errors` a|`true` `false` @@ -46,9 +65,8 @@ a|`true` `false` |`false` |defines whether urls that has redirect to the one that has already been included in the table are excluded from the table + |=== -==== Required properties -* `web-application.main-page-url` - defines main application page URL .Usage example ---- @@ -60,12 +78,19 @@ Examples: `FROM_HEADLESS_CRAWLING` transformer generates table based on the results of headless crawling. +IMPORTANT: The use of `web-application.main-page-url` property for setting of main page for crawling is deprecated and will be removed in VIVIDUS 0.7.0, pelase see either `mainPageUrl` transformer parameter or `transformer.from-headless-crawling.main-page-url` property. + [cols="1,3", options="header"] |=== |Parameter Name |Description +|`mainPageUrl` +a|main application page URL, used as initial seed URL that is fetched by the crawler to extract new URLs in it and follow them for crawling. + +IMPORTANT: The main page url value defined by this parameter overrides the value defined by the `transformer.from-headless-crawling.main-page-url` property. + |`column` |The column name in the generated table. @@ -81,6 +106,13 @@ Examples: 4+^.^|_General_ +|`transformer.from-headless-crawling.main-page-url` +|`URL` +| +a|main application page URL, used as initial seed URL that is fetched by the crawler to extract new URLs in it and follow them for crawling. + +IMPORTANT: The main page url value defined by this property gets overriden by the value defined in `mainPageUrl` transformer parameter. + |`transformer.from-headless-crawling.seed-relative-urls` |Comma-separated list of values | @@ -229,10 +261,6 @@ transformer.from-headless-crawling.http.headers.x-vercel-protection-bypass=1fac2 |=== -==== Required properties - -* `{main-page-url}` - defines main application page URL - .Usage example ---- Examples: diff --git a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/AbstractFetchingUrlsTableTransformer.java b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/AbstractFetchingUrlsTableTransformer.java index 1d15ff37a0..1fd12b0c52 100644 --- a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/AbstractFetchingUrlsTableTransformer.java +++ b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/AbstractFetchingUrlsTableTransformer.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ import org.vividus.transformer.ExtendedTableTransformer; import org.vividus.ui.web.configuration.WebApplicationConfiguration; import org.vividus.util.ExamplesTableProcessor; +import org.vividus.util.UriUtils; public abstract class AbstractFetchingUrlsTableTransformer implements ExtendedTableTransformer { @@ -43,6 +44,7 @@ public abstract class AbstractFetchingUrlsTableTransformer implements ExtendedTa private WebApplicationConfiguration webApplicationConfiguration; private HttpRedirectsProvider httpRedirectsProvider; private boolean filterRedirects; + private URI mainPageUrl; @Override public String transform(String tableAsString, TableParsers tableParsers, TableProperties properties) @@ -101,6 +103,24 @@ private String build(Set urls, TableProperties properties) .buildExamplesTableFromColumns(List.of(columnName), List.of(urlsList), properties); } + protected URI getMainApplicationPageUri(TableProperties properties) + { + URI uri = Optional.ofNullable(properties.getProperties().getProperty("mainPageUrl")) + .map(UriUtils::createUri) + .orElse(mainPageUrl); + + if (uri == null) + { + uri = webApplicationConfiguration.getMainApplicationPageUrl(); + logger.atWarn().addArgument("web-application.main-page-url") + .log("The use of {} property for setting of main page for crawling is deprecated and will " + + "be removed in VIVIDUS 0.7.0, pelase see official documentation for corresponding" + + " replacements."); + } + + return uri; + } + public void setWebApplicationConfiguration(WebApplicationConfiguration webApplicationConfiguration) { this.webApplicationConfiguration = webApplicationConfiguration; @@ -111,13 +131,13 @@ public void setHttpRedirectsProvider(HttpRedirectsProvider httpRedirectsProvider this.httpRedirectsProvider = httpRedirectsProvider; } - protected URI getMainApplicationPageUri() + public void setFilterRedirects(boolean filterRedirects) { - return webApplicationConfiguration.getMainApplicationPageUrl(); + this.filterRedirects = filterRedirects; } - public void setFilterRedirects(boolean filterRedirects) + public void setMainPageUrl(URI mainPageUrl) { - this.filterRedirects = filterRedirects; + this.mainPageUrl = mainPageUrl; } } diff --git a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformer.java b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformer.java index 15fd827427..858b25c959 100644 --- a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformer.java +++ b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformer.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,10 @@ import java.net.URI; import java.util.Set; -import java.util.function.Supplier; -import com.google.common.base.Suppliers; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.apache.commons.lang3.StringUtils; import org.jbehave.core.model.ExamplesTable.TableProperties; @@ -52,23 +53,22 @@ public class HeadlessCrawlerTableTransformer extends AbstractFetchingUrlsTableTr private String excludeUrlsRegex; private String excludeExtensionsRegex; - private final Supplier> urlsProvider = Suppliers.memoize(() -> - { - if (!StringUtils.isEmpty(excludeExtensionsRegex)) - { - LOGGER.warn(DEPRECATION_MESSAGE); - } - - URI mainApplicationPage = getMainApplicationPageUri(); - CrawlController controller = crawlControllerFactory.createCrawlController(mainApplicationPage); + private final LoadingCache> crawledUrlsCache = CacheBuilder.newBuilder() + .build(new CacheLoader<>() + { + @Override + public Set load(URI mainApplicationPage) + { + CrawlController controller = crawlControllerFactory.createCrawlController(mainApplicationPage); - addSeeds(mainApplicationPage, controller); + addSeeds(mainApplicationPage, controller); - LinkCrawlerData linkCrawlerData = new LinkCrawlerData(); - controller.start(new LinkCrawlerFactory(linkCrawlerData, excludeUrlsRegex), NUMBER_OF_CRAWLERS); - Set absoluteUrls = linkCrawlerData.getAbsoluteUrls(); - return filterResults(absoluteUrls.stream()); - }); + LinkCrawlerData linkCrawlerData = new LinkCrawlerData(); + controller.start(new LinkCrawlerFactory(linkCrawlerData, excludeUrlsRegex), NUMBER_OF_CRAWLERS); + Set absoluteUrls = linkCrawlerData.getAbsoluteUrls(); + return filterResults(absoluteUrls.stream()); + } + }); private void addSeeds(URI mainApplicationPage, CrawlController controller) { @@ -102,7 +102,13 @@ private void addSeed(CrawlController controller, String pageUrl) @Override protected Set fetchUrls(TableProperties properties) { - return urlsProvider.get(); + if (!StringUtils.isEmpty(excludeExtensionsRegex)) + { + LOGGER.warn(DEPRECATION_MESSAGE); + } + + URI mainApplicationPage = getMainApplicationPageUri(properties); + return crawledUrlsCache.getUnchecked(mainApplicationPage); } public void setCrawlControllerFactory(ICrawlControllerFactory crawlControllerFactory) diff --git a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/SiteMapTableTransformer.java b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/SiteMapTableTransformer.java index 7692993ddf..39c2968f67 100644 --- a/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/SiteMapTableTransformer.java +++ b/vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/SiteMapTableTransformer.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ public Set fetchUrls(TableProperties properties) .orElse(ignoreErrors); String siteMapRelativeUrl = properties.getMandatoryNonBlankProperty("siteMapRelativeUrl", String.class); - URI mainApplicationPage = getMainApplicationPageUri(); + URI mainApplicationPage = getMainApplicationPageUri(properties); Set urls = siteMaps.stream().filter( s -> s.mainAppPage().equals(mainApplicationPage) && s.siteMapRelativeUrl().equals(siteMapRelativeUrl)) .findFirst().orElseGet(() -> diff --git a/vividus-plugin-web-app-to-rest-api/src/main/resources/properties/defaults/default.properties b/vividus-plugin-web-app-to-rest-api/src/main/resources/properties/defaults/default.properties index 00de9983d8..c1b72460ad 100644 --- a/vividus-plugin-web-app-to-rest-api/src/main/resources/properties/defaults/default.properties +++ b/vividus-plugin-web-app-to-rest-api/src/main/resources/properties/defaults/default.properties @@ -4,9 +4,11 @@ sitemap.parser.site-url= sitemap.parser.base-url= sitemap.parser.follow-redirects=true +transformer.from-sitemap.main-page-url= transformer.from-sitemap.ignore-errors=false transformer.from-sitemap.filter-redirects=false +transformer.from-headless-crawling.main-page-url= transformer.from-headless-crawling.filter-redirects=false transformer.from-headless-crawling.seed-relative-urls= transformer.from-headless-crawling.exclude-extensions-regex= diff --git a/vividus-plugin-web-app-to-rest-api/src/main/resources/vividus-plugin/spring.xml b/vividus-plugin-web-app-to-rest-api/src/main/resources/vividus-plugin/spring.xml index e4e7cb589e..37c73c3054 100644 --- a/vividus-plugin-web-app-to-rest-api/src/main/resources/vividus-plugin/spring.xml +++ b/vividus-plugin-web-app-to-rest-api/src/main/resources/vividus-plugin/spring.xml @@ -73,6 +73,7 @@ + + diff --git a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/FetchingUrlsTableTransformerTests.java b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/FetchingUrlsTableTransformerTests.java index 59e27a8acb..ec8c2c0945 100644 --- a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/FetchingUrlsTableTransformerTests.java +++ b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/FetchingUrlsTableTransformerTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,11 +18,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import java.net.URI; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.commons.lang3.StringUtils; import org.jbehave.core.configuration.Keywords; import org.jbehave.core.model.ExamplesTable.TableProperties; import org.jbehave.core.steps.ParameterConverters; @@ -31,9 +37,11 @@ import org.junit.jupiter.params.provider.CsvSource; import org.vividus.ui.web.configuration.AuthenticationMode; import org.vividus.ui.web.configuration.WebApplicationConfiguration; +import org.vividus.util.UriUtils; class FetchingUrlsTableTransformerTests { + private static final URI PAGE_URI = UriUtils.createUri("https://example.com"); private static final String URLS = "urls"; private static final String COLUMN = "column"; private final TestTransformer transformer = new TestTransformer(); @@ -70,11 +78,48 @@ void shouldHandleInvalidInputs(String propertiesAsString, String tableAsString, @Test void testTransformWithoutMainApplicationPageUrl() { + TableProperties props = new TableProperties(StringUtils.EMPTY, new Keywords(), new ParameterConverters()); transformer.setWebApplicationConfiguration(new WebApplicationConfiguration(null, AuthenticationMode.URL)); - var exception = assertThrows(IllegalArgumentException.class, transformer::getMainApplicationPageUri); + var exception = assertThrows(IllegalArgumentException.class, + () -> transformer.getMainApplicationPageUri(props)); assertEquals("URL of the main application page should be non-blank", exception.getMessage()); } + @Test + void shouldReturnMainAppUrlFromTransformerParameter() + { + WebApplicationConfiguration webAppCfg = mock(); + transformer.setWebApplicationConfiguration(webAppCfg); + transformer.setMainPageUrl(null); + TableProperties props = new TableProperties("mainPageUrl=%s".formatted(PAGE_URI), new Keywords(), + new ParameterConverters()); + assertEquals(PAGE_URI, transformer.getMainApplicationPageUri(props)); + verifyNoInteractions(webAppCfg); + } + + @Test + void shouldReturnMainAppUrlFromMainPageUrlProperty() + { + WebApplicationConfiguration webAppCfg = mock(); + transformer.setWebApplicationConfiguration(webAppCfg); + transformer.setMainPageUrl(PAGE_URI); + TableProperties props = new TableProperties(StringUtils.EMPTY, new Keywords(), new ParameterConverters()); + assertEquals(PAGE_URI, transformer.getMainApplicationPageUri(props)); + verifyNoInteractions(webAppCfg); + } + + @Test + void shouldReturnMainAppUrlFromWebConfiguration() + { + WebApplicationConfiguration webAppCfg = mock(); + when(webAppCfg.getMainApplicationPageUrl()).thenReturn(PAGE_URI); + transformer.setWebApplicationConfiguration(webAppCfg); + transformer.setMainPageUrl(null); + TableProperties props = new TableProperties(StringUtils.EMPTY, new Keywords(), new ParameterConverters()); + assertEquals(PAGE_URI, transformer.getMainApplicationPageUri(props)); + verify(webAppCfg).getMainApplicationPageUrl(); + } + private static final class TestTransformer extends AbstractFetchingUrlsTableTransformer { @Override diff --git a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java index 89fee314c5..7db5652e69 100644 --- a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java +++ b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ import java.util.Set; import java.util.stream.Stream; +import com.github.valfirst.slf4jtest.LoggingEvent; import com.github.valfirst.slf4jtest.TestLogger; import com.github.valfirst.slf4jtest.TestLoggerFactory; import com.github.valfirst.slf4jtest.TestLoggerFactoryExtension; @@ -124,7 +125,7 @@ void testFetchUrlsSuccessfully(String mainAppPageRelativeUrl, Set seedRe Set urls = testFetchUrls(mainAppPageRelativeUrl, expectedSeedRelativeUrls); assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); verifyNoInteractions(redirectsProvider); - assertThat(logger.getLoggingEvents(), is(List.of())); + assertThat(logger.getLoggingEvents(), is(List.of(getMainAppPageWarn()))); } @Test @@ -135,7 +136,8 @@ void testFetchUrlsWithDeprecatedExcludeExtensionsRegexProperty() throws IOExcept assertThat(logger.getLoggingEvents(), is(List.of(warn( "Property `transformer.from-headless-crawling.exclude-extensions-regex` is deprecated and will " + "be removed in VIVIDUS 0.7.0. " - + "Please use `transformer.from-headless-crawling.exclude-urls-regex` instead.")))); + + "Please use `transformer.from-headless-crawling.exclude-urls-regex` instead."), + getMainAppPageWarn()))); } @Test @@ -159,7 +161,7 @@ void shouldTreatInvalidStatusCodeAsNoRedirects() throws IOException, Interrupted when(redirectsProvider.getRedirects(outgoingURI)).thenThrow(httpResponseException); Set urls = testFetchUrls(ROOT, List.of(PATH2, SLASH_PATH3)); assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); - assertThat(logger.getLoggingEvents(), is(List.of(warn(httpResponseException, + assertThat(logger.getLoggingEvents(), is(List.of(getMainAppPageWarn(), warn(httpResponseException, "Exception during redirects receiving")))); } @@ -288,6 +290,13 @@ private CrawlController mockCrawlerControllerFactory(String mainAppPage) return crawlController; } + private LoggingEvent getMainAppPageWarn() + { + return warn("The use of {} property for setting of main page for crawling is deprecated and will be removed" + + " in VIVIDUS 0.7.0, pelase see official documentation for corresponding replacements.", + "web-application.main-page-url"); + } + private TableProperties buildTableProperties() { return new TableProperties("", keywords, parameterConverters); diff --git a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/SiteMapTableTransformerTests.java b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/SiteMapTableTransformerTests.java index e4fce81374..f680b01610 100644 --- a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/SiteMapTableTransformerTests.java +++ b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/SiteMapTableTransformerTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ import com.github.valfirst.slf4jtest.TestLogger; import com.github.valfirst.slf4jtest.TestLoggerFactory; +import com.github.valfirst.slf4jtest.TestLoggerFactoryExtension; import org.apache.hc.client5.http.HttpResponseException; import org.apache.hc.core5.http.HttpStatus; @@ -51,7 +52,7 @@ import crawlercommons.sitemaps.SiteMapURL; -@ExtendWith(MockitoExtension.class) +@ExtendWith({ MockitoExtension.class, TestLoggerFactoryExtension.class }) class SiteMapTableTransformerTests { private static final String SOME_URL = "http://www.some.url"; @@ -181,7 +182,11 @@ void testThrowHttpResponseException() throws SiteMapParseException, IOException var actual = siteMapTableTransformer.fetchUrls(createTableProperties()); assertEquals(Set.of(OUTGOING_ABSOLUT_URL), actual); assertThat(logger.getLoggingEvents(), - is(List.of(warn(httpResponseException, "Exception during redirects receiving")))); + is(List.of( + warn("The use of {} property for setting of main page for crawling is deprecated and will be " + + "removed in VIVIDUS 0.7.0, pelase see official documentation for corresponding " + + "replacements.", "web-application.main-page-url"), + warn(httpResponseException, "Exception during redirects receiving")))); } @Test diff --git a/vividus-tests/src/main/resources/story/integration/TableTransformers.story b/vividus-tests/src/main/resources/story/integration/TableTransformers.story index ba696505ee..07c4a12e27 100644 --- a/vividus-tests/src/main/resources/story/integration/TableTransformers.story +++ b/vividus-tests/src/main/resources/story/integration/TableTransformers.story @@ -276,6 +276,13 @@ Then `` matches `.*links.*` Examples: {transformer=FROM_HEADLESS_CRAWLING, column=relativeUrl} + +Scenario: Verify FROM_HEADLESS_CRAWLING transformer passing URL through transformer parameter +Then `` matches `.*links.*` +Examples: +{transformer=FROM_HEADLESS_CRAWLING, column=relativeUrl, mainPageUrl=$\{vividus-test-site-url\}} + + Scenario: Verify FROM_SITEMAP transformer When I initialize scenario variable `sitemapTransformerTable` with values: {transformer=FROM_SITEMAP, siteMapRelativeUrl=/sitemap.xml, column=sitemapUrl}