scraper: Fix TmwVRnet scraper not picking up subsequent pages (xbapps…

…#1510)
vt-idiot · Nov 24, 2023 · f234eb3 · f234eb3
1 parent e6f38b6
commit f234eb3
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/pkg/scrape/tmwvrnet.go b/pkg/scrape/tmwvrnet.go
@@ -99,9 +99,11 @@ func TmwVRnet(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha
 		out <- sc
 	})
 
-	siteCollector.OnHTML(`div.pagination__element.next a`, func(e *colly.HTMLElement) {
-		pageURL := e.Request.AbsoluteURL(e.Attr("href"))
-		siteCollector.Visit(pageURL)
+	siteCollector.OnHTML(`a.pagination-element__link`, func(e *colly.HTMLElement) {
+		if strings.Contains(e.Text, "Next") {
+			pageURL := e.Request.AbsoluteURL(e.Attr("href"))
+			siteCollector.Visit(pageURL)
+		}
 	})
 
 	siteCollector.OnHTML(`div.thumb-photo`, func(e *colly.HTMLElement) {