chore(website): fix crawl establish domain removal [#233]

spider-rs · Dec 1, 2024 · f184914 · f184914
1 parent f8a9fca
commit f184914
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 33 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.13.88"
+version = "2.13.90"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider/src/page.rs b/spider/src/page.rs
@@ -223,11 +223,11 @@ pub fn push_link<A: PartialEq + Eq + std::hash::Hash + From<String>>(
         let mut abs = convert_abs_path(b, href);
         let scheme = abs.scheme();
 
-        if scheme == "https" || scheme == "http" {
-            if let Some(link_map) = links_pages {
-                link_map.insert(A::from(abs.as_str().to_string()));
-            }
+        if let Some(link_map) = links_pages {
+            link_map.insert(A::from(abs.as_str().to_string()));
+        }
 
+        if scheme == "https" || scheme == "http" {
             let host_name = abs.host_str();
             let mut can_process = parent_host_match(
                 host_name,
@@ -570,21 +570,38 @@ impl Page {
                     res.content_length().unwrap_or(DEFAULT_BYTE_CAPACITY) as usize,
                 );
 
-                if url != res.url().as_str() {
-                    let domain = res.url().as_str();
-                    let mut url = Box::new(CaseInsensitiveString::new(&url));
+                let target_url = res.url().as_str();
 
-                    modify_selectors(
-                        prior_domain,
-                        domain,
-                        domain_parsed,
-                        &mut url,
-                        selectors,
-                        AllowedDomainTypes::new(r_settings.subdomains, r_settings.tld),
-                    );
+                // handle redirects
+                if url != target_url {
+                    let mut url = Box::new(CaseInsensitiveString::new(&url));
+                    let end_target_slash = target_url.ends_with("/");
+                    let main_slash = url.ends_with("/");
+
+                    let exact_match = end_target_slash
+                        && !main_slash
+                        && target_url[..target_url.len() - 1] == *url
+                        || !end_target_slash && main_slash && url[..url.len() - 1] == *target_url;
+
+                    if !exact_match {
+                        modify_selectors(
+                            prior_domain,
+                            target_url,
+                            domain_parsed,
+                            &mut url,
+                            selectors,
+                            AllowedDomainTypes::new(r_settings.subdomains, r_settings.tld),
+                        );
+                    }
                 };
 
-                let base = domain_parsed.as_deref();
+                // always use a base url.
+                let base = if domain_parsed.is_none() {
+                    prior_domain
+                } else {
+                    domain_parsed
+                }
+                .as_deref();
 
                 let parent_host = &selectors.1[0];
                 // the host schemes

diff --git a/spider/src/website.rs b/spider/src/website.rs
@@ -6,7 +6,7 @@ use crate::configuration::{
 use crate::features::chrome_common::RequestInterceptConfiguration;
 use crate::packages::robotparser::parser::RobotFileParser;
 use crate::page::{Page, PageLinkBuildSettings};
-use crate::utils::abs::parse_absolute_url;
+use crate::utils::abs::{convert_abs_url, parse_absolute_url};
 use crate::utils::{
     emit_log, emit_log_shutdown, setup_website_selectors, spawn_set, spawn_task, AllowedDomainTypes,
 };
@@ -1288,12 +1288,19 @@ impl Website {
                 &page_links_settings,
                 &mut links,
                 Some(&mut links_ssg),
-                &mut domain_parsed,
+                &mut domain_parsed, // original domain
                 &mut self.domain_parsed,
                 &mut links_pages,
             )
             .await;
 
+            if self.domain_parsed.is_none() {
+                if let Some(mut domain_parsed) = domain_parsed.take() {
+                    convert_abs_url(&mut domain_parsed);
+                    self.domain_parsed.replace(domain_parsed);
+                }
+            }
+
             let mut retry_count = self.configuration.retry;
             let domains_caseless = &self.configuration.external_domains_caseless;
 
@@ -1366,9 +1373,6 @@ impl Website {
 
             if self.configuration.return_page_links {
                 page.page_links = links_pages.filter(|pages| !pages.is_empty()).map(Box::new);
-                if let Some(page_links) = page.page_links.as_mut() {
-                    page_links.extend(links_ssg.clone());
-                }
             }
 
             links.extend(links_ssg);

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.13.88"
+version = "2.13.90"
 rust-version = "1.70"
 authors = [
     "j-mendez <jeff@spider.cloud>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.13.88"
+version = "2.13.90"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.13.88"
+version = "2.13.90"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.13.88"
+version = "2.13.90"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.13.88"
+version = "2.13.90"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]