chore(page): fix subdomain entry point handling root (#146)

examples/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_examples"
-version = "1.46.4"
+version = "1.46.5"
 authors = ["madeindjs <contact@rousseau-alexandre.fr>", "j-mendez <jeff@a11ywatch.com>"]
 description = "Multithreaded web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -22,7 +22,7 @@ htr = "0.5.27"
 flexbuffers = "2.0.0"
 
 [dependencies.spider]
-version = "1.46.4"
+version = "1.46.5"
 path = "../spider"
 features = ["serde"]
 
@@ -61,7 +61,7 @@ path = "callback.rs"
 [[example]]
 name = "sitemap"
 path = "sitemap.rs"
-features = ["sitemap"]
+required-features = ["spider/sitemap"]
 
 [[example]]
 name = "configuration"
@@ -70,6 +70,4 @@ path = "configuration.rs"
 [[example]]
 name = "budget"
 path = "budget.rs"
-
-[features]
-default = ["spider/budget", "spider/sync"]
+required-features = ["spider/budget", "spider/sync"]

examples/download.rs

@@ -30,7 +30,7 @@ async fn main() {
     website.scrape().await;
 
     for page in website.get_pages().unwrap().iter() {
-        let download_file = page.get_url().clone();
+        let download_file = page.get_url();
         let download_file = download_file.replace(website_name, "");
         let download_file = download_file.replace(".", "-");
         let download_file = download_file.replace("/", "-");

examples/download_to_react.rs

@@ -39,7 +39,7 @@ async fn main() {
     match website.get_pages() {
         Some(pages) => {
             for page in pages.iter() {
-                let download_file = page.get_url().clone();
+                let download_file = page.get_url();
                 let download_file = download_file.replace(website_name, "");
                 let download_file = download_file.replace(".", "-");
                 let download_file = download_file.replace("/", "-");

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "1.46.4"
+version = "1.46.5"
 authors = ["madeindjs <contact@rousseau-alexandre.fr>", "j-mendez <jeff@a11ywatch.com>"]
 description = "The fastest web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"

spider/README.md

@@ -16,7 +16,7 @@ This is a basic async example crawling a web page, add spider to your `Cargo.tom
 
 ```toml
 [dependencies]
-spider = "1.46.4"
+spider = "1.46.5"
 ```
 
 And then the code:
@@ -87,7 +87,7 @@ We have a couple optional feature flags. Regex blacklisting, jemaloc backend, gl
 
 ```toml
 [dependencies]
-spider = { version = "1.46.4", features = ["regex", "ua_generator"] }
+spider = { version = "1.46.5", features = ["regex", "ua_generator"] }
 ```
 
 1. `ua_generator`: Enables auto generating a random real User-Agent.
@@ -116,7 +116,7 @@ Move processing to a worker, drastically increases performance even if worker is
 
 ```toml
 [dependencies]
-spider = { version = "1.46.4", features = ["decentralized"] }
+spider = { version = "1.46.5", features = ["decentralized"] }
 ```
 
 ```sh
@@ -136,7 +136,7 @@ Use the subscribe method to get a broadcast channel.
 
 ```toml
 [dependencies]
-spider = { version = "1.46.4", features = ["sync"] }
+spider = { version = "1.46.5", features = ["sync"] }
 ```
 
 ```rust,no_run
@@ -166,7 +166,7 @@ Allow regex for blacklisting routes
 
 ```toml
 [dependencies]
-spider = { version = "1.46.4", features = ["regex"] }
+spider = { version = "1.46.5", features = ["regex"] }
 ```
 
 ```rust,no_run
@@ -193,7 +193,7 @@ If you are performing large workloads you may need to control the crawler by ena
 
 ```toml
 [dependencies]
-spider = { version = "1.46.4", features = ["control"] }
+spider = { version = "1.46.5", features = ["control"] }
 ```
 
 ```rust

spider/src/page.rs

@@ -312,7 +312,13 @@ impl Page {
                             let mut abs = self.abs_path(href);
                             let host_name = abs.host_str();
                             let mut can_process = match host_name {
-                                Some(host) => parent_host.ends_with(host),
+                                Some(host) => {
+                                    if base_domain.is_empty() {
+                                        parent_host.eq(&host)
+                                    } else {
+                                        parent_host.ends_with(host)
+                                    }
+                                }
                                 _ => false,
                             };
                             if !can_process
@@ -409,7 +415,13 @@ impl Page {
                 let mut abs = self.abs_path(href.inner());
                 let host_name = abs.host_str();
                 let mut can_process = match host_name {
-                    Some(host) => parent_host.ends_with(host),
+                    Some(host) => {
+                        if base_domain.is_empty() {
+                            parent_host.eq(&host)
+                        } else {
+                            parent_host.ends_with(host)
+                        }
+                    }
                     _ => false,
                 };
                 if !can_process && host_name.is_some() && !self.external_domains_caseless.is_empty()
@@ -494,7 +506,13 @@ impl Page {
 
                                 // determine if the crawl can continue based on host match
                                 let mut can_process = match abs.host_str() {
-                                    Some(host) => parent_host.ends_with(host),
+                                    Some(host) => {
+                                        if base_domain.is_empty() {
+                                            parent_host.eq(&host)
+                                        } else {
+                                            parent_host.ends_with(host)
+                                        }
+                                    }
                                     _ => false,
                                 };
 
@@ -567,7 +585,13 @@ impl Page {
                         let mut abs = self.abs_path(href);
 
                         let can_process = match abs.host_str() {
-                            Some(host) => parent_host.ends_with(host),
+                            Some(host) => {
+                                if base_domain.is_empty() {
+                                    parent_host.eq(&host)
+                                } else {
+                                    parent_host.ends_with(host)
+                                }
+                            }
                             _ => false,
                         };
 

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "1.46.4"
+version = "1.46.5"
 authors = ["madeindjs <contact@rousseau-alexandre.fr>", "j-mendez <jeff@a11ywatch.com>"]
 description = "The fastest web crawler CLI written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -26,7 +26,7 @@ quote = "1.0.18"
 failure_derive = "0.1.8"
 
 [dependencies.spider]
-version = "1.46.4"
+version = "1.46.5"
 path = "../spider"
 
 [[bin]]

spider_worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "1.46.4"
+version = "1.46.5"
 authors = ["madeindjs <contact@rousseau-alexandre.fr>", "j-mendez <jeff@a11ywatch.com>"]
 description = "The fastest web crawler as a worker or proxy."
 repository = "https://github.com/spider-rs/spider"
@@ -22,7 +22,7 @@ lazy_static = "1.4.0"
 env_logger = "0.10.0"
 
 [dependencies.spider]
-version = "1.46.4"
+version = "1.46.5"
 path = "../spider"
 features = ["serde", "flexbuffers"]