feat(redirect): add transparent top redirect handling

spider-rs · Dec 14, 2023 · 33953c2 · 33953c2
1 parent 91cae27
commit 33953c2
Show file tree

Hide file tree

Showing 7 changed files with 166 additions and 57 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_examples"
-version = "1.60.12"
+version = "1.60.13"
 authors = ["madeindjs <contact@rousseau-alexandre.fr>", "j-mendez <jeff@a11ywatch.com>"]
 description = "Multithreaded web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"
@@ -22,7 +22,7 @@ htr = "0.5.27"
 flexbuffers = "2.0.0"
 
 [dependencies.spider]
-version = "1.60.12"
+version = "1.60.13"
 path = "../spider"
 features = ["serde"]
 

diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "1.60.12"
+version = "1.60.13"
 authors = ["madeindjs <contact@rousseau-alexandre.fr>", "j-mendez <jeff@a11ywatch.com>"]
 description = "The fastest web crawler written in Rust."
 repository = "https://github.com/spider-rs/spider"

diff --git a/spider/README.md b/spider/README.md
@@ -16,7 +16,7 @@ This is a basic async example crawling a web page, add spider to your `Cargo.tom
 
 ```toml
 [dependencies]
-spider = "1.60.12"
+spider = "1.60.13"
 ```
 
 And then the code:
@@ -91,7 +91,7 @@ We have a couple optional feature flags. Regex blacklisting, jemaloc backend, gl
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["regex", "ua_generator"] }
+spider = { version = "1.60.13", features = ["regex", "ua_generator"] }
 ```
 
 1. `ua_generator`: Enables auto generating a random real User-Agent.
@@ -125,7 +125,7 @@ Move processing to a worker, drastically increases performance even if worker is
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["decentralized"] }
+spider = { version = "1.60.13", features = ["decentralized"] }
 ```
 
 ```sh
@@ -145,7 +145,7 @@ Use the subscribe method to get a broadcast channel.
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["sync"] }
+spider = { version = "1.60.13", features = ["sync"] }
 ```
 
 ```rust,no_run
@@ -175,7 +175,7 @@ Allow regex for blacklisting routes
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["regex"] }
+spider = { version = "1.60.13", features = ["regex"] }
 ```
 
 ```rust,no_run
@@ -202,7 +202,7 @@ If you are performing large workloads you may need to control the crawler by ena
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["control"] }
+spider = { version = "1.60.13", features = ["control"] }
 ```
 
 ```rust
@@ -272,7 +272,7 @@ Use cron jobs to run crawls continuously at anytime.
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["sync", "cron"] }
+spider = { version = "1.60.13", features = ["sync", "cron"] }
 ```
 
 ```rust,no_run
@@ -310,7 +310,7 @@ Connecting to Chrome can be done using the ENV variable `CHROME_URL`, if no conn
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["chrome"] }
+spider = { version = "1.60.13", features = ["chrome"] }
 ```
 
 You can use `website.crawl_concurrent_raw` to perform a crawl without chromium when needed. Use the feature flag `chrome_headed` to enable headful browser usage if needed to debug.
@@ -322,7 +322,7 @@ Intelligently run crawls using HTTP and JavaScript Rendering when needed. The be
 
 ```toml
 [dependencies]
-spider = { version = "1.60.12", features = ["smart"] }
+spider = { version = "1.60.13", features = ["smart"] }
 ```
 
 ```rust,no_run