chore(crawler): add full analytics ignore

spider-rs · Dec 3, 2024 · e54443b · e54443b
1 parent 2417953
commit e54443b
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 13 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.13.100"
+version = "2.14.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.13.100"
+version = "2.14.0"
 rust-version = "1.70"
 authors = [
     "j-mendez <jeff@spider.cloud>"

diff --git a/spider_chrome/src/handler/network.rs b/spider_chrome/src/handler/network.rs
@@ -31,6 +31,7 @@ lazy_static::lazy_static! {
             "d3.js",
             "app.js",
             "main.js",
+            "index.js",
             // Verified 3rd parties for request
             "https://m.stripe.network/inner.html",
             "https://m.stripe.network/out-4.5.43.js",
@@ -83,12 +84,16 @@ lazy_static::lazy_static! {
     static ref URL_IGNORE_TRIE: Trie = {
         let mut trie = Trie::new();
         let patterns = [
+            "https://pagead2.googlesyndication.com",
+            "https://googleads.g.doubleclick.net",
             "https://www.google-analytics.com",
             "https://www.googletagmanager.com",
             "https://px.ads.linkedin.com",
             "https://connect.facebook.net",
-            "https://analytics.twitter.com",
             "https://ads.twitter.com",
+            "https://cdn.segment.com",
+            "https://analytics.",
+            "http://analytics.",
             "sc.omtrdc.net",
             "doubleclick.net",
             "hotjar.com",

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.13.100"
+version = "2.14.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.13.100"
+version = "2.14.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.13.100"
+version = "2.14.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.13.100"
+version = "2.14.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]