added support for conserve-memory flag for folder searches

Cargo.toml

@@ -1,7 +1,7 @@
 workspace = { members = [ "tests","tests/obfuscate_access_log_ips"] }
 [package]
 name = "ngxav"
-version = "0.4.7"
+version = "0.4.8"
 edition = "2021"
 license = "MIT"
 description = "Search through NGINX logs with advanced filters and support for displaying analytics about your selected log entries"

src/main.rs

@@ -37,7 +37,9 @@ fn main() {
             .build_global()
             .unwrap();
     }
-    if !args.conserve_memory.is_none() && args.conserve_memory.unwrap() == true {
+    let file_md = metadata(args.file.clone()).unwrap();
+    if !args.conserve_memory.is_none() && args.conserve_memory.unwrap() == true && file_md.is_file()
+    {
         if let Ok(lines) = read_line_by_line(args.file) {
             let mut occurrences: HashMap<String, bool> = HashMap::new();
             for line in lines.flatten() {
@@ -57,9 +59,13 @@ fn main() {
         return;
     }
     let mut lines = Vec::new();
-    let file_md = metadata(args.file.clone()).unwrap();
     if file_md.is_dir() {
-        lines = utils::read_folder::read_folder(args.file);
+        if args.conserve_memory.is_some() && args.conserve_memory.unwrap() == true {
+            utils::read_folder_conserve_memory::read_folder_conserve_memory(args.file, args.unique);
+            return;
+        } else {
+            lines = utils::read_folder::read_folder(args.file);
+        }
     } else {
         lines = lines_from_file(args.file).expect("should read");
     }

src/utils/mod.rs

@@ -5,6 +5,7 @@ pub mod parse_line;
 pub mod parse_nginx_time_format;
 pub mod parse_user_agent;
 pub mod read_folder;
+pub mod read_folder_conserve_memory;
 pub mod session_analytics;
 pub mod session_unique;
 pub mod sessionize;

src/utils/read_folder_conserve_memory.rs

@@ -0,0 +1,65 @@
+use crate::utils::keep_line::keep_line;
+use crate::utils::parse_line::parse_line;
+use flate2::read::GzDecoder;
+use std::collections::HashMap;
+use std::fs;
+use std::fs::metadata;
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+
+pub fn read_folder_conserve_memory(file_path: String, isUnique: Option<bool>) {
+    println!("{}", "step zero");
+    let mut paths: Vec<_> = fs::read_dir(file_path).unwrap().collect();
+    paths.sort_by_key(|x| {
+        metadata(x.as_ref().unwrap().path().to_str().unwrap())
+            .unwrap()
+            .modified()
+            .unwrap()
+    });
+    println!("{}", "step one");
+    let mut occurrences: HashMap<String, bool> = HashMap::new();
+    for path in paths {
+        let p: String = path.unwrap().path().to_str().unwrap().to_string();
+        if p.contains(".gz") {
+            let file = File::open(p).expect("Ooops.");
+            let reader = BufReader::new(GzDecoder::new(file));
+            for r in reader.lines() {
+                match r {
+                    Ok(line) => {
+                        let ip: String =
+                            line.clone().split(" ").collect::<Vec<&str>>()[0].to_string();
+                        if keep_line(&parse_line(&line), true) {
+                            if isUnique.is_some() && isUnique.unwrap() == true {
+                                if !occurrences.contains_key(&ip) {
+                                    println!("{}", line.clone() + "\n");
+                                    occurrences.insert(ip, true);
+                                }
+                            } else {
+                                println!("{}", line.clone() + "\n");
+                            }
+                        }
+                    }
+                    Err(error) => {
+                        eprintln!("Error reading line: {}", error);
+                    }
+                }
+            }
+        } else {
+            let file = File::open(p).expect("Ooops.");
+            let reader = BufReader::new(file).lines();
+            for line in reader.flatten() {
+                let ip: String = line.clone().split(" ").collect::<Vec<&str>>()[0].to_string();
+                if keep_line(&parse_line(&line), true) {
+                    if isUnique.is_some() && isUnique.unwrap() == true {
+                        if !occurrences.contains_key(&ip) {
+                            println!("{}", line.clone() + "\n");
+                            occurrences.insert(ip, true);
+                        }
+                    } else {
+                        println!("{}", line.clone() + "\n");
+                    }
+                }
+            }
+        }
+    }
+}