feat: added flag to parse all lines instead of skipping some (#78)

bc-m · Sep 23, 2024 · 7b2dd74 · 7b2dd74
1 parent aeebe07
commit 7b2dd74
Show file tree

Hide file tree

Showing 14 changed files with 120 additions and 34 deletions.
diff --git a/src/custom_menu_catalog.rs b/src/custom_menu_catalog.rs
@@ -6,12 +6,14 @@ use quick_xml::reader::Reader;
 
 use crate::utils::xml_utils::skip_element;
 use crate::utils::{initialize_out_dir, write_xml_element_to_file};
+use crate::Flags;
 
 pub fn xml_explode_custom_menu_catalog<R: Read + BufRead>(
     reader: &mut Reader<R>,
     _: &BytesStart,
     out_dir_path: &Path,
     fm_file_name: &str,
+    flags: &Flags,
 ) {
     let out_dir_path = out_dir_path.join("custom_menus").join(fm_file_name);
     initialize_out_dir(&out_dir_path);
@@ -29,7 +31,7 @@ pub fn xml_explode_custom_menu_catalog<R: Read + BufRead>(
                 depth += 1;
                 if depth == 2 {
                     if e.name().as_ref() == b"CustomMenu" {
-                        write_xml_element_to_file(reader, &e, &out_dir_path, 4);
+                        write_xml_element_to_file(reader, &e, &out_dir_path, 4, flags);
                         depth -= 1;
                         continue;
                     } else {

diff --git a/src/custom_menu_set_catalog.rs b/src/custom_menu_set_catalog.rs
@@ -6,12 +6,14 @@ use quick_xml::reader::Reader;
 
 use crate::utils::xml_utils::skip_element;
 use crate::utils::{initialize_out_dir, write_xml_element_to_file};
+use crate::Flags;
 
 pub fn xml_explode_custom_menu_set_catalog<R: Read + BufRead>(
     reader: &mut Reader<R>,
     _: &BytesStart,
     out_dir_path: &Path,
     fm_file_name: &str,
+    flags: &Flags,
 ) {
     let out_dir_path = out_dir_path.join("custom_menu_sets").join(fm_file_name);
     initialize_out_dir(&out_dir_path);
@@ -29,7 +31,7 @@ pub fn xml_explode_custom_menu_set_catalog<R: Read + BufRead>(
                 depth += 1;
                 if depth == 3 {
                     if e.name().as_ref() == b"CustomMenuSet" {
-                        write_xml_element_to_file(reader, &e, &out_dir_path, 5);
+                        write_xml_element_to_file(reader, &e, &out_dir_path, 5, flags);
                         depth -= 1;
                         continue;
                     } else {

diff --git a/src/extended_privileges_catalog.rs b/src/extended_privileges_catalog.rs
@@ -6,12 +6,14 @@ use quick_xml::reader::Reader;
 
 use crate::utils::xml_utils::skip_element;
 use crate::utils::{initialize_out_dir, write_xml_element_to_file};
+use crate::Flags;
 
 pub fn xml_explode_extended_privileges_catalog<R: Read + BufRead>(
     reader: &mut Reader<R>,
     _: &BytesStart,
     out_dir_path: &Path,
     fm_file_name: &str,
+    flags: &Flags,
 ) {
     let out_dir_path = out_dir_path.join("extended_privileges").join(fm_file_name);
     initialize_out_dir(&out_dir_path);
@@ -29,7 +31,7 @@ pub fn xml_explode_extended_privileges_catalog<R: Read + BufRead>(
                 depth += 1;
                 if depth == 3 {
                     if e.name().as_ref() == b"ExtendedPrivilege" {
-                        write_xml_element_to_file(reader, &e, &out_dir_path, 5);
+                        write_xml_element_to_file(reader, &e, &out_dir_path, 5, flags);
                         depth -= 1;
                         continue;
                     } else {

diff --git a/src/external_data_source_catalog.rs b/src/external_data_source_catalog.rs
@@ -8,12 +8,14 @@ use crate::utils::xml_utils::{
     cdata_element_to_string, end_element_to_string, start_element_to_string, text_element_to_string,
 };
 use crate::utils::{create_dir, write_xml_file};
+use crate::Flags;
 
 pub fn xml_extract_external_data_sources<R: Read + BufRead>(
     reader: &mut Reader<R>,
     start: &BytesStart,
     out_dir_path: &Path,
     fm_file_name: &str,
+    flags: &Flags,
 ) {
     let out_dir_path = out_dir_path.join("external_data_sources");
     create_dir(&out_dir_path);
@@ -43,6 +45,7 @@ pub fn xml_extract_external_data_sources<R: Read + BufRead>(
                         &out_dir_path,
                         fm_file_name,
                         &external_data_source_info,
+                        flags,
                     );
                     break;
                 }
@@ -63,7 +66,12 @@ pub fn xml_extract_external_data_sources<R: Read + BufRead>(
     }
 }
 
-fn write_external_data_sources_to_file(output_dir: &Path, fm_file_name: &str, content: &str) {
+fn write_external_data_sources_to_file(
+    output_dir: &Path,
+    fm_file_name: &str,
+    content: &str,
+    flags: &Flags,
+) {
     let output_file_path = output_dir.join(format!("{}.xml", fm_file_name));
-    write_xml_file(&output_file_path, content, 3);
+    write_xml_file(&output_file_path, content, 3, flags);
 }
diff --git a/src/layout_catalog.rs b/src/layout_catalog.rs
@@ -12,6 +12,7 @@ use crate::utils::xml_utils::{
     start_element_to_string, text_element_to_string,
 };
 use crate::utils::{initialize_out_dir, write_xml_file};
+use crate::Flags;
 use crate::{escape_filename, join_scope_id_and_name};
 
 #[derive(Debug, Default)]
@@ -27,6 +28,7 @@ pub fn xml_explode_layout_catalog<R: Read + BufRead>(
     _: &BytesStart,
     out_dir_path: &Path,
     fm_file_name: &str,
+    flags: &Flags,
 ) {
     let out_dir_path = out_dir_path.join("layouts").join(fm_file_name);
     initialize_out_dir(&out_dir_path);
@@ -106,7 +108,7 @@ pub fn xml_explode_layout_catalog<R: Read + BufRead>(
                     .push_str(end_element_to_string(&e).as_str());
 
                 if depth == 1 && local_name_to_string(e.name().as_ref()) == "Layout" {
-                    write_layout_to_file(&out_dir_path, &layout_info)
+                    write_layout_to_file(&out_dir_path, &layout_info, flags)
                 }
             }
             Ok(Event::CData(e)) => {
@@ -131,7 +133,7 @@ pub fn xml_explode_layout_catalog<R: Read + BufRead>(
     }
 }
 
-fn write_layout_to_file(dir_path: &Path, layout: &LayoutInfo) {
+fn write_layout_to_file(dir_path: &Path, layout: &LayoutInfo, flags: &Flags) {
     let layout_filename = join_scope_id_and_name(layout.id.as_str(), layout.name.as_str());
     let layout_filename = escape_filename(&layout_filename);
 
@@ -148,5 +150,5 @@ fn write_layout_to_file(dir_path: &Path, layout: &LayoutInfo) {
         .unwrap_or_else(|err| panic!("Error creating directory {}: {}", output_dir.display(), err));
 
     let output_file_path = output_dir.join(format!("{}.xml", layout_filename));
-    write_xml_file(&output_file_path, &layout.content, 4);
+    write_xml_file(&output_file_path, &layout.content, 4, flags);
 }
diff --git a/src/main.rs b/src/main.rs
@@ -54,6 +54,14 @@ struct Args {
 
     /// The target directory to write output
     target: PathBuf,
+
+    /// Parse all lines (or skip less important ones to reduce noise)
+    #[arg(short, long)]
+    all_lines: bool,
+}
+
+struct Flags {
+    parse_all_lines: bool,
 }
 
 fn main() -> Result<()> {
@@ -62,6 +70,9 @@ fn main() -> Result<()> {
     let args = Args::parse();
     let in_dir = args.source;
     let out_dir = args.target;
+    let flags = Flags {
+        parse_all_lines: args.all_lines,
+    };
 
     valid_dir_or_throw(&in_dir)?;
     valid_dir_or_throw(&out_dir)?;
@@ -76,7 +87,7 @@ fn main() -> Result<()> {
 
     // Process XML files in parallel
     paths.par_iter().for_each(|path| {
-        match explode_xml(path, &out_dir) {
+        match explode_xml(path, &out_dir, &flags) {
             Ok(_) => {}
             Err(err) => {
                 let file_name = path.file_name().unwrap().to_str().unwrap();
@@ -105,7 +116,11 @@ fn valid_dir_or_throw(dir_path: &PathBuf) -> Result<(), Error> {
     }
 }
 
-fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(), Error> {
+fn explode_xml(
+    fm_export_file_path: &PathBuf,
+    out_dir_path: &Path,
+    flags: &Flags,
+) -> Result<(), Error> {
     let start = Instant::now();
     let fm_export_file_name = fm_export_file_path.file_name().unwrap().to_str().unwrap();
 
@@ -162,6 +177,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -172,6 +188,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 out_dir_path,
                                 &fm_file_name,
                                 &table_name_id_map,
+                                flags,
                             );
                             continue;
                         }
@@ -191,6 +208,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 out_dir_path,
                                 &fm_file_name,
                                 &script_id_path_map,
+                                flags,
                             );
                             continue;
                         }
@@ -204,6 +222,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -213,6 +232,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -230,6 +250,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -239,6 +260,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -248,11 +270,18 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
                         b"ThemeCatalog" => {
-                            xml_explode_theme_catalog(&mut reader, &e, out_dir_path, &fm_file_name);
+                            xml_explode_theme_catalog(
+                                &mut reader,
+                                &e,
+                                out_dir_path,
+                                &fm_file_name,
+                                flags,
+                            );
                             continue;
                         }
                         b"PrivilegeSetsCatalog" => {
@@ -261,6 +290,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -270,6 +300,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -279,6 +310,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -288,6 +320,7 @@ fn explode_xml(fm_export_file_path: &PathBuf, out_dir_path: &Path) -> Result<(),
                                 &e,
                                 out_dir_path,
                                 &fm_file_name,
+                                flags,
                             );
                             continue;
                         }
@@ -336,7 +369,6 @@ fn escape_filename(filename: &str) -> String {
 }
 
 fn should_skip_line(line: &str) -> bool {
-    // de-noise TODO: make optional
     if line.contains("<TagList></TagList>") {
         return true;
     };
@@ -379,6 +411,9 @@ mod tests {
         let snapshot_dir = Path::new("./tests/snapshots");
         let input_dir = Path::new("./tests/xml");
         let output_dir = Path::new("./out");
+        let flags = Flags {
+            parse_all_lines: false,
+        };
         let _ = remove_dir_all(output_dir);
 
         let mut settings = insta::Settings::clone_current();
@@ -393,7 +428,7 @@ mod tests {
             .collect::<Vec<_>>();
 
         for path in paths {
-            explode_xml(&path, output_dir)
+            explode_xml(&path, output_dir, &flags)
                 .unwrap_or_else(|_| panic!("Error processing file '{}'", path.display()));
         }
 

diff --git a/src/privilege_sets_catalog.rs b/src/privilege_sets_catalog.rs
@@ -6,12 +6,14 @@ use quick_xml::reader::Reader;
 
 use crate::utils::xml_utils::skip_element;
 use crate::utils::{initialize_out_dir, write_xml_element_to_file};
+use crate::Flags;
 
 pub fn xml_explode_privilege_set_catalog<R: Read + BufRead>(
     reader: &mut Reader<R>,
     _: &BytesStart,
     out_dir_path: &Path,
     fm_file_name: &str,
+    flags: &Flags,
 ) {
     let out_dir_path = out_dir_path.join("privilege_sets").join(fm_file_name);
     initialize_out_dir(&out_dir_path);
@@ -29,7 +31,7 @@ pub fn xml_explode_privilege_set_catalog<R: Read + BufRead>(
                 depth += 1;
                 if depth == 3 {
                     if e.name().as_ref() == b"PrivilegeSet" {
-                        write_xml_element_to_file(reader, &e, &out_dir_path, 5);
+                        write_xml_element_to_file(reader, &e, &out_dir_path, 5, flags);
                         depth -= 1;
                         continue;
                     } else {

diff --git a/src/relationship_catalog.rs b/src/relationship_catalog.rs
@@ -10,6 +10,7 @@ use crate::utils::xml_utils::{
     start_element_to_string, text_element_to_string,
 };
 use crate::utils::{initialize_out_dir, write_xml_file};
+use crate::Flags;
 use crate::{escape_filename, join_scope_id_and_name};
 
 #[derive(Debug, Default)]
@@ -25,6 +26,7 @@ pub fn xml_explode_relationship_catalog<R: Read + BufRead>(
     _: &BytesStart,
     out_dir_path: &Path,
     fm_file_name: &str,
+    flags: &Flags,
 ) {
     let out_dir_path = out_dir_path.join("relationships").join(fm_file_name);
     initialize_out_dir(&out_dir_path);
@@ -95,7 +97,7 @@ pub fn xml_explode_relationship_catalog<R: Read + BufRead>(
                     .push_str(end_element_to_string(&e).as_str());
 
                 if depth == 1 && local_name_to_string(e.name().as_ref()) == "Relationship" {
-                    write_relationship_to_file(&out_dir_path, &relationship_info);
+                    write_relationship_to_file(&out_dir_path, &relationship_info, flags);
                     relationship_info.id.clear();
                     relationship_info.left.clear();
                     relationship_info.right.clear();
@@ -133,12 +135,12 @@ pub fn xml_explode_relationship_catalog<R: Read + BufRead>(
     }
 }
 
-fn write_relationship_to_file(output_dir: &Path, relationship: &RelationshipInfo) {
+fn write_relationship_to_file(output_dir: &Path, relationship: &RelationshipInfo, flags: &Flags) {
     let relationship_filename = join_scope_id_and_name(
         relationship.id.as_str(),
         format!("[{}] - [{}]", relationship.left, relationship.right).as_str(),
     );
     let relationship_filename = escape_filename(&relationship_filename).replace('.', "_");
     let output_file_path = output_dir.join(format!("{}.xml", relationship_filename));
-    write_xml_file(&output_file_path, &relationship.content, 4);
+    write_xml_file(&output_file_path, &relationship.content, 4, flags);
 }