Move to using a binary file format

Cargo.toml

@@ -48,3 +48,4 @@ libc = "0.2"
 flate2 = "1.0.26"
 tar = "0.4.38"
 infer = "0.13.0"
+bincode = "1.3.3"

src/data.rs

@@ -16,13 +16,12 @@ pub mod grv_perf_events;
 pub mod constants;
 
 use anyhow::Result;
-use crate::InitParams;
+use crate::{APERF_FILE_FORMAT, InitParams};
 use crate::visualizer::GetData;
 use chrono::prelude::*;
 use cpu_utilization::{CpuUtilization, CpuUtilizationRaw};
 use log::trace;
 use serde::{Deserialize, Serialize};
-use serde_yaml::{self};
 use std::fs::{File, OpenOptions};
 use vmstat::{Vmstat, VmstatRaw};
 use diskstats::{Diskstats, DiskstatsRaw};
@@ -63,7 +62,7 @@ impl DataType {
 
     pub fn init_data_type(&mut self, param: InitParams) -> Result<()> {
         trace!("Initializing data type...");
-        let name = format!("{}_{}.yaml", self.file_name, param.time_str);
+        let name = format!("{}_{}.{}", self.file_name, param.time_str, APERF_FILE_FORMAT);
         let full_path = format!("{}/{}", param.dir_name, name);
 
         self.file_name = name;
@@ -95,9 +94,9 @@ impl DataType {
     }
 
     pub fn print_to_file(&mut self) -> Result<()> {
-        trace!("Printing to YAML file...");
+        trace!("Printing to file...");
         let file_handle = self.file_handle.as_ref().unwrap();
-        serde_yaml::to_writer(file_handle.try_clone()?, &self.data)?;
+        bincode::serialize_into(file_handle.try_clone()?, &self.data)?;
         Ok(())
     }
 }
@@ -245,8 +244,6 @@ mod tests {
     use super::{Data, DataType, TimeEnum};
     use crate::InitParams;
     use chrono::prelude::*;
-    use serde::Deserialize;
-    use serde_yaml::{self};
     use std::fs;
     use std::path::Path;
 
@@ -300,14 +297,19 @@ mod tests {
         assert!(Path::new(&dt.full_path).exists());
         assert!(dt.print_to_file().unwrap() == ());
 
-        for document in serde_yaml::Deserializer::from_reader(dt.file_handle.unwrap()) {
-            let v = Data::deserialize(document).expect("File read error");
-            match v {
-                Data::CpuUtilizationRaw(ref value) => {
-                    assert!(value.data.is_empty());
-                }
-                _ => assert!(true),
-            }
+        loop {
+            match bincode::deserialize_from::<_, Data>(dt.file_handle.as_ref().unwrap()) {
+                Ok(v) => {
+                    match v {
+                        Data::CpuUtilizationRaw(ref value) => assert!(value.data.is_empty()),
+                        _ => assert!(false),
+                    }
+                },
+                Err(e) => match *e {
+                    bincode::ErrorKind::Io(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
+                    _ => assert!(false),
+                },
+            };
         }
         fs::remove_file(dt.full_path).unwrap();
         fs::remove_dir_all(dt.dir_name).unwrap();

src/data/interrupts.rs

@@ -347,7 +347,6 @@ mod tests {
     use crate::data::{CollectData, Data, ProcessedData};
     use crate::visualizer::{DataVisualizer, GetData};
     use crate::get_file;
-    use serde::Deserialize;
 
     #[test]
     fn test_collect_data() {
@@ -440,11 +439,14 @@ mod tests {
     #[test]
     fn test_process_raw_data() {
         let mut raw_data = Vec::new();
-        let file = get_file("test/aperf_2022-01-01_01_01_01/".to_string(), "interrupts".to_string()).unwrap();
-        for document in serde_yaml::Deserializer::from_reader(file) {
-            let v = Data::deserialize(document);
-            raw_data.push(v.unwrap());
-        }
+        let file = get_file("test/aperf_2023-07-26_18_37_43/".to_string(), "interrupts".to_string()).unwrap();
+        match bincode::deserialize_from::<_, Data>(file) {
+            Ok(v) => raw_data.push(v),
+            Err(e) => match *e {
+                bincode::ErrorKind::Io(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => assert!(true),
+                e => assert!(false, "{:#?}", e),
+            },
+        };
         let mut dv = DataVisualizer::new(
             ProcessedData::InterruptData(InterruptData::new()),
             String::new(),
@@ -455,9 +457,9 @@ mod tests {
         let processed_data = dv.data.process_raw_data(raw_data[0].clone()).unwrap();
         match processed_data {
             ProcessedData::InterruptData(ref value) => {
-                assert!(value.interrupt_data[0].interrupt_line == InterruptLine::InterruptNr(123), "{:#?}", value);
-                assert!(value.interrupt_data[0].interrupt_type == "PCI".to_string(), "Invalid interrupt type");
-                assert!(value.interrupt_data[0].interrupt_device == "device".to_string(), "Invalid interrupt device");
+                assert!(value.interrupt_data[0].interrupt_line == InterruptLine::InterruptNr(1), "{:#?}", value);
+                assert!(value.interrupt_data[0].interrupt_type == "IO-APIC".to_string(), "Invalid interrupt type");
+                assert!(value.interrupt_data[0].interrupt_device == "i8042".to_string(), "Invalid interrupt device");
             }
             _ => assert!(false, "Invalid data type in interrupts"),
         }

src/lib.rs

@@ -18,6 +18,8 @@ use thiserror::Error;
 use timerfd::{SetTimeFlags, TimerFd, TimerState};
 use flate2::{Compression, write::GzEncoder};
 
+pub static APERF_FILE_FORMAT: &str = "bin";
+
 #[derive(Error, Debug)]
 pub enum PDError {
     #[error("Error getting JavaScript file for {}", .0)]
@@ -114,17 +116,17 @@ impl PerformanceData {
         let _ret = fs::create_dir(self.init_params.dir_name.clone())?;
 
         /*
-         * Create a meta_data.yaml to hold the InitParams that was used by the collector.
+         * Create a meta_data file to hold the InitParams that was used by the collector.
          * This will help when we visualize the data and we don't have to guess these values.
          */
-        let meta_data_path = format!("{}/meta_data.yaml", self.init_params.dir_name.clone());
+        let meta_data_path = format!("{}/meta_data.{}", self.init_params.dir_name.clone(), APERF_FILE_FORMAT);
         let meta_data_handle = fs::OpenOptions::new()
             .create(true)
             .write(true)
             .open(meta_data_path.clone())
             .expect("Could not create meta-data file");
 
-        serde_yaml::to_writer(meta_data_handle, &self.init_params)?;
+        bincode::serialize_into(meta_data_handle, &self.init_params)?;
 
         for (_name, datatype) in self.collectors.iter_mut() {
             datatype.init_data_type(self.init_params.clone())?;
@@ -382,7 +384,7 @@ impl Default for InitParams {
 
 #[cfg(test)]
 mod tests {
-    use super::{InitParams, PerformanceData};
+    use super::{InitParams, PerformanceData, APERF_FILE_FORMAT};
     use std::fs;
     use std::path::Path;
 
@@ -407,7 +409,7 @@ mod tests {
         pd.set_params(params.clone());
         pd.init_collectors().unwrap();
         assert!(Path::new(&pd.init_params.dir_name).exists());
-        let full_path = format!("{}/meta_data.yaml", params.dir_name.clone());
+        let full_path = format!("{}/meta_data.{}", params.dir_name.clone(), APERF_FILE_FORMAT);
         assert!(Path::new(&full_path).exists());
         fs::remove_dir_all(pd.init_params.dir_name).unwrap();
     }

src/visualizer.rs

@@ -1,6 +1,5 @@
 use anyhow::Result;
 use crate::{data::Data, data::ProcessedData, get_file, PDError};
-use serde::Deserialize;
 use std::{collections::HashMap, fs::File};
 use log::debug;
 
@@ -49,9 +48,15 @@ impl DataVisualizer {
         }
         debug!("Processing raw data for: {}", self.api_name);
         let mut raw_data = Vec::new();
-        for document in serde_yaml::Deserializer::from_reader(self.file_handle.as_ref().unwrap()) {
-            let v = Data::deserialize(document);
-            raw_data.push(v?);
+        loop {
+            match bincode::deserialize_from::<_, Data>(self.file_handle.as_ref().unwrap()) {
+                Ok(v) => raw_data.push(v),
+                Err(e) => match *e {
+                    // EOF
+                    bincode::ErrorKind::Io(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
+                    e => panic!("Error when Deserializing {} data {}", self.api_name, e),
+                },
+            };
         }
         let mut data = Vec::new();
         for value in raw_data {
@@ -112,7 +117,7 @@ mod tests {
             "cpu_utilization".to_string(),
         );
         assert!(
-            dv.init_visualizer("test/aperf_2022-01-01_01_01_01/".to_string(), "test".to_string()).unwrap() == ()
+            dv.init_visualizer("test/aperf_2023-07-26_18_37_43/".to_string(), "test".to_string()).unwrap() == ()
         );
         assert!(dv.process_raw_data("test".to_string()).unwrap() == ());
         let ret = dv.get_data("test".to_string(), "run=test&get=values&key=aggregate".to_string()).unwrap();