Auto merge of #719 - Mark-Simulacrum:shrink-html-report, r=Mark-Simul…

…acrum

Update tera/minifier & cut some memory allocations

Cargo.toml

@@ -2,8 +2,8 @@
 name = "crater"
 version = "0.1.0"
 edition = "2021"
-
 build = "build.rs"
+default-run = "crater"
 
 [profile.dev]
 opt-level = 0
@@ -24,7 +24,7 @@ http = "0.2"
 hyper = "0.14"
 lazy_static = "1.0"
 mime = "0.3.1"
-minifier = { version = "0.2", features = ["html"] }
+minifier = { version = "0.3", features = ["html"] }
 r2d2 = "0.8.2"
 r2d2_sqlite = "0.22.0"
 rusqlite = { version = "0.29.0", features = ["chrono", "functions", "bundled"] }
@@ -38,7 +38,7 @@ serde_regex = "1.1.0"
 clap = { version = "4", features = ["derive"] }
 tar = "0.4.36"
 tempfile = "3.0.0"
-tera = "0.11.7"
+tera = "1.19.1"
 toml = "0.8.6"
 url = "2"
 walkdir = "2"
@@ -60,7 +60,7 @@ tokio = "1.24"
 aws-sdk-s3 = "1.7"
 aws-config = { version = "1", features = ["behavior-version-latest"] }
 thiserror = "1.0.38"
-nix = { version = "0.27.1", features = ["mman"] }
+nix = { version = "0.27.1", features = ["mman", "resource"] }
 
 [dev-dependencies]
 assert_cmd = "2.0.4"

src/assets.rs

@@ -145,7 +145,7 @@ fn build_tera_cache() -> Fallible<Tera> {
 }
 
 #[allow(unused_variables)]
-pub fn render_template<C: Serialize>(name: &str, context: &C) -> Fallible<String> {
+pub fn render_template<C: Serialize>(name: &str, context: C) -> Fallible<String> {
     // On debug builds the cache is rebuilt every time to pick up changed templates
     let tera_owned: Tera;
     let tera;
@@ -161,7 +161,8 @@ pub fn render_template<C: Serialize>(name: &str, context: &C) -> Fallible<String
         tera = &TERA_CACHE;
     }
 
+    let tera_context = tera::Context::from_serialize(context)?;
     Ok(tera
-        .render(name, context)
+        .render(name, &tera_context)
         .map_err(|e| failure::format_err!("{:?}", e))?)
 }

src/bin/test-report.rs

@@ -0,0 +1,78 @@
+use crater::experiments::ExperimentDBRecord;
+use crater::report::ReportWriter;
+use crater::results::EncodingType;
+use crater::{config::Config, db::QueryUtils};
+use failure::Fallible;
+use mime::{self, Mime};
+use std::{borrow::Cow, fmt, path::Path};
+
+fn main() {
+    let mut env = env_logger::Builder::new();
+    env.filter_module("test_report", log::LevelFilter::Info);
+    env.filter_module("crater", log::LevelFilter::Info);
+    env.filter_module("rustwide", log::LevelFilter::Info);
+    if let Ok(content) = std::env::var("RUST_LOG") {
+        env.parse_filters(&content);
+    }
+    rustwide::logging::init_with(env.build());
+    let config: Config = toml::from_str(&std::fs::read_to_string("config.toml").unwrap()).unwrap();
+    let db = crater::db::Database::open_at(std::path::Path::new("crater.db")).unwrap();
+    let experiments = db
+        .query("SELECT * FROM experiments;", [], |r| {
+            ExperimentDBRecord::from_row(r)
+        })
+        .unwrap();
+    let experiments: Vec<_> = experiments
+        .into_iter()
+        .map(|record| record.into_experiment())
+        .collect::<Fallible<_>>()
+        .unwrap();
+    let ex = experiments.iter().find(|e| e.name == "pr-118920").unwrap();
+    let rdb = crater::results::DatabaseDB::new(&db);
+
+    log::info!("Getting crates...");
+
+    let crates = ex.get_crates(&db).unwrap();
+    let writer = NullWriter;
+
+    log::info!("Starting report generation...");
+    log::info!(
+        "@ {:?}",
+        nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF)
+            .unwrap()
+            .max_rss()
+    );
+    crater::report::gen(&rdb, ex, &crates, &writer, &config, false).unwrap();
+    log::info!(
+        "@ {:?}",
+        nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF)
+            .unwrap()
+            .max_rss()
+    );
+}
+
+#[derive(Debug)]
+struct NullWriter;
+
+impl ReportWriter for NullWriter {
+    fn write_bytes<P: AsRef<Path>>(
+        &self,
+        _path: P,
+        _b: &[u8],
+        _mime: &Mime,
+        _encoding_type: EncodingType,
+    ) -> Fallible<()> {
+        // no-op
+        Ok(())
+    }
+    fn write_string<P: AsRef<Path>>(&self, _path: P, _s: Cow<str>, _mime: &Mime) -> Fallible<()> {
+        // no-op
+        Ok(())
+    }
+}
+
+impl fmt::Display for NullWriter {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}

src/db/mod.rs

@@ -6,6 +6,7 @@ use r2d2::{CustomizeConnection, Pool};
 use r2d2_sqlite::SqliteConnectionManager;
 use rusqlite::types::ToSql;
 use rusqlite::{Connection, Row, Transaction};
+use std::path::Path;
 use std::sync::Arc;
 use std::time::Instant;
 use tempfile::NamedTempFile;
@@ -55,6 +56,11 @@ impl Database {
         Database::new(SqliteConnectionManager::file(path), None)
     }
 
+    pub fn open_at(path: &Path) -> Fallible<Self> {
+        std::fs::create_dir_all(&*WORK_DIR)?;
+        Database::new(SqliteConnectionManager::file(path), None)
+    }
+
     #[cfg(test)]
     pub fn temp() -> Fallible<Self> {
         let tempfile = NamedTempFile::new()?;
@@ -191,6 +197,24 @@ pub trait QueryUtils {
         })
     }
 
+    fn query_row<T, F: FnOnce(&Row) -> Fallible<T>>(
+        &self,
+        sql: &str,
+        params: impl rusqlite::Params,
+        func: F,
+    ) -> Fallible<Option<T>> {
+        self.with_conn(|conn| {
+            self.trace(sql, || {
+                let mut prepared = conn.prepare(sql)?;
+                let mut rows = prepared.query(params)?;
+                if let Ok(Some(row)) = rows.next() {
+                    return Ok(Some(func(row)?));
+                }
+                Ok(None)
+            })
+        })
+    }
+
     fn trace<T, F: FnOnce() -> T>(&self, sql: &str, f: F) -> T {
         let start = Instant::now();
         let res = f();

src/experiments.rs

@@ -659,7 +659,7 @@ impl Experiment {
     }
 }
 
-struct ExperimentDBRecord {
+pub struct ExperimentDBRecord {
     name: String,
     mode: String,
     cap_lints: String,
@@ -680,7 +680,7 @@ struct ExperimentDBRecord {
 }
 
 impl ExperimentDBRecord {
-    fn from_row(row: &Row) -> rusqlite::Result<Self> {
+    pub fn from_row(row: &Row) -> rusqlite::Result<Self> {
         Ok(ExperimentDBRecord {
             name: row.get("name")?,
             mode: row.get("mode")?,
@@ -702,7 +702,7 @@ impl ExperimentDBRecord {
         })
     }
 
-    fn into_experiment(self) -> Fallible<Experiment> {
+    pub fn into_experiment(self) -> Fallible<Experiment> {
         Ok(Experiment {
             name: self.name,
             toolchains: [self.toolchain_start.parse()?, self.toolchain_end.parse()?],

src/report/html.rs

@@ -256,10 +256,6 @@ fn write_report<W: ReportWriter>(
     };
 
     info!("generating {}", to);
-    let rendered = assets::render_template("report/results.html", &context)
-        .context("rendering template report/results.html")?;
-    let html = minifier::html::minify(&rendered);
-    dest.write_string(to, html.into(), &mime::TEXT_HTML)?;
 
     if output_templates {
         dest.write_string(
@@ -269,6 +265,11 @@ fn write_report<W: ReportWriter>(
         )?;
     }
 
+    let rendered = assets::render_template("report/results.html", context)
+        .context("rendering template report/results.html")?;
+    let html = minifier::html::minify(&rendered);
+    dest.write_string(to, html.into(), &mime::TEXT_HTML)?;
+
     Ok(())
 }
 

src/results/db.rs

@@ -179,22 +179,13 @@ impl<'a> ReadResults for DatabaseDB<'a> {
         toolchain: &Toolchain,
         krate: &Crate,
     ) -> Fallible<Option<TestResult>> {
-        let result: Option<String> = self
-            .db
-            .query(
-                "SELECT result FROM results \
+        Ok(self.db.query_row(
+            "SELECT result FROM results \
                  WHERE experiment = ?1 AND toolchain = ?2 AND crate = ?3 \
                  LIMIT 1;",
-                [&ex.name, &toolchain.to_string(), &krate.id()],
-                |row| row.get("result"),
-            )?
-            .pop();
-
-        if let Some(res) = result {
-            Ok(Some(res.parse()?))
-        } else {
-            Ok(None)
-        }
+            [&ex.name, &toolchain.to_string(), &krate.id()],
+            |row| Ok(row.get_ref("result")?.as_str()?.parse::<TestResult>()?),
+        )?)
     }
 }