Unify HTML and text versions of the dashboard (rust-lang#539)

* Add litani structs for deserialization and use them

* Remove compiletest code to use litani runs

* Update dashboard docs

* Fix format

Cargo.lock

@@ -944,6 +944,8 @@ dependencies = [
  "Inflector",
  "pulldown-cmark 0.8.0",
  "rustdoc",
+ "serde",
+ "serde_json",
  "walkdir",
 ]
 

rmc-docs/src/dashboard.md

@@ -1,6 +1,6 @@
 # RMC Dashboard
 
-The [RMC Dashboard](./dashboard/index.html) is a testing tool based on [Compiletest](https://rustc-dev-guide.rust-lang.org/tests/intro.html) and [Litani](https://github.com/awslabs/aws-build-accumulator).
+The [RMC Dashboard](./dashboard/index.html) is a testing tool based on [Litani](https://github.com/awslabs/aws-build-accumulator).
 
 The purpose of the dashboard to show the level of support in RMC for all Rust features.
 To this end, we use Rust code snippet examples from the following general Rust documentation books:
@@ -36,3 +36,7 @@ If an example shows one red bar, it is considered a failed example that cannot b
 
 The [RMC Dashboard](./dashboard/index.html) is automatically updated whenever
 a PR gets merged into RMC.
+
+> **Tip:** In addition, we publish a [text version of the dashboard](./dashboard/dashboard.txt)
+> while we work on adding more features to [Litani](https://github.com/awslabs/aws-build-accumulator).
+> The [text-based dashboard](./dashboard/dashboard.txt) displays the same results in hierarchical way.

src/tools/dashboard/Cargo.toml

@@ -11,3 +11,5 @@ Inflector = "0.11.4"
 pulldown-cmark = { version = "0.8.0", default-features = false }
 rustdoc = { path = "../../librustdoc" }
 walkdir = "2.3.2"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"

src/tools/dashboard/src/books.rs

@@ -7,7 +7,7 @@ extern crate rustc_span;
 
 use crate::{
     dashboard,
-    litani::Litani,
+    litani::{Litani, LitaniPipeline, LitaniRun},
     util::{self, FailStep, TestProps},
 };
 use inflector::cases::{snakecase::to_snake_case, titlecase::to_title_case};
@@ -17,16 +17,15 @@ use rustdoc::{
     doctest::Tester,
     html::markdown::{ErrorCodes, Ignore, LangString},
 };
+use serde_json;
 use std::{
     collections::{HashMap, HashSet},
-    env,
     ffi::OsStr,
     fmt::Write,
     fs,
-    io::{BufRead, BufReader},
+    io::BufReader,
     iter::FromIterator,
     path::{Path, PathBuf},
-    process::{Command, Stdio},
 };
 use walkdir::WalkDir;
 
@@ -356,35 +355,6 @@ fn paths_to_string(paths: HashSet<PathBuf>) -> String {
     f
 }
 
-/// Runs `compiletest` on the `suite` and logs the results to `log_path`.
-fn run_examples(suite: &str, log_path: &Path) {
-    // Before executing this program, `cargo` populates the environment with
-    // build configs. `x.py` respects those configs, causing a recompilation
-    // of `rustc`. This is not a desired behavior, so we remove those configs.
-    let filtered_env: HashMap<String, String> = env::vars()
-        .filter(|&(ref k, _)| {
-            !(k.contains("CARGO") || k.contains("LD_LIBRARY_PATH") || k.contains("RUST"))
-        })
-        .collect();
-    // Create the log's parent directory (if it does not exist).
-    fs::create_dir_all(log_path.parent().unwrap()).unwrap();
-    let mut cmd = Command::new([".", "x.py"].iter().collect::<PathBuf>());
-    cmd.args([
-        "test",
-        suite,
-        "-i",
-        "--stage",
-        "1",
-        "--test-args",
-        "--logfile",
-        "--test-args",
-        log_path.to_str().unwrap(),
-    ]);
-    cmd.env_clear().envs(filtered_env);
-    cmd.stdout(Stdio::null());
-    cmd.spawn().unwrap().wait().unwrap();
-}
-
 /// Creates a new [`Tree`] from `path`, and a test `result`.
 fn tree_from_path(mut path: Vec<String>, result: bool) -> dashboard::Tree {
     assert!(path.len() > 0, "Error: `path` must contain at least 1 element.");
@@ -405,42 +375,45 @@ fn tree_from_path(mut path: Vec<String>, result: bool) -> dashboard::Tree {
     tree
 }
 
-/// Parses and generates a dashboard from the log output of `compiletest` in
-/// `path`.
-fn parse_log(path: &Path) -> dashboard::Tree {
+/// Parses a `litani` run and generates a dashboard tree from it
+fn parse_litani_output(path: &Path) -> dashboard::Tree {
     let file = fs::File::open(path).unwrap();
     let reader = BufReader::new(file);
+    let run: LitaniRun = serde_json::from_reader(reader).unwrap();
     let mut tests =
         dashboard::Tree::new(dashboard::Node::new(String::from("dashboard"), 0, 0), vec![]);
-    for line in reader.lines() {
-        let (ns, l) = parse_log_line(&line.unwrap());
+    let pipelines = run.get_pipelines();
+    for pipeline in pipelines {
+        let (ns, l) = parse_log_line(&pipeline);
         tests = dashboard::Tree::merge(tests, tree_from_path(ns, l)).unwrap();
     }
     tests
 }
 
-/// Parses a line in the log output of `compiletest` and returns a pair containing
+/// Parses a `litani` pipeline and returns a pair containing
 /// the path to a test and its result.
-fn parse_log_line(line: &str) -> (Vec<String>, bool) {
-    // Each line has the format `<result> [rmc] <path>`. Extract <result> and
-    // <path>.
-    let splits: Vec<_> = line.split(" [rmc] ").map(String::from).collect();
-    let l = if splits[0].as_str() == "ok" { true } else { false };
-    let mut ns: Vec<_> = splits[1].split(&['/', '.'][..]).map(String::from).collect();
-    // Remove unnecessary `.rs` suffix.
+fn parse_log_line(pipeline: &LitaniPipeline) -> (Vec<String>, bool) {
+    let l = pipeline.get_status();
+    let name = pipeline.get_name();
+    let mut ns: Vec<String> = name.split(&['/', '.'][..]).map(String::from).collect();
+    // Remove unnecessary items from the path until "dashboard"
+    let dash_index = ns.iter().position(|item| item == "dashboard").unwrap();
+    ns.drain(..dash_index);
+    // Remove unnecessary "rs" suffix.
     ns.pop();
     (ns, l)
 }
 
-/// Display the dashboard in the terminal.
-fn display_terminal_dashboard(dashboard: dashboard::Tree) {
-    println!(
-        "# of tests: {}\t✔️ {}\t❌ {}",
+/// Format and write a text version of the dashboard
+fn generate_text_dashboard(dashboard: dashboard::Tree, path: &Path) {
+    let dashboard_str = format!(
+        "# of tests: {}\t✔️ {}\t❌ {}\n{}",
         dashboard.data.num_pass + dashboard.data.num_fail,
         dashboard.data.num_pass,
-        dashboard.data.num_fail
+        dashboard.data.num_fail,
+        dashboard
     );
-    println!("{}", dashboard);
+    fs::write(&path, dashboard_str).expect("Error: Unable to write dashboard results");
 }
 
 /// Runs examples using Litani build.
@@ -468,9 +441,9 @@ fn litani_run_tests() {
 /// Extracts examples from the Rust books, run them through RMC, and displays
 /// their results in a terminal dashboard.
 pub fn generate_dashboard() {
-    let build_dir = &env::var("BUILD_DIR").unwrap();
-    let triple = &env::var("TRIPLE").unwrap();
-    let log_path: PathBuf = [build_dir, triple, "dashboard", "log"].iter().collect();
+    let litani_log: PathBuf = ["build", "output", "latest", "run.json"].iter().collect();
+    let text_dash: PathBuf =
+        ["build", "output", "latest", "html", "dashboard.txt"].iter().collect();
     // Parse the chapter/section hierarchy for the books.
     let mut map = HashMap::new();
     map.extend(parse_reference_hierarchy());
@@ -480,13 +453,10 @@ pub fn generate_dashboard() {
     // Extract examples from the books, pre-process them, and save them
     // following the partial hierarchy in map.
     extract_examples(map);
-    // Pre-process the examples before running them through `compiletest`.
-    // Run `compiletest` on the examples.
-    run_examples("dashboard", &log_path);
-    // Parse `compiletest` log file.
-    let dashboard = parse_log(&log_path);
-    // Display the terminal dashboard.
-    display_terminal_dashboard(dashboard);
-    // Generate Litani's HTML dashboard.
+    // Generate Litani's HTML dashboard
     litani_run_tests();
+    // Parse Litani's output
+    let dashboard = parse_litani_output(&litani_log);
+    // Generate text dashboard
+    generate_text_dashboard(dashboard, &text_dash);
 }

src/tools/dashboard/src/litani.rs

@@ -3,10 +3,140 @@
 //! Utilities to interact with the `Litani` build accumulator.
 
 use pulldown_cmark::escape::StrWrite;
+use serde::Deserialize;
 use std::collections::HashMap;
 use std::path::Path;
 use std::process::{Child, Command};
 
+/// Data structure representing a full `litani` run.
+/// The same representation is used to represent a run
+/// in the `run.json` (cache) file generated by `litani`
+///
+/// Deserialization is performed automatically for most
+/// attributes in such files, but it may require you to
+/// extend it if advanced features are used (e.g., pools)
+#[derive(Debug, Deserialize)]
+pub struct LitaniRun {
+    aux: Option<HashMap<String, String>>,
+    project: String,
+    version: String,
+    version_major: u32,
+    version_minor: u32,
+    version_patch: u32,
+    release_candidate: bool,
+    run_id: String,
+    start_time: String,
+    parallelism: LitaniParalellism,
+    latest_symlink: Option<String>,
+    end_time: String,
+    pipelines: Vec<LitaniPipeline>,
+}
+
+impl LitaniRun {
+    pub fn get_pipelines(self) -> Vec<LitaniPipeline> {
+        self.pipelines
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct LitaniParalellism {
+    trace: Vec<LitaniTrace>,
+    max_paralellism: Option<u32>,
+    n_proc: u32,
+}
+
+#[derive(Debug, Deserialize)]
+struct LitaniTrace {
+    running: u32,
+    finished: u32,
+    total: u32,
+    time: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct LitaniPipeline {
+    name: String,
+    ci_stages: Vec<LitaniStage>,
+    url: String,
+    status: String,
+}
+
+impl LitaniPipeline {
+    pub fn get_name(&self) -> &String {
+        &self.name
+    }
+
+    pub fn get_status(&self) -> bool {
+        match self.status.as_str() {
+            "fail" => false,
+            "success" => true,
+            _ => panic!("pipeline status is not \"fail\" nor \"success\""),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct LitaniStage {
+    jobs: Vec<LitaniJob>,
+    progress: u32,
+    complete: bool,
+    status: String,
+    url: String,
+    name: String,
+}
+
+// Some attributes in litani's `jobs` are not always included
+// or they are null, so we use `Option<...>` to deserialize them
+#[derive(Debug, Deserialize)]
+struct LitaniJob {
+    wrapper_arguments: LitaniWrapperArguments,
+    complete: bool,
+    start_time: Option<String>,
+    timeout_reached: Option<bool>,
+    command_return_code: Option<i32>,
+    memory_trace: Option<HashMap<String, String>>,
+    loaded_outcome_dict: Option<HashMap<String, String>>,
+    outcome: Option<String>,
+    wrapper_return_code: Option<i32>,
+    stdout: Option<Vec<String>>,
+    stderr: Option<Vec<String>>,
+    end_time: Option<String>,
+    duration_str: Option<String>,
+    duration: Option<u32>,
+}
+
+// Some attributes in litani's `wrapper_arguments` are not always included
+// or they are null, so we use `Option<...>` to deserialize them
+#[derive(Debug, Deserialize)]
+struct LitaniWrapperArguments {
+    subcommand: String,
+    verbose: bool,
+    very_verbose: bool,
+    inputs: Vec<String>,
+    command: String,
+    outputs: Option<Vec<String>>,
+    pipeline_name: String,
+    ci_stage: String,
+    cwd: Option<String>,
+    timeout: Option<u32>,
+    timeout_ok: Option<bool>,
+    timeout_ignore: Option<bool>,
+    ignore_returns: Option<bool>,
+    ok_returns: Vec<String>,
+    outcome_table: Option<HashMap<String, String>>,
+    interleave_stdout_stderr: bool,
+    stdout_file: Option<String>,
+    stderr_file: Option<String>,
+    pool: Option<u32>,
+    description: String,
+    profile_memory: bool,
+    profile_memory_interval: u32,
+    phony_outputs: Option<Vec<String>>,
+    tags: Option<String>,
+    status_file: String,
+    job_id: String,
+}
+
 /// Data structure representing a `Litani` build.
 pub struct Litani {
     /// A buffer of the `spawn`ed Litani jobs so far. `Litani` takes some time