From 79eb37e193f90c6777b7a993677d934dd30b8eaf Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 11 Sep 2024 12:32:38 -0700 Subject: [PATCH] Support multi-line run/fail directives in CLI tests Also add a `readme.wat` test explainer as to what's going on. --- Cargo.lock | 1 + Cargo.toml | 1 + tests/cli.rs | 157 ++++++++++++++---------- tests/cli/readme.wat | 76 ++++++++++++ tests/cli/readme.wat.print.stdout | 3 + tests/cli/readme.wat.should-fail.stderr | 1 + 6 files changed, 172 insertions(+), 67 deletions(-) create mode 100644 tests/cli/readme.wat create mode 100644 tests/cli/readme.wat.print.stdout create mode 100644 tests/cli/readme.wat.should-fail.stderr diff --git a/Cargo.lock b/Cargo.lock index 95e51ef058..05dc757d11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1760,6 +1760,7 @@ dependencies = [ "cpp_demangle", "env_logger", "gimli 0.30.0", + "indexmap 2.4.0", "is_executable", "libtest-mimic", "log", diff --git a/Cargo.toml b/Cargo.toml index 45318fb1fd..a65bf4d555 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -178,6 +178,7 @@ tempfile = "3.1" wast = { path = 'crates/wast' } pretty_assertions = { workspace = true } libtest-mimic = { workspace = true } +indexmap = { workspace = true } [[test]] name = "cli" diff --git a/tests/cli.rs b/tests/cli.rs index 96427e1240..6ecca33fe0 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1,31 +1,12 @@ //! A test suite to test the `wasm-tools` CLI itself. //! -//! This test suite will look for `*.wat` files in the `tests/cli/**` directory, -//! recursively. Each wat file must have a directive of the form: -//! -//! ;; RUN: ... -//! -//! where `...` is a space-separate set of command to pass to the `wasm-tools` -//! CLI. The `%` argument is replaced with the path to the current file. For -//! example: -//! -//! ;; RUN: dump % -//! -//! would execute `wasm-tools dump the-current-file.wat`. The `cli` directory -//! additionally contains `*.stdout` and `*.stderr` files to assert the output -//! of the subcommand. Files are not present if the stdout/stderr are empty. -//! -//! This also supports a limited form of piping along the lines of: -//! -//! ;; RUN: strip % | objdump -//! -//! where a `|` will execute the first subcommand and pipe its stdout into the -//! stdin of the next command. -//! -//! Use `BLESS=1` in the environment to auto-update expectation files. Be sure -//! to look at the diff! +//! This test suite will look for `*.wat` and `*.wit` files in the +//! `tests/cli/**` directory, recursively. For more information about supported +//! directives and features of this test suite see the `tests/cli/readme.wat` +//! file which has an explanatory comment at the top for what's going on. -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{bail, Context, Result}; +use indexmap::IndexMap; use libtest_mimic::{Arguments, Trial}; use pretty_assertions::StrComparison; use std::env; @@ -59,54 +40,96 @@ fn main() { libtest_mimic::run(&args, trials).exit(); } -fn wasm_tools_exe() -> Command { - Command::new(env!("CARGO_BIN_EXE_wasm-tools")) -} - fn run_test(test: &Path, bless: bool) -> Result<()> { let contents = std::fs::read_to_string(test)?; - let (line, should_fail) = contents + + let mut directives = contents .lines() - .filter_map(|l| { - let run = l.strip_prefix(";; RUN: ").or(l.strip_prefix("// RUN: ")); - let fail = l.strip_prefix(";; FAIL: ").or(l.strip_prefix("// FAIL: ")); - run.map(|l| (l, false)).or(fail.map(|l| (l, true))) - }) - .next() - .ok_or_else(|| anyhow!("no line found with `;; RUN: ` directive"))?; - - let mut cmd = wasm_tools_exe(); - let mut stdin = None; - let tempdir = TempDir::new()?; - for arg in line.split_whitespace() { - let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap()); - if arg == "|" { - let output = execute(&mut cmd, stdin.as_deref(), false)?; - stdin = Some(output.stdout); - cmd = wasm_tools_exe(); - } else if arg == "%" { - cmd.arg(test); - } else { - cmd.arg(arg); + .enumerate() + .filter(|(_, l)| !l.is_empty()) + .filter_map(|(i, l)| { + l.strip_prefix("// ") + .or(l.strip_prefix(";; ")) + .map(|l| (i + 1, l)) + }); + + let mut commands = IndexMap::new(); + + while let Some((i, line)) = directives.next() { + let run = line.strip_prefix("RUN"); + let fail = line.strip_prefix("FAIL"); + let (directive, should_fail) = match run.map(|l| (l, false)).or(fail.map(|l| (l, true))) { + Some(pair) => pair, + None => continue, + }; + let (cmd, name) = match directive.strip_prefix("[") { + Some(prefix) => match prefix.find("]:") { + Some(i) => (&prefix[i + 2..], &prefix[..i]), + None => bail!("line {i}: failed to find `]:` after `[`"), + }, + None => match directive.strip_prefix(":") { + Some(cmd) => (cmd, ""), + None => bail!("line {i}: failed to find `:` after `RUN` or `FAIL`"), + }, + }; + let mut cmd = cmd.to_string(); + while cmd.ends_with("\\") { + cmd.pop(); + match directives.next() { + Some((_, line)) => cmd.push_str(line), + None => bail!("line {i}: directive ends in `\\` but nothing on next line"), + } + } + + match commands.insert(name, (cmd, should_fail)) { + Some(_) => bail!("line {i}: duplicate directive named {name:?}"), + None => {} } } - let output = execute(&mut cmd, stdin.as_deref(), should_fail)?; - let extension = test.extension().unwrap().to_str().unwrap(); - assert_output( - bless, - &output.stdout, - &test.with_extension(&format!("{extension}.stdout")), - &tempdir, - ) - .context("failed to check stdout expectation (auto-update with BLESS=1)")?; - assert_output( - bless, - &output.stderr, - &test.with_extension(&format!("{extension}.stderr")), - &tempdir, - ) - .context("failed to check stderr expectation (auto-update with BLESS=1)")?; + if commands.is_empty() { + bail!("failed to find `// RUN: ...` or `// FAIL: ...` at the top of this file"); + } + let exe = Path::new(env!("CARGO_BIN_EXE_wasm-tools")); + let tempdir = TempDir::new_in(exe.parent().unwrap())?; + for (name, (line, should_fail)) in commands { + let mut cmd = Command::new(exe); + let mut stdin = None; + for arg in line.split_whitespace() { + let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap()); + if arg == "|" { + let output = execute(&mut cmd, stdin.as_deref(), false)?; + stdin = Some(output.stdout); + cmd = Command::new(exe); + } else if arg == "%" { + cmd.arg(test); + } else { + cmd.arg(arg); + } + } + + let output = execute(&mut cmd, stdin.as_deref(), should_fail)?; + let extension = test.extension().unwrap().to_str().unwrap(); + let extension = if name.is_empty() { + extension.to_string() + } else { + format!("{extension}.{name}") + }; + assert_output( + bless, + &output.stdout, + &test.with_extension(&format!("{extension}.stdout")), + &tempdir, + ) + .context("failed to check stdout expectation (auto-update with BLESS=1)")?; + assert_output( + bless, + &output.stderr, + &test.with_extension(&format!("{extension}.stderr")), + &tempdir, + ) + .context("failed to check stderr expectation (auto-update with BLESS=1)")?; + } Ok(()) } diff --git a/tests/cli/readme.wat b/tests/cli/readme.wat new file mode 100644 index 0000000000..600aae76c5 --- /dev/null +++ b/tests/cli/readme.wat @@ -0,0 +1,76 @@ +;; This is intended to be a self-documenting test which explains what's +;; possible in directives for this test suite in `tests/cli/*`. The purpose of +;; this test suite is to make it as easy as dropping a file in this directory to +;; test the `wasm-tools` CLI tool and its subcommands. The test file itself is +;; generally the input to the test and what's being tested will be present in +;; comments at the top of the file with directives. +;; +;; All test directives must come in comments at the start of the file: +;; +;; RUN: validate % +;; +;; The `RUN` prefix indicates that the specified `wasm-tools` subcommand should +;; be executed. It's possible to have more than one test in a file by having +;; named directives such as: +;; +;; RUN[validate-again]: validate % +;; +;; Directive names must be unique, so using `validate-again` would not be valid. +;; Additionally you can't use an unprefixed directive more than once so using +;; `RUN: ...` here again would not be allowed for example. +;; +;; You can also use the `FAIL` directive to indicate that the subcommand should +;; fail rather than succeed. +;; +;; FAIL[should-fail]: validate % --features=-simd +;; +;; As you can see directives can have comments around them. Directives are +;; identified as comment lines starting with `RUN` or `FAIL`. +;; +;; Within directives there are a few feature. First as you've seen the `%` value +;; will be substituted with the current filename which means: +;; +;; RUN[subst]: validate % +;; +;; means to run `wasm-tools validate tests/cli/readme.wat` and test the result +;; is successful. +;; +;; You can additionally use `|` to pipe commands together by feeding the stdout +;; of the previous command into the stdin of the next command. +;; +;; RUN[pipe]: print % | validate +;; +;; Note that when piping commands the intermediate commands before the final +;; one, in this case `print` being the intermediate, must all succeed. +;; +;; Tests also assert the stdout/stderr of the command being tested. For example +;; if printing is tested: +;; +;; RUN[print]: print % +;; +;; then this tests that `tests/cli/readme.wat.print.stdout` is the result of +;; `wasm-tools print tests/cli/readme.wat`. Note that this can be tedious to +;; update so you can use the environment variable `BLESS=1` to automatically +;; update all test assertions. This can then be reviewed after the test is +;; passing for accuracy. +;; +;; Each test additionally can have a temporary directory available to it which +;; is accessible with the `%tmpdir` substitution. For example: +;; +;; RUN[tmpdir]: print % -o %tmpdir/foo.wat | validate %tmpdir/foo.wat +;; +;; Note that temporary directories are persisted across tests in the same file, +;; but different files all get different temporary directories. +;; +;; You can also split commands across multiple lines: +;; +;; RUN[multiline]: print % | \ +;; validate +;; +;; here the `\` character is deleted and the next line is concatenated. + + +;; this is the contents of the test, mostly empty in this case. +(module + (type (func (result v128))) +) diff --git a/tests/cli/readme.wat.print.stdout b/tests/cli/readme.wat.print.stdout new file mode 100644 index 0000000000..c8cd2b5ed9 --- /dev/null +++ b/tests/cli/readme.wat.print.stdout @@ -0,0 +1,3 @@ +(module + (type (;0;) (func (result v128))) +) diff --git a/tests/cli/readme.wat.should-fail.stderr b/tests/cli/readme.wat.should-fail.stderr new file mode 100644 index 0000000000..45ead379a2 --- /dev/null +++ b/tests/cli/readme.wat.should-fail.stderr @@ -0,0 +1 @@ +error: SIMD support is not enabled (at offset 0xb)