Skip to content

Commit

Permalink
Implement should_panic and improved test diagnostics
Browse files Browse the repository at this point in the history
  • Loading branch information
@udoprog
udoprog committed Jun 25, 2023
1 parent 21239b1 commit 294390f
Show file tree
Hide file tree
Showing 6 changed files with 212 additions and 108 deletions.
229 changes: 131 additions & 98 deletions crates/rune/src/cli/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ use crate::cli::naming::Naming;
use crate::compile::{ItemBuf, FileSourceLoader};
use crate::modules::capture_io::CaptureIo;
use crate::runtime::{Value, Vm, VmError, VmResult, UnitFn};
use crate::doc::TestParams;
use crate::{Hash, Sources, Unit, Diagnostics, Source};
use crate::termcolor::{WriteColor, ColorSpec, Color};

#[derive(Parser, Debug, Clone)]
pub(super) struct Flags {
Expand Down Expand Up @@ -66,9 +68,11 @@ pub(super) async fn run<'p, I>(
where
I: IntoIterator<Item = EntryPoint<'p>>,
{
let colors = Colors::new();

let start = Instant::now();

let mut failures = 0usize;
let mut build_errors = 0usize;
let mut executed = 0usize;

let capture = crate::modules::capture_io::CaptureIo::new();
Expand All @@ -78,8 +82,6 @@ where
let mut cases = Vec::new();
let mut naming = Naming::default();

let mut build_error = false;

let mut include_std = false;

for opt in &flags.options {
Expand Down Expand Up @@ -126,7 +128,7 @@ where
diagnostics.emit(&mut io.stdout.lock(), &sources)?;

if diagnostics.has_error() || flags.warnings_are_errors && diagnostics.has_warning() {
build_error = true;
build_errors = build_errors.wrapping_add(1);
continue;
}

Expand All @@ -136,7 +138,7 @@ where
doc_visitors.push(doc_visitor);

for (hash, item) in functions.into_functions() {
cases.push(TestCase::new(hash, item, unit.clone(), sources.clone()));
cases.push(TestCase::new(hash, item, unit.clone(), sources.clone(), TestParams::default()));
}
}

Expand Down Expand Up @@ -174,7 +176,7 @@ where
diagnostics.emit(&mut io.stdout.lock(), &sources)?;

if diagnostics.has_error() || flags.warnings_are_errors && diagnostics.has_warning() {
build_error = true;
build_errors = build_errors.wrapping_add(1);
continue;
}

Expand All @@ -186,176 +188,207 @@ where
bail!("Compiling source did not result in a function at offset 0");
};

cases.push(TestCase::new(hash, test.item.clone(), unit.clone(), sources.clone()));
cases.push(TestCase::new(hash, test.item.clone(), unit.clone(), sources.clone(), test.params));
}
}

let runtime = Arc::new(context.runtime());
let mut failed = Vec::new();

for case in &mut cases {
let mut vm = Vm::new(runtime.clone(), case.unit.clone());

let total = cases.len();

for mut case in cases {
executed = executed.wrapping_add(1);

let success = case.execute(io, &mut vm, flags.quiet, Some(&capture)).await?;
let mut vm = Vm::new(runtime.clone(), case.unit.clone());
case.execute(&mut vm, &capture).await?;

if flags.quiet {
if case.outcome.is_ok() {
write!(io.stdout, ".")?;
continue;
} else {
write!(io.stdout, "f")?;
}
} else {
if case.outcome.is_ok() {
case.emit(io, &colors)?;
continue;
}
}

if !success {
failures = failures.wrapping_add(1);
failed.push(case);

if flags.fail_fast {
break;
}
if flags.fail_fast {
break;
}
}

if flags.quiet {
writeln!(io.stdout)?;
}

for case in &cases {
case.emit(io)?;
let failures = failed.len();

for case in failed {
case.emit(io, &colors)?;
}

let elapsed = start.elapsed();

writeln!(
io.stdout,
"Executed {} tests with {} failures ({} skipped) in {:.3} seconds",
"Executed {} tests with {} failures ({} skipped, {} build errors) in {:.3} seconds",
executed,
failures,
cases.len() - executed,
total - executed,
build_errors,
elapsed.as_secs_f64()
)?;

if !build_error && failures == 0 {
if build_errors == 0 && failures == 0 {
Ok(ExitCode::Success)
} else {
Ok(ExitCode::Failure)
}
}

#[derive(Debug)]
enum FailureReason {
Crash(VmError),
ReturnedNone,
ReturnedErr { output: Box<[u8]>, error: Value },
enum Outcome {
Ok,
Panic(VmError),
ExpectedPanic,
None,
Err(Value),
}

impl Outcome {
fn is_ok(&self) -> bool {
matches!(self, Outcome::Ok)
}
}

struct TestCase {
hash: Hash,
item: ItemBuf,
unit: Arc<Unit>,
sources: Arc<Sources>,
outcome: Option<FailureReason>,
buf: Vec<u8>,
params: TestParams,
outcome: Outcome,
output: Vec<u8>,
}

impl TestCase {
fn new(hash: Hash, item: ItemBuf, unit: Arc<Unit>, sources: Arc<Sources>) -> Self {
fn new(hash: Hash, item: ItemBuf, unit: Arc<Unit>, sources: Arc<Sources>, params: TestParams) -> Self {
Self {
hash,
item,
unit,
sources,
outcome: None,
buf: Vec::new(),
params,
outcome: Outcome::Ok,
output: Vec::new(),
}
}

async fn execute(
&mut self,
io: &mut Io<'_>,
vm: &mut Vm,
quiet: bool,
capture_io: Option<&CaptureIo>,
) -> Result<bool> {
if !quiet {
write!(io.stdout, "{} ", self.item)?;
}

capture_io: &CaptureIo,
) -> Result<()> {
let result = match vm.execute(self.hash, ()) {
Ok(mut execution) => execution.async_complete().await,
Err(err) => VmResult::Err(err),
};

if let Some(capture_io) = capture_io {
let _ = capture_io.drain_into(&mut self.buf);
}
capture_io.drain_into(&mut self.output)?;

self.outcome = match result {
VmResult::Ok(v) => match v {
Value::Result(result) => match result.take()? {
Ok(..) => None,
Err(error) => Some(FailureReason::ReturnedErr {
output: self.buf.as_slice().into(),
error,
}),
Ok(..) => Outcome::Ok,
Err(error) => Outcome::Err(error),
},
Value::Option(option) => match *option.borrow_ref()? {
Some(..) => None,
None => Some(FailureReason::ReturnedNone),
Some(..) => Outcome::Ok,
None => Outcome::None,
},
_ => None,
_ => Outcome::Ok,
},
VmResult::Err(e) => Some(FailureReason::Crash(e)),
VmResult::Err(e) => {
Outcome::Panic(e)
}
};

if quiet {
match &self.outcome {
Some(FailureReason::Crash(..)) => {
writeln!(io.stdout, "F")?;
}
Some(FailureReason::ReturnedErr { .. }) => {
write!(io.stdout, "f")?;
}
Some(FailureReason::ReturnedNone { .. }) => {
write!(io.stdout, "n")?;
}
None => {
write!(io.stdout, ".")?;
}
}
} else {
match &self.outcome {
Some(FailureReason::Crash(..)) => {
writeln!(io.stdout, "failed")?;
}
Some(FailureReason::ReturnedErr { .. }) => {
writeln!(io.stdout, "returned error")?;
}
Some(FailureReason::ReturnedNone { .. }) => {
writeln!(io.stdout, "returned none")?;
}
None => {
writeln!(io.stdout, "passed")?;
}
if self.params.should_panic {
if matches!(self.outcome, Outcome::Panic(..)) {
self.outcome = Outcome::Ok;
} else {
self.outcome = Outcome::ExpectedPanic;
}
}

self.buf.clear();
Ok(self.outcome.is_none())
Ok(())
}

fn emit(&self, io: &mut Io<'_>) -> Result<()> {
if let Some(outcome) = &self.outcome {
match outcome {
FailureReason::Crash(err) => {
writeln!(io.stdout, "----------------------------------------")?;
writeln!(io.stdout, "Test: {}\n", self.item)?;
err.emit(io.stdout, &self.sources)?;
}
FailureReason::ReturnedNone { .. } => {}
FailureReason::ReturnedErr { output, error, .. } => {
writeln!(io.stdout, "----------------------------------------")?;
writeln!(io.stdout, "Test: {}\n", self.item)?;
writeln!(io.stdout, "Error: {:?}\n", error)?;
writeln!(io.stdout, "-- output --")?;
io.stdout.write_all(output)?;
writeln!(io.stdout, "-- end of output --")?;
}
fn emit(self, io: &mut Io<'_>, colors: &Colors) -> Result<()> {
write!(io.stdout, "Test: {}: ", self.item)?;

match &self.outcome {
Outcome::Panic(error) => {
io.stdout.set_color(&colors.error)?;
writeln!(io.stdout, "Panicked")?;
io.stdout.reset()?;

error.emit(io.stdout, &self.sources)?;
}
Outcome::ExpectedPanic => {
io.stdout.set_color(&colors.error)?;
writeln!(io.stdout, "Expected panic because of `should_panic`, but ran without issue")?;
io.stdout.reset()?;
}
Outcome::Err(error) => {
io.stdout.set_color(&colors.error)?;
write!(io.stdout, "Returned Err: ")?;
io.stdout.reset()?;
writeln!(io.stdout, "{:?}", error)?;
}
Outcome::None => {
io.stdout.set_color(&colors.error)?;
writeln!(io.stdout, "Returned None")?;
io.stdout.reset()?;
}
Outcome::Ok => {
io.stdout.set_color(&colors.passed)?;
writeln!(io.stdout, "Ok")?;
io.stdout.reset()?;
}
}

if !self.outcome.is_ok() && !self.output.is_empty() {
writeln!(io.stdout, "-- output --")?;
io.stdout.write_all(&self.output)?;
writeln!(io.stdout, "-- end of output --")?;
}

Ok(())
}
}

struct Colors {
error: ColorSpec,
passed: ColorSpec,
}

impl Colors {
fn new() -> Self {
let mut this = Self {
error: ColorSpec::new(),
passed: ColorSpec::new(),
};

this.error.set_fg(Some(Color::Red));
this.passed.set_fg(Some(Color::Green));
this
}
}
2 changes: 1 addition & 1 deletion crates/rune/src/doc.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ mod context;
pub(self) use self::context::Context;

mod artifacts;
pub(crate) use self::artifacts::Artifacts;
pub(crate) use self::artifacts::{TestParams, Artifacts};

mod templating;

Expand Down
2 changes: 2 additions & 0 deletions crates/rune/src/doc/artifacts.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ use anyhow::{Context as _, Error, Result};
pub(crate) struct TestParams {
/// If the test should not run.
pub(crate) no_run: bool,
/// If the test should panic.
pub(crate) should_panic: bool,
}

/// A discovered test.
Expand Down
4 changes: 4 additions & 0 deletions crates/rune/src/doc/build/markdown.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,10 @@ where
params.no_run = true;
continue;
}
"should_panic" => {
params.should_panic = true;
continue;
}
RUNE_TOKEN => {
(RUNE_TOKEN, RUST_TOKEN, true)
}
Expand Down
1 change: 1 addition & 0 deletions crates/rune/src/modules/option.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub fn module() -> Result<Module, ContextError> {
module.associated_function("transpose", transpose_impl)?;
module.associated_function("unwrap", unwrap_impl)?;
module.associated_function("unwrap_or", Option::<Value>::unwrap_or)?;
module.associated_function("ok_or", Option::<Value>::ok_or::<Value>)?;
module.function_meta(unwrap_or_else)?;
module.associated_function(Protocol::INTO_ITER, option_iter)?;
Ok(module)
Expand Down
Loading

0 comments on commit 294390f

Please sign in to comment.