Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Change presentation of test results #113

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion utest/shared/src/main/scala/utest/asserts/Asserts.scala
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ object Asserts {
failures match{
case Seq() => () // nothing failed, do nothing
case Seq(failure) => throw failure
case multipleFailures => throw new MultipleErrors(multipleFailures:_*)
case multipleFailures => throw MultipleErrors(multipleFailures:_*)
}
}

Expand Down
9 changes: 6 additions & 3 deletions utest/shared/src/main/scala/utest/asserts/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@ package object asserts extends utest.asserts.Asserts[DummyTypeclass]{

type AssertEntry[T] = (String, (TestValue => Unit) => T)

def renderTestValue(testValue: TestValue) = {
val TestValue(name, tpe, value) = testValue
s"${name.magenta}: ${tpe.toString.yellow} = ${value.toString.blue}"
}

/**
* Shorthand to quickly throw a utest.AssertionError, together with all the
* macro-debugging goodness
*/
def assertError(msgPrefix: String, logged: Seq[TestValue], cause: Throwable = null) = {
throw AssertionError(
msgPrefix + Option(cause).fold("")(e => s"\ncaused by: $e") + logged.map{
case TestValue(name, tpe, thing) => s"\n$name: $tpe = $thing"
}.mkString,
msgPrefix + Option(cause).fold("")(e => s"\ncaused by: ${e.toString.red}") + logged.map(v => "\n" + renderTestValue(v)).mkString,
logged,
cause
)
Expand Down
54 changes: 35 additions & 19 deletions utest/shared/src/main/scala/utest/framework/Formatter.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package utest
package framework
//import acyclic.file
import scala.util.{Failure, Success}
import utest.ColorStrings

/**
* Default implementation of [[Formatter]], also used by the default SBT test
Expand All @@ -24,34 +25,49 @@ trait Formatter {
offset: Result => String = _ => ""): String = {


val cutUnit = r.value match{
case Success(()) => "Success"
case Success(v) =>
"Success " + ("\n"+v).replace("\n", "\n" + offset(r) + formatStartColor(true))
case Failure(e) => errorFormatter(e)
val durationStr = Console.RESET + (r.milliDuration.toString + " ms").faint
val cutUnit = r.value match {
case Success(()) => durationStr
case Success(v: Any) =>
val offsetStr = offset(r)
durationStr + " \n" + offsetStr + v.toString.blue
case Failure(e) => "\n" + errorFormatter(e)
}

val truncUnit =
if (cutUnit.length <= formatTruncate) cutUnit
else cutUnit.take(formatTruncate) + "..."

if (formatColor) formatStartColor(r.value.isSuccess) + truncUnit + formatEndColor else truncUnit
if (formatColor) formatStartColor(success = r.value.isSuccess) + truncUnit + formatEndColor else truncUnit
}

def formatSingle(path: Seq[String], r: Result): Option[String] = Some{
path.map("." + _).mkString + "\t\t" + prettyTruncate(
r,
e => s"${("\n"+e.toString).replace("\n", "\n" + (" " * r.name.length) + "\t\t" + formatStartColor(false))}"
)
}
private val successIcon = "\u25C9".green
private val failureIcon = "\u25C9".red
def format(results: Tree[Result]): Option[String] = Some {
def errorFormatter(ex: Throwable): String = {
val causation = Option(ex.getCause) match {
case Some(cause) =>
" caused by: " + cause.toString.red
case None =>
""
}

def format(results: Tree[Result]): Option[String] = Some{
def errorFormatter(ex: Throwable): String =
s"Failure('$ex'${Option(ex.getCause).fold("")(cause => s" caused by '$cause'")})"
val errStr = ex.toString
val assertPrefix = "utest.AssertionError:"
if (errStr.startsWith(assertPrefix)) {
assertPrefix.bold.white.redBg + errStr.stripPrefix(assertPrefix)
} else {
errStr.bold.white.redBg + "\n" + causation
}
}

results.map(r =>
r.name + "\t\t" + prettyTruncate(r, errorFormatter, r => " " * r.name.length)
).reduce(_ + _.map("\n" + _).mkString.replace("\n", "\n "))
results.map(result => {
val isFailure = result.value.isFailure
val icon = if (isFailure) failureIcon else successIcon
val nameSegment = " " + result.name + " "
val testNameText = if (isFailure) nameSegment.bold.white.redBg else nameSegment.bold
icon + testNameText + prettyTruncate(result, errorFormatter, r => " ")
}
).reduce(_ + _.map("\n" + _).mkString.replace("\n", "\n "))
}
}

9 changes: 5 additions & 4 deletions utest/shared/src/main/scala/utest/framework/Model.scala
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,22 @@ import scala.concurrent.Future
import utest.PlatformShims

case class TestPath(value: Seq[String])
object TestPath{
object TestPath {
@reflect.internal.annotations.compileTimeOnly(
"TestPath is only available within a uTest suite, and not outside."
)
implicit def synthetic: TestPath = ???
}
object Test{

object Test {
/**
* Creates a test from a set of children, a name a a [[TestThunKTree]].
* Creates a test from a set of children, a name a a [[TestThunkTree]].
* Generally called by the [[TestSuite.apply]] macro and doesn't need to
* be called manually.
*/
def create(tests: (String, (String, TestThunkTree) => Tree[Test])*)
(name: String, testTree: TestThunkTree): Tree[Test] = {
new Tree(
Tree(
new Test(name, testTree),
tests.map{ case (k, v) => v(k, testTree) }
)
Expand Down
19 changes: 16 additions & 3 deletions utest/shared/src/main/scala/utest/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,23 @@ import scala.concurrent.duration._
* Created by haoyi on 1/24/14.
*/
package object utest {
implicit val retryInterval = new RetryInterval(100.millis)
implicit val retryMax = new RetryMax(1.second)
implicit val retryInterval = RetryInterval(100.millis)
implicit val retryMax = RetryMax(1.second)
import scala.language.experimental.macros

implicit class ColorStrings(s: String) {
private def wrap(ansiColorSequence: String) = ansiColorSequence + s + Console.RESET
def blue = wrap(Console.BLUE)
def green = wrap(Console.GREEN)
def red = wrap(Console.RED)
def redBg = wrap(Console.RED_B)
def bold = wrap(Console.BOLD)
def yellow = wrap(Console.YELLOW)
def white = wrap(Console.WHITE)
def magenta = wrap(Console.MAGENTA)
def faint = wrap("\u001b[2m")
}

type Show = asserts.Show
/**
* Extension methods to allow you to create tests via the "omg"-{ ... }
Expand Down Expand Up @@ -47,7 +60,7 @@ package object utest {
* Extension methods on `Tree[Test]` in order to conveniently run the tests
* and aggregate the results
*/
implicit def toTestSeq(t: Tree[Test]) = new TestTreeSeq(t)
implicit def toTestSeq(t: Tree[Test]): TestTreeSeq = new TestTreeSeq(t)


/**
Expand Down
13 changes: 6 additions & 7 deletions utest/shared/src/main/scala/utest/runner/BaseRunner.scala
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package utest
package runner
//import acyclic.file
import scala.concurrent.ExecutionContext

import sbt.testing._
import utest.TestSuite

import scala.util.Failure

import org.scalajs.testinterface.TestUtils
Expand Down Expand Up @@ -49,15 +50,15 @@ abstract class BaseRunner(val args: Array[String],
})
}

val title = s"Starting Suite " + name
val title = s"Starting Suite " + name.bold.red
val dashes = "-" * ((80 - title.length) / 2)
loggers.foreach(_.info(dashes + title + dashes))

val (indices, found) = suite.tests.resolve(selector)

addTotal(found.length)

implicit val ec =
implicit val ec: ExecutionContext =
if (utest.framework.ArgParse.find("--parallel", _.toBoolean, false, true)(args)){
scala.concurrent.ExecutionContext.global
}else{
Expand All @@ -68,18 +69,16 @@ abstract class BaseRunner(val args: Array[String],
(subpath, s) => {
if(s.value.isSuccess) incSuccess() else incFailure()

val str = suite.formatSingle(selector ++ subpath, s)
handleEvent(new OptionalThrowable(), Status.Success)
str.foreach{msg => loggers.foreach(_.info(name + "" + msg))}
s.value match{
s.value match {
case Failure(e) =>
handleEvent(new OptionalThrowable(e), Status.Failure)
// Trim the stack trace so all the utest internals don't get shown,
// since the user probably doesn't care about those anyway
e.setStackTrace(
e.getStackTrace.takeWhile(_.getClassName != "utest.framework.TestThunkTree")
)
addFailure(name + "" + str.getOrElse(""))
addFailure(name)
addTrace(
if (e.isInstanceOf[SkippedOuterFailure]) ""
else e.getStackTrace.map(_.toString).mkString("\n")
Expand Down
34 changes: 17 additions & 17 deletions utest/shared/src/main/scala/utest/runner/MasterRunner.scala
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ final class MasterRunner(args: Array[String],
if (!results.compareAndSet(old, r :: old)) addResult(r)
}

@tailrec final def addFailure(r: String): Unit = {
@tailrec def addFailure(r: String): Unit = {
val old = failures.get()
if (!failures.compareAndSet(old, r :: old)) addFailure(r)
}
@tailrec final def addTrace(r: String): Unit = {
@tailrec def addTrace(r: String): Unit = {
val old = traces.get()
if (!traces.compareAndSet(old, r :: old)) addTrace(r)
}
Expand All @@ -49,25 +49,25 @@ final class MasterRunner(args: Array[String],

val body = results.get.mkString("\n")

val failureMsg = if (failures.get() == Nil) ""
else Seq(
Console.RED + "Failures:",
failures.get()
.zip(traces.get())
// We pre-pending to a list, so need to reverse to make the order correct
.reverse
// ignore those with an empty trace, e.g. utest.SkippedOuterFailures,
// since those are generally just spam (we already can see the outer failure)
.collect{case (f, t) if t != "" => f + ("\n" + t).replace("\n", "\n"+Console.RED)}
.mkString("\n")
).mkString("\n")
val failureMsg = "" // if (failures.get() == Nil) ""
// else Seq(
// Console.RED + "Failures:",
// failures.get()
// .zip(traces.get())
// // We pre-pending to a list, so need to reverse to make the order correct
// .reverse
// // ignore those with an empty trace, e.g. utest.SkippedOuterFailures,
// // since those are generally just spam (we already can see the outer failure)
// .collect{case (f, t) if t != "" => f + ("\n" + t).replace("\n", "\n"+Console.RED)}
// .mkString("\n")
// ).mkString("\n")
Seq(
header,
body,
failureMsg,
s"Tests: $total",
s"Passed: $success",
s"Failed: $failure"
s"Tests: ".bold + totalCount.toString.blue,
s"Passed: ".bold + successCount.toString.blue,
s"Failed: ".bold + (if (failureCount == 0) failureCount.toString.green else s" $failureCount ".redBg.bold.white)
).mkString("\n")
}

Expand Down
8 changes: 5 additions & 3 deletions utest/shared/src/test/scala/test/utest/AssertsTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,11 @@ object AssertsTests extends utest.TestSuite{
"Logging didn't capture the locals properly " + logged
)

* - Predef.assert(
e.toString.contains("y: String = 2") && e.toString.contains("x: Int = 1"),
"Logging doesn't display local values properly " + e.toString
val errStr = e.toString
val yRendered = asserts.renderTestValue(TestValue("y", "String", 2))
val xRendered = asserts.renderTestValue(TestValue("x", "Int", 1))
"Logging doesn't display local values properly" - assert(
errStr.contains(yRendered) && e.toString.contains(xRendered)
)

* - Predef.assert(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package test.utest
import utest._

object DisablePrint2Tests extends utest.TestSuite{
override def formatSingle(path: Seq[String], res: utest.framework.Result) = None
override def formatColor = false
def tests = this{
'hello{
Expand Down