More summary improvements. (#122)

* Output info about filtered files.

* Supported JSON-based sub-reasons of NotSupported

* Refactored Summary.Elem

* Rename elements in Summary

yet --> notsupported

for other elements, e.g. timeout
timeouts -> timeout
timeout --> timeout_count

* Include static filter info in summary

* Refactored TestFilter.getFilters

* Initialize summary for parseTest.

Previously didn't initialize summary for parseTest. This fixes that.

Also some stylistic changes e.g. snake_case -> camelCase

* Refactored Summary.toString using helpers of Appender

* Removed unnecessary file

* Refactored ProgressBar and Summary for more readable info

* Fixed bug in Summary.toSring

* Apply suggested changes

* Use Reasonpath instead of List[String]

Co-authored-by: jhnaldo <jhpjhp0223@gmail.com>

src/main/resources/manuals/default/test262/filtered.json

@@ -304,7 +304,7 @@
     "built-ins/AsyncGeneratorPrototype/return/return-state-completed-broken-promise",
     "built-ins/AsyncGeneratorPrototype/return/return-suspendedStart-broken-promise"
   ],
-  "yet tests": [
+  "not yet categorized tests": [
     "built-ins/Array/prototype/sort/stability-2048-elements",
     "built-ins/AsyncFromSyncIteratorPrototype/next/iterator-result-rejected",
     "built-ins/AsyncFromSyncIteratorPrototype/throw/iterator-result",
@@ -365,6 +365,7 @@
     "built-ins/Promise/all/iter-arg-is-string-resolve",
     "built-ins/Promise/allSettled/iter-arg-is-string-resolve",
     "built-ins/Promise/any/iter-arg-is-empty-string-reject",
+    "built-ins/RegExp/lookBehind/sliced-strings",
     "built-ins/Set/prototype/entries/returns-iterator",
     "built-ins/Set/prototype/entries/returns-iterator-empty",
     "built-ins/Set/prototype/values/returns-iterator",
@@ -391,6 +392,15 @@
     "built-ins/String/prototype/S15.5.4_A2",
     "built-ins/String/prototype/concat/S15.5.4.6_A3",
     "built-ins/String/prototype/constructor/S15.5.4.1_A1_T2",
+    "built-ins/String/prototype/matchAll/length",
+    "built-ins/String/prototype/matchAll/name",
+    "built-ins/String/prototype/matchAll/prop-desc",
+    "built-ins/String/prototype/matchAll/regexp-is-null",
+    "built-ins/String/prototype/matchAll/regexp-is-undefined",
+    "built-ins/String/prototype/matchAll/regexp-matchAll-invocation",
+    "built-ins/String/prototype/matchAll/regexp-matchAll-not-callable",
+    "built-ins/String/prototype/matchAll/regexp-prototype-has-no-matchAll",
+    "built-ins/String/prototype/matchAll/this-val-non-obj-coercible",
     "built-ins/Symbol/prototype/Symbol.toPrimitive/redefined-symbol-wrapper-ordinary-toprimitive",
     "built-ins/Symbol/prototype/Symbol.toPrimitive/removed-symbol-wrapper-ordinary-toprimitive",
     "language/computed-property-names/class/static/generator-prototype",

src/main/scala/esmeta/error/NotSupported.scala

@@ -1,4 +1,9 @@
 package esmeta.error
 
-// not supported errors
-case class NotSupported(msg: String) extends ESMetaError(msg, "NotSupported")
+/** not supported errors */
+case class NotSupported(reasonPath: NotSupported.ReasonPath)
+  extends ESMetaError(s"${reasonPath.mkString("/")}", "NotSupported")
+object NotSupported:
+  type Reason = String
+  type ReasonPath = List[Reason]
+  def apply(reason: Reason): NotSupported = NotSupported(List(reason))

src/main/scala/esmeta/error/Test262Error.scala

@@ -1,14 +1,14 @@
 package esmeta.error
 
 import esmeta.LINE_SEP
-import esmeta.util.SummaryElem
+import esmeta.util.Summary
 
 sealed abstract class Test262Error(msg: String)
   extends ESMetaError(msg, s"Test262Error")
 
-// Test262 failure
-case class Test262Fail(fails: SummaryElem)
+/** Test262 failure */
+case class Test262Fail(fails: Summary.Elem)
   extends Test262Error(
     s"${fails.size} tests are failed:" + LINE_SEP +
-    fails.elements.toList.sorted.mkString(LINE_SEP),
+    fails.all.sorted.mkString(LINE_SEP),
   )

src/main/scala/esmeta/interpreter/Interpreter.scala

@@ -245,7 +245,7 @@ class Interpreter(
         case v           => throw NoNt(nt, v)
       st.allocList(eval(ast).asAst.getItems(name).map(AstValue(_)))
     case EYet(msg) =>
-      throw NotSupported(msg)
+      throw NotSupported(List("metalanguage", msg))
     case EContains(list, elem, field) =>
       val l = eval(list).getList(list, st)
       val e = eval(elem)
@@ -666,7 +666,7 @@ object Interpreter {
         Str(ESValueParser.parseTRVTemplateTail(str))
       case (_, "Contains") => Bool(false)
       case ("RegularExpressionLiteral", name) =>
-        throw NotSupported(s"RegularExpressionLiteral.$sdoName")
+        throw NotSupported(List("RegExp", sdoName))
       case _ =>
         throw InvalidAstProp(lex, Str(sdoName))
     }

src/main/scala/esmeta/phase/Test262Test.scala

@@ -41,7 +41,7 @@ case object Test262Test extends Phase[CFG, Summary] {
     )
 
     // if summary has failed test case, throws an exception
-    if (summary.fail > 0) throw Test262Fail(summary.fails)
+    if (summary.failCount > 0) throw Test262Fail(summary.fail)
 
     // return summary
     summary

src/main/scala/esmeta/state/util/Stringifier.scala

@@ -40,7 +40,7 @@ class Stringifier(detail: Boolean, location: Boolean) {
       app :> "context: " >> st.context
       given Rule[List[String]] = iterableRule("[", ", ", "]")
       app :> "call-stack: "
-      app.wrapIterable("[", "]")(st.callStack)
+      app.wrapIterable("[", ",", "]")(st.callStack)
       app :> "globals: " >> st.globals
       app :> "heap: " >> st.heap
     }

src/main/scala/esmeta/test262/Test262.scala

@@ -3,6 +3,7 @@ package esmeta.test262
 import esmeta.*
 import esmeta.cfg.CFG
 import esmeta.error.{NotSupported, InvalidExit, UnexpectedParseResult}
+import esmeta.error.NotSupported.*
 import esmeta.es.*
 import esmeta.es.util.*
 import esmeta.interpreter.Interpreter
@@ -70,17 +71,19 @@ case class Test262(
   def getProgressBar(
     name: String,
     targetTests: List[Test],
+    removed: Iterable[(Test, ReasonPath)] = Nil,
     useProgress: Boolean = false,
     useErrorHandler: Boolean = true,
   ): ProgressBar[Test] = ProgressBar(
     msg = s"Run Test262 $name tests",
     iterable = targetTests,
+    notSupported = removed,
     getName = (test, _) => test.relName,
     errorHandler = (e, summary, name) =>
       if (useErrorHandler) e match
-        case NotSupported(msg)   => summary.yets += s"$name - $msg"
-        case _: TimeoutException => summary.timeouts += name
-        case e: Throwable        => summary.fails += s"$name - ${e.getMessage}"
+        case NotSupported(reasons) => summary.notSupported.add(name, reasons)
+        case _: TimeoutException   => summary.timeout.add(name)
+        case e: Throwable          => summary.fail.add(name, e.getMessage)
       else throw e,
     verbose = useProgress,
   )
@@ -106,6 +109,7 @@ case class Test262(
     val progressBar = getProgressBar(
       name = "eval",
       targetTests = targetTests,
+      removed = removed,
       useProgress = useProgress,
       useErrorHandler = multiple,
     )
@@ -121,7 +125,6 @@ case class Test262(
     logForTests(
       name = "eval",
       progressBar = progressBar,
-      removed = removed,
       postSummary = if (useCoverage) cov.toString else "",
       log = log && multiple,
     )(
@@ -158,14 +161,14 @@ case class Test262(
     val progressBar = getProgressBar(
       name = "parse",
       targetTests = targetTests,
+      removed = removed,
       useProgress = useProgress,
     )
 
     // run tests with logging
     logForTests(
       name = "parse",
       progressBar = progressBar,
-      removed = removed,
       log = log,
     )(
       // check parsing result with its corresponding code
@@ -211,7 +214,6 @@ case class Test262(
   private def logForTests(
     name: String,
     progressBar: ProgressBar[Test],
-    removed: Map[String, List[Test]],
     postSummary: => String = "",
     log: Boolean = false,
   )(
@@ -223,31 +225,19 @@ case class Test262(
 
     // setting for logging
     if (log)
-      println(s"- Logging to $logDir...")
       mkdir(logDir)
       dumpFile(spec.versionString, s"$logDir/ecma262-version")
       dumpFile(ESMeta.currentVersion, s"$logDir/esmeta-version")
-      summary.timeouts.setPath(s"$logDir/timeout.log")
-      summary.yets.setPath(s"$logDir/yet.log")
-      summary.fails.setPath(s"$logDir/fail.log")
-      summary.passes.setPath(s"$logDir/pass.log")
 
     // run tests
     for (test <- progressBar) check(test)
 
     // logging after tests
     if (log)
-      summary.close
-      val removed_total = removed.foldLeft(0)(_ + _._2.length)
+      summary.dumpTo(logDir)
       val summaryStr =
-        s"- total: ${summary.total + removed_total}$LINE_SEP" +
-        s"- removed: ${removed_total}$LINE_SEP" +
-        removed.foldLeft("") {
-          case (acc, (s, i)) => acc + s"  - $s: ${i.length}$LINE_SEP"
-        }
-        + (if (postSummary.isEmpty) s"$summary$LINE_SEP"
-           else
-             s"$summary$LINE_SEP$postSummary$LINE_SEP")
+        if (postSummary.isEmpty) s"$summary"
+        else s"$summary$LINE_SEP$postSummary"
       dumpFile(s"Test262 $name test summary", summaryStr, s"$logDir/summary")
 }
 object Test262 extends Git(TEST262_DIR)

src/main/scala/esmeta/test262/util/ManualConfig.scala

@@ -7,7 +7,6 @@ import java.io.File
 
 case class ManualConfig(
   filtered: Map[String, List[String]],
-  inProgress: Map[String, List[String]],
   supportedFeatures: List[String],
 )
 object ManualConfig {
@@ -20,11 +19,8 @@ object ManualConfig {
     val filtered = fileMap
       .get("filtered.json")
       .fold(Map())(readJson[Map[String, List[String]]])
-    val inProgress = fileMap
-      .get("in-progress.json")
-      .fold(Map())(readJson[Map[String, List[String]]])
     val supportedFeatures = fileMap
       .get("supported-features.json")
       .fold(Nil)(readJson[List[String]])
-    ManualConfig(filtered, inProgress, supportedFeatures)
+    ManualConfig(filtered, supportedFeatures)
 }