File-grained NotSupported error (#126)

* More generous parsing of test262 metadata.

Now parsing logic of metadata of test262 test can also handle features
separated just with comma.

* Iterate filtered.json to initialize TestFilter.

* Refactored TestFilter

* Classify YetObj with NOT_SUPPORTED_FEATURES

* Update supported-features.json

They were not supported but included in supported-features.json.

* Intro enum for categories of not supported reasons

* Delte unnecessary unapply

Co-authored-by: jhnaldo <jhpjhp0223@gmail.com>

src/main/resources/manuals/default/test262/supported-features.json

@@ -4,23 +4,7 @@
     "Array.prototype.flat",
     "Array.prototype.flatMap",
     "Array.prototype.values",
-    "ArrayBuffer",
-    "Atomics",
     "BigInt",
-    "DataView",
-    "DataView.prototype.getFloat32",
-    "DataView.prototype.getFloat64",
-    "DataView.prototype.getInt16",
-    "DataView.prototype.getInt32",
-    "DataView.prototype.getInt8",
-    "DataView.prototype.getUint16",
-    "DataView.prototype.getUint32",
-    "DataView.prototype.setUint8",
-    "Float32Array",
-    "Float64Array",
-    "Int16Array",
-    "Int32Array",
-    "Int8Array",
     "Map",
     "Object.fromEntries",
     "Object.hasOwn",
@@ -29,13 +13,7 @@
     "Promise.allSettled",
     "Promise.any",
     "Promise.prototype.finally",
-    "Proxy",
-    "Reflect",
-    "Reflect.construct",
-    "Reflect.set",
-    "Reflect.setPrototypeOf",
     "Set",
-    "SharedArrayBuffer",
     "String.fromCodePoint",
     "String.prototype.at",
     "String.prototype.endsWith",
@@ -59,12 +37,6 @@
     "Symbol.toPrimitive",
     "Symbol.toStringTag",
     "Symbol.unscopables",
-    "TypedArray",
-    "TypedArray.prototype.at",
-    "Uint16Array",
-    "Uint32Array",
-    "Uint8Array",
-    "Uint8ClampedArray",
     "WeakMap",
     "WeakSet",
     "align-detached-buffer-semantics-with-web-reality",
@@ -106,10 +78,6 @@
     "optional-catch-binding",
     "optional-chaining",
     "proxy-missing-checks",
-    "regexp-dotall",
-    "regexp-lookbehind",
-    "regexp-named-groups",
-    "regexp-unicode-property-escapes",
     "rest-parameters",
     "string-trimming",
     "super",

src/main/scala/esmeta/analyzer/package.scala

@@ -4,6 +4,7 @@ import esmeta.analyzer.domain.*
 import esmeta.analyzer.util.*
 import esmeta.cfg.{CFG, Node}
 import esmeta.error.*
+import esmeta.error.NotSupported.given
 import esmeta.es.Initialize
 import esmeta.ir.*
 import esmeta.state.*

src/main/scala/esmeta/error/NotSupported.scala

@@ -6,4 +6,24 @@ case class NotSupported(reasonPath: NotSupported.ReasonPath)
 object NotSupported:
   type Reason = String
   type ReasonPath = List[Reason]
-  def apply(reason: Reason): NotSupported = NotSupported(List(reason))
+  def apply(category: Category)(reasonPath: ReasonPath): NotSupported =
+    NotSupported(category.name :: reasonPath)
+
+  /** implicit conversion from reasons to reason paths */
+  given Conversion[Reason, ReasonPath] = List(_)
+
+  /** constants for categories of not supported features */
+  enum Category(val name: String) {
+    case Harness extends Category("harness")
+    case Internationalisation extends Category("internationalisation")
+    case Annex extends Category("annex")
+    case Negative extends Category("negative")
+    case NonStrict extends Category("non-strict")
+    case Module extends Category("module")
+    case Fixture extends Category("fixture")
+    case Feature extends Category("feature")
+    case Metalanguage extends Category("metalanguage")
+    case Long extends Category("long")
+    case Wrong extends Category("wrong")
+    case YetCategorized extends Category("yet-categorized")
+  }

src/main/scala/esmeta/interpreter/Interpreter.scala

@@ -4,6 +4,8 @@ import esmeta.{EVAL_LOG_DIR, LINE_SEP}
 import esmeta.analyzer.*
 import esmeta.cfg.*
 import esmeta.error.*
+import esmeta.error.NotSupported.{*, given}
+import esmeta.error.NotSupported.Category.*
 import esmeta.ir.{Func => IRFunc, *}
 import esmeta.es.*
 import esmeta.parser.{ESParser, ESValueParser}
@@ -245,7 +247,7 @@ class Interpreter(
         case v           => throw NoNt(nt, v)
       st.allocList(eval(ast).asAst.getItems(name).map(AstValue(_)))
     case EYet(msg) =>
-      throw NotSupported(List("metalanguage", msg))
+      throw NotSupported(Metalanguage)(List(msg))
     case EContains(list, elem, field) =>
       val l = eval(list).getList(list, st)
       val e = eval(elem)
@@ -666,7 +668,7 @@ object Interpreter {
         Str(ESValueParser.parseTRVTemplateTail(str))
       case (_, "Contains") => Bool(false)
       case ("RegularExpressionLiteral", name) =>
-        throw NotSupported(List("RegExp", sdoName))
+        throw NotSupported(Feature)(List("RegExp"))
       case _ =>
         throw InvalidAstProp(lex, Str(sdoName))
     }
@@ -810,10 +812,13 @@ object Interpreter {
       case (MOp.Sinh, List(Math(x)))  => Math(sinh(x.toDouble))
       case (MOp.Tanh, List(Math(x)))  => Math(tanh(x.toDouble))
       case (MOp.Acos, List(Math(x)))  => Math(acos(x.toDouble))
-      case (MOp.Acosh, List(Math(x))) => throw NotSupported("acosh")
-      case (MOp.Asinh, List(Math(x))) => throw NotSupported("asinh")
-      case (MOp.Atanh, List(Math(x))) => throw NotSupported("atanh")
-      case (MOp.Asin, List(Math(x)))  => Math(asin(x.toDouble))
+      case (MOp.Acosh, List(Math(x))) =>
+        throw NotSupported(Metalanguage)("acosh")
+      case (MOp.Asinh, List(Math(x))) =>
+        throw NotSupported(Metalanguage)("asinh")
+      case (MOp.Atanh, List(Math(x))) =>
+        throw NotSupported(Metalanguage)("atanh")
+      case (MOp.Asin, List(Math(x))) => Math(asin(x.toDouble))
       case (MOp.Atan2, List(Math(x), Math(y))) =>
         Math(atan2(x.toDouble, y.toDouble))
       case (MOp.Atan, List(Math(x)))  => Math(atan(x.toDouble))

src/main/scala/esmeta/state/Heap.scala

@@ -2,6 +2,8 @@ package esmeta.state
 
 import esmeta.cfg.*
 import esmeta.error.*
+import esmeta.error.NotSupported.{*, given}
+import esmeta.error.NotSupported.Category.*
 import esmeta.ir.{Func => IRFunc, *}
 import esmeta.es.builtin.*
 import esmeta.util.BaseUtils.*
@@ -16,14 +18,14 @@ case class Heap(
   /** getters */
   def apply(addr: Addr): Obj =
     map.getOrElse(addr, throw UnknownAddr(addr)) match
-      case YetObj(_, msg) => throw NotSupported(msg)
+      case YetObj(_, msg) => throw NotSupported(Feature)(msg)
       case obj            => obj
   def apply(addr: Addr, key: PureValue): Value = apply(addr) match
     case _ if addr == NamedAddr(INTRINSICS) => Heap.getIntrinsics(key)
     case (s: SymbolObj)                     => s(key)
     case (m: MapObj)                        => m(key)
     case (l: ListObj)                       => l(key)
-    case YetObj(_, msg)                     => throw NotSupported(msg)
+    case YetObj(_, msg)                     => throw NotSupported(Feature)(msg)
 
   /** setters */
   def update(addr: Addr, prop: PureValue, value: Value): this.type =

src/main/scala/esmeta/test262/Test.scala

@@ -117,7 +117,7 @@ object Test {
           val featuresn =
             if (line contains "features:") line.dropWhile(_ != '[') match {
               case "" => List()
-              case s  => s.tail.takeWhile(_ != ']').split(", ").toList
+              case s => s.tail.takeWhile(_ != ']').split(",").map(_.trim).toList
             }
             else features
           (

src/main/scala/esmeta/test262/util/ManualConfig.scala

@@ -7,6 +7,7 @@ import java.io.File
 
 case class ManualConfig(
   filtered: Map[String, List[String]],
+  yetCategorized: List[String],
   supportedFeatures: List[String],
 )
 object ManualConfig {
@@ -17,10 +18,13 @@ object ManualConfig {
       if jsonFilter(file.getName)
     } yield file.getName -> file.toString).toMap
     val filtered = fileMap
-      .get("filtered.json")
+      .get("categorized.json")
       .fold(Map())(readJson[Map[String, List[String]]])
+    val yetCategorized = fileMap
+      .get("yet-categorized.json")
+      .fold(Nil)(readJson[List[String]])
     val supportedFeatures = fileMap
       .get("supported-features.json")
       .fold(Nil)(readJson[List[String]])
-    ManualConfig(filtered, supportedFeatures)
+    ManualConfig(filtered, yetCategorized, supportedFeatures)
 }

src/main/scala/esmeta/test262/util/TestFilter.scala

@@ -1,11 +1,11 @@
 package esmeta.test262.util
 
-import esmeta.*
 import esmeta.spec.Spec
 import esmeta.util.*
 import esmeta.util.BaseUtils.*
 import esmeta.util.SystemUtils.*
 import esmeta.error.NotSupported.*
+import esmeta.error.NotSupported.Category.*
 import esmeta.test262.{*, given}
 import io.circe.*, io.circe.syntax.*
 import java.io.*
@@ -21,7 +21,7 @@ case class TestFilter(spec: Spec) {
   ): (List[Test], Iterable[(Test, ReasonPath)]) =
     var removedMap: Vector[(Test, ReasonPath)] = Vector()
     val targetTests = getFilters(withYet, features.toSet).foldLeft(tests) {
-      case (tests, (desc, filter)) =>
+      case (tests, filter) =>
         for {
           test <- tests
           if filter(test) match
@@ -34,99 +34,70 @@ case class TestFilter(spec: Spec) {
   private def getFilters(
     withYet: Boolean,
     features: Set[String],
-  ): List[(String, Test => Option[ReasonPath])] = List(
-    lift("harness" -> (_.relName.startsWith("harness"))),
-    lift("internationalisation" -> (_.relName.startsWith("intl"))),
-    lift(
-      "annex" -> (test =>
-        test.relName.startsWith("annex") ||
-        test.relName.contains("__proto__"),
-      ),
+  ): List[Test => Option[ReasonPath]] = List(
+    // harness files
+    Harness -> ((test: Test) => test.relName.startsWith("harness")),
+    // tests for internationalisation
+    Internationalisation -> ((test: Test) => test.relName.startsWith("intl")),
+    // tests for appendices, including web browsers
+    Annex -> ((test: Test) =>
+      test.relName.startsWith("annex") ||
+      test.relName.contains("__proto__"),
     ),
-    liftReason(
-      "not-supported-features" -> (_.features.find(!features.contains(_))),
+    // tests for negative result, including parsing errors
+    Negative -> ((test: Test) => test.negative.isDefined),
+    // tests for non-strict mode
+    NonStrict -> ((test: Test) =>
+      test.flags.contains("noStrict") ||
+      test.flags.contains("raw"),
     ),
-    lift(
-      "non-strict" -> (test =>
-        test.flags.contains("noStrict") ||
-        test.flags.contains("raw") ||
-        manualNonstrict.contains(removedExt(test.relName)),
-      ),
+    // tests for modules
+    Module -> ((test: Test) =>
+      test.flags.contains("module") ||
+      test.relName.startsWith("language/module-code/") ||
+      test.relName.startsWith("language/import/") ||
+      test.relName.startsWith("language/expressions/dynamic-import/") ||
+      test.relName.startsWith("language/expressions/import.meta/"),
     ),
-    lift(
-      "module" -> (test =>
-        test.flags.contains("module") ||
-        test.relName.startsWith("language/module-code/") ||
-        test.relName.startsWith("language/import/") ||
-        test.relName.startsWith("language/expressions/dynamic-import/") ||
-        test.relName.startsWith("language/expressions/import.meta/"),
-      ),
+    // fixtures
+    Fixture -> ((test: Test) => removedExt(test.relName).endsWith("_FIXTURE")),
+    // tests for language features not supported in ESMeta
+    Feature -> ((test: Test) => test.features.find(!features.contains(_))),
+    // manually filtered tests not yet categorized based on failure reasons
+    YetCategorized -> ((test: Test) =>
+      !withYet && manualYetCategorized.contains(removedExt(test.relName)),
     ),
-    lift(
-      "negative-errors" -> (test =>
-        test.negative.isDefined ||
-        manualEarlyError.contains(removedExt(test.relName)),
-      ),
-    ),
-    lift(
-      "inessential-builtin-objects" -> (test =>
-        test.flags.contains("CanBlockIsFalse") ||
-        test.flags.contains("CanBlockIsTrue") ||
-        !test.locales.isEmpty,
-      ),
-    ),
-    lift(
-      "non-tests" -> (test => manualNonTest.contains(removedExt(test.relName))),
-    ),
-    lift(
-      "wrong-tests" -> (test => wrongTest.contains(removedExt(test.relName))),
-    ),
-    lift("longTest" -> (test => longTest.contains(removedExt(test.relName)))),
-    lift("yet" -> (test => !withYet && yets.contains(removedExt(test.relName)))),
+    // manually filtered tests categorized based on failure reasons
+    test => manualFilterMap.get(removedExt(test.relName)),
   )
 
-  private def lift(
-    pair: (String, Test => Boolean),
-  ): (String, Test => Option[ReasonPath]) =
-    val (name, filter) = pair
-    val lifted = (x: Test) => if (filter(x)) Some(List(name)) else None
-    name -> lifted
+  given liftBool: Conversion[
+    (Category, Test => Boolean),
+    Test => Option[ReasonPath],
+  ] = pair =>
+    val (category, filter) = pair
+    val lifted = (x: Test) => if (filter(x)) Some(List(category.name)) else None
+    lifted
 
-  private def liftReason(
-    pair: (String, Test => Option[Reason]),
-  ): (String, Test => Option[ReasonPath]) =
-    val (name, filter) = pair
-    name -> (test => filter(test).map(List(name, _)))
+  given liftReason: Conversion[
+    (Category, Test => Option[Reason]),
+    Test => Option[ReasonPath],
+  ] = pair =>
+    val (category, filter) = pair
+    test => filter(test).map(List(category.name, _))
 
   lazy val manualConfig = spec.manualInfo.test262
 
+  lazy val manualFilterMap = (for {
+    (reason, tests) <- manualConfig.filtered
+    test <- tests
+  } yield test -> List(reason)).toMap
+
+  lazy val manualYetCategorized = manualConfig.yetCategorized.toSet
+
   /** language features in Test262
     * @url
     *   https://github.com/tc39/test262/blob/main/features.txt
     */
   lazy val languageFeatures = manualConfig.supportedFeatures
-
-  /** manually filtered out non-strict mode tests */
-  lazy val manualNonstrict =
-    manualConfig.filtered.getOrElse("non-strict tests", Nil).toSet
-
-  /** manually filtered out tests for EarlyErorr */
-  lazy val manualEarlyError =
-    manualConfig.filtered.getOrElse("early errors", Nil).toSet
-
-  /** manually filtered out non test files */
-  lazy val manualNonTest =
-    manualConfig.filtered.getOrElse("non tests", Nil).toSet
-
-  /** manually filtered out long tests */
-  lazy val longTest =
-    manualConfig.filtered.getOrElse("long tests", Nil).toSet
-
-  /** manually filtered out wrong test262 tests */
-  lazy val wrongTest =
-    manualConfig.filtered.getOrElse("wrong tests", Nil).toSet
-
-  /** manually filtered out not yet supported tests */
-  lazy val yets =
-    manualConfig.filtered.getOrElse("not yet categorized tests", Nil).toSet
 }