Leaking internal type parameter reference in GADT casting

[Linyxus]

Compiler version

3.2.0-RC1

Minimized code

trait Tag { val data: Int }

enum EQ[A, B]:
  case Refl[C]() extends EQ[C, C]

def foo[T, B <: Tag](ev: EQ[T, B], x: T) = ev match
  case EQ.Refl() =>
    val i: Int = x.data

Output (click arrow to expand)

error when pickling type B(param)2
error when pickling tree B(param)2
error when pickling tree x.$asInstanceOf[B(param)2]
error when pickling tree x.$asInstanceOf[B(param)2].data
error when pickling tree val i: Int = x.$asInstanceOf[B(param)2].data
error when pickling tree {
  val i: Int = x.$asInstanceOf[B(param)2].data
  ()
}
error when pickling tree case EQ.Refl.unapply[T]():EQ.Refl[T] => 
  val i: Int = x.$asInstanceOf[B(param)2].data
  ()
error when pickling tree ev match 
  {
    case EQ.Refl.unapply[T]():EQ.Refl[T] => 
      val i: Int = x.$asInstanceOf[B(param)2].data
      ()
  }
error when pickling tree def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = 
  ev match 
    {
      case EQ.Refl.unapply[T]():EQ.Refl[T] => 
        val i: Int = x.$asInstanceOf[B(param)2].data
        ()
    }
error when pickling tree () extends Object() { this: gadt-selection$package.type =>
  private def writeReplace(): AnyRef = new scala.runtime.ModuleSerializationProxy(classOf[gadt-selection$package.type])
  def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = 
    ev match 
      {
        case EQ.Refl.unapply[T]():EQ.Refl[T] => 
          val i: Int = x.$asInstanceOf[B(param)2].data
          ()
      }
}
error when pickling tree @SourceFile("issues/gadt-selection.scala") final module class gadt-selection$package() extends Object() { this: gadt-selection$package.type =>
  private def writeReplace(): AnyRef = new scala.runtime.ModuleSerializationProxy(classOf[gadt-selection$package.type])
  def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = 
    ev match 
      {
        case EQ.Refl.unapply[T]():EQ.Refl[T] => 
          val i: Int = x.$asInstanceOf[B(param)2].data
          ()
      }
}
error when pickling tree package <empty> {
  final lazy module val gadt-selection$package: gadt-selection$package = new gadt-selection$package()
  @SourceFile("issues/gadt-selection.scala") final module class gadt-selection$package() extends Object() { this: gadt-selection$package.type =>
    private def writeReplace(): AnyRef = new scala.runtime.ModuleSerializationProxy(classOf[gadt-selection$package.type])
    def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = 
      ev match 
        {
          case EQ.Refl.unapply[T]():EQ.Refl[T] => 
            val i: Int = x.$asInstanceOf[B(param)2].data
            ()
        }
  }
}
exception occurred while compiling issues/gadt-selection.scala
java.lang.AssertionError: assertion failed: orphan parameter reference: TypeParamRef(B(param)2) while compiling issues/gadt-selection.scala
Exception in thread "main" java.lang.AssertionError: assertion failed: orphan parameter reference: TypeParamRef(B(param)2)
	at scala.runtime.Scala3RunTime$.assertFailed(Scala3RunTime.scala:8)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleNewType(TreePickler.scala:291)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleType(TreePickler.scala:160)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:609)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTpt(TreePickler.scala:318)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$4$$anonfun$1(TreePickler.scala:440)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$4(TreePickler.scala:440)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$4(TreePickler.scala:441)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:441)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:415)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTreeUnlessEmpty(TreePickler.scala:321)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$1(TreePickler.scala:338)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$adapted$1(TreePickler.scala:340)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef(TreePickler.scala:340)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:550)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$9$$anonfun$1(TreePickler.scala:474)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$9(TreePickler.scala:474)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$8(TreePickler.scala:474)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:474)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$13(TreePickler.scala:501)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$12(TreePickler.scala:501)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:501)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$12$$anonfun$1(TreePickler.scala:497)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$12(TreePickler.scala:497)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$11(TreePickler.scala:498)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:498)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTreeUnlessEmpty(TreePickler.scala:321)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$1(TreePickler.scala:338)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$adapted$1(TreePickler.scala:340)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef(TreePickler.scala:340)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:565)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleStats$$anonfun$2(TreePickler.scala:365)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleStats(TreePickler.scala:365)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$26(TreePickler.scala:591)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$24(TreePickler.scala:592)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:592)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$1(TreePickler.scala:335)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef$$anonfun$adapted$1(TreePickler.scala:340)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleDef(TreePickler.scala:340)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:567)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleStats$$anonfun$2(TreePickler.scala:365)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleStats(TreePickler.scala:365)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$29(TreePickler.scala:607)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree$$anonfun$adapted$27(TreePickler.scala:607)
	at scala.Function0.apply$mcV$sp(Function0.scala:39)
	at dotty.tools.dotc.core.tasty.TreePickler.withLength(TreePickler.scala:58)
	at dotty.tools.dotc.core.tasty.TreePickler.pickleTree(TreePickler.scala:607)
	at dotty.tools.dotc.core.tasty.TreePickler.pickle$$anonfun$1(TreePickler.scala:779)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.core.tasty.TreePickler.pickle(TreePickler.scala:779)
	at dotty.tools.dotc.transform.Pickler.run$$anonfun$1$$anonfun$1(Pickler.scala:72)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.transform.Pickler.run$$anonfun$1(Pickler.scala:110)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.immutable.List.foreach(List.scala:333)
	at dotty.tools.dotc.transform.Pickler.run(Pickler.scala:110)
	at dotty.tools.dotc.core.Phases$Phase.runOn$$anonfun$1(Phases.scala:311)
	at scala.collection.immutable.List.map(List.scala:246)
	at dotty.tools.dotc.core.Phases$Phase.runOn(Phases.scala:312)
	at dotty.tools.dotc.transform.Pickler.runOn(Pickler.scala:115)
	at dotty.tools.dotc.Run.runPhases$1$$anonfun$1(Run.scala:234)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1328)
	at dotty.tools.dotc.Run.runPhases$1(Run.scala:245)
	at dotty.tools.dotc.Run.compileUnits$$anonfun$1(Run.scala:253)
	at dotty.tools.dotc.Run.compileUnits$$anonfun$adapted$1(Run.scala:262)
	at dotty.tools.dotc.util.Stats$.maybeMonitored(Stats.scala:68)
	at dotty.tools.dotc.Run.compileUnits(Run.scala:262)
	at dotty.tools.dotc.Run.compileSources(Run.scala:186)
	at dotty.tools.dotc.Run.compile(Run.scala:170)
	at dotty.tools.dotc.Driver.doCompile(Driver.scala:35)
	at dotty.tools.dotc.Driver.process(Driver.scala:195)
	at dotty.tools.dotc.Driver.process(Driver.scala:163)
	at dotty.tools.dotc.Driver.process(Driver.scala:175)
	at dotty.tools.dotc.Driver.main(Driver.scala:205)
	at dotty.tools.dotc.Main.main(Main.scala)
[error] Nonzero exit code returned from runner: 1
[error] (scala3-compiler / Compile / runMain) Nonzero exit code returned from runner: 1
[error] Total time: 1 s, completed Jun 27, 2022, 7:43:30 PM

As seen in the following typed tree, the internal type of GADT constraint solver B(param)2 is leaked into the GADT casting, crashing the pickler.

def foo[T >: Nothing <: Any, B >: Nothing <: Tag](ev: EQ[T, B], x: T): Unit = 
  ev match 
    {
      case EQ.Refl.unapply[T]():EQ.Refl[T] => 
        val i: Int = x.$asInstanceOf[B(param)2].data
        ()
    }

The cause of this behavior:

• When typing the selection x.data we try to GADT-approximate x: T and check whether it matches the SelectionProto.
• During this, we will call GADTConstraint.approximation, which will call ConstraintHandling.approximation here.
• As seen here, if the type variable has been instantiated to another type parameter, the internal representation is returned directly and back in GADTConstraint we do not externalize it.
• This leaks the internal type and breaks the pickler.

This is a bug discovered when I am trying to solve #14776. I am working on a PR fixing both issues.

[dwijnand]

This is a bug discovered when I am trying to solve #14776. I am working on a PR fixing this one along with #14776.

Typo.