Rollback constraints in compareAppliedTypeParamRef (#22339)

This PR adds a `rollbackConstraintsUnless` in
`compareAppliedTypeParamRef`. It is required in case the call to
`directionalIsSubType` introduces constraints and the call to
`directionalRecur` returns `false`.

Fixes #20519

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -1244,8 +1244,9 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
                 tl => otherTycon.appliedTo(bodyArgs(tl)))
             else
               otherTycon
-          (assumedTrue(tycon) || directionalIsSubType(tycon, adaptedTycon)) &&
-          directionalRecur(adaptedTycon.appliedTo(args), other)
+          rollbackConstraintsUnless:
+            (assumedTrue(tycon) || directionalIsSubType(tycon, adaptedTycon))
+              && directionalRecur(adaptedTycon.appliedTo(args), other)
         }
       }
     end compareAppliedTypeParamRef

tests/pos/20519.scala

@@ -0,0 +1,10 @@
+class Box[T](val value: T)
+
+def boo[F[_], A](e: F[Box[A]]): F[A] = ???
+
+type Result[G[_], B] = G[Box[B]]
+
+def main =
+  val b: Result[Option, Int] = ???
+  val c = boo(b)
+  c: Option[Int]

tests/pos/20519b.scala

@@ -0,0 +1,16 @@
+trait TCl[F[_]]
+
+def boo[F[_], A](e: F[Option[A]], ev: TCl[F]): Unit = ()
+
+type Result[F[_], A] = F[Option[A]]
+
+@main def main =
+  summon[Result[Option, Int] =:= Option[Option[Int]]]
+
+  val ev = new TCl[Option] {}
+
+  val b: Result[Option, Int] = None
+  boo(b, ev)
+
+  val b2: Option[Option[Int]] = None
+  boo(b2, ev)

tests/pos/20519c.scala

@@ -0,0 +1,20 @@
+object Main {
+  trait TCl[F[_]]
+
+  implicit class Stx[F[_], A](e: F[Option[A]]) {
+    def boo(implicit ev: TCl[F]): Unit = ()
+  }
+
+  type Result[F[_], A] = F[Option[A]]
+
+  implicit val t: TCl[Option] = new TCl[Option] {}
+
+  def main(args: Array[String]): Unit = {
+    val b: Result[Option, Int] = None
+    b.boo
+
+    // works without the alias:
+    val b2: Option[Option[Int]] = None
+    b2.boo
+  }
+}