Fix mapping TypeMaps over annotations (#15948)

Avoids orphan parameters when pickling

Fixes #15922

I am not sure about the status of the test in custome-args/captures. Should it
pass or be rejected? But in any case it does not crash anymore.

compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala

@@ -56,7 +56,7 @@ class TreeTypeMap(
   /** Replace occurrences of `This(oldOwner)` in some prefix of a type
    *  by the corresponding `This(newOwner)`.
    */
-  private val mapOwnerThis = new TypeMap {
+  private val mapOwnerThis = new TypeMap with cc.CaptureSet.IdempotentCaptRefMap {
     private def mapPrefix(from: List[Symbol], to: List[Symbol], tp: Type): Type = from match {
       case Nil => tp
       case (cls: ClassSymbol) :: from1 => mapPrefix(from1, to.tail, tp.substThis(cls, to.head.thisType))

compiler/src/dotty/tools/dotc/cc/CaptureSet.scala

@@ -523,7 +523,7 @@ object CaptureSet:
 
     private def mapIsIdempotent = tm.isInstanceOf[IdempotentCaptRefMap]
 
-    assert(ccAllowUnsoundMaps || mapIsIdempotent)
+    assert(ccAllowUnsoundMaps || mapIsIdempotent, tm.getClass)
 
     private def whereCreated(using Context): String =
       if stack == null then ""

compiler/src/dotty/tools/dotc/core/Annotations.scala

@@ -1,4 +1,5 @@
-package dotty.tools.dotc
+package dotty.tools
+package dotc
 package core
 
 import Symbols._, Types._, Contexts._, Constants._
@@ -58,7 +59,7 @@ object Annotations {
             if tm.isRange(x) then x
             else
               val tp1 = tm(tree.tpe)
-              foldOver(if tp1 =:= tree.tpe then x else tp1, tree)
+              foldOver(if tp1 frozen_=:= tree.tpe then x else tp1, tree)
         val diff = findDiff(NoType, args)
         if tm.isRange(diff) then EmptyAnnotation
         else if diff.exists then derivedAnnotation(tm.mapOver(tree))
@@ -69,7 +70,7 @@ object Annotations {
       val args = arguments
       if args.isEmpty then false
       else tree.existsSubTree {
-        case id: Ident => id.tpe match
+        case id: Ident => id.tpe.stripped match
           case TermParamRef(tl1, _) => tl eq tl1
           case _ => false
         case _ => false

compiler/test/dotc/pos-test-pickling.blacklist

@@ -19,6 +19,7 @@ i6507b.scala
 i12299a.scala
 i13871.scala
 i15181.scala
+i15922.scala
 
 # Tree is huge and blows stack for printing Text
 i7034.scala

tests/pos-custom-args/captures/i15922.scala

@@ -0,0 +1,14 @@
+trait Cap { def use(): Int }
+type Id[X] = [T] -> (op: X => T) -> T
+def mkId[X](x: X): Id[X] = [T] => (op: X => T) => op(x)
+
+def withCap[X](op: ({*} Cap) => X): X = {
+  val cap: {*} Cap = new Cap { def use() = { println("cap is used"); 0 } }
+  val result = op(cap)
+  result
+}
+
+def leaking(c: {*} Cap): Id[{c} Cap] = mkId(c)
+
+def test =
+  val bad = withCap(leaking)

tests/pos/i15922.scala

@@ -0,0 +1,14 @@
+trait Cap:
+  type M
+class Id[X]
+
+object Test:
+  def withCap[X](op: Cap => X): X = ???
+
+  class retains1(xs: Any*) extends annotation.StaticAnnotation
+
+  def leaking1(c: Cap): Id[Cap @retains1(c)] = ??? // used to crash with orphan parameter on pickling
+  def leaking2(c: Cap): Id[c.type] = ???
+
+  val bad1 = withCap(leaking1)
+  val bad2 = withCap(leaking2)