support macro defs in TASTy

project/DottySupport.scala

@@ -12,7 +12,7 @@ import sbt.librarymanagement.{
   * Settings to support validation of TastyUnpickler against the release of dotty with the matching TASTy version
   */
 object TastySupport {
-  val supportedTASTyRelease = "0.25.0-RC1" // TASTy version 23
+  val supportedTASTyRelease = "0.25.0-RC2" // TASTy version 23
   val dottyCompiler = "ch.epfl.lamp" % "dotty-compiler_0.25" % supportedTASTyRelease
 }
 

src/compiler/scala/tools/nsc/tasty/TastyModes.scala

@@ -22,6 +22,8 @@ object TastyModes {
   final val ReadParents: TastyMode    = TastyMode(1 << 0)
   final val ReadAnnotation: TastyMode = TastyMode(1 << 1)
   final val OuterTerm: TastyMode      = TastyMode(1 << 2)
+  final val ReadMacro: TastyMode      = TastyMode(1 << 3)
+  final val IndexBody: TastyMode      = TastyMode(1 << 4)
 
   case class TastyMode(val toInt: Int) extends AnyVal { mode =>
 
@@ -39,6 +41,8 @@ object TastyModes {
         if (mode.is(ReadParents)) sb += "ReadParents"
         if (mode.is(ReadAnnotation)) sb += "ReadAnnotation"
         if (mode.is(OuterTerm)) sb += "OuterTerm"
+        if (mode.is(ReadMacro)) sb += "ReadMacro"
+        if (mode.is(IndexBody)) sb += "IndexBody"
         sb.mkString("|")
       }
     }

src/compiler/scala/tools/nsc/tasty/TreeUnpickler.scala

@@ -521,16 +521,11 @@ class TreeUnpickler[Tasty <: TastyUniverse](
       sym.setAnnotations(annotFns.map(_(sym)))
       ctx.owner match {
         case cls if cls.isClass && canEnterInClass =>
+          if (ctx.mode.is(IndexBody) && ctx.isLatentCandidate(sym))
+            ctx.registerLatent(sym)
           val decl = if (flags.is(Object) && isClass) sym.sourceObject else sym
-          val decls = cls.rawInfo.decls
-          if (allowsOverload(decl)) {
-            if (ctx.canEnterOverload(decl)) {
-              decls.enter(decl)
-            }
-          }
-          else {
-            decls.enterIfNew(decl)
-          }
+          if (ctx.canEnter(decl))
+            ctx.enter(cls, decl)
         case _ =>
       }
       registerSym(start, sym)
@@ -722,11 +717,12 @@ class TreeUnpickler[Tasty <: TastyUniverse](
         val localCtx = ctx.withOwner(sym)
         tag match {
           case DEFDEF =>
-            val unsupported = completer.tastyFlagSet &~ (Extension | Inline | Macro | Exported)
+            val unsupported = completer.tastyFlagSet &~ (Extension | Inline | Exported | Erased)
             unsupportedWhen(unsupported.hasFlags, s"flags on $sym: ${showTasty(unsupported)}")
             if (completer.tastyFlagSet.is(Extension)) ctx.log(s"$tname is a Scala 3 extension method.")
-            unsupportedWhen(completer.tastyFlagSet.is(Inline, butNot = Macro), s"inline $sym")
-            unsupportedWhen(completer.tastyFlagSet.is(Inline | Macro), s"macro $sym")
+            unsupportedWhen(completer.tastyFlagSet.is(Inline), s"${if (sym.is(Macro)) "" else "inline "}$sym")
+            val isMacroDef = completer.tastyFlagSet.is(Erased) && sym.is(Macro)
+            unsupportedWhen(completer.tastyFlagSet.is(Erased) && !isMacroDef, s"erased $sym")
             val isCtor = sym.isClassConstructor
             val typeParams = {
               if (isCtor) {
@@ -739,6 +735,10 @@ class TreeUnpickler[Tasty <: TastyUniverse](
             }
             val vparamss = readParamss(localCtx)
             val tpt = readTpt()(localCtx)
+            if (isMacroDef) {
+              val impl = tpd.Macro(readTerm()(ctx.addMode(ReadMacro)))
+              sym.addAnnotation(symbolTable.AnnotationInfo(symbolTable.definitions.MacroTastyImplAnnotation.tpe, List(impl), Nil))
+            }
             val valueParamss = normalizeIfConstructor(vparamss.map(_.map(symFromNoCycle)), isCtor)
             val resType = effectiveResultType(sym, typeParams, tpt.tpe)
             ctx.setInfo(sym, defn.DefDefType(if (isCtor) Nil else typeParams, valueParamss, resType))
@@ -803,8 +803,10 @@ class TreeUnpickler[Tasty <: TastyUniverse](
       // ** MEMBERS **
       ctx.log(s"$symAddr Template: indexing members of $cls:")
       val bodyIndexer = fork
+      val bodyCtx = ctx.addMode(IndexBody)
       while (bodyIndexer.reader.nextByte != DEFDEF) bodyIndexer.skipTree() // skip until primary ctor
-      bodyIndexer.indexStats(end)
+      bodyIndexer.indexStats(end)(bodyCtx)
+      bodyCtx.enterLatents()
 
       // ** PARENTS **
       ctx.log(s"$symAddr Template: adding parents of $cls:")
@@ -990,7 +992,7 @@ class TreeUnpickler[Tasty <: TastyUniverse](
               if (alias != untpd.EmptyTree) alias // only for opaque type alias
               else tpd.TypeBoundsTree(lo, hi)
             case BLOCK =>
-              if (inParentCtor) {
+              if (inParentCtor | ctx.mode.is(ReadMacro)) {
                 val exprReader = fork
                 skipTree()
                 until(end)(skipTree()) //val stats = readStats(ctx.owner, end)

src/compiler/scala/tools/nsc/tasty/bridge/ContextOps.scala

@@ -18,6 +18,7 @@ import scala.reflect.io.AbstractFile
 import scala.tools.tasty.{TastyName, TastyFlags}, TastyFlags._, TastyName.ObjectName
 import scala.tools.nsc.tasty.{TastyUniverse, TastyModes, SafeEq}, TastyModes._
 import scala.reflect.internal.MissingRequirementError
+import scala.collection.mutable
 
 trait ContextOps { self: TastyUniverse =>
   import self.{symbolTable => u}, u.{internal => ui}
@@ -67,6 +68,25 @@ trait ContextOps { self: TastyUniverse =>
     final def ignoreAnnotations: Boolean = u.settings.YtastyNoAnnotations
     final def verboseDebug: Boolean = u.settings.debug
 
+    def isScala3Macro(sym: Symbol): Boolean = isScala3Inline(sym) && sym.is(Macro)
+    def isScala3Inline(sym: Symbol): Boolean = sym.completer.tastyFlagSet.is(Inline)
+    def isScala2Macro(sym: Symbol): Boolean = sym.completer.tastyFlagSet.is(Erased) && sym.is(Macro)
+
+    def isLatentCandidate(sym: Symbol): Boolean = isScala3Inline(sym) || isScala2Macro(sym)
+
+    def canEnter(decl: Symbol): Boolean = !isScala3Macro(decl)
+    def enter(clazz: Symbol, decl: Symbol): Unit = enter(clazz.rawInfo.decls, decl)
+    private[ContextOps] def enter(decls: u.Scope, decl: Symbol): Unit = {
+      if (allowsOverload(decl)) {
+        if (canEnterOverload(decl)) {
+          decls.enter(decl)
+        }
+      }
+      else {
+        decls.enterIfNew(decl)
+      }
+    }
+
     def canEnterOverload(decl: Symbol): Boolean = {
       !(decl.isModule && isSymbol(findObject(decl.name)))
     }
@@ -92,6 +112,9 @@ trait ContextOps { self: TastyUniverse =>
     def source: AbstractFile
     def mode: TastyMode
 
+    def registerLatent(sym: Symbol): Unit
+    def enterLatents(): Unit
+
     private final def loadingMirror: u.Mirror = u.mirrorThatLoaded(owner)
 
     final def requiredPackage(fullname: TastyName): Symbol = {
@@ -390,11 +413,46 @@ trait ContextOps { self: TastyUniverse =>
   final class InitialContext(val topLevelClass: Symbol, val source: AbstractFile) extends Context {
     def mode: TastyMode = EmptyTastyMode
     def owner: Symbol = topLevelClass.owner
+    def registerLatent(sym: Symbol): Unit = ()
+    def enterLatents(): Unit = ()
   }
 
   final class FreshContext(val owner: Symbol, val outer: Context, val mode: TastyMode) extends Context {
     private[this] var mySource: AbstractFile = null
+    private[this] var myLatentDefs: mutable.ArrayBuffer[Symbol] = null
+    private[this] var myMacros: mutable.ArrayBuffer[Symbol] = null
     def atSource(source: AbstractFile): this.type = { mySource = source ; this }
     def source: AbstractFile = if (mySource == null) outer.source else mySource
+    def registerLatent(sym: Symbol): Unit = {
+      if (isScala2Macro(sym)) {
+        val macros = {
+          if (myMacros == null) myMacros = mutable.ArrayBuffer.empty
+          myMacros
+        }
+        macros += sym
+      } else {
+        val defs = {
+          if (myLatentDefs == null) myLatentDefs = mutable.ArrayBuffer.empty
+          myLatentDefs
+        }
+        defs += sym
+      }
+    }
+    def enterLatents(): Unit = {
+      for {
+        owner  <- Option.when(owner.isClass)(owner)
+        defs   <- Option(myLatentDefs)
+      } {
+        val macros = Option(myMacros).getOrElse(mutable.ArrayBuffer.empty)
+        val decls = owner.rawInfo.decls
+        for (d <- defs if !macros.exists(_.name == d.name)) {
+          enter(decls, d)
+        }
+        defs.clear()
+        macros.clear()
+      }
+      myLatentDefs = null
+      myMacros = null
+    }
   }
 }

src/compiler/scala/tools/nsc/tasty/bridge/FlagOps.scala

@@ -21,7 +21,7 @@ trait FlagOps { self: TastyUniverse =>
 
   object FlagSets {
     val TastyOnlyFlags: TastyFlagSet = (
-      Erased | Internal | Inline | InlineProxy | Opaque | Extension | Given | Exported | Macro | SuperTrait | Enum
+      Erased | Internal | Inline | InlineProxy | Opaque | Extension | Given | Exported | SuperTrait | Enum
       | Open | ParamAlias
     )
     val TermParamOrAccessor: TastyFlagSet = Param | ParamSetter
@@ -42,6 +42,7 @@ trait FlagOps { self: TastyUniverse =>
     if (tflags.is(Case)) flags |= Flag.CASE
     if (tflags.is(Implicit)) flags |= ModifierFlags.IMPLICIT
     if (tflags.is(Lazy)) flags |= Flag.LAZY
+    if (tflags.is(Macro)) flags |= Flag.MACRO
     if (tflags.is(Override)) flags |= Flag.OVERRIDE
     if (tflags.is(Static)) flags |= ModifierFlags.STATIC
     if (tflags.is(Object)) flags |= Flags.MODULE
@@ -76,7 +77,6 @@ trait FlagOps { self: TastyUniverse =>
         case Extension   => "<extension>"
         case Given       => "given"
         case Exported    => "<exported>"
-        case Macro       => "<tastymacro>"
         case SuperTrait  => "<supertrait>"
         case Enum        => "enum"
         case Open        => "open"

src/compiler/scala/tools/nsc/tasty/bridge/SymbolOps.scala

@@ -94,7 +94,7 @@ trait SymbolOps { self: TastyUniverse =>
   }
 
   private def hasType(member: Symbol)(implicit ctx: Context) = {
-    ctx.mode.is(ReadAnnotation) || (member.rawInfo `ne` u.NoType)
+    ctx.mode.is(ReadAnnotation) || ctx.mode.is(ReadMacro) && (member.info `ne` u.NoType) || (member.rawInfo `ne` u.NoType)
   }
 
   private def errorMissing[T](space: Type, tname: TastyName)(implicit ctx: Context) = {

src/compiler/scala/tools/nsc/tasty/bridge/TreeOps.scala

@@ -15,6 +15,7 @@ package scala.tools.nsc.tasty.bridge
 import scala.tools.nsc.tasty.TastyUniverse
 
 import scala.tools.tasty.TastyName
+import scala.reflect.internal.Flags
 
 
 trait TreeOps { self: TastyUniverse =>
@@ -56,6 +57,28 @@ trait TreeOps { self: TastyUniverse =>
       u.TypeTree(defn.LambdaFromParams(tparams, body.tpe))
     }
 
+    def Macro(impl: Tree): Tree = impl match {
+      case tree @ u.TypeApply(qual, args) =>
+        u.TypeApply(Macro(qual), args).setType(tree.tpe)
+      case tree @ u.Select(pre, sel) =>
+        val sym = if (sel.isTermName) tree.tpe.termSymbol else tree.tpe.typeSymbol
+        u.Select(Macro(pre), sym).setType(tree.tpe)
+      case tree: u.TypeTree if tree.tpe.prefix !== u.NoType =>
+        val sym = tree.tpe match {
+          case u.SingleType(_, sym) => sym
+          case u.TypeRef(_, sym, _) => sym
+        }
+        if (tree.tpe.prefix === u.NoPrefix && (sym.hasFlag(Flags.PACKAGE) && !sym.isPackageObjectOrClass || sym.isLocalToBlock)) {
+          if (sym.isLocalToBlock) u.Ident(sym).setType(tree.tpe)
+          else u.This(sym).setType(tree.tpe)
+        }
+        else {
+          u.Select(Macro(u.TypeTree(tree.tpe.prefix)), sym).setType(tree.tpe)
+        }
+      case tree =>
+        tree
+    }
+
     def Typed(expr: Tree, tpt: Tree): Tree = u.Typed(expr, tpt).setType(tpt.tpe)
 
     def Apply(fun: Tree, args: List[Tree]): Tree = u.Apply(fun, args).setType(fnResult(fun.tpe))

src/compiler/scala/tools/nsc/tasty/bridge/TypeOps.scala

@@ -21,6 +21,7 @@ import scala.util.chaining._
 
 import scala.collection.mutable
 import scala.reflect.internal.Flags
+import scala.tools.tasty.TastyName.QualifiedName
 
 trait TypeOps { self: TastyUniverse =>
   import self.{symbolTable => u}, u.{internal => ui}

src/compiler/scala/tools/nsc/typechecker/Macros.scala

@@ -282,6 +282,10 @@ trait Macros extends MacroRuntimes with Traces with Helpers {
     macroImplBindingCache.getOrElseUpdate(macroDef,
       macroDef.getAnnotation(MacroImplAnnotation) collect {
         case AnnotationInfo(_, List(pickle), _) => MacroImplBinding.unpickle(pickle)
+      } orElse {
+        macroDef.getAnnotation(MacroTastyImplAnnotation) collect {
+          case AnnotationInfo(_, List(unpickled), _) => MacroImplBinding.unpickle(MacroImplBinding.pickle(unpickled))
+        }
       }
     )
   }

src/reflect/scala/reflect/internal/Definitions.scala

@@ -566,6 +566,34 @@ trait Definitions extends api.StandardDefinitions {
          def MacroContextWeakTypeTagClass = BlackboxContextClass.map(sym => getTypeMember(sym, tpnme.WeakTypeTag))
          def MacroContextTreeType         = BlackboxContextClass.map(sym => getTypeMember(sym, tpnme.Tree))
     lazy val MacroImplAnnotation          = requiredClass[scala.reflect.macros.internal.macroImpl]
+    lazy val MacroInternalPackage         = MacroImplAnnotation.owner.suchThat(_.isPackageClass)
+
+    /** Implementation of a class that is identical to `scala.reflect.macros.internal.macroImpl` but only exists at compileTime
+     */
+    lazy val MacroTastyImplAnnotation = {
+      val TastyMacroImpl = MacroInternalPackage.newClassSymbol(tpnme.TastyMacroImpl, NoPosition)
+      TastyMacroImpl.setPrivateWithin(ScalaPackage)
+      TastyMacroImpl.setInfoAndEnter(ClassInfoType(AnnotationClass.tpe :: Nil, newScope, TastyMacroImpl))
+      val unpickledMacroImpl = TermName("unpickledMacroImpl")
+      // getter
+      val unpickledMacroImplMeth = TastyMacroImpl.newMethod(unpickledMacroImpl, newFlags = STABLE | ACCESSOR | PARAMACCESSOR)
+      unpickledMacroImplMeth.setInfo(internal.nullaryMethodType(AnyTpe)).markAllCompleted()
+      TastyMacroImpl.info.decls enter unpickledMacroImplMeth
+      // field
+      val unpickledMacroImplField = TastyMacroImpl.newValue(unpickledMacroImpl, newFlags = PRIVATE | LOCAL | PARAMACCESSOR)
+      unpickledMacroImplField.setInfo(AnyTpe).markAllCompleted()
+      TastyMacroImpl.info.decls enter unpickledMacroImplField
+      // ctor
+      val ctor = TastyMacroImpl.newConstructor(NoPosition)
+      val param = ctor.newValueParameter(unpickledMacroImpl).setInfo(AnyTpe)
+      ctor.setInfo(MethodType(param :: Nil, TastyMacroImpl.tpe)).markAllCompleted()
+      TastyMacroImpl.info.decls enter ctor
+      TastyMacroImpl.addAnnotation(
+        sym = CompileTimeOnlyAttr,
+        arg = Literal(Constant("TastyMacroImpl is an implementation detail of unpickling TASTy"))
+      )
+      TastyMacroImpl.markAllCompleted()
+    }
 
     lazy val StringContextClass           = requiredClass[scala.StringContext]
     lazy val StringContextModule          = requiredModule[scala.StringContext.type]

test/tasty/neg-move-macros/pre-a/Position.scala

@@ -0,0 +1,3 @@
+package tastytest
+
+final case class Position(sourceName: String, line: Int)

test/tasty/neg-move-macros/pre-a/package.scala

@@ -0,0 +1,16 @@
+import scala.util.Random
+
+import scala.reflect.macros.blackbox.Context
+
+package object tastytest {
+
+  object Macros {
+    def posImpl(c: Context): c.Expr[Position] = {
+      import c.universe._
+      val fileName = c.enclosingPosition.source.path.split('/').last
+      val line = c.enclosingPosition.line
+      c.Expr(q"new Position($fileName, $line)")
+    }
+  }
+
+}

test/tasty/neg-move-macros/pre-b/Position.scala

@@ -0,0 +1,14 @@
+package tastytest
+
+import scala.reflect.macros.blackbox.Context
+
+final case class Position(sourceName: String, line: Int)
+
+object Position {
+  def posImpl(c: Context): c.Expr[Position] = {
+    import c.universe._
+    val fileName = c.enclosingPosition.source.path.split('/').last
+    val line = c.enclosingPosition.line
+    c.Expr(q"new Position($fileName, $line)")
+  }
+}

test/tasty/neg-move-macros/pre-b/package.scala

@@ -0,0 +1,9 @@
+import scala.util.Random
+
+package object tastytest {
+
+  object Macros {
+
+  }
+
+}

test/tasty/neg-move-macros/src-2/TestMacroCompat.check

@@ -0,0 +1,4 @@
+TestMacroCompat_fail.scala:7: error: can't find term required by object tastytest.MacroCompat: tastytest.`package`.Macros.posImpl; perhaps it is missing from the classpath.
+    val result = MacroCompat.testCase("foo")(pos)
+                                             ^
+1 error

test/tasty/neg-move-macros/src-2/TestMacroCompat_fail.scala

@@ -0,0 +1,11 @@
+package tastytest
+
+object TestMacroCompat {
+  import MacroCompat._
+
+  def test = {
+    val result = MacroCompat.testCase("foo")(pos)
+    println(result)
+  }
+
+}

test/tasty/neg-move-macros/src-3/MacroCompat.scala

@@ -0,0 +1,24 @@
+package tastytest
+
+import scala.language.experimental.macros
+
+object MacroCompat {
+
+  implicit def pos: Position = macro Macros.posImpl // implemented in test/tasty/run/pre/tastytest/package.scala
+  implicit inline def pos: Position = ${ Macros3.posImpl }
+
+  def testCase(test: => Any)(using Position): String =
+    s"${String.valueOf(test)} @@ ${summon[Position]}"
+
+  object Macros3 {
+    import quoted._
+
+    def posImpl(using qctx: QuoteContext): Expr[Position] = {
+      import qctx.tasty.{given _}
+      val name = qctx.tasty.rootPosition.sourceFile.jpath.getFileName.toString
+      val line = qctx.tasty.rootPosition.startLine + 1
+      '{ Position(${Expr(name)}, ${Expr(line)}) }
+    }
+  }
+
+}