Bump zinc to 1.4.4 (#1094)

It bumps the version of Zinc but it does not take advantage of the new Zinc features: VirtualFiles and Stamps.

A few remarks:

* The zinc compiler-interface now depends on the Scala library. It cannot be loaded globally in the ZincWorkerModule anymore. So it is loaded once for each scala version.
* Zinc cannot reuse the compiler bridges compiled by older version of Mill. So I classified them by the version of Zinc. The compiler bridge is now stored in out/mill/scalalib/ZincWorkerModule/worker/dest/zinc-<zincVersion>/<scalaVersion>.
* The current version of Zinc is incompatible with 2.12.0 but not with 2.12.1.
* The compiler bridge now depends on its resources to load itself

Commits:

* Bump zinc to 1.4.4

* Override scala-library in compiler-interface classpath

* Copy resources when compiling bridge

* Classify Zinc workers by zinc version

Each version of zinc comes with its own compiler bridges on every
version of Scala. Given that two different versions of Mill can use
two different versions of Zinc we must classify zinc workers by zinc
version (in addition to scala version).

* Zinc 1.4.x is incompatible with 2.12.0

* Add error message for Scala 2.12.0

Pull request: #1094

build.sc

@@ -78,7 +78,7 @@ object Deps {
   val sourcecode = ivy"com.lihaoyi::sourcecode:0.2.1"
   val upickle = ivy"com.lihaoyi::upickle:1.2.2"
   val utest = ivy"com.lihaoyi::utest:0.7.5"
-  val zinc = ivy"org.scala-sbt::zinc:1.4.0-M1"
+  val zinc = ivy"org.scala-sbt::zinc:1.4.4"
   val bsp = ivy"ch.epfl.scala:bsp4j:2.0.0-M13"
   val jarjarabrams = ivy"com.eed3si9n.jarjarabrams::jarjar-abrams-core:0.3.0"
 }
@@ -309,6 +309,25 @@ object scalalib extends MillModule {
     def testArgs = T{Seq(
       "-DMILL_SCALA_WORKER=" + runClasspath().map(_.path).mkString(",")
     )}
+
+    override def generatedSources = T{
+      val dest = T.ctx.dest
+      val artifacts = T.traverse(dev.moduleDeps)(_.publishSelfDependency)()
+      os.write(dest / "Versions.scala",
+        s"""package mill.scalalib.worker
+           |
+           |/**
+           | * Dependency versions.
+           | * Generated from mill in build.sc.
+           | */
+           |object Versions {
+           |  /** Version of Zinc. */
+           |  val zinc = "${Deps.zinc.dep.version}"
+           |}
+           |
+           |""".stripMargin)
+      super.generatedSources() ++ Seq(PathRef(dest))
+    }
   }
 }
 

contrib/proguard/test/src/ProguardTests.scala

@@ -19,7 +19,7 @@ object ProguardTests extends TestSuite {
     override def millSourcePath: os.Path =
       TestUtil.getSrcPathBase() / millOuterCtx.enclosing.split('.')
 
-    override def scalaVersion = "2.12.0"
+    override def scalaVersion = "2.12.1"
 
     def proguardContribClasspath = T{
       mill.modules.Util.millProjectModule("MILL_PROGUARD_LIB", "mill-contrib-proguard", repositoriesTask())
@@ -54,7 +54,7 @@ object ProguardTests extends TestSuite {
         val javaVersion = sys.props("java.version")
         val acceptFailure = isGithubActions && javaVersion.startsWith("11")
         try {
-
+          
           val Right((path, _)) = eval.apply(proguard.proguard)
           assert(os.exists(path.path))
 

scalalib/src/ZincWorkerModule.scala

@@ -1,14 +1,13 @@
 package mill.scalalib
 
-import coursier.core.Repository
+import coursier.Repository
 import mill.Agg
 import mill.T
 import mill.api.{Ctx, FixSizedCache, KeyedLockedCache}
 import mill.define.{Command, Discover, ExternalModule, Worker}
 import mill.scalalib.Lib.resolveDependencies
 import mill.scalalib.api.Util.{isBinaryBridgeAvailable, isDotty, isDottyOrScala3}
 import mill.scalalib.api.ZincWorkerApi
-import mill.util.JsonFormatters._
 
 object ZincWorkerModule extends ExternalModule with ZincWorkerModule with CoursierModule {
   lazy val millDiscover = Discover[this.type]
@@ -43,7 +42,6 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou
       case _ => 1
     }
     ctx.log.debug(s"ZinkWorker: using cache size ${jobs}")
-    val cp = compilerInterfaceClasspath()
     val cl = mill.api.ClassLoader.create(
       classpath().map(_.path.toNIO.toUri.toURL).toVector,
       getClass.getClassLoader
@@ -64,7 +62,7 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou
       .newInstance(
         Left((
           T.ctx(),
-          (x: String, y: String) => scalaCompilerBridgeJar(x, y, cp, repositoriesTask()).asSuccess.get.value
+          (x: String, y: String) => scalaCompilerBridgeJar(x, y, repositoriesTask()).asSuccess.get.value
         )),
         mill.scalalib.api.Util.grepJar(_, "scala-library", _, sources = false),
         mill.scalalib.api.Util.grepJar(_, "scala-compiler", _, sources = false),
@@ -76,7 +74,6 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou
 
   def scalaCompilerBridgeJar(scalaVersion: String,
                              scalaOrganization: String,
-                             compileClassPath: Agg[mill.api.PathRef],
                              repositories: Seq[Repository]) = {
     val (scalaVersion0, scalaBinaryVersion0) = scalaVersion match {
       case _ => (scalaVersion, mill.scalalib.api.Util.scalaBinaryVersion(scalaVersion))
@@ -98,39 +95,57 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou
       }
     val useSources = !isBinaryBridgeAvailable(scalaVersion)
 
-    resolveDependencies(
+    val bridgeJar = resolveDependencies(
       repositories,
       Lib.depToDependency(_, scalaVersion0),
       Seq(bridgeDep),
-      useSources
-    ).map{deps =>
-      val cp = if (useSources) Some(compileClassPath.map(_.path).toArray) else None
-      val res = mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, useSources)
-      (cp, res)
+      useSources,
+      Some(overrideScalaLibrary(scalaVersion, scalaOrganization))
+    ).map( deps =>
+      mill.scalalib.api.Util.grepJar(deps.map(_.path), bridgeName, bridgeVersion, useSources)
+    )
+
+    if (useSources) {
+      for {
+        jar <- bridgeJar
+        classpath <- compilerInterfaceClasspath(scalaVersion, scalaOrganization, repositories)
+      } yield (Some(classpath.map(_.path).toArray), jar)
+    } else {
+      bridgeJar.map((None, _))
     }
   }
 
-  def compilerInterfaceClasspath = T{
+  def compilerInterfaceClasspath(scalaVersion: String,
+                                 scalaOrganization: String,
+                                 repositories: Seq[Repository]) = {
     resolveDependencies(
-      repositoriesTask(),
+      repositories,
       Lib.depToDependency(_, "2.12.4", ""),
       Seq(ivy"org.scala-sbt:compiler-interface:${Versions.zinc}"),
-      ctx = Some(implicitly[mill.util.Ctx.Log])
+      // Since Zinc 1.4.0, the compiler-interface depends on the Scala library
+      // We need to override it with the scalaVersion and scalaOrganization of the module
+      mapDependencies = Some(overrideScalaLibrary(scalaVersion, scalaOrganization))
     )
   }
 
+  def overrideScalaLibrary(scalaVersion: String, scalaOrganization: String)
+                          (dep: coursier.Dependency): coursier.Dependency = {
+    if (dep.module.name.value == "scala-library") {
+      dep.withModule(dep.module.withOrganization(coursier.Organization(scalaOrganization)))
+        .withVersion(scalaVersion)
+    } else dep
+  }
+
   override def prepareOffline(): Command[Unit] = T.command {
     super.prepareOffline()
     classpath()
-    compilerInterfaceClasspath()
     // worker()
     ()
   }
 
   def prepareOfflineCompiler(scalaVersion: String, scalaOrganization: String): Command[Unit] = T.command {
     classpath()
-    val cp = compilerInterfaceClasspath()
-    scalaCompilerBridgeJar(scalaVersion, scalaOrganization, cp, repositoriesTask())
+    scalaCompilerBridgeJar(scalaVersion, scalaOrganization, repositoriesTask())
     ()
   }
 

scalalib/worker/src/ZincWorkerImpl.scala

@@ -2,23 +2,23 @@ package mill.scalalib.worker
 
 import java.io.File
 import java.util.Optional
-
 import mill.api.Loose.Agg
 import mill.api.{BuildProblemReporter, KeyedLockedCache, PathRef, Problem, ProblemPosition, Severity}
 import mill.scalalib.api.Util.{grepJar, isDotty, isDottyOrScala3, isScala3, scalaBinaryVersion}
 import mill.scalalib.api.{CompilationResult, ZincWorkerApi}
 import sbt.internal.inc._
 import sbt.internal.util.{ConsoleAppender, ConsoleOut}
 import sbt.util.LogExchange
+import xsbti.{PathBasedFile, VirtualFile}
 import xsbti.compile.{CompilerCache => _, FileAnalysisStore => _, ScalaInstance => _, _}
 
 import scala.ref.WeakReference
 
-case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClasspathEntryLookup {
-  override def analysis(classpathEntry: File): Optional[CompileAnalysis] =
+case class MockedLookup(am: VirtualFile => Optional[CompileAnalysis]) extends PerClasspathEntryLookup {
+  override def analysis(classpathEntry: VirtualFile): Optional[CompileAnalysis] =
     am(classpathEntry)
 
-  override def definesClass(classpathEntry: File): DefinesClass =
+  override def definesClass(classpathEntry: VirtualFile): DefinesClass =
     Locate.definesClass(classpathEntry)
 }
 
@@ -85,7 +85,8 @@ class ZincWorkerImpl(compilerBridge: Either[
     // Zinc does not have an entry point for Java-only compilation, so we need
     // to make up a dummy ScalaCompiler instance.
     val scalac = ZincUtil.scalaCompiler(
-      new ScalaInstance("", null, null, dummyFile, dummyFile, new Array(0), Some("")), null,
+      new ScalaInstance("", null, null, dummyFile, dummyFile, new Array(0), Some("")),
+      dummyFile,
       classpathOptions // this is used for javac too
     )
 
@@ -133,7 +134,14 @@ class ZincWorkerImpl(compilerBridge: Either[
                         compilerJars: Array[File],
                         compilerBridgeClasspath: Array[os.Path],
                         compilerBridgeSourcesJar: os.Path): Unit = {
-    val compileLog = compileDest / "compile-log.txt"
+    if (scalaVersion == "2.12.0") {
+      // The Scala 2.10.0 compiler fails on compiling the compiler bridge
+      throw new IllegalArgumentException(
+        "The current version of Zinc is incompatible with Scala 2.12.0.\n" +
+          "Use Scala 2.12.1 or greater (2.12.12 is recommended)."
+      )
+    }
+
     ctx0.log.info("Compiling compiler interface...")
 
     os.makeDir.all(workingDir)
@@ -142,7 +150,14 @@ class ZincWorkerImpl(compilerBridge: Either[
     val sourceFolder = mill.api.IO.unpackZip(compilerBridgeSourcesJar)(workingDir)
     val classloader = mill.api.ClassLoader.create(compilerJars.map(_.toURI.toURL), null)(ctx0)
 
-    val sources = os.walk(sourceFolder.path).filter(a => a.ext == "scala" || a.ext == "java")
+    val (sources, resources) =
+      os.walk(sourceFolder.path).filter(os.isFile)
+        .partition(a => a.ext == "scala" || a.ext == "java")
+
+    resources.foreach { res =>
+      val dest = compileDest / res.relativeTo(sourceFolder.path)
+      os.move(res, dest, replaceExisting = true, createFolders = true)
+    }
 
     val argsArray = Array[String](
       "-d", compileDest.toString,
@@ -174,7 +189,7 @@ class ZincWorkerImpl(compilerBridge: Either[
     compilerBridge match {
       case Right(compiled) => compiled(scalaVersion)
       case Left((ctx0, bridgeProvider)) =>
-        val workingDir = ctx0.dest / scalaVersion
+        val workingDir = ctx0.dest / s"zinc-${Versions.zinc}" / scalaVersion
         val lock = synchronized(compilerBridgeLocks.getOrElseUpdate(scalaVersion, new Object()))
         val compiledDest = workingDir / 'compiled
         lock.synchronized{
@@ -423,9 +438,13 @@ class ZincWorkerImpl(compilerBridge: Either[
     }
     val analysisMap0 = upstreamCompileOutput.map(_.swap).toMap
 
-    def analysisMap(f: File): Optional[CompileAnalysis] = {
-      analysisMap0.get(os.Path(f)) match{
-        case Some(zincPath) => FileAnalysisStore.binary(zincPath.toIO).get().map[CompileAnalysis](_.getAnalysis)
+    def analysisMap(f: VirtualFile): Optional[CompileAnalysis] = {
+      val analysisFile = f match {
+        case pathBased: PathBasedFile => analysisMap0.get(os.Path(pathBased.toPath))
+        case _ => None
+      }
+      analysisFile match {
+        case Some(zincPath) => FileAnalysisStore.binary(zincPath.toIO).get().map(_.getAnalysis)
         case None => Optional.empty[CompileAnalysis]
       }
     }
@@ -437,15 +456,21 @@ class ZincWorkerImpl(compilerBridge: Either[
       if (compileToJar) ctx.dest / "classes.jar"
       else ctx.dest / "classes"
 
-    val zincIOFile = zincFile.toIO
-    val classesIODir = classesDir.toIO
+    val store = FileAnalysisStore.binary(zincFile.toIO)
 
-    val store = FileAnalysisStore.binary(zincIOFile)
+    val converter = PlainVirtualFileConverter.converter
+    val classpath = (compileClasspath.iterator ++ Some(classesDir))
+      .map(path => converter.toVirtualFile(path.toNIO))
+      .toArray
+    val virtualSources = sources.iterator
+      .map(path => converter.toVirtualFile(path.toNIO))
+      .toArray
 
     val inputs = ic.inputs(
-      classpath = classesIODir +: compileClasspath.map(_.toIO).toArray,
-      sources = sources.toArray.map(_.toIO),
-      classesDirectory = classesIODir,
+      classpath = classpath,
+      sources = virtualSources,
+      classesDirectory = classesDir.toNIO,
+      earlyJarPath = None,
       scalacOptions = scalacOptions.toArray,
       javacOptions = javacOptions.toArray,
       maxErrors = 10,
@@ -455,18 +480,23 @@ class ZincWorkerImpl(compilerBridge: Either[
       setup = ic.setup(
         lookup,
         skip = false,
-        zincIOFile,
+        zincFile.toNIO,
         new FreshCompilerCache,
         IncOptions.of(),
         newReporter,
         None,
+        None,
         Array()
       ),
       pr = {
         val prev = store.get()
-        PreviousResult.of(prev.map(_.getAnalysis), prev.map(_.getMiniSetup))
+        PreviousResult.of(
+          prev.map(_.getAnalysis): Optional[CompileAnalysis],
+          prev.map(_.getMiniSetup): Optional[MiniSetup])
       },
-      temporaryClassesDirectory = java.util.Optional.empty()
+      temporaryClassesDirectory = java.util.Optional.empty(),
+      converter = converter,
+      stampReader = Stamps.timeWrapBinaryStamps(converter)
     )
 
     try {