Switch to the original fork of zinc and remove pipelining.

In order to make upgrading versions easier we decided to switch back to sbt/zinc instead of a custom fork. The original reason for using the fork was to enable build pipelining, but that itself has been implemented in the orginal fork. Unfortunately, to make the migration easier we needed to remove build pipelining for now to later add it with the default mechanism. Related to scalacenter#1383 The benchmarks are being run to make sure we are not regressing in performance.
tgodzik · Jan 3, 2022 · 7194388 · 7194388
1 parent 3688839
commit 7194388
Show file tree

Hide file tree

Showing 41 changed files with 184 additions and 1,318 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,7 +1,3 @@
-[submodule "zinc"]
-	path = zinc
-	url = https://github.com/scalacenter/zinc.git
-	branch = loop
 [submodule "nailgun"]
 	path = nailgun
 	url = https://github.com/scalacenter/nailgun.git

diff --git a/backend/src/main/scala/bloop/BloopClassFileManager.scala b/backend/src/main/scala/bloop/BloopClassFileManager.scala
@@ -1,28 +1,29 @@
 package bloop
 
-import bloop.io.{Paths => BloopPaths}
 import bloop.io.AbsolutePath
-import bloop.tracing.BraveTracer
 import bloop.io.ParallelOps
 import bloop.io.ParallelOps.CopyMode
+import bloop.io.{Paths => BloopPaths}
+import bloop.reporter.Reporter
+import bloop.tracing.BraveTracer
+import monix.eval.Task
+import xsbti.compile.ClassFileManager
+import xsbti.compile.PreviousResult
 
 import java.io.File
+import java.io.IOException
+import java.nio.file.CopyOption
+import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
-
+import java.nio.file.StandardCopyOption
 import scala.collection.mutable
-
-import xsbti.compile.ClassFileManager
-import monix.eval.Task
-import bloop.reporter.Reporter
-import xsbti.compile.PreviousResult
-import java.nio.file.Files
-import java.io.IOException
-import scala.util.Try
 import scala.util.Failure
 import scala.util.Success
+import scala.util.Try
 
 final class BloopClassFileManager(
+    backupDir0: Path,
     inputs: CompileInputs,
     outPaths: CompileOutPaths,
     allGeneratedRelativeClassFilePaths: mutable.HashMap[String, File],
@@ -38,9 +39,16 @@ final class BloopClassFileManager(
   private[this] val newClassesDirPath = newClassesDir.toString
   private[this] val dependentClassFilesLinks = new mutable.HashSet[Path]()
   private[this] val weakClassFileInvalidations = new mutable.HashSet[Path]()
+  private[this] val generatedFiles = new mutable.HashSet[File]
 
   // Supported compile products by the class file manager
   private[this] val supportedCompileProducts = List(".sjsir", ".nir", ".tasty")
+  // Files backed up during compilation
+  private[this] val movedFiles = new mutable.HashMap[File, File]
+
+  private val backupDir = backupDir0.normalize
+  backupDir.toFile.delete()
+  Files.createDirectories(backupDir)
 
   /**
    * Returns the set of all invalidated class files.
@@ -129,7 +137,6 @@ final class BloopClassFileManager(
         }
       }
     }
-
     allInvalidatedClassFilesForProject.++=(classes)
 
     val invalidatedExtraCompileProducts = classes.flatMap { classFile =>
@@ -142,12 +149,26 @@ final class BloopClassFileManager(
       }
     }
 
+    // Idea taken from the default TransactionalClassFileManager in zinc
+    // https://github.com/sbt/zinc/blob/c18637c1b30f8ab7d1f702bb98301689ec75854b/internal/zinc-core/src/main/scala/sbt/internal/inc/ClassFileManager.scala#L183
+    val toBeBackedUp = (classes ++ invalidatedExtraCompileProducts).filter(c =>
+      !movedFiles.contains(c) && !generatedFiles(c)
+    )
+    for (c <- toBeBackedUp)
+      if (c.exists)
+        movedFiles.put(c, move(c)).foreach(move)
+
+    classes.foreach { f =>
+      if (f.exists()) f.delete()
+    }
+
     allInvalidatedExtraCompileProducts.++=(invalidatedExtraCompileProducts)
   }
 
   def generated(generatedClassFiles: Array[File]): Unit = {
     memoizedInvalidatedClassFiles = null
     generatedClassFiles.foreach { generatedClassFile =>
+      generatedFiles += generatedClassFile
       val newClassFile = generatedClassFile.getAbsolutePath
       val relativeClassFilePath = newClassFile.replace(newClassesDirPath, "")
       allGeneratedRelativeClassFilePaths.put(relativeClassFilePath, generatedClassFile)
@@ -167,6 +188,7 @@ final class BloopClassFileManager(
           allInvalidatedExtraCompileProducts.-=(productAssociatedToClassFile)
       }
     }
+
   }
 
   def complete(success: Boolean): Unit = {
@@ -200,6 +222,22 @@ final class BloopClassFileManager(
         }
       )
     } else {
+      /* Restore all files from backuped last successful compilation to make sure
+       * that they are still available.
+       */
+      for {
+        (orig, tmp) <- movedFiles
+      } {
+        if (tmp.exists) {
+          if (!orig.getParentFile.exists) {
+            Files.createDirectory(orig.getParentFile.toPath())
+          }
+          Files.move(tmp.toPath(), orig.toPath())
+        }
+      }
+      backupDir.toFile().delete()
+      ()
+
       // Delete all compilation products generated in the new classes directory
       val deleteNewDir = Task { BloopPaths.delete(AbsolutePath(newClassesDir)); () }.memoize
       backgroundTasksForFailedCompilation.+=(
@@ -245,6 +283,12 @@ final class BloopClassFileManager(
       )
     }
   }
+
+  private def move(c: File): File = {
+    val target = Files.createTempFile(backupDir, "bloop", ".class").toFile
+    Files.move(c.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING)
+    target
+  }
 }
 
 object BloopClassFileManager {

diff --git a/backend/src/main/scala/bloop/CompileMode.scala b/backend/src/main/scala/bloop/CompileMode.scala
diff --git a/backend/src/main/scala/bloop/CompileProducts.scala b/backend/src/main/scala/bloop/CompileProducts.scala
@@ -35,6 +35,5 @@ case class CompileProducts(
     resultForDependentCompilationsInSameRun: PreviousResult,
     resultForFutureCompilationRuns: PreviousResult,
     invalidatedCompileProducts: Set[File],
-    generatedRelativeClassFilePaths: Map[String, File],
-    definedMacroSymbols: Array[String]
+    generatedRelativeClassFilePaths: Map[String, File]
 )
diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala
@@ -13,8 +13,6 @@ import bloop.tracing.BraveTracer
 import bloop.logging.{ObservedLogger, Logger}
 import bloop.reporter.{ProblemPerPhase, ZincReporter}
 import bloop.util.{AnalysisUtils, UUIDUtil, CacheHashCode}
-import bloop.CompileMode.Pipelined
-import bloop.CompileMode.Sequential
 
 import xsbti.compile._
 import xsbti.T2
@@ -23,7 +21,6 @@ import sbt.util.InterfaceUtil
 import sbt.internal.inc.Analysis
 import sbt.internal.inc.bloop.BloopZincCompiler
 import sbt.internal.inc.{FreshCompilerCache, InitialChanges, Locate}
-import sbt.internal.inc.bloop.internal.StopPipelining
 import sbt.internal.inc.{ConcreteAnalysisContents, FileAnalysisStore}
 
 import scala.concurrent.Promise
@@ -55,7 +52,6 @@ case class CompileInputs(
     previousCompilerResult: Compiler.Result,
     reporter: ZincReporter,
     logger: ObservedLogger[Logger],
-    mode: CompileMode,
     dependentResults: Map[File, PreviousResult],
     cancelPromise: Promise[Unit],
     tracer: BraveTracer,
@@ -260,6 +256,7 @@ object Compiler {
 
     def newFileManager: ClassFileManager = {
       new BloopClassFileManager(
+        Files.createTempDirectory("bloop"),
         compileInputs,
         compileOut,
         allGeneratedRelativeClassFilePaths,
@@ -300,7 +297,6 @@ object Compiler {
         .withClasspath(classpath)
         .withScalacOptions(optionsWithoutFatalWarnings)
         .withJavacOptions(inputs.javacOptions)
-        .withClasspathOptions(inputs.classpathOptions)
         .withOrder(inputs.compileOrder)
     }
 
@@ -344,19 +340,12 @@ object Compiler {
 
     import ch.epfl.scala.bsp
     import scala.util.{Success, Failure}
-    val mode = compileInputs.mode
     val reporter = compileInputs.reporter
 
     def cancel(): Unit = {
       // Complete all pending promises when compilation is cancelled
       logger.debug(s"Cancelling compilation from ${readOnlyClassesDirPath} to ${newClassesDirPath}")
       compileInputs.cancelPromise.trySuccess(())
-      mode match {
-        case _: Sequential => ()
-        case Pipelined(completeJava, finishedCompilation, _, _, _) =>
-          completeJava.trySuccess(())
-          finishedCompilation.tryFailure(CompileExceptions.FailedOrCancelledPromise)
-      }
 
       // Always report the compilation of a project no matter if it's completed
       reporter.reportCancelledCompilation()
@@ -379,7 +368,16 @@ object Compiler {
     val uniqueInputs = compileInputs.uniqueInputs
     reporter.reportStartCompilation(previousProblems)
     BloopZincCompiler
-      .compile(inputs, mode, reporter, logger, uniqueInputs, newFileManager, cancelPromise, tracer)
+      .compile(
+        inputs,
+        reporter,
+        logger,
+        uniqueInputs,
+        newFileManager,
+        cancelPromise,
+        tracer,
+        classpathOptions
+      )
       .materialize
       .doOnCancel(Task(cancel()))
       .map {
@@ -417,9 +415,9 @@ object Compiler {
                 val invalidatedExtraProducts =
                   allInvalidatedExtraCompileProducts.iterator.map(_.toPath).toSet
                 val invalidatedInThisProject = invalidatedClassFiles ++ invalidatedExtraProducts
-                val blacklist = invalidatedInThisProject ++ readOnlyCopyBlacklist.iterator
+                val denyList = invalidatedInThisProject ++ readOnlyCopyBlacklist.iterator
                 val config =
-                  ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, blacklist)
+                  ParallelOps.CopyConfiguration(5, CopyMode.ReplaceIfMetadataMismatch, denyList)
                 val lastCopy = ParallelOps.copyDirectories(config)(
                   readOnlyClassesDir,
                   clientClassesDir.underlying,
@@ -444,7 +442,6 @@ object Compiler {
           }
 
           val isNoOp = previousAnalysis.contains(analysis)
-          val definedMacroSymbols = mode.oracle.collectDefinedMacroSymbols
           if (isNoOp) {
             // If no-op, return previous result with updated classpath hashes
             val noOpPreviousResult = {
@@ -460,8 +457,7 @@ object Compiler {
               noOpPreviousResult,
               noOpPreviousResult,
               Set(),
-              Map.empty,
-              definedMacroSymbols
+              Map.empty
             )
 
             val backgroundTasks = new CompileBackgroundTasks {
@@ -578,8 +574,7 @@ object Compiler {
               resultForDependentCompilationsInSameRun,
               resultForFutureCompilationRuns,
               allInvalidated.toSet,
-              allGeneratedProducts,
-              definedMacroSymbols
+              allGeneratedProducts
             )
 
             Result.Success(
@@ -599,7 +594,6 @@ object Compiler {
           reporter.reportEndCompilation()
 
           cause match {
-            case f: StopPipelining => Result.Blocked(f.failedProjectNames)
             case f: xsbti.CompileFailed =>
               // We cannot guarantee reporter.problems == f.problems, so we aggregate them together
               val reportedProblems = reporter.allProblemsPerPhase.toList