[WIP] Parallelize the compiler via two-pass compilation

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -216,13 +216,9 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
           for (binary <- ctx.compilationUnit.pickled.get(claszSymbol.asClass)) {
             val store = if (mirrorC ne null) mirrorC else plainC
             val tasty =
-              if (ctx.settings.YemitTasty.value) {
-                val outTastyFile = getFileForClassfile(outF, store.name, ".tasty")
-                val outstream = new DataOutputStream(outTastyFile.bufferedOutput)
-                try outstream.write(binary)
-                finally outstream.close()
+              if (!ctx.settings.YemitTastyInClass.value) {
                 // TASTY attribute is created but 0 bytes are stored in it.
-                // A TASTY attribute has length 0 if and only if the .tasty file exists.
+                // A TASTY attribute has length 0 if and only if the .tasty file exists (created in the Pickler phase).
                 Array.empty[Byte]
               } else {
                 // Create an empty file to signal that a tasty section exist in the corresponding .class

compiler/src/dotty/tools/dotc/Driver.scala

@@ -1,4 +1,5 @@
-package dotty.tools.dotc
+package dotty.tools
+package dotc
 
 import dotty.tools.FatalError
 import config.CompilerCommand
@@ -9,6 +10,12 @@ import util.DotClass
 import reporting._
 import scala.util.control.NonFatal
 import fromtasty.TASTYCompiler
+import io.VirtualDirectory
+
+import java.util.concurrent.Executors
+import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.duration.Duration
+import scala.util.{Success, Failure}
 
 /** Run the Dotty compiler.
  *
@@ -130,8 +137,86 @@ class Driver extends DotClass {
    *                    if compilation succeeded.
    */
   def process(args: Array[String], rootCtx: Context): Reporter = {
+    def compile(files: List[String], ctx: Context) = doCompile(newCompiler(ctx), files)(ctx)
+
     val (fileNames, ctx) = setup(args, rootCtx)
-    doCompile(newCompiler(ctx), fileNames)(ctx)
+    val parallelism = {
+      val p = ctx.settings.parallelism.value(ctx)
+      if (p != 1 && (
+          ctx.settings.YemitTastyInClass.value(ctx) ||
+          ctx.settings.YtestPickler.value(ctx) ||
+          ctx.settings.fromTasty.value(ctx))) {
+        ctx.warning("Parallel compilation disabled due to incompatible setting.")
+        1
+      }
+      else if (p == 0)
+        Runtime.getRuntime().availableProcessors
+      else
+        p
+    }
+    if (parallelism == 1)
+      compile(fileNames, ctx)
+    else {
+      val tastyOutlinePath = new VirtualDirectory("<tasty outline>")
+
+      // First pass: generate .tasty outline files
+      val firstPassCtx = ctx.fresh
+        .setSetting(ctx.settings.outputDir, tastyOutlinePath)
+        .setSetting(ctx.settings.YemitTastyOutline, true)
+        .setSbtCallback(null) // Do not run the sbt-specific phases in this pass
+        .setCompilerCallback(null) // TODO: Change the CompilerCallback API to handle two-pass compilation?
+
+      compile(fileNames, firstPassCtx)
+
+      val scalaFileNames = fileNames.filterNot(_.endsWith(".java"))
+      if (!firstPassCtx.reporter.hasErrors && scalaFileNames.nonEmpty) {
+        // Second pass: split the list of files into $parallelism groups,
+        // compile each group independently.
+
+
+        val maxGroupSize = Math.ceil(scalaFileNames.length.toDouble / parallelism).toInt
+        val fileGroups = scalaFileNames.grouped(maxGroupSize).toList
+        val compilers = fileGroups.length
+
+        // Needed until https://github.com/sbt/zinc/pull/410 is merged.
+        val synchronizedSbtCallback =
+          if (rootCtx.sbtCallback != null)
+            new sbt.SynchronizedAnalysisCallback(rootCtx.sbtCallback)
+          else
+            null
+
+        def secondPassCtx = {
+          // TODO: figure out which parts of rootCtx we can safely reuse exactly.
+          val baseCtx = initCtx.fresh
+            .setSettings(rootCtx.settingsState)
+            .setReporter(new StoreReporter(rootCtx.reporter))
+            .setSbtCallback(synchronizedSbtCallback)
+            .setCompilerCallback(rootCtx.compilerCallback)
+
+          val (_, ctx) = setup(args, baseCtx)
+          ctx.fresh.setSetting(ctx.settings.priorityclasspath, tastyOutlinePath)
+        }
+
+        val executor = Executors.newFixedThreadPool(compilers)
+        implicit val ec = ExecutionContext.fromExecutor(executor)
+
+        val futureReporters = Future.sequence(fileGroups.map(fileGroup => Future {
+          // println("#Compiling: " + fileGroup.mkString(" "))
+          val reporter = compile(fileGroup, secondPassCtx)
+          // println("#Done: " + fileGroup.mkString(" "))
+          reporter
+        })).andThen {
+          case Success(reporters) =>
+            reporters.foreach(_.flush()(firstPassCtx))
+          case Failure(ex) =>
+            ex.printStackTrace
+            firstPassCtx.error(s"Exception during parallel compilation: ${ex.getMessage}")
+        }
+        Await.ready(futureReporters, Duration.Inf)
+        executor.shutdown()
+      }
+      firstPassCtx.reporter
+    }
   }
 
   def main(args: Array[String]): Unit = {

compiler/src/dotty/tools/dotc/Run.scala

@@ -152,9 +152,9 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
 
     compiling = true
 
-    // If testing pickler, make sure to stop after pickling phase:
+    // If testing pickler or generating tasty outlines, make sure to stop after pickling phase:
     val stopAfter =
-      if (ctx.settings.YtestPickler.value) List("pickler")
+      if (ctx.settings.YtestPickler.value || ctx.settings.YemitTastyOutline.value) List("pickler")
       else ctx.settings.YstopAfter.value
 
     val pluginPlan = ctx.addPluginPhases(ctx.phasePlan)
@@ -271,4 +271,4 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
     myUnits = null
     myUnitsCached = null
   }
-}
+}

compiler/src/dotty/tools/dotc/config/PathResolver.scala

@@ -180,7 +180,6 @@ class PathResolver(implicit ctx: Context) {
     case "extdirs"            => settings.extdirs.value
     case "classpath" | "cp"   => settings.classpath.value
     case "sourcepath"         => settings.sourcepath.value
-    case "priorityclasspath"  => settings.priorityclasspath.value
   }
 
   /** Calculated values based on any given command line options, falling back on
@@ -194,7 +193,8 @@ class PathResolver(implicit ctx: Context) {
     def javaUserClassPath   = if (useJavaClassPath) Defaults.javaUserClassPath else ""
     def scalaBootClassPath  = cmdLineOrElse("bootclasspath", Defaults.scalaBootClassPath)
     def scalaExtDirs        = cmdLineOrElse("extdirs", Defaults.scalaExtDirs)
-    def priorityClassPath   = cmdLineOrElse("priorityclasspath", "")
+    def priorityClassPath   = Option(settings.priorityclasspath.value)
+
     /** Scaladoc doesn't need any bootstrapping, otherwise will create errors such as:
      * [scaladoc] ../scala-trunk/src/reflect/scala/reflect/macros/Reifiers.scala:89: error: object api is not a member of package reflect
      * [scaladoc] case class ReificationException(val pos: reflect.api.PositionApi, val msg: String) extends Throwable(msg)
@@ -224,7 +224,7 @@ class PathResolver(implicit ctx: Context) {
     // Assemble the elements!
     // priority class path takes precedence
     def basis = List[Traversable[ClassPath]](
-      classesInExpandedPath(priorityClassPath),     // 0. The priority class path (for testing).
+      priorityClassPath.map(ClassPathFactory.newClassPath), // 0. The priority class path (for testing).
       classesInPath(javaBootClassPath),             // 1. The Java bootstrap class path.
       contentsOfDirsInPath(javaExtDirs),            // 2. The Java extension class path.
       classesInExpandedPath(javaUserClassPath),     // 3. The Java application class path.
@@ -249,7 +249,7 @@ class PathResolver(implicit ctx: Context) {
       |  userClassPath        = %s
       |  sourcePath           = %s
       |}""".trim.stripMargin.format(
-        scalaHome, ppcp(priorityClassPath),
+        scalaHome, ppcp(priorityClassPath.map(_.path).getOrElse("")),
         ppcp(javaBootClassPath), ppcp(javaExtDirs), ppcp(javaUserClassPath),
         useJavaClassPath,
         ppcp(scalaBootClassPath), ppcp(scalaExtDirs), ppcp(userClassPath),

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -22,7 +22,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val classpath = PathSetting("-classpath", "Specify where to find user class files.", defaultClasspath) withAbbreviation "-cp"
   val outputDir = OutputSetting("-d", "directory|jar", "destination for generated classfiles.",
     new PlainDirectory(Directory(".")))
-  val priorityclasspath = PathSetting("-priorityclasspath", "class path that takes precedence over all other paths (or testing only)", "")
+  val priorityclasspath = OutputSetting("-priorityclasspath", "directory|jar", "class path that takes precedence over all other paths (for testing only)", null)
 
   /** Other settings */
   val deprecation = BooleanSetting("-deprecation", "Emit warning and location for usages of deprecated APIs.")
@@ -46,6 +46,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val rewrite = OptionSetting[Rewrites]("-rewrite", "When used in conjunction with -language:Scala2 rewrites sources to migrate to new syntax")
   val silentWarnings = BooleanSetting("-nowarn", "Silence all warnings.")
   val fromTasty = BooleanSetting("-from-tasty", "Compile classes from tasty in classpath. The arguments are used as class names.")
+  val parallelism = IntSetting("-parallelism", "Number of parallel threads, 0 to use all cores.", 0)
 
   /** Decompiler settings */
   val printTasty = BooleanSetting("-print-tasty", "Prints the raw tasty.")
@@ -90,7 +91,8 @@ class ScalaSettings extends Settings.SettingGroup {
   val YdebugNames = BooleanSetting("-Ydebug-names", "Show internal representation of names")
   val YtermConflict = ChoiceSetting("-Yresolve-term-conflict", "strategy", "Resolve term conflicts", List("package", "object", "error"), "error")
   val Ylog = PhasesSetting("-Ylog", "Log operations during")
-  val YemitTasty = BooleanSetting("-Yemit-tasty", "Generate tasty in separate *.tasty file.")
+  val YemitTastyInClass = BooleanSetting("-Yemit-tasty-in-class", "Generate tasty in the .class file and add an empty *.hasTasty file.")
+  val YemitTastyOutline = BooleanSetting("-Yemit-tasty-outline", "Generate outline .tasty files and stop compilation after Pickler.")
   val YlogClasspath = BooleanSetting("-Ylog-classpath", "Output information about what classpath is being applied.")
   val YdisableFlatCpCaching  = BooleanSetting("-YdisableFlatCpCaching", "Do not cache flat classpath representation of classpath elements from jars across compiler instances.")
 

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -349,6 +349,11 @@ class Definitions {
       val methodNames = ScalaValueTypes.map(TreeGen.wrapArrayMethodName) + nme.wrapRefArray
       methodNames.map(ScalaPredefModule.requiredMethodRef(_).symbol)
     })
+    // A cache for the tree `Predef.???`
+    // TODO: Check if this actually matters for performance
+    val Predef_undefinedTree = new PerRun[ast.tpd.Tree]({ implicit ctx =>
+      ast.tpd.ref(defn.Predef_undefinedR)
+    })
 
   lazy val ScalaRuntimeModuleRef = ctx.requiredModuleRef("scala.runtime.ScalaRunTime")
   def ScalaRuntimeModule(implicit ctx: Context) = ScalaRuntimeModuleRef.symbol
@@ -427,10 +432,10 @@ class Definitions {
   def ArrayModule(implicit ctx: Context) = ArrayModuleType.symbol.moduleClass.asClass
 
 
-  lazy val UnitType: TypeRef = valueTypeRef("scala.Unit", BoxedUnitType, java.lang.Void.TYPE, UnitEnc, nme.specializedTypeNames.Void)
+  lazy val UnitType: TypeRef = valueTypeRef("scala.Unit", java.lang.Void.TYPE, UnitEnc, nme.specializedTypeNames.Void)
   def UnitClass(implicit ctx: Context) = UnitType.symbol.asClass
   def UnitModuleClass(implicit ctx: Context) = UnitType.symbol.asClass.linkedClass
-  lazy val BooleanType = valueTypeRef("scala.Boolean", BoxedBooleanType, java.lang.Boolean.TYPE, BooleanEnc, nme.specializedTypeNames.Boolean)
+  lazy val BooleanType = valueTypeRef("scala.Boolean", java.lang.Boolean.TYPE, BooleanEnc, nme.specializedTypeNames.Boolean)
   def BooleanClass(implicit ctx: Context) = BooleanType.symbol.asClass
     lazy val Boolean_notR   = BooleanClass.requiredMethodRef(nme.UNARY_!)
     def Boolean_! = Boolean_notR.symbol
@@ -449,13 +454,13 @@ class Definitions {
     })
     def Boolean_!= = Boolean_neqeqR.symbol
 
-  lazy val ByteType: TypeRef = valueTypeRef("scala.Byte", BoxedByteType, java.lang.Byte.TYPE, ByteEnc, nme.specializedTypeNames.Byte)
+  lazy val ByteType: TypeRef = valueTypeRef("scala.Byte", java.lang.Byte.TYPE, ByteEnc, nme.specializedTypeNames.Byte)
   def ByteClass(implicit ctx: Context) = ByteType.symbol.asClass
-  lazy val ShortType: TypeRef = valueTypeRef("scala.Short", BoxedShortType, java.lang.Short.TYPE, ShortEnc, nme.specializedTypeNames.Short)
+  lazy val ShortType: TypeRef = valueTypeRef("scala.Short", java.lang.Short.TYPE, ShortEnc, nme.specializedTypeNames.Short)
   def ShortClass(implicit ctx: Context) = ShortType.symbol.asClass
-  lazy val CharType: TypeRef = valueTypeRef("scala.Char", BoxedCharType, java.lang.Character.TYPE, CharEnc, nme.specializedTypeNames.Char)
+  lazy val CharType: TypeRef = valueTypeRef("scala.Char", java.lang.Character.TYPE, CharEnc, nme.specializedTypeNames.Char)
   def CharClass(implicit ctx: Context) = CharType.symbol.asClass
-  lazy val IntType: TypeRef = valueTypeRef("scala.Int", BoxedIntType, java.lang.Integer.TYPE, IntEnc, nme.specializedTypeNames.Int)
+  lazy val IntType: TypeRef = valueTypeRef("scala.Int", java.lang.Integer.TYPE, IntEnc, nme.specializedTypeNames.Int)
   def IntClass(implicit ctx: Context) = IntType.symbol.asClass
     lazy val Int_minusR   = IntClass.requiredMethodRef(nme.MINUS, List(IntType))
     def Int_- = Int_minusR.symbol
@@ -471,7 +476,7 @@ class Definitions {
     def Int_>= = Int_geR.symbol
     lazy val Int_leR   = IntClass.requiredMethodRef(nme.LE, List(IntType))
     def Int_<= = Int_leR.symbol
-  lazy val LongType: TypeRef = valueTypeRef("scala.Long", BoxedLongType, java.lang.Long.TYPE, LongEnc, nme.specializedTypeNames.Long)
+  lazy val LongType: TypeRef = valueTypeRef("scala.Long", java.lang.Long.TYPE, LongEnc, nme.specializedTypeNames.Long)
   def LongClass(implicit ctx: Context) = LongType.symbol.asClass
     lazy val Long_XOR_Long = LongType.member(nme.XOR).requiredSymbol(
       x => (x is Method) && (x.info.firstParamTypes.head isRef defn.LongClass)
@@ -486,9 +491,9 @@ class Definitions {
     lazy val Long_divR   = LongClass.requiredMethodRef(nme.DIV, List(LongType))
     def Long_/ = Long_divR.symbol
 
-  lazy val FloatType: TypeRef = valueTypeRef("scala.Float", BoxedFloatType, java.lang.Float.TYPE, FloatEnc, nme.specializedTypeNames.Float)
+  lazy val FloatType: TypeRef = valueTypeRef("scala.Float", java.lang.Float.TYPE, FloatEnc, nme.specializedTypeNames.Float)
   def FloatClass(implicit ctx: Context) = FloatType.symbol.asClass
-  lazy val DoubleType: TypeRef = valueTypeRef("scala.Double", BoxedDoubleType, java.lang.Double.TYPE, DoubleEnc, nme.specializedTypeNames.Double)
+  lazy val DoubleType: TypeRef = valueTypeRef("scala.Double", java.lang.Double.TYPE, DoubleEnc, nme.specializedTypeNames.Double)
   def DoubleClass(implicit ctx: Context) = DoubleType.symbol.asClass
 
   lazy val BoxedUnitType: TypeRef = ctx.requiredClassRef("scala.runtime.BoxedUnit")
@@ -1043,7 +1048,7 @@ class Definitions {
   lazy val Function2SpecializedParamTypes: collection.Set[TypeRef] =
     Set(IntType, LongType, DoubleType)
   lazy val Function0SpecializedReturnTypes: collection.Set[TypeRef] =
-    ScalaNumericValueTypeList.toSet + UnitType + BooleanType
+    ScalaValueTypes
   lazy val Function1SpecializedReturnTypes: collection.Set[TypeRef] =
     Set(UnitType, BooleanType, IntType, FloatType, LongType, DoubleType)
   lazy val Function2SpecializedReturnTypes: collection.Set[TypeRef] =
@@ -1115,27 +1120,43 @@ class Definitions {
   }
 
   lazy val ScalaNumericValueTypeList = List(
-    ByteType, ShortType, CharType, IntType, LongType, FloatType, DoubleType)
-
-  private lazy val ScalaNumericValueTypes: collection.Set[TypeRef] = ScalaNumericValueTypeList.toSet
-  private lazy val ScalaValueTypes: collection.Set[TypeRef] = ScalaNumericValueTypes + UnitType + BooleanType
-  private lazy val ScalaBoxedTypes = ScalaValueTypes map (t => boxedTypes(t.name))
+    ByteType,
+    ShortType,
+    CharType,
+    IntType,
+    LongType,
+    FloatType,
+    DoubleType
+  )
+  private lazy val ScalaNumericValueTypes = ScalaNumericValueTypeList.toSet
+
+  private lazy val ScalaValueTypeMap = Map(
+    ByteType   -> BoxedByteType,
+    ShortType  -> BoxedShortType,
+    CharType   -> BoxedCharType,
+    IntType    -> BoxedIntType,
+    LongType   -> BoxedLongType,
+    FloatType  -> BoxedFloatType,
+    DoubleType -> BoxedDoubleType,
+    UnitType    -> BoxedUnitType,
+    BooleanType -> BoxedBooleanType
+  )
+  private lazy val ScalaValueTypes = ScalaValueTypeMap.keySet
+  private lazy val ScalaBoxedTypes = ScalaValueTypeMap.values.toSet
 
   val ScalaNumericValueClasses = new PerRun[collection.Set[Symbol]](implicit ctx => ScalaNumericValueTypes.map(_.symbol))
   val ScalaValueClasses        = new PerRun[collection.Set[Symbol]](implicit ctx => ScalaValueTypes.map(_.symbol))
   val ScalaBoxedClasses        = new PerRun[collection.Set[Symbol]](implicit ctx => ScalaBoxedTypes.map(_.symbol))
 
-  private val boxedTypes = mutable.Map[TypeName, TypeRef]()
   private val valueTypeEnc = mutable.Map[TypeName, PrimitiveClassEnc]()
   private val typeTags = mutable.Map[TypeName, Name]().withDefaultValue(nme.specializedTypeNames.Object)
 
 //  private val unboxedTypeRef = mutable.Map[TypeName, TypeRef]()
 //  private val javaTypeToValueTypeRef = mutable.Map[Class[_], TypeRef]()
 //  private val valueTypeNamesToJavaType = mutable.Map[TypeName, Class[_]]()
 
-  private def valueTypeRef(name: String, boxed: TypeRef, jtype: Class[_], enc: Int, tag: Name): TypeRef = {
+  private def valueTypeRef(name: String, jtype: Class[_], enc: Int, tag: Name): TypeRef = {
     val vcls = ctx.requiredClassRef(name)
-    boxedTypes(vcls.name) = boxed
     valueTypeEnc(vcls.name) = enc
     typeTags(vcls.name) = tag
 //    unboxedTypeRef(boxed.name) = vcls
@@ -1145,7 +1166,7 @@ class Definitions {
   }
 
   /** The type of the boxed class corresponding to primitive value type `tp`. */
-  def boxedType(tp: Type)(implicit ctx: Context): TypeRef = boxedTypes(scalaClassName(tp))
+  def boxedType(tp: TypeRef)(implicit ctx: Context): TypeRef = ScalaValueTypeMap(tp)
 
   /** The JVM tag for `tp` if it's a primitive, `java.lang.Object` otherwise. */
   def typeTag(tp: Type)(implicit ctx: Context): Name = typeTags(scalaClassName(tp))

compiler/src/dotty/tools/dotc/core/Mode.scala

@@ -98,4 +98,7 @@ object Mode {
   /** Read comments from definitions when unpickling from TASTY */
   val ReadComments = newMode(21, "ReadComments")
 
+  /** We are in the rhs of an inline definition */
+  val InlineRHS = newMode(22, "InlineRHS")
+
 }

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -191,7 +191,7 @@ object SymbolLoaders {
   }
 
   def needCompile(bin: AbstractFile, src: AbstractFile) =
-    src.lastModified >= bin.lastModified
+    src.lastModified >= bin.lastModified && !bin.isVirtual
 
   /** Load contents of a package
    */

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -319,6 +319,7 @@ object Types {
     final def isAlias: Boolean = this.isInstanceOf[TypeAlias]
 
     /** Is this a MethodType which is from Java */
+    // FIXME: Why is this needed? Can't we just check is(JavaDefined) ?
     def isJavaMethod: Boolean = false
 
     /** Is this a MethodType which has implicit parameters */