Improve build times of large projects

.github/actions/build-project/action.yaml

@@ -113,7 +113,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-history.outputs.can-skip-build != 'true'
       with:
-        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.2.1"
+        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.2.2"
         options: -v ${{ github.workspace }}:/opencb/ -e  ELASTIC_USERNAME=${{ inputs.elastic-user }} -e ELASTIC_PASSWORD=${{ inputs.elastic-password }}
         run: |
           ConfigFile="/opencb/.github/workflows/buildConfig.json"

.github/actions/setup-build/action.yaml

@@ -72,7 +72,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-published.outputs.is-compiler-published == 'false'
       with:
-        image: "virtuslab/scala-community-build-compiler-builder:v0.2.1"
+        image: "virtuslab/scala-community-build-compiler-builder:v0.2.2"
         options: -v ${{ github.workspace }}/compiler:/compiler/
         run: |
           Version="${{ steps.calc-version.outputs.effective-scala-version }}"

.github/workflows/buildSingle.yaml

@@ -0,0 +1,68 @@
+name: "Open CB: single project manual build"
+run-name: ${{ inputs.build-name != '' && inputs.build-name || format('{0} - Scala {1} @ {2} / {3} {4}{5}', inputs.project-name, inputs.published-scala-version != '' && inputs.published-scala-version || 'snapshot', inputs.repository-url, inputs.repository-branch, inputs.extra-scalac-options != '' && format('extraScalacOptions={0} ', inputs.extra-scalac-options) || '', inputs.disabled-scalac-options != '' && format('disabledScalacOptions={0}', inputs.disabled-scalac-options) || '' ) }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      build-name:
+        type: string
+        description: "Custom name of the job in GitHub Actions"
+        default: ""
+      project-name:
+        type: string
+        description: "Name of a project to build using GitHub coordinates <org>/<repo> eg. VirtusLab/scala-cli"
+      published-scala-version:
+        type: string
+        description: "Published Scala version to use, if empty new version of compiler would be build with default name based on the selected repository"
+      repository-url:
+        type: string
+        description: "GitHub repository URL for compiler to build, ignored when published-scala-version is defined"
+        default: "lampepfl/dotty"
+      repository-branch:
+        type: string
+        description: "GitHub repository branch for compiler to build, ignored when published-scala-version is defined"
+        default: "main"
+      extra-scalac-options:
+        type: string
+        description: "List of scalacOptions which should be used when building projects. Multiple entires should be seperated by a single comma character `,`"
+        default: ""
+      disabled-scalac-options:
+        type: string
+        description: "List of scalacOptions which should be filtered out when building projects."
+        default: ""
+jobs:
+  setup-build:
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    outputs:
+      scala-version: ${{ steps.setup.outputs.scala-version }}
+      maven-repo-url: ${{ steps.setup.outputs.maven-repo-url }}
+    steps:
+      - name: "Git Checkout"
+        uses: actions/checkout@v3
+      - name: "Setup build"
+        uses: ./.github/actions/setup-build
+        id: setup
+        with:
+          scala-version: ${{ inputs.published-scala-version }}
+          repository-url: ${{ inputs.repository-url }}
+          repository-branch: ${{ inputs.repository-branch }}
+
+  build-project:
+    runs-on: ubuntu-22.04
+    needs: [setup-build]
+    continue-on-error: false
+    timeout-minutes: 60
+    steps:
+      - name: "Git Checkout"
+        uses: actions/checkout@v3
+      - name: "Build project"
+        uses: ./.github/actions/build-project
+        with:
+          project-name: ${{ inputs.project-name }}
+          extra-scalac-options: ${{ inputs.extra-scalac-options }}
+          disabled-scalac-options: ${{ inputs.disabled-scalac-options }}
+          scala-version: ${{ needs.setup-build.outputs.scala-version }}
+          maven-repo-url: ${{ needs.setup-build.outputs.maven-repo-url }}
+          elastic-user: ${{ secrets.OPENCB_ELASTIC_USER }}
+          elastic-password: ${{ secrets.OPENCB_ELASTIC_PSWD }}

README.md

@@ -93,19 +93,19 @@ eval $(minikube -p minikube docker-env)
 Most likely you'll need to build the base image only once (it doesn't get modified too often but building it takes quite a lot of time), e.g.:
 
 ```shell
-scripts/build-builder-base.sh v0.2.1
+scripts/build-builder-base.sh v0.2.2
 ```
 
 Build all the remaining images
 
 ```shell
-scripts/build-quick.sh v0.2.1
+scripts/build-quick.sh v0.2.2
 ```
 
 or (re)build each image separately e.g.
 
 ```shell
-scripts/build-mvn-repo.sh v0.2.1
+scripts/build-mvn-repo.sh v0.2.2
 ```
 
 ### Deploying and debugging in k8s

cli/scb-cli.scala

@@ -3,7 +3,7 @@
 //> using lib "org.json4s::json4s-native:4.0.6"
 //> using lib "com.lihaoyi::requests:0.8.0"
 //> using lib "com.lihaoyi::os-lib:0.9.1"
-//> using lib "io.get-coursier:coursier_2.13:2.1.0"
+//> using lib "io.get-coursier:coursier_2.13:2.1.3"
 //> using lib "com.goyeau::kubernetes-client:0.9.0"
 //> using lib "org.slf4j:slf4j-simple:2.0.7"
 //> using lib "com.github.scopt::scopt:4.1.0"
@@ -31,13 +31,13 @@ class FailedProjectException(msg: String)
     with NoStackTrace
 
 val communityBuildVersion =
-  sys.props.getOrElse("communitybuild.version", "v0.2.1")
+  sys.props.getOrElse("communitybuild.version", "v0.2.2")
 private val CBRepoName = "VirtusLab/community-build3"
 val projectBuilderUrl =
   s"https://raw.githubusercontent.com/$CBRepoName/master/project-builder"
-lazy val communityBuildDir = sys.props
-  .get("communitybuild.local.dir")
-  .map(os.Path(_))
+lazy val communityBuildLocalDir =
+  sys.props.get("communitybuild.local.dir").map(os.Path(_))
+lazy val communityBuildDir = communityBuildLocalDir
   .getOrElse(
     gitCheckout(s"https://github.com/$CBRepoName.git", None)(os.temp.dir())
   )
@@ -237,12 +237,13 @@ object BuildInfo:
     given StringManifest: Manifest[String] =
       scala.reflect.ManifestFactory.classType(classOf[String])
     def prepareBuildPlan(): JValue =
+      println(config.customRun.projectName)
       val args = Seq[os.Shellable](
         /* scalaBinaryVersion = */ 3,
         /* minStartsCount = */ 0,
         /* maxProjectsCount = */ 0,
         /* requiredProjects = */ config.customRun.projectName,
-        /* configsPath = */ communityBuildDir / "coordinator" / "configs",
+        /* configsPath = */ communityBuildDir / "coordinator" / "configs"
       )
       val javaProps =
         Seq("--java-prop", "opencb.coordinator.reproducer-mode=true")
@@ -254,10 +255,10 @@ object BuildInfo:
 
     val buildPlan = prepareBuildPlan() match
       case JArray(plan) => plan.filter(_ != JArray(Nil))
-      case t => sys.error("Unexpected build plan input: " + t)
+      case t            => sys.error("Unexpected build plan input: " + t)
     val projects = for
       case JArray(buildStage) <- buildPlan.take(1)
-       // There should be only 1 stage
+      // There should be only 1 stage
       project <- buildStage.take(1) // There should be only 1 project
       // Config is an object, though be default would be decoded to None when we expect Option[String]
       // We don't care about its content so we treat it as opaque string value
@@ -575,12 +576,14 @@ class MinikubeReproducer(using config: Config, build: BuildInfo):
       }
       .iterateUntil(_.nonEmpty)
       .map(_.get)
-      .recoverWith{
+      .recoverWith {
         case ex: Exception if retries > 0 =>
-           Console.err.println(s"Failed to get container state, retry with backoff, reason: ${ex}")
+          Console.err.println(
+            s"Failed to get container state, retry with backoff, reason: ${ex}"
+          )
           Thread.sleep(1000)
           getContainerState(retries - 1)
-      } 
+      }
 
     def waitForStart =
       for
@@ -630,7 +633,7 @@ class MinikubeReproducer(using config: Config, build: BuildInfo):
         Thread.sleep(1000)
         getLogs()
       }
-      val stdout=  getLogs()
+      val stdout = getLogs()
 
       logsFile
     }
@@ -974,7 +977,7 @@ class LocalReproducer(using config: Config, build: BuildInfo):
         MillReproducer(projectDir, logsFile)
       else if os.exists(projectDir / "build.sbt") then
         SbtReproducer(projectDir, logsFile)
-      else sys.error("Unsupported build tool")
+      else ScalaCliReproducer(projectDir, logsFile)
     try
       val redirectMessage =
         if config.redirectLogs then s", logs redirected to $logsFile" else ""
@@ -1047,6 +1050,45 @@ class LocalReproducer(using config: Config, build: BuildInfo):
         case _                  => None
       }
 
+  class ScalaCliReproducer(projectDir: os.Path, logsFile: os.Path)(using
+      project: ProjectInfo,
+      build: BuildInfo
+  ) extends BuildToolReproducer:
+    override def prepareBuild(): Unit = ()
+    override def runBuild(): Unit = {
+        val logsOutput =
+          if !config.redirectLogs then os.Inherit
+          else os.PathAppendRedirect(logsFile)
+        val effectiveConfig = project.params.config.getOrElse("{}")
+        val projectbuilderDirOrURI = communityBuildLocalDir
+          .map(_ / "project-builder")
+          .getOrElse(projectBuilderUrl)
+        os.proc(
+          "scala-cli",
+          "run",
+          s"${projectbuilderDirOrURI}/scala-cli/build.scala",
+          "--",
+          projectDir,
+          effectiveScalaVersion,
+          effectiveConfig
+        ).call(
+          check = false,
+          cwd = projectDir,
+          stdout = logsOutput,
+          stderr = logsOutput
+        )
+      }.exitCode match {
+        case 0 =>
+          println(
+            s"Sucessfully finished build for project ${project.id} (${project.projectName})"
+          )
+        case code =>
+          System.err.println(    s"Failed to run the build, for details check logs in $logsFile")
+          throw FailedProjectException(
+            s"Build for project ${project.id} (${project.projectName}) failed with exit code $code"
+          )
+    }
+
   class SbtReproducer(projectDir: os.Path, logsFile: os.Path)(using
       project: ProjectInfo,
       build: BuildInfo

coordinator/configs/filtered-projects.txt

@@ -232,11 +232,10 @@ tabdulradi:mazboot:.*
 markehammons:slinc:.*
 softwaremill:ox:.*
 
-# scala-cli project
+# Not a real project
 alexarchambault:scalacon-2022-demo:.*
 alexarchambault:scalacon2022-demo:.*
-kordyjan:typelogger:.*
-kuceramartin:tyqu:.*
+# Uses deprecated scala-cli directives
 quafadas:scautable:.*
 
 # bleep projects

coordinator/configs/projects-config.conf

@@ -469,11 +469,14 @@ softwaremill_sttp {
   ]
 }
 softwaremill_tapir {
+  java.version=17
   sbt.options=["-J-Xmx6g", "-J-Xss2M", "-J-XX:+UseG1GC"]
+  sbt.commands=["set vertxServerZio1.jvm(true)/Compile/doc/sources := Nil"]
   projects.overrides {
     tapir-datadog-metrics.tests = compile-only
     tapir-http4s-client.tests = compile-only
     tapir-sttp-client-ws-zio1.tests = compile-only  
+    tapir-sttp-client.tests = compile-only
     // Deadlocks when executing in minikube
     tapir-vertx-server.tests = compile-only
   }

coordinator/configs/slow-projects.txt

@@ -1,10 +1,3 @@
-akka/akka
-avast/scala-server-toolkit
 disneystreaming/smithy4s
-http4s/http4s
-kamon-io/kamon
-scalatest/scalatest
-softwaremill/sttp
 softwaremill/tapir
-thoughtworksinc/dsl.scala
-zio/zio
+thoughtworksinc/dsl.scala

coordinator/src/main/scala/build.scala

@@ -1,9 +1,9 @@
 //> using scala "3.2"
 //> using lib "com.novocode:junit-interface:0.11"
-//> using lib "org.jsoup:jsoup:1.15.4"
+//> using lib "org.jsoup:jsoup:1.16.1"
 //> using lib "org.json4s::json4s-native:4.0.6"
 //> using lib "org.json4s::json4s-ext:4.0.6"
-//> using lib "com.github.pureconfig::pureconfig-core:0.17.2"
+//> using lib "com.github.pureconfig::pureconfig-core:0.17.3"
 //> using lib "com.lihaoyi::os-lib:0.9.1"
 //> using lib "com.lihaoyi::requests:0.8.0"
 

k8s/mvn-repo.yaml

@@ -20,7 +20,7 @@ spec:
           secretName: mvn-repo-keystore
       containers:
       - name: mvn-repo
-        image: virtuslab/scala-community-build-mvn-repo:v0.2.1
+        image: virtuslab/scala-community-build-mvn-repo:v0.2.2
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8081

project-builder/build-revision.sh

@@ -43,9 +43,9 @@ elif [ -f "repo/build.sbt" ]; then
   echo "sbt project found: ${isSbtProject}"
   $scriptDir/sbt/prepare-project.sh repo "$enforcedSbtVersion" "$scalaVersion" "$projectConfig"
   $scriptDir/sbt/build.sh repo "$scalaVersion" "$version" "$targets" "$mvnRepoUrl" "$projectConfig" "$extraScalacOptions" "$disabledScalacOption"
-
 else
-  echo "Unknown project build tool, project layout:"
+  echo "Not found sbt or mill build files, assuming scala-cli project"
   ls -l repo/
+  scala-cli $scriptDir/scala-cli/build.scala -- repo "$scalaVersion" "$projectConfig"
   exit 1
 fi