Update project builders

.github/actions/build-project/action.yaml

@@ -88,7 +88,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-history.outputs.can-skip-build != 'true'
       with:
-        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.2.4"
+        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.2.5"
         options: -v ${{ github.workspace }}:/opencb/ -v ${{ github.workspace }}/github_rsa:/root/.ssh/id_rsa:ro
         run: |
           DefaultConfig='{"memoryRequestMb":4096}'
@@ -119,7 +119,7 @@ runs:
           mv build-status.txt /opencb/
           mv build-summary.txt /opencb/
 
-    - name: Check status status
+    - name: Check status
       id: check-status
       shell: bash
       if: steps.check-history.outputs.can-skip-build != 'true'
@@ -168,7 +168,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-history.outputs.can-skip-build != 'true'
       with:
-        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.2.4"
+        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.2.5"
         options: -v ${{ github.workspace }}:/opencb/ -e  ELASTIC_USERNAME=${{ inputs.elastic-user }} -e ELASTIC_PASSWORD=${{ inputs.elastic-password }}
         run: |
           ConfigFile="/opencb/.github/workflows/buildConfig.json"

.github/actions/setup-build/action.yaml

@@ -72,7 +72,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-published.outputs.is-compiler-published == 'false'
       with:
-        image: "virtuslab/scala-community-build-compiler-builder:v0.2.4"
+        image: "virtuslab/scala-community-build-compiler-builder:v0.2.5"
         options: -v ${{ github.workspace }}/compiler:/compiler/
         run: |
           Version="${{ steps.calc-version.outputs.effective-scala-version }}"

.github/workflows/buildSingle.yaml

@@ -48,6 +48,7 @@ jobs:
           repository-url: ${{ inputs.repository-url }}
           repository-branch: ${{ inputs.repository-branch }}
 
+  # This name is required to ensure we can get the build-url.
   build-project:
     runs-on: ubuntu-22.04
     needs: [setup-build]

.github/workflows/ci-build-imaages.yaml

@@ -0,0 +1,27 @@
+name: Infra CI - publish docker images
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        type: string
+        description: "Custom tag to publish container image version"
+        required: true
+jobs:
+  publish-images:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v3
+
+      - name: Login to container image repository
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build all images
+        run: scripts/build-all.sh ${{ inputs.version }}
+
+      - name: Push images to image registry
+        run: scripts/publish-all.sh ${{ inputs.version }}

README.md

@@ -93,19 +93,19 @@ eval $(minikube -p minikube docker-env)
 Most likely you'll need to build the base image only once (it doesn't get modified too often but building it takes quite a lot of time), e.g.:
 
 ```shell
-scripts/build-builder-base.sh v0.2.4
+scripts/build-builder-base.sh v0.2.5
 ```
 
 Build all the remaining images
 
 ```shell
-scripts/build-quick.sh v0.2.4
+scripts/build-quick.sh v0.2.5
 ```
 
 or (re)build each image separately e.g.
 
 ```shell
-scripts/build-mvn-repo.sh v0.2.4
+scripts/build-mvn-repo.sh v0.2.5
 ```
 
 ### Deploying and debugging in k8s

builder-base/Dockerfile

@@ -1,4 +1,4 @@
-FROM lampepfl/dotty:2021-03-22
+FROM lampepfl/dotty:2023-06-21-2
 
 ARG JDK_VERSION
 
@@ -19,5 +19,5 @@ ENV PATH="$JAVA_HOME/bin:$PATH"
 WORKDIR /build
 COPY . /build/
 
-ENV SBT_VERSIONS="1.6.2 1.7.0 1.7.1 1.7.2 1.7.3 1.8.0 1.8.1 1.8.2"
+ENV SBT_VERSIONS="1.7.3 1.8.2 1.9.1"
 RUN ./warmup.sh

builder-base/warmup.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-scala_version=3.2.1 # TODO
+scala_version=3.3.0 # TODO
 
 mkdir warm_up
 

cli/scb-cli.scala

@@ -31,7 +31,7 @@ class FailedProjectException(msg: String)
     with NoStackTrace
 
 val communityBuildVersion =
-  sys.props.getOrElse("communitybuild.version", "v0.2.4")
+  sys.props.getOrElse("communitybuild.version", "v0.2.5")
 private val CBRepoName = "VirtusLab/community-build3"
 val projectBuilderUrl =
   s"https://raw.githubusercontent.com/$CBRepoName/master/project-builder"
@@ -834,11 +834,7 @@ object MinikubeReproducer:
       ),
       command = Seq("/build/build-revision.sh"),
       args = args,
-      tty = true,
-      resources = ResourceRequirements(
-        requests = Map("memory" -> Quantity("4Gi")),
-        limits = Map("memory" -> Quantity("7Gi"))
-      )
+      tty = true
     )
 
   import javax.net.ssl.*

coordinator/configs/projects-config.conf

@@ -7,9 +7,7 @@ ablearthy_tdjson-bind.tests = disabled # cannot build native library
 akka_akka {
   // Based on Scala3 managed community build
   tests = compile-only
-  memory-request-mb = 6200
   sbt {
-    options = ["-J-Xmx6144m"]
     commands = [
       "set every targetSystemJdk := true",
       """set actor/Compile/scalacOptions -= "-Xfatal-warnings"""",
@@ -41,7 +39,6 @@ armanbilge_feral.projects.exclude = [
   "com.armanbilge%feral-lambda-api-gateway-proxy-http4s"
 ]
 armanbilge_gcp4s.tests = compile-only
-armanbilge_saslprep.sbt.options = ["-J-Xss2M"]
 ashwinbhaskar_sight-scala.sbt.commands = [
    """set root/Test/unmanagedSources/excludeFilter ~= { _ || "IntegrationTest.scala" }"""
 ]
@@ -67,15 +64,17 @@ babylonhealth_lit-fhir.tests = disabled // does not compile
 benhutchison_scalaswingcontrib.tests = disabled // Does not compile with any Scala version
 bilal-fazlani_akka-http-jwt-auth.tests = compile-only
 bilal-fazlani_circe-json-schema.tests = compile-only // Missing test files at runtime
-bot4s_zmatrix.sbt.commands = [
-  "excludeLibraryDependency com.github.ghik:zerowaste_{scalaVersion}",
-]
+bot4s_zmatrix {
+  sbt.commands = [
+    "excludeLibraryDependency com.github.ghik:zerowaste_{scalaVersion}"
+  ]
+}
 
 carueda_tscfg.sbt.commands = ["genCode"]
 cheleb_zio-pravega {
   tests = compile-only // uses Docker
   sbt.commands = [
-    """set pravega/Test/unmanagedSources/excludeFilter ~= {_ || "CreateResourcesExample.scala" }""", // error in source
+    """set pravega/Test/unmanagedSources/excludeFilter ~= {_ || "CreateResourcesExample.scala" }""", // error in sourceu
     "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}",
     "removeScalacOptionsStartingWith -P:wartremover"
   ]
@@ -274,6 +273,12 @@ japgolly_test-state {
 }
 japgolly_clear-config.tests = compile-only
 jbwheatley_pact4s.sbt.commands = ["""set shared/scalacOptions --= Seq("-Wconf:cat=deprecation:i", "-Xfatal-warnings") """]
+jchapuis_fs2-kafka-mock{
+  sbt.commands = [
+    "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}",
+    "removeScalacOptionsStartingWith -P:wartremover"
+  ]
+}
 joan38_kubernetes-client.tests = compile-only
 jobial-io_scase{
   projects.exclude=["io.jobial%scase-lambda-scala3-example"] 
@@ -346,7 +351,7 @@ losizm_scamper{
 
 markehammons_slinc {
   tests = compile-only
-  java.version = 19
+  java.version = 20
 }
 mateuszkubuszok_pipez{
   source-patches = [
@@ -380,9 +385,6 @@ mvv_sager.sbt.commands = ["disableFatalWarnings"]
 mvv_typine.sbt.commands = ["disableFatalWarnings"]
 
 naoh87_lettucef.tests = compile-only
-nickburkard_aws-cdk-scala {
-  memory-request-mb = 6148
-}
 noelwelsh_mads.sbt.commands = ["disableFatalWarnings"]
 
 ohze_couchbase-scala.tests = compile-only
@@ -420,6 +422,9 @@ polyvariant_colorize-scala {
 polyvariant_scodec-java-classfile {
   sbt.commands = ["excludeLibraryDependency org.polyvariant:better-tostring_{scalaVersion}"]
 }
+polyvariant_smithy4s-caliban {
+  sbt.commands = ["excludeLibraryDependency org.polyvariant:better-tostring_{scalaVersion}"]
+}
 polyvariant_treesitter4s {
   sbt.commands = ["excludeLibraryDependency org.polyvariant:better-tostring_{scalaVersion}"]
 }
@@ -502,7 +507,6 @@ scalapy_scalapy.tests = compile-only
 scalapy_python-native-libs.tests = compile-only
 scalaz_scalaz {
   sbt {
-    options = ["-J-Xmx6g"]
     commands = ["set every unidoc/unidocAllSources := Nil"] // in Scala 3.1.0+ gets into infinite loop when run in containers
   }
 }
@@ -537,12 +541,11 @@ sirthias_parboiled2.tests = compile-only
 sksamuel_elastic4s.tests = compile-only
 sky-uk_kafka-topic-loader.tests = compile-only // timeouts
 softwaremill_ox {
-  java.version=19
+  java.version=20
   sbt.options = ["-J--enable-preview" "-J--add-modules jdk.incubator.concurrent"]
 }
 softwaremill_sttp {
   sbt {
-    options=["-J-Xmx5g"]
     commands=["set every closeClassLoaders := false"]
   }
   tests = compile-only
@@ -553,7 +556,7 @@ softwaremill_sttp {
 }
 softwaremill_tapir {
   java.version=17
-  sbt.options=["-J-Xmx6g", "-J-Xss2M", "-J-XX:+UseG1GC"]
+  sbt.options=["-J-XX:+UseG1GC"]
   sbt.commands=["set vertxServerZio1.jvm(true)/Compile/doc/sources := Nil"]
   projects.overrides {
     tapir-datadog-metrics.tests = compile-only
@@ -564,6 +567,9 @@ softwaremill_tapir {
     tapir-vertx-server.tests = compile-only
   }
 }
+softwaremill_tapir-loom {
+  java.version=20
+}
 svroonland_rezilience.tests = compile-only
 svroonland_zio-kinesis.tests = compile-only
 svroonland_zio-amqp.tests = compile-only
@@ -581,7 +587,6 @@ taig_scala-linguist.projects.exclude=["io.taig%scala-linguist-graalvm-ruby"] //
 taig_scala-pygments.projects.exclude=["io.taig%scala-pygments-graalvm-python"]
 takezoe_solr-scala-client.tests = compile-only
 testcontainers_testcontainers-scala{
-  memory-request-mb = 5120
   java.version = 17
   tests = compile-only
 }
@@ -735,9 +740,6 @@ vladkopanev_cats-saga.source-patches = [
     replace-with = "bookCar.compensateIfFail"
   }
 ]
-wartremover_wartremover {
-  memory-request-mb = 7168
-}
 
 xuwei-k_httpz.tests = compile-only
 xuwei-k_wartremover-scalikejdbc {
@@ -751,7 +753,6 @@ y-yu_slick-bulk-insert.tests = compile-only
 zengularity_benji.tests = compile-only
 zio_izumi-reflect.tests = compile-only
 zio_zio-config {
-  memory-request-mb = 7168
   projects.overrides = {
     // Problem with ZIO deps
     zio-config-magnolia.tests = disabled 
@@ -768,7 +769,6 @@ zio_zio-logging {
   sbt.commands = [disableFatalWarnings]
 }
 zio_zio-protoquill {
-  memory-request-mb = 3072
   sbt.commands=[
     "set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"
   ]

coordinator/configs/replaced-projects.txt

@@ -8,3 +8,10 @@ outr/perfolation outr/perfolation master
 #1.0.0
 scala/nanotest-strawman scala/nanotest-strawman main
 eed3si9n/verify eed3si9n/verify main
+
+#2.13/2.13
+sellmerfud/optparse sellmerfud/optparse master
+
+# Unmaintained
+findify/flink-adt WojciechMazur/flink-ad fix/compilation-scala-3.3.x
+findify/flink-scala-api flink-extended/flink-scala-api master

k8s/mvn-repo.yaml

@@ -20,7 +20,7 @@ spec:
           secretName: mvn-repo-keystore
       containers:
       - name: mvn-repo
-        image: virtuslab/scala-community-build-mvn-repo:v0.2.4
+        image: virtuslab/scala-community-build-mvn-repo:v0.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8081

project-builder/sbt/build.sh

@@ -37,7 +37,7 @@ fi
 cd $repoDir
 
 # GithHub actions workers have maximally 7GB of RAM
-memorySettings=("-J-Xmx7G" "-J-Xms4G")
+memorySettings=("-J-Xmx7G" "-J-Xms4G" "-J-Xss8M")
 
 # Don't set version if not publishing
 setVersionCmd="setPublishVersion $version"

scripts/build-all.sh

@@ -7,10 +7,10 @@ if [ $# -ne 1 ]; then
 fi
 
 VERSION="$1"
-export PREV_CB_VERSION="v0.2.4"
+export PREV_CB_VERSION="v0.2.5"
 
 javaDefault=11
-javaAccessoryVersions=(8 17 19)
+javaAccessoryVersions=(8 17 20)
 scriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 
 if [[ ! -z "${BUILD_ONLY_DEFAULT_JDK}" ]]; then

scripts/build-builder-base.sh

@@ -31,8 +31,8 @@ case $JDK_VERSION in
   jdkDistro="17.0.5-tem "
   ;;
 
-"19")
-  jdkDistro="19.0.1-tem"
+"20")
+  jdkDistro="20.0.1-tem"
   ;;
 
 *)

scripts/publish-all.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -e
+
+if [ $# -ne 1 ]; then
+  echo "Wrong number of script arguments. Expected <revision>"
+  exit 1
+fi
+
+VERSION="$1"
+
+javaVersions=(8 11 17 20)
+scriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+BUILDER_BASE=virtuslab/scala-community-build-builder-base
+PROJECT_BUILDER=virtuslab/scala-community-build-project-builder
+MVN_REPO=virtuslab/scala-community-build-mvn-repo
+COMPILER_BUILDER=virtuslab/scala-community-build-compiler-builder
+
+# JDK-specifc images
+for image in $BUILDER_BASE $PROJECT_BUILDER; do
+  for javaVersion in "${javaVersions[@]}"; do
+    JDK_VERSION=jdk$javaVersion-$VERSION
+    JDK_LATEST=jdk$javaVersion-latest
+    docker tag $image:$JDK_VERSION $image:$JDK_LATEST
+    for tag in $JDK_VERSION $JDK_LATEST; do
+      docker push $image:$tag
+    done
+
+  done
+done
+
+# Single-JDK images
+for image in $MVN_REPO $COMPILER_BUILDER; do
+  docker tag $image:$VERSION $image:latest
+  for tag in $VERSION latest; do
+    docker push $image:$tag
+  done
+done

scripts/utils.sh

@@ -9,7 +9,7 @@ function scbk() {
 
 function checkJavaVersion() {
   version="$1"
-  supportedVersions=(8 11 17 19)
+  supportedVersions=(8 11 17 20)
 
   if [ -z "$version" ]; then
     echo >&2 "Java version has to be set"