besom-cfg initial checkin to monorepo

.github/workflows/ci.yml

@@ -85,3 +85,6 @@ jobs:
 
     - name: Integration test
       run: scala-cli --power test integration-tests
+
+    - name: Test besom-cfg
+      run: just test-cfg

Justfile

@@ -1,6 +1,7 @@
 # Big idea behind using a Justfile is so that we can have modules like in sbt.
 
 besom-version := `cat version.txt`
+besom-cfg-version := `cat besom-cfg/version.txt`
 is-snapshot := if "{{besom-version}}" =~ '.*-SNAPSHOT' { "true" } else { "false" }
 no-bloop-ci := if env_var_or_default('CI', "") == "true" { "--server=false" } else { "" }
 
@@ -35,8 +36,10 @@ default:
 # Aggregate tasks
 ####################
 
+# TODO aggregate tasks do not incorporate besom-cfg module (with the exception of clean-all)
+
 # Cleans everything
-clean-all: clean-json clean-sdk clean-auto clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-test-templates clean-test-examples clean-test-markdown
+clean-all: clean-json clean-sdk clean-auto clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-cfg clean-test-templates clean-test-examples clean-test-markdown
 
 # Compiles everything
 compile-all: compile-json compile-sdk compile-auto compile-codegen compile-scripts compile-compiler-plugin build-language-plugin
@@ -280,6 +283,63 @@ publish-language-plugins-all: package-language-plugins-all
 	just publish-language-plugin windows arm64
 	just publish-language-plugin windows amd64
 
+####################
+# Besom CFG
+####################
+
+# Compiles besom-cfg lib module
+compile-cfg-lib: publish-local-json publish-local-core
+	scala-cli --power compile besom-cfg/lib --suppress-experimental-feature-warning
+
+# Compiles besom-cfg k8s module
+compile-cfg-k8s: publish-local-cfg-lib
+	just cli packages local kubernetes:4.10.0
+	scala-cli --power compile besom-cfg/k8s --suppress-experimental-feature-warning
+
+# Compiles all besom-cfg modules
+compile-cfg: compile-cfg-lib compile-cfg-k8s
+
+# Publishes locally besom-cfg lib module
+publish-local-cfg-lib:
+	scala-cli --power publish local besom-cfg/lib --project-version {{besom-cfg-version}} --suppress-experimental-feature-warning
+
+# Publishes locally besom-cfg k8s module
+publish-local-cfg-k8s: compile-cfg-k8s
+	scala-cli --power publish local besom-cfg/k8s --project-version {{besom-cfg-version}} --suppress-experimental-feature-warning
+
+# Publishes locally all besom-cfg modules
+publish-local-cfg: publish-local-cfg-lib publish-local-cfg-k8s
+
+# Publishes besom-cfg lib module to Maven
+publish-maven-cfg-lib:
+	scala-cli --power publish besom-cfg/lib --project-version {{besom-cfg-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning
+
+# Publishes besom-cfg k8s module to Maven
+publish-maven-cfg-k8s:
+	scala-cli --power publish besom-cfg/k8s --project-version {{besom-cfg-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning
+
+# Tests besom-cfg lib module
+test-cfg-lib: compile-cfg-lib
+	scala-cli --power test besom-cfg/lib --suppress-experimental-feature-warning
+
+# Tests besom-cfg k8s module
+test-cfg-k8s: publish-local-cfg-lib compile-cfg-k8s
+	scala-cli --power test besom-cfg/k8s --suppress-experimental-feature-warning
+
+# Runs all tests of besom-cfg
+test-cfg: test-cfg-lib test-cfg-k8s
+
+# Cleans besom-cfg-lib build
+clean-cfg-lib:
+	scala-cli clean besom-cfg/lib
+
+# Cleans besom-cfg-k8s build
+clean-cfg-k8s:
+	scala-cli clean besom-cfg/k8s
+
+# Cleans all besom-cfg builds
+clean-cfg: clean-cfg-lib clean-cfg-k8s
+
 ####################
 # Codegen
 ####################

besom-cfg/k8s/.scalafmt.conf

@@ -0,0 +1,11 @@
+version = 3.5.2
+runner.dialect = scala3
+project.git = true
+align = most
+align.openParenCallSite = false
+align.openParenDefnSite = false
+align.tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="]
+indent.defnSite = 2
+maxColumn = 140
+
+rewrite.scala3.insertEndMarkerMinLines = 40

besom-cfg/k8s/project.scala

@@ -0,0 +1,23 @@
+//> using scala 3.3.3
+
+//> using dep com.lihaoyi::os-lib::0.9.3
+//> using dep org.virtuslab::besom-cfg:0.2.0-SNAPSHOT
+//> using dep org.virtuslab::besom-kubernetes::4.10.0-core.0.4-SNAPSHOT
+//> using dep com.lihaoyi::fansi::0.5.0
+//> using dep com.lihaoyi::fastparse:3.1.0
+
+//> using test.resourceDir ./src/test/resources
+
+//> using test.dep com.lihaoyi::pprint:0.6.6
+//> using test.dep org.scalameta::munit:1.0.0-M11
+
+//> using publish.name "besom-cfg-k8s"
+//> using publish.organization "org.virtuslab"
+//> using publish.url "https://github.com/VirtusLab/besom"
+//> using publish.vcs "github:VirtusLab/besom"
+//> using publish.license "Apache-2.0"
+//> using publish.repository "central"
+//> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy"
+//> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ"
+//> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban"
+//> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak"

besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala

@@ -0,0 +1,172 @@
+package besom.cfg.k8s
+
+import besom.cfg.internal.*
+import besom.types.{Input, Context, Output}
+import besom.cfg.*
+import besom.json.*
+import besom.cfg.containers.*
+import besom.api.kubernetes.core.v1.inputs.*
+
+import scala.util.*
+import scala.quoted.*
+import besom.cfg.k8s.syntax.*
+
+// this is besom-cfg-kubernetes entrypoint
+
+object syntax:
+  extension (s: Struct)
+    def foldedToEnvVarArgs(using Context): Output[List[EnvVarArgs]] =
+      s.foldToEnv.map(_.map { case (k, v) => EnvVarArgs(name = k, value = v) })
+
+object ConfiguredContainerArgs:
+
+  private val NL = System.lineSeparator()
+
+  inline def apply[C <: Struct](
+    name: String,
+    image: String,
+    configuration: C,
+    args: Input.Optional[List[Input[String]]] = None,
+    command: Input.Optional[List[Input[String]]] = None,
+    env: Input.Optional[List[Input[EnvVarArgs]]] = None,
+    envFrom: Input.Optional[List[Input[EnvFromSourceArgs]]] = None,
+    imagePullPolicy: Input.Optional[String] = None,
+    lifecycle: Input.Optional[LifecycleArgs] = None,
+    livenessProbe: Input.Optional[ProbeArgs] = None,
+    ports: Input.Optional[List[Input[ContainerPortArgs]]] = None,
+    readinessProbe: Input.Optional[ProbeArgs] = None,
+    resizePolicy: Input.Optional[List[Input[ContainerResizePolicyArgs]]] = None,
+    resources: Input.Optional[ResourceRequirementsArgs] = None,
+    restartPolicy: Input.Optional[String] = None,
+    securityContext: Input.Optional[SecurityContextArgs] = None,
+    startupProbe: Input.Optional[ProbeArgs] = None,
+    stdin: Input.Optional[Boolean] = None,
+    stdinOnce: Input.Optional[Boolean] = None,
+    terminationMessagePath: Input.Optional[String] = None,
+    terminationMessagePolicy: Input.Optional[String] = None,
+    tty: Input.Optional[Boolean] = None,
+    volumeDevices: Input.Optional[List[Input[VolumeDeviceArgs]]] = None,
+    volumeMounts: Input.Optional[List[Input[VolumeMountArgs]]] = None,
+    workingDir: Input.Optional[String] = None
+  )(using ctx: Context) = ${
+    applyImpl(
+      'name,
+      'image,
+      'configuration,
+      'args,
+      'command,
+      'env,
+      'envFrom,
+      'imagePullPolicy,
+      'lifecycle,
+      'livenessProbe,
+      'ports,
+      'readinessProbe,
+      'resizePolicy,
+      'resources,
+      'restartPolicy,
+      'securityContext,
+      'startupProbe,
+      'stdin,
+      'stdinOnce,
+      'terminationMessagePath,
+      'terminationMessagePolicy,
+      'tty,
+      'volumeDevices,
+      'volumeMounts,
+      'workingDir,
+      'ctx
+    )
+  }
+
+  def applyImpl[C <: Struct: Type](
+    name: Expr[String],
+    image: Expr[String],
+    configuration: Expr[C],
+    args: Expr[Input.Optional[List[Input[String]]]],
+    command: Expr[Input.Optional[List[Input[String]]]],
+    env: Expr[Input.Optional[List[Input[EnvVarArgs]]]],
+    envFrom: Expr[Input.Optional[List[Input[EnvFromSourceArgs]]]],
+    imagePullPolicy: Expr[Input.Optional[String]],
+    lifecycle: Expr[Input.Optional[LifecycleArgs]],
+    livenessProbe: Expr[Input.Optional[ProbeArgs]],
+    ports: Expr[Input.Optional[List[Input[ContainerPortArgs]]]],
+    readinessProbe: Expr[Input.Optional[ProbeArgs]],
+    resizePolicy: Expr[Input.Optional[List[Input[ContainerResizePolicyArgs]]]],
+    resources: Expr[Input.Optional[ResourceRequirementsArgs]],
+    restartPolicy: Expr[Input.Optional[String]],
+    securityContext: Expr[Input.Optional[SecurityContextArgs]],
+    startupProbe: Expr[Input.Optional[ProbeArgs]],
+    stdin: Expr[Input.Optional[Boolean]],
+    stdinOnce: Expr[Input.Optional[Boolean]],
+    terminationMessagePath: Expr[Input.Optional[String]],
+    terminationMessagePolicy: Expr[Input.Optional[String]],
+    tty: Expr[Input.Optional[Boolean]],
+    volumeDevices: Expr[Input.Optional[List[Input[VolumeDeviceArgs]]]],
+    volumeMounts: Expr[Input.Optional[List[Input[VolumeMountArgs]]]],
+    workingDir: Expr[Input.Optional[String]],
+    context: Expr[Context]
+  )(using Quotes): Expr[ContainerArgs] =
+    import quotes.reflect.*
+
+    val contName = name.value match
+      case None        => report.errorAndAbort("Container name has to be a literal!", name)
+      case Some(value) => value
+
+    val dockerImage = image.value match
+      case None        => report.errorAndAbort("Image name has to be a literal!", image)
+      case Some(value) => value
+
+    val schema = getDockerImageMetadata(dockerImage) match
+      case Left(throwable) => report.errorAndAbort(s"Failed to get metadata for image $dockerImage:$NL${pprint(throwable)}", image)
+      case Right(schema)   => schema
+
+    Diff.performDiff(schema, configuration) match
+      case Left(prettyDiff) => // TODO maybe strip all the ansi codes if in CI?
+        report.errorAndAbort(
+          s"Configuration provided for container $contName ($dockerImage) is invalid:$NL$NL$prettyDiff",
+          configuration
+        )
+
+      case Right(()) =>
+        val envExpr = '{
+          val envOutput                 = ${ env }.asOptionOutput()(using ${ context })
+          val conf                      = ${ configuration }
+          val configurationAsEnvVarArgs = conf.foldedToEnvVarArgs(using $context)
+          envOutput.zip(configurationAsEnvVarArgs).map {
+            case (Some(envVarArgsList), envVarArgsListFromConf) => envVarArgsList ++ envVarArgsListFromConf
+            case (None, envVarArgsListFromConf)                 => envVarArgsListFromConf
+          }
+        }
+
+        '{
+          ContainerArgs(
+            args = $args,
+            command = $command,
+            env = $envExpr,
+            envFrom = $envFrom,
+            image = $image,
+            imagePullPolicy = $imagePullPolicy,
+            lifecycle = $lifecycle,
+            livenessProbe = $livenessProbe,
+            name = ${ Expr(contName) },
+            ports = $ports,
+            readinessProbe = $readinessProbe,
+            resizePolicy = $resizePolicy,
+            resources = $resources,
+            restartPolicy = $restartPolicy,
+            securityContext = $securityContext,
+            startupProbe = $startupProbe,
+            stdin = $stdin,
+            stdinOnce = $stdinOnce,
+            terminationMessagePath = $terminationMessagePath,
+            terminationMessagePolicy = $terminationMessagePolicy,
+            tty = $tty,
+            volumeDevices = $volumeDevices,
+            volumeMounts = $volumeMounts,
+            workingDir = $workingDir
+          )(using $context)
+        }
+    end match
+  end applyImpl
+end ConfiguredContainerArgs

besom-cfg/k8s/src/main/scala/containers.scala

@@ -0,0 +1,63 @@
+package besom.cfg.containers
+
+// this should be a separate package, base for all container integrations
+
+import besom.cfg.internal.Schema
+import besom.json.*
+import scala.util.Try
+
+val cacheDir = sys.props.get("java.io.tmpdir").getOrElse("/tmp")
+
+def sanitizeImageName(image: String): String =
+  image
+    .replace("/", "_")
+    .replace(":", "_")
+
+def fetchFromCache(image: String): Option[String] =
+  if image.endsWith(":latest") then None
+  else
+    val sanitized = sanitizeImageName(image)
+    os.makeDir.all(os.Path(s"$cacheDir/besom-cfg"))
+    Try(os.read(os.Path(s"$cacheDir/besom-cfg/$sanitized"))).toOption
+
+def saveToCache(image: String, content: String): Unit =
+  if !image.endsWith(":latest") then
+    val sanitized = sanitizeImageName(image)
+    os.makeDir.all(os.Path(s"$cacheDir/besom-cfg"))
+    os.write.over(os.Path(s"$cacheDir/besom-cfg/$sanitized"), content)
+
+def resolveMetadataFromImage(image: String): String =
+  lazy val sbtNativePackagerFormatCall =
+    os
+      .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", "/opt/docker/lib/*", "besom.cfg.SummonConfiguration")
+      .call(check = false)
+
+  lazy val customDockerFormatCall =
+    os
+      .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", "/app/main", "besom.cfg.SummonConfiguration")
+      .call(check = false)
+
+  if sbtNativePackagerFormatCall.exitCode == 0 then sbtNativePackagerFormatCall.out.text().trim()
+  else if customDockerFormatCall.exitCode == 0 then customDockerFormatCall.out.text().trim()
+  else throw RuntimeException(s"Failed to get configuration from $image")
+
+def getDockerImageMetadata(image: String): Either[Throwable, Schema] =
+  Try {
+    // 1. cache result per image in /tmp DONE
+    // 2. verify the version of the library used, fail macro if we are older than it
+    // 3. parse the json to correct structure DONE
+    // next:
+    // - support different image setups, autodetect which one is used somehow? somewhat DONE
+    // - cp argument should be configurable
+    val json = fetchFromCache(image) match {
+      case Some(cachedJson) => cachedJson
+      case None =>
+        val json = resolveMetadataFromImage(image)
+
+        saveToCache(image, json)
+
+        json
+    }
+
+    summon[JsonFormat[Schema]].read(json.parseJson)
+  }.toEither