Towards Auto API

Implement APIs:
- add Stack
- add LocalWorkspace
- add Workspace

Add Language gRPC service for Auto API
Add language.proto for Auto API

Justfile

@@ -35,16 +35,16 @@ default:
 ####################
 
 # Cleans everything
-clean-all: clean-json clean-sdk clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-test-templates clean-test-examples clean-test-markdown
+clean-all: clean-json clean-sdk clean-auto clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-test-templates clean-test-examples clean-test-markdown
 
 # Compiles everything
-compile-all: compile-json compile-sdk compile-codegen compile-compiler-plugin build-language-plugin compile-scripts
+compile-all: compile-json compile-sdk compile-auto compile-codegen compile-compiler-plugin build-language-plugin compile-scripts
 
 # Tests everything
-test-all: test-json test-sdk test-codegen test-integration test-templates test-examples test-markdown
+test-all: test-json test-sdk test-auto test-codegen test-integration test-templates test-examples test-markdown
 
 # Publishes everything locally
-publish-local-all: publish-local-json publish-local-sdk publish-local-codegen install-language-plugin
+publish-local-all: publish-local-json publish-local-sdk publish-local-auto publish-local-codegen install-language-plugin
 
 # Publishes everything to Maven
 publish-maven-all: publish-maven-json publish-maven-sdk publish-maven-codegen
@@ -182,6 +182,31 @@ publish-local-json:
 publish-maven-json:
 	scala-cli --power publish besom-json --project-version {{besom-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning
 
+
+####################
+# Auto
+####################
+
+# Compiles auto module
+compile-auto: publish-local-core
+    scala-cli --power compile auto --suppress-experimental-feature-warning
+
+# Runs tests for auto module
+test-auto: compile-auto
+    scala-cli --power test auto --suppress-experimental-feature-warning
+
+# Cleans auto module
+clean-auto:
+    scala-cli --power clean auto
+
+# Publishes locally auto module
+publish-local-auto: test-auto
+    scala-cli --power publish local auto --project-version {{besom-version}} --suppress-experimental-feature-warning
+
+# Publishes auto module
+publish-maven-auto: test-auto
+    scala-cli --power publish auto --project-version {{besom-version}} {{publish-maven-auth-options}}
+
 ####################
 # Language plugin
 ####################

auto/project.scala

@@ -0,0 +1,24 @@
+//> using scala 3.3.1
+//> using options -java-output-version:11 -encoding:utf-8
+//> using options -deprecation -feature -Werror -Wunused:all -language:noAutoTupling
+
+//> using dep org.virtuslab::besom-json:0.2.1-SNAPSHOT
+//> using dep org.virtuslab::besom-core:0.2.1-SNAPSHOT
+//> using dep org.virtuslab::scala-yaml:0.0.8
+//> using dep com.lihaoyi::os-lib:0.9.3
+//> using dep com.lihaoyi::os-lib-watch:0.9.1
+//> using dep commons-io:commons-io:2.15.1
+//> using dep org.eclipse.jgit:org.eclipse.jgit:6.8.0.202311291450-r
+//> using dep org.eclipse.jgit:org.eclipse.jgit.ssh.jsch:6.8.0.202311291450-r
+//> using dep org.slf4j:slf4j-nop:2.0.5
+
+//> using test.dep "org.scalameta::munit:1.0.0-M10"
+
+//> using publish.name "besom-auto"
+//> using publish.organization "org.virtuslab"
+//> using publish.url "https://github.com/VirtusLab/besom"
+//> using publish.vcs "github:VirtusLab/besom"
+//> using publish.license "Apache-2.0"
+//> using publish.repository "central"
+//> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy"
+//> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak"

auto/src/main/scala/besom/auto/AutoError.scala

@@ -0,0 +1,142 @@
+package besom.auto
+
+import scala.util.matching.Regex
+
+@SerialVersionUID(1L)
+sealed abstract class BaseAutoError(message: Option[String], cause: Option[Throwable])
+    extends Exception(message.orElse(cause.map(_.toString)).orNull, cause.orNull)
+    with Product
+    with Serializable
+
+@SerialVersionUID(1L)
+case class AutoError(message: Option[String], cause: Option[Throwable]) extends BaseAutoError(message, cause)
+object AutoError:
+  def apply(message: String)                   = new AutoError(Some(message), None)
+  def apply(message: String, cause: Throwable) = new AutoError(Some(message), Some(cause))
+  def apply(cause: Throwable)                  = new AutoError(None, Some(cause))
+
+@SerialVersionUID(1L)
+case class ShellAutoError(
+  message: Option[String],
+  cause: Option[Throwable],
+  exitCode: Int,
+  stdout: String,
+  stderr: String,
+  command: Seq[String],
+  envVars: Map[String, String]
+) extends BaseAutoError(message, cause):
+  def withMessage(message: String): ShellAutoError = copy(message = this.message.map(message + "; " + _).orElse(Some(message)))
+
+  /** Returns true if the error was a result of selecting a stack that does not exist.
+    *
+    * @return
+    *   `true` if the error was due to a non-existent stack.
+    */
+  def isSelectStack404Error: Boolean =
+    val regex: Regex = "no stack named.*found".r
+    regex.findFirstIn(stderr).isDefined
+
+  /** Returns true if the error was a result of a conflicting update locking the stack.
+    *
+    * @return
+    *   `true` if the error was due to a conflicting update.
+    */
+  def isConcurrentUpdateError: Boolean =
+    val conflictText             = "[409] Conflict: Another update is currently in progress."
+    val localBackendConflictText = "the stack is currently locked by"
+    stderr.contains(conflictText) || stderr.contains(localBackendConflictText)
+
+  /** Returns true if the error was a result of creating a stack that already exists.
+    *
+    * @return
+    *   `true` if the error was due to a stack that already exists.
+    */
+  def isCreateStack409Error: Boolean =
+    val regex: Regex = "stack.*already exists".r
+    regex.findFirstIn(stderr).isDefined
+
+  /** Returns true if the pulumi core engine encountered an error (most likely a bug).
+    *
+    * @return
+    *   `true` if the error was due to an unexpected engine error.
+    */
+  def isUnexpectedEngineError: Boolean =
+    stdout.contains("The Pulumi CLI encountered a fatal error. This is a bug!")
+
+end ShellAutoError
+object ShellAutoError:
+  def apply(message: String, exitCode: Int, stdout: String, stderr: String, command: Seq[String], envVars: Map[String, String]) =
+    new ShellAutoError(
+      Some(msg(Some(message), None, exitCode, stdout, stderr, command, envVars)),
+      None,
+      exitCode,
+      stdout,
+      stderr,
+      command,
+      envVars
+    )
+  def apply(
+    message: String,
+    cause: Throwable,
+    exitCode: Int,
+    stdout: String,
+    stderr: String,
+    command: Seq[String],
+    envVars: Map[String, String]
+  ) =
+    new ShellAutoError(
+      Some(msg(Some(message), Some(cause), exitCode, stdout, stderr, command, envVars)),
+      Some(cause),
+      exitCode,
+      stdout,
+      stderr,
+      command,
+      envVars
+    )
+  def apply(cause: Throwable, exitCode: Int, stdout: String, stderr: String, command: Seq[String], envVars: Map[String, String]) =
+    new ShellAutoError(
+      Some(msg(None, Some(cause), exitCode, stdout, stderr, command, envVars)),
+      Some(cause),
+      exitCode,
+      stdout,
+      stderr,
+      command,
+      envVars
+    )
+  def apply(exitCode: Int, stdout: String, stderr: String, command: Seq[String], envVars: Map[String, String]) =
+    new ShellAutoError(
+      Some(msg(None, None, exitCode, stdout, stderr, command, envVars)),
+      None,
+      exitCode,
+      stdout,
+      stderr,
+      command,
+      envVars
+    )
+
+  private def msg(
+    message: Option[String],
+    cause: Option[Throwable],
+    exitCode: Int,
+    stdout: String,
+    stderr: String,
+    command: Seq[String],
+    envVars: Map[String, String]
+  ): String = {
+    s"""|${message.map(_ + "\n").getOrElse("")}${cause.map("cause: " + _.getMessage + "\n").getOrElse("")}
+        |command: ${redacted(command)}
+        |code: $exitCode
+        |stdout: 
+        |$stdout
+        |stderr: 
+        |$stderr
+        |env:
+        |${envVars.map { case (k, v) => s"  $k=$v" }.mkString("\n")}
+        |""".stripMargin
+  }
+
+  private def redacted(command: Seq[String]) =
+    val parts = if command.contains("-secret") then command.take(2) :+ "...[REDACTED]" else command
+    parts.mkString(" ")
+
+end ShellAutoError

auto/src/main/scala/besom/auto/Git.scala

@@ -0,0 +1,141 @@
+package besom.auto
+
+import besom.util.*
+import com.jcraft.jsch.JSch
+import org.eclipse.jgit.api.{CloneCommand, TransportConfigCallback}
+import org.eclipse.jgit.lib.ObjectId
+import org.eclipse.jgit.revwalk.RevWalk
+import org.eclipse.jgit.transport.ssh.jsch.JschConfigSessionFactory
+import org.eclipse.jgit.transport.{CredentialsProvider, RefSpec, SshTransport, UsernamePasswordCredentialsProvider}
+
+import java.util.Collections
+import scala.util.Using
+
+object Git:
+  private val RefPrefix       = "refs/"
+  private val RefHeadPrefix   = RefPrefix + "heads/"
+  private val RefTagPrefix    = RefPrefix + "tags/"
+  private val RefRemotePrefix = RefPrefix + "remotes/"
+
+  private class RichCloneCommand(private var depth: Option[Int] = None) extends CloneCommand:
+    def getCredentialsProvider: CredentialsProvider         = credentialsProvider
+    def getTransportConfigCallback: TransportConfigCallback = transportConfigCallback
+    def getDepth: Option[Int]                               = depth
+    override def setDepth(depth: Int): CloneCommand =
+      val cmd = super.setDepth(depth)
+      this.depth = Some(depth)
+      cmd
+
+  def setupGitRepo(workDir: os.Path, repoArgs: GitRepo): Either[Exception, os.Path] =
+    try
+      val cloneCommand = new RichCloneCommand()
+
+      cloneCommand
+        .setRemote("origin") // be explicit so we can require it in remote refs
+        .setURI(repoArgs.url)
+        .setDirectory(workDir.toIO)
+
+      if repoArgs.shallow then
+        cloneCommand
+          .setDepth(1)
+          .setCloneAllBranches(false)
+
+      repoArgs.auth match
+        case GitAuth.UsernameAndPassword(username, password) =>
+          cloneCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password))
+        case GitAuth.PersonalAccessToken(token) =>
+          // With Personal Access Token the username for use with a PAT can be
+          // *anything* but an empty string so we are setting this to 'git'
+          cloneCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider("git", token))
+        case GitAuth.SSHPrivateKey(key, passphrase) =>
+          val sshSessionFactory = new JschConfigSessionFactory():
+            override protected def configureJSch(jsch: JSch): Unit =
+              jsch.removeAllIdentity()
+              jsch.addIdentity("key", key.getBytes(), null /* no pub key */, passphrase.asOption.getOrElse(null).getBytes())
+          cloneCommand.setTransportConfigCallback { transport =>
+            transport.asInstanceOf[SshTransport].setSshSessionFactory(sshSessionFactory)
+          }
+        case GitAuth.SSHPrivateKeyPath(keyPath, passphrase) =>
+          val sshSessionFactory = new JschConfigSessionFactory():
+            override protected def configureJSch(jsch: JSch): Unit =
+              jsch.removeAllIdentity()
+              jsch.addIdentity(keyPath, passphrase.asOption.getOrElse(null).getBytes())
+          cloneCommand.setTransportConfigCallback { transport =>
+            transport.asInstanceOf[SshTransport].setSshSessionFactory(sshSessionFactory)
+          }
+        case NotProvided => // do nothing
+      repoArgs.branch match
+        case branch: String =>
+          // `Git.cloneRepository` will do appropriate fetching given a branch name. We must deal with
+          // different varieties, since people have been advised to use these as a workaround while only
+          // "refs/heads/<default>" worked.
+          //
+          // If a reference name is not supplied, then clone will fetch all refs (and all objects
+          // referenced by those), and checking out a commit later will work as expected.
+          val ref = {
+            try
+              val refSpec = RefSpec(branch)
+              if refSpec.matchSource(RefRemotePrefix)
+              then
+                refSpec.getDestination match
+                  case s"origin/$branch" => s"$RefHeadPrefix/$branch"
+                  case _                 => throw AutoError("a remote ref must begin with 'refs/remote/origin/', but got: '$branch'")
+              else if refSpec.matchSource(RefTagPrefix) then
+                branch // looks like `refs/tags/v1.0.0` -- respect this even though the field is `.Branch`
+              else if !refSpec.matchSource(RefHeadPrefix) then
+                s"$RefHeadPrefix/$branch" // not a remote, not refs/heads/branch; treat as a simple branch name
+              else
+                // already looks like a full branch name or tag, so use as is
+                refSpec.toString
+            catch case e: IllegalArgumentException => throw AutoError(s"Invalid branch name: '$branch'", e)
+          }
+          cloneCommand.setBranchesToClone(Collections.singletonList(ref))
+
+        case NotProvided => // do nothing
+      end match
+      // NOTE: pulumi has a workaround here for Azure DevOps requires, we might add if needed
+      val git        = cloneCommand.call()
+      val repository = git.getRepository
+
+      repoArgs.commitHash match
+        case commitHash: String =>
+          // ensure that the commit has been fetched
+          val fetchCommand = git
+            .fetch()
+            .setRemote("origin")
+            .setRefSpecs(new RefSpec(s"$commitHash:$commitHash"))
+            .setCredentialsProvider(cloneCommand.getCredentialsProvider)
+            .setTransportConfigCallback(cloneCommand.getTransportConfigCallback)
+
+          cloneCommand.getDepth.foreach {
+            fetchCommand.setDepth(_)
+          }
+          val _ = fetchCommand.call()
+
+          // If a commit hash is provided, then we must check it out explicitly. Otherwise, the
+          // repository will be in a detached HEAD state, and the commit hash will be the only
+          // commit in the repository.
+          val commitId = ObjectId.fromString(commitHash)
+          Using.resource(new RevWalk(repository)) { revWalk =>
+            val commit = revWalk.parseCommit(commitId)
+            val _ = git
+              .checkout()
+              .setName(commit.getName)
+              .setForced(true) // this method guarantees 'git --force' semantics
+              .call()
+          }
+        case NotProvided => // do nothing
+      end match
+
+      val finalWorkDir =
+        if repoArgs.projectPath.asOption.nonEmpty then
+          val projectPath = os.rel / repoArgs.projectPath.asOption.get
+          workDir / projectPath
+        else workDir
+
+      Right(finalWorkDir)
+
+    catch case e: Exception => Left(e)
+
+  end setupGitRepo
+end Git