Downloading of latest engine (#3412)

* Added fetching of latest engine

* Added caching of engine versions

* Added fixture for latest opentofu

* Documentation update

* Blank lines removal

* Update docs/_docs/02_features/engine.md

Co-authored-by: Yousif Akbar <11247449+yhakbar@users.noreply.github.com>

* Update docs/_docs/02_features/engine.md

Co-authored-by: Yousif Akbar <11247449+yhakbar@users.noreply.github.com>

---------

Co-authored-by: Yousif Akbar <11247449+yhakbar@users.noreply.github.com>

config/catalog.go

@@ -44,8 +44,8 @@ func (cfg *CatalogConfig) String() string {
 	return fmt.Sprintf("Catalog{URLs = %v}", cfg.URLs)
 }
 
-func (cfg *CatalogConfig) normalize(cofnigPath string) {
-	configDir := filepath.Dir(cofnigPath)
+func (cfg *CatalogConfig) normalize(configPath string) {
+	configDir := filepath.Dir(configPath)
 
 	// transform relative paths to absolute ones
 	for i, url := range cfg.URLs {

docs/_docs/02_features/engine.md

@@ -19,15 +19,21 @@ To try it out, all you need to do is include the following in your `terragrunt.h
 ```hcl
 engine {
   source  = "github.com/gruntwork-io/terragrunt-engine-opentofu"
-  version = "v0.0.5"
+  version = "v0.0.7"
 }
 ```
 
 This example leverages the official OpenTofu engine, [publicly available on GitHub](https://github.com/gruntwork-io/terragrunt-engine-opentofu).
 
 This engine currently leverages the locally available installation of the `tofu` binary, just like Terragrunt does by default without use of engine configurations. It provides a convenient example of how to build engines for Terragrunt.
 
-In the future, this plugin will expand in capability to include additional features and configurations.
+In the future, this engine will expand in capability to include additional features and configurations.
+
+Due to the fact that this functionality is still experimental, and not recommended for general production usage, set the following environment variable to opt-in to this functionality:
+
+```sh
+export TG_EXPERIMENTAL_ENGINE=1
+```
 
 ### Use Cases
 
@@ -45,7 +51,7 @@ e.g.
 
 ### HTTPS Sources
 
-Use an HTTP(S) URL to specify the path to the plugin:
+Use an HTTP(S) URL to specify the path to the engine:
 
 ```hcl
 engine {
@@ -67,9 +73,9 @@ engine {
 ### Parameters
 
 * `source`: (Required) The source of the plugin. Multiple engine approaches are supported, including GitHub repositories, HTTP(S) paths, and local absolute paths.
-* `version`: (Required for GitHub) The version of the plugin to download from GitHub releases.
+* `version`: The version of the engine to download from GitHub releases, if not specified, the latest release is always downloaded.
 * `type`: (Optional) Currently, the only supported type is `rpc`.
-* `meta`: (Optional) A block for setting plugin-specific metadata. This can include various configuration settings required by the plugin.
+* `meta`: (Optional) A block for setting engine-specific metadata. This can include various configuration settings required by the engine.
 
 ### Caching
 
@@ -91,22 +97,16 @@ To disable this feature, set the environment variable:
 export TG_ENGINE_SKIP_CHECK=0 
 ```
 
-Due to the fact that this functionality is still experimental, and not recommended for general production usage, set the following environment variable to opt-in to this functionality:
-
-```sh
-export TG_EXPERIMENTAL_ENGINE=1
-```
-
 ### Engine Metadata
 
 The `meta` block is used to pass metadata to the engine. This metadata can be used to configure the engine or pass additional information to the engine.
 
-The metadata block is a map of key-value pairs. Plugins can read the information passed via the metadata map to configure themselves or to pass additional information to the engine.
+The metadata block is a map of key-value pairs. Engines can read the information passed via the metadata map to configure themselves.
 
 ```hcl
 engine {
-   source  = "/home/users/iac-engines/my-custom-plugin"
-   # Optionally set metadata for the plugin.
+   source  = "/home/users/iac-engines/my-custom-engine"
+   # Optionally set metadata for the engine.
    meta = { 
      key_1 = ["value1", "value2"]
      key_2 = "1.6.0"

engine/engine.go

@@ -8,6 +8,7 @@ import (
 	goErrors "errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -16,6 +17,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gruntwork-io/terragrunt/internal/cache"
+
 	"github.com/hashicorp/go-getter"
 	"github.com/mholt/archiver/v3"
 
@@ -46,8 +49,10 @@ const (
 	ChecksumFileNameFormat                           = "terragrunt-iac-%s_%s_%s_SHA256SUMS"
 	EngineCachePathEnv                               = "TG_ENGINE_CACHE_PATH"
 	EngineSkipCheckEnv                               = "TG_ENGINE_SKIP_CHECK"
+	defaultEngineRepoRoot                            = "github.com/"
 	TerraformCommandContextKey      engineClientsKey = iota
 	LocksContextKey                 engineLocksKey   = iota
+	LatestVersionsContextKey        engineLocksKey   = iota
 )
 
 type engineClientsKey byte
@@ -130,6 +135,7 @@ func WithEngineValues(ctx context.Context) context.Context {
 
 	ctx = context.WithValue(ctx, TerraformCommandContextKey, &sync.Map{})
 	ctx = context.WithValue(ctx, LocksContextKey, util.NewKeyLocks())
+	ctx = context.WithValue(ctx, LatestVersionsContextKey, cache.NewCache[string]("engineVersions"))
 
 	return ctx
 }
@@ -147,6 +153,18 @@ func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error
 		return nil
 	}
 
+	// identify engine version if not specified
+	if len(e.Version) == 0 {
+		if !strings.Contains(e.Source, "://") {
+			tag, err := lastReleaseVersion(ctx, opts)
+			if err != nil {
+				return errors.WithStackTrace(err)
+			}
+
+			e.Version = tag
+		}
+	}
+
 	path, err := engineDir(e)
 	if err != nil {
 		return errors.WithStackTrace(err)
@@ -226,6 +244,53 @@ func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error
 	return nil
 }
 
+func lastReleaseVersion(ctx context.Context, opts *options.TerragruntOptions) (string, error) {
+	url := fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", strings.TrimPrefix(opts.Engine.Source, defaultEngineRepoRoot))
+
+	versionCache, err := engineVersionsCacheFromContext(ctx)
+
+	if err != nil {
+		return "", errors.WithStackTrace(err)
+	}
+
+	if val, found := versionCache.Get(ctx, url); found {
+		return val, nil
+	}
+
+	type release struct {
+		Tag string `json:"tag_name"`
+	}
+	// query tag from https://api.github.com/repos/{owner}/{repo}/releases/latest
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+
+	if err != nil {
+		return "", errors.WithStackTrace(err)
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+
+	if err != nil {
+		return "", errors.WithStackTrace(err)
+	}
+
+	defer resp.Body.Close() //nolint:errcheck
+	body, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		return "", errors.WithStackTrace(err)
+	}
+
+	var r release
+	if err := json.Unmarshal(body, &r); err != nil {
+		return "", errors.WithStackTrace(err)
+	}
+
+	versionCache.Put(ctx, url, r.Tag)
+
+	return r.Tag, nil
+}
+
 func extractArchive(opts *options.TerragruntOptions, downloadFile string, engineFile string) error {
 	if !isArchiveByHeader(downloadFile) {
 		opts.Logger.Info("Downloaded file is not an archive, no extraction needed")
@@ -383,6 +448,20 @@ func downloadLocksFromContext(ctx context.Context) (*util.KeyLocks, error) {
 	return result, nil
 }
 
+func engineVersionsCacheFromContext(ctx context.Context) (*cache.Cache[string], error) {
+	val := ctx.Value(LatestVersionsContextKey)
+	if val == nil {
+		return nil, errors.WithStackTrace(goErrors.New("failed to fetch engine versions cache from context"))
+	}
+
+	result, ok := val.(*cache.Cache[string])
+	if !ok {
+		return nil, errors.WithStackTrace(goErrors.New("failed to cast engine versions cache from context"))
+	}
+
+	return result, nil
+}
+
 // IsEngineEnabled returns true if the experimental engine is enabled.
 func IsEngineEnabled() bool {
 	ok, _ := strconv.ParseBool(os.Getenv(EnableExperimentalEngineEnvName)) //nolint:errcheck

test/fixtures/engine/opentofu-latest-run-all/app1/main.tf

@@ -0,0 +1,5 @@
+
+resource "local_file" "test" {
+  content  = "app1"
+  filename = "${path.module}/test.txt"
+}

test/fixtures/engine/opentofu-latest-run-all/app1/terragrunt.hcl

@@ -0,0 +1,3 @@
+include {
+  path   = find_in_parent_folders()
+}

test/fixtures/engine/opentofu-latest-run-all/app2/main.tf

@@ -0,0 +1,5 @@
+
+resource "local_file" "test" {
+  content  = "app1"
+  filename = "${path.module}/test.txt"
+}

test/fixtures/engine/opentofu-latest-run-all/app2/terragrunt.hcl

@@ -0,0 +1,3 @@
+include {
+  path   = find_in_parent_folders()
+}

test/fixtures/engine/opentofu-latest-run-all/app3/main.tf

@@ -0,0 +1,5 @@
+
+resource "local_file" "test" {
+  content  = "app1"
+  filename = "${path.module}/test.txt"
+}

test/fixtures/engine/opentofu-latest-run-all/app3/terragrunt.hcl

@@ -0,0 +1,3 @@
+include {
+  path   = find_in_parent_folders()
+}

test/fixtures/engine/opentofu-latest-run-all/app4/main.tf

@@ -0,0 +1,5 @@
+
+resource "local_file" "test" {
+  content  = "app1"
+  filename = "${path.module}/test.txt"
+}

test/fixtures/engine/opentofu-latest-run-all/app4/terragrunt.hcl

@@ -0,0 +1,3 @@
+include {
+  path   = find_in_parent_folders()
+}

test/fixtures/engine/opentofu-latest-run-all/app5/main.tf

@@ -0,0 +1,5 @@
+
+resource "local_file" "test" {
+  content  = "app1"
+  filename = "${path.module}/test.txt"
+}

test/fixtures/engine/opentofu-latest-run-all/app5/terragrunt.hcl

@@ -0,0 +1,3 @@
+include {
+  path   = find_in_parent_folders()
+}

test/fixtures/engine/opentofu-latest-run-all/terragrunt.hcl

@@ -0,0 +1,4 @@
+engine {
+  // use latest OpenTofu engine to do basic validation of implementation
+  source  = "github.com/gruntwork-io/terragrunt-engine-opentofu"
+}

test/integration_engine_test.go

@@ -20,10 +20,11 @@ import (
 )
 
 const (
-	testFixtureLocalEngine    = "fixtures/engine/local-engine"
-	testFixtureRemoteEngine   = "fixtures/engine/remote-engine"
-	testFixtureOpenTofuEngine = "fixtures/engine/opentofu-engine"
-	testFixtureOpenTofuRunAll = "fixtures/engine/opentofu-run-all"
+	testFixtureLocalEngine          = "fixtures/engine/local-engine"
+	testFixtureRemoteEngine         = "fixtures/engine/remote-engine"
+	testFixtureOpenTofuEngine       = "fixtures/engine/opentofu-engine"
+	testFixtureOpenTofuRunAll       = "fixtures/engine/opentofu-run-all"
+	testFixtureOpenTofuLatestRunAll = "fixtures/engine/opentofu-latest-run-all"
 
 	envVarExperimental = "TG_EXPERIMENTAL_ENGINE"
 )
@@ -200,6 +201,22 @@ func TestEngineDisableChecksumCheck(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestEngineOpentofuLatestRunAll(t *testing.T) {
+	t.Setenv(envVarExperimental, "1")
+
+	cleanupTerraformFolder(t, testFixtureOpenTofuLatestRunAll)
+	tmpEnvPath := copyEnvironment(t, testFixtureOpenTofuLatestRunAll)
+	rootPath := util.JoinPath(tmpEnvPath, testFixtureOpenTofuLatestRunAll)
+
+	stdout, _, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all apply -no-color -auto-approve --terragrunt-non-interactive --terragrunt-forward-tf-stdout --terragrunt-working-dir %s", rootPath))
+	require.NoError(t, err)
+
+	assert.Contains(t, stdout, "resource \"local_file\" \"test\"")
+	assert.Contains(t, stdout, "filename             = \"./test.txt\"\n")
+	assert.Contains(t, stdout, "Tofu Shutdown completed")
+	assert.Contains(t, stdout, "Apply complete!")
+}
+
 func setupEngineCache(t *testing.T) (string, string) {
 	// create temporary folder
 	cacheDir := t.TempDir()