From 08f8b75bf27fbf0e5f8a7767c23eda5bfe789721 Mon Sep 17 00:00:00 2001 From: james0209 Date: Fri, 3 May 2024 18:50:56 +0100 Subject: [PATCH 1/7] WIP: Add support for custom download URLs --- releases/exact_version.go | 13 ++++++++----- releases/latest_version.go | 13 ++++++++----- releases/releases_test.go | 6 +++--- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/releases/exact_version.go b/releases/exact_version.go index d288471..990f294 100644 --- a/releases/exact_version.go +++ b/releases/exact_version.go @@ -36,7 +36,10 @@ type ExactVersion struct { // instead of built-in pubkey to verify signature of downloaded checksums ArmoredPublicKey string - apiBaseURL string + // CustomURL is an optional field that specifies a custom URL to download the product from. + // This can be useful in environments where access to the default HashiCorp releases site (releases.hashicorp.com) is blocked. + // If CustomURL is set, the product will be downloaded from this URL instead of the default site. + CustomURL string logger *log.Logger pathsToRemove []string } @@ -102,8 +105,8 @@ func (ev *ExactVersion) Install(ctx context.Context) (string, error) { ev.log().Printf("will install into dir at %s", dstDir) rels := rjson.NewReleases() - if ev.apiBaseURL != "" { - rels.BaseURL = ev.apiBaseURL + if ev.CustomURL != "" { + rels.BaseURL = ev.CustomURL } rels.SetLogger(ev.log()) installVersion := ev.Version @@ -124,8 +127,8 @@ func (ev *ExactVersion) Install(ctx context.Context) (string, error) { if ev.ArmoredPublicKey != "" { d.ArmoredPublicKey = ev.ArmoredPublicKey } - if ev.apiBaseURL != "" { - d.BaseURL = ev.apiBaseURL + if ev.CustomURL != "" { + d.BaseURL = ev.CustomURL } licenseDir := "" diff --git a/releases/latest_version.go b/releases/latest_version.go index c4b458b..4cc5cf4 100644 --- a/releases/latest_version.go +++ b/releases/latest_version.go @@ -36,7 +36,10 @@ type LatestVersion struct { // instead of built-in pubkey to verify signature of downloaded checksums ArmoredPublicKey string - apiBaseURL string + // CustomURL is an optional field that specifies a custom URL to download the product from. + // This can be useful in environments where access to the default HashiCorp releases site (releases.hashicorp.com) is blocked. + // If CustomURL is set, the product will be downloaded from this URL instead of the default site. + CustomURL string logger *log.Logger pathsToRemove []string } @@ -98,8 +101,8 @@ func (lv *LatestVersion) Install(ctx context.Context) (string, error) { lv.log().Printf("will install into dir at %s", dstDir) rels := rjson.NewReleases() - if lv.apiBaseURL != "" { - rels.BaseURL = lv.apiBaseURL + if lv.CustomURL != "" { + rels.BaseURL = lv.CustomURL } rels.SetLogger(lv.log()) versions, err := rels.ListProductVersions(ctx, lv.Product.Name) @@ -125,8 +128,8 @@ func (lv *LatestVersion) Install(ctx context.Context) (string, error) { if lv.ArmoredPublicKey != "" { d.ArmoredPublicKey = lv.ArmoredPublicKey } - if lv.apiBaseURL != "" { - d.BaseURL = lv.apiBaseURL + if lv.CustomURL != "" { + d.BaseURL = lv.CustomURL } licenseDir := "" if lv.Enterprise != nil { diff --git a/releases/releases_test.go b/releases/releases_test.go index 132a92d..c8a56cb 100644 --- a/releases/releases_test.go +++ b/releases/releases_test.go @@ -61,7 +61,7 @@ func TestLatestVersion_basic(t *testing.T) { lv := &LatestVersion{ Product: product.Terraform, ArmoredPublicKey: getTestPubKey(t), - apiBaseURL: testutil.NewTestServer(t, mockApiRoot).URL, + CustomURL: testutil.NewTestServer(t, mockApiRoot).URL, } lv.SetLogger(testutil.TestLogger()) @@ -95,7 +95,7 @@ func TestLatestVersion_prereleases(t *testing.T) { Product: product.Terraform, IncludePrereleases: true, ArmoredPublicKey: getTestPubKey(t), - apiBaseURL: testutil.NewTestServer(t, mockApiRoot).URL, + CustomURL: testutil.NewTestServer(t, mockApiRoot).URL, } lv.SetLogger(testutil.TestLogger()) @@ -167,7 +167,7 @@ func BenchmarkExactVersion(b *testing.B) { Product: product.Terraform, Version: version.Must(version.NewVersion("0.14.11")), ArmoredPublicKey: getTestPubKey(b), - apiBaseURL: testutil.NewTestServer(b, mockApiRoot).URL, + CustomURL: testutil.NewTestServer(b, mockApiRoot).URL, InstallDir: installDir, } ev.SetLogger(testutil.TestLogger()) From bc32c69803bb37a2b453b6e8af85fab99e87d798 Mon Sep 17 00:00:00 2001 From: james0209 Date: Fri, 3 May 2024 20:36:22 +0100 Subject: [PATCH 2/7] Add note about index.json --- releases/exact_version.go | 1 + releases/latest_version.go | 1 + 2 files changed, 2 insertions(+) diff --git a/releases/exact_version.go b/releases/exact_version.go index 990f294..61b55f3 100644 --- a/releases/exact_version.go +++ b/releases/exact_version.go @@ -39,6 +39,7 @@ type ExactVersion struct { // CustomURL is an optional field that specifies a custom URL to download the product from. // This can be useful in environments where access to the default HashiCorp releases site (releases.hashicorp.com) is blocked. // If CustomURL is set, the product will be downloaded from this URL instead of the default site. + // Note: The directory structure of the custom URL must match the HashiCorp releases site (including the index.json files). CustomURL string logger *log.Logger pathsToRemove []string diff --git a/releases/latest_version.go b/releases/latest_version.go index 4cc5cf4..51ea8d5 100644 --- a/releases/latest_version.go +++ b/releases/latest_version.go @@ -39,6 +39,7 @@ type LatestVersion struct { // CustomURL is an optional field that specifies a custom URL to download the product from. // This can be useful in environments where access to the default HashiCorp releases site (releases.hashicorp.com) is blocked. // If CustomURL is set, the product will be downloaded from this URL instead of the default site. + // Note: The directory structure of the custom URL must match the HashiCorp releases site (including the index.json files). CustomURL string logger *log.Logger pathsToRemove []string From 8183073b020741616fcae8b04eea31e5c6852497 Mon Sep 17 00:00:00 2001 From: james0209 Date: Sat, 4 May 2024 13:17:30 +0100 Subject: [PATCH 3/7] Adjust comment --- internal/releasesjson/downloader.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/releasesjson/downloader.go b/internal/releasesjson/downloader.go index bb749f3..01ea7c8 100644 --- a/internal/releasesjson/downloader.go +++ b/internal/releasesjson/downloader.go @@ -62,8 +62,9 @@ func (d *Downloader) DownloadAndUnpack(ctx context.Context, pv *ProductVersion, archiveURL := pb.URL if d.BaseURL != "" { - // ensure that absolute download links from mocked responses - // are still pointing to the mock server if one is set + // If custom URL is set, use that instead of the one from the JSON. + // If using a custom URL, it may be because access to the HashiCorp Releases API is restricted. + // Also ensure that absolute download links from mocked responses are still pointing to the mock server if one is set. baseURL, err := url.Parse(d.BaseURL) if err != nil { return "", err From 8fe960093aa86deabb6caaa5d6a59e40538fcc15 Mon Sep 17 00:00:00 2001 From: james0209 Date: Sat, 4 May 2024 17:08:15 +0100 Subject: [PATCH 4/7] Add flag to CLI --- cmd/hc-install/cmd_install.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/hc-install/cmd_install.go b/cmd/hc-install/cmd_install.go index eb61493..9afa1d0 100644 --- a/cmd/hc-install/cmd_install.go +++ b/cmd/hc-install/cmd_install.go @@ -43,6 +43,8 @@ Usage: hc-install install [options] -version Defaults to current working directory. -log-file Path to file where logs will be written. /dev/stdout or /dev/stderr can be used to log to STDOUT/STDERR. + -custom-url Custom URL to download the product from. + URL Endpoint must have same structure as HashiCorp releases. ` return strings.TrimSpace(helpText) } @@ -52,6 +54,7 @@ func (c *InstallCommand) Run(args []string) int { version string installDirPath string logFilePath string + customURL string ) fs := flag.NewFlagSet("install", flag.ExitOnError) @@ -59,6 +62,7 @@ func (c *InstallCommand) Run(args []string) int { fs.StringVar(&version, "version", "", "version of product to install") fs.StringVar(&installDirPath, "path", "", "path to directory where production will be installed") fs.StringVar(&logFilePath, "log-file", "", "path to file where logs will be written") + fs.StringVar(&customURL, "custom-url", "", "custom URL to download the product from") if err := fs.Parse(args); err != nil { return 1 @@ -99,7 +103,7 @@ Option flags must be provided before the positional argument`) logger = log.New(f, "[DEBUG] ", log.LstdFlags|log.Lshortfile|log.Lmicroseconds) } - installedPath, err := c.install(product, version, installDirPath, logger) + installedPath, err := c.install(product, version, installDirPath, customURL, logger) if err != nil { msg := fmt.Sprintf("failed to install %s@%s: %v", product, version, err) c.Ui.Error(msg) @@ -110,7 +114,7 @@ Option flags must be provided before the positional argument`) return 0 } -func (c *InstallCommand) install(project, tag, installDirPath string, logger *log.Logger) (string, error) { +func (c *InstallCommand) install(project, tag, installDirPath, customURL string, logger *log.Logger) (string, error) { msg := fmt.Sprintf("hc-install: will install %s@%s", project, tag) c.Ui.Info(msg) @@ -133,6 +137,7 @@ func (c *InstallCommand) install(project, tag, installDirPath string, logger *lo }, Version: v, InstallDir: installDirPath, + CustomURL: customURL, } ctx := context.Background() From aef66b3892656177b77b72e50209b5b09442c2ee Mon Sep 17 00:00:00 2001 From: james0209 Date: Tue, 7 May 2024 12:13:59 +0100 Subject: [PATCH 5/7] Address review comment --- cmd/hc-install/cmd_install.go | 9 ++------- releases/exact_version.go | 15 +++++++-------- releases/latest_version.go | 15 +++++++-------- releases/releases_test.go | 6 +++--- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/cmd/hc-install/cmd_install.go b/cmd/hc-install/cmd_install.go index 9afa1d0..eb61493 100644 --- a/cmd/hc-install/cmd_install.go +++ b/cmd/hc-install/cmd_install.go @@ -43,8 +43,6 @@ Usage: hc-install install [options] -version Defaults to current working directory. -log-file Path to file where logs will be written. /dev/stdout or /dev/stderr can be used to log to STDOUT/STDERR. - -custom-url Custom URL to download the product from. - URL Endpoint must have same structure as HashiCorp releases. ` return strings.TrimSpace(helpText) } @@ -54,7 +52,6 @@ func (c *InstallCommand) Run(args []string) int { version string installDirPath string logFilePath string - customURL string ) fs := flag.NewFlagSet("install", flag.ExitOnError) @@ -62,7 +59,6 @@ func (c *InstallCommand) Run(args []string) int { fs.StringVar(&version, "version", "", "version of product to install") fs.StringVar(&installDirPath, "path", "", "path to directory where production will be installed") fs.StringVar(&logFilePath, "log-file", "", "path to file where logs will be written") - fs.StringVar(&customURL, "custom-url", "", "custom URL to download the product from") if err := fs.Parse(args); err != nil { return 1 @@ -103,7 +99,7 @@ Option flags must be provided before the positional argument`) logger = log.New(f, "[DEBUG] ", log.LstdFlags|log.Lshortfile|log.Lmicroseconds) } - installedPath, err := c.install(product, version, installDirPath, customURL, logger) + installedPath, err := c.install(product, version, installDirPath, logger) if err != nil { msg := fmt.Sprintf("failed to install %s@%s: %v", product, version, err) c.Ui.Error(msg) @@ -114,7 +110,7 @@ Option flags must be provided before the positional argument`) return 0 } -func (c *InstallCommand) install(project, tag, installDirPath, customURL string, logger *log.Logger) (string, error) { +func (c *InstallCommand) install(project, tag, installDirPath string, logger *log.Logger) (string, error) { msg := fmt.Sprintf("hc-install: will install %s@%s", project, tag) c.Ui.Info(msg) @@ -137,7 +133,6 @@ func (c *InstallCommand) install(project, tag, installDirPath, customURL string, }, Version: v, InstallDir: installDirPath, - CustomURL: customURL, } ctx := context.Background() diff --git a/releases/exact_version.go b/releases/exact_version.go index 61b55f3..2d04e7b 100644 --- a/releases/exact_version.go +++ b/releases/exact_version.go @@ -36,11 +36,10 @@ type ExactVersion struct { // instead of built-in pubkey to verify signature of downloaded checksums ArmoredPublicKey string - // CustomURL is an optional field that specifies a custom URL to download the product from. - // This can be useful in environments where access to the default HashiCorp releases site (releases.hashicorp.com) is blocked. - // If CustomURL is set, the product will be downloaded from this URL instead of the default site. + // ApiBaseURL is an optional field that specifies a custom URL to download the product from. + // If ApiBaseURL is set, the product will be downloaded from this base URL instead of the default site. // Note: The directory structure of the custom URL must match the HashiCorp releases site (including the index.json files). - CustomURL string + ApiBaseURL string logger *log.Logger pathsToRemove []string } @@ -106,8 +105,8 @@ func (ev *ExactVersion) Install(ctx context.Context) (string, error) { ev.log().Printf("will install into dir at %s", dstDir) rels := rjson.NewReleases() - if ev.CustomURL != "" { - rels.BaseURL = ev.CustomURL + if ev.ApiBaseURL != "" { + rels.BaseURL = ev.ApiBaseURL } rels.SetLogger(ev.log()) installVersion := ev.Version @@ -128,8 +127,8 @@ func (ev *ExactVersion) Install(ctx context.Context) (string, error) { if ev.ArmoredPublicKey != "" { d.ArmoredPublicKey = ev.ArmoredPublicKey } - if ev.CustomURL != "" { - d.BaseURL = ev.CustomURL + if ev.ApiBaseURL != "" { + d.BaseURL = ev.ApiBaseURL } licenseDir := "" diff --git a/releases/latest_version.go b/releases/latest_version.go index 51ea8d5..ad092fa 100644 --- a/releases/latest_version.go +++ b/releases/latest_version.go @@ -36,11 +36,10 @@ type LatestVersion struct { // instead of built-in pubkey to verify signature of downloaded checksums ArmoredPublicKey string - // CustomURL is an optional field that specifies a custom URL to download the product from. - // This can be useful in environments where access to the default HashiCorp releases site (releases.hashicorp.com) is blocked. - // If CustomURL is set, the product will be downloaded from this URL instead of the default site. + // ApiBaseURL is an optional field that specifies a custom URL to download the product from. + // If ApiBaseURL is set, the product will be downloaded from this base URL instead of the default site. // Note: The directory structure of the custom URL must match the HashiCorp releases site (including the index.json files). - CustomURL string + ApiBaseURL string logger *log.Logger pathsToRemove []string } @@ -102,8 +101,8 @@ func (lv *LatestVersion) Install(ctx context.Context) (string, error) { lv.log().Printf("will install into dir at %s", dstDir) rels := rjson.NewReleases() - if lv.CustomURL != "" { - rels.BaseURL = lv.CustomURL + if lv.ApiBaseURL != "" { + rels.BaseURL = lv.ApiBaseURL } rels.SetLogger(lv.log()) versions, err := rels.ListProductVersions(ctx, lv.Product.Name) @@ -129,8 +128,8 @@ func (lv *LatestVersion) Install(ctx context.Context) (string, error) { if lv.ArmoredPublicKey != "" { d.ArmoredPublicKey = lv.ArmoredPublicKey } - if lv.CustomURL != "" { - d.BaseURL = lv.CustomURL + if lv.ApiBaseURL != "" { + d.BaseURL = lv.ApiBaseURL } licenseDir := "" if lv.Enterprise != nil { diff --git a/releases/releases_test.go b/releases/releases_test.go index c8a56cb..0d0512e 100644 --- a/releases/releases_test.go +++ b/releases/releases_test.go @@ -61,7 +61,7 @@ func TestLatestVersion_basic(t *testing.T) { lv := &LatestVersion{ Product: product.Terraform, ArmoredPublicKey: getTestPubKey(t), - CustomURL: testutil.NewTestServer(t, mockApiRoot).URL, + ApiBaseURL: testutil.NewTestServer(t, mockApiRoot).URL, } lv.SetLogger(testutil.TestLogger()) @@ -95,7 +95,7 @@ func TestLatestVersion_prereleases(t *testing.T) { Product: product.Terraform, IncludePrereleases: true, ArmoredPublicKey: getTestPubKey(t), - CustomURL: testutil.NewTestServer(t, mockApiRoot).URL, + ApiBaseURL: testutil.NewTestServer(t, mockApiRoot).URL, } lv.SetLogger(testutil.TestLogger()) @@ -167,7 +167,7 @@ func BenchmarkExactVersion(b *testing.B) { Product: product.Terraform, Version: version.Must(version.NewVersion("0.14.11")), ArmoredPublicKey: getTestPubKey(b), - CustomURL: testutil.NewTestServer(b, mockApiRoot).URL, + ApiBaseURL: testutil.NewTestServer(b, mockApiRoot).URL, InstallDir: installDir, } ev.SetLogger(testutil.TestLogger()) From c607da5ea4e2758ffc90bbec9c88c0aacb86a855 Mon Sep 17 00:00:00 2001 From: james0209 Date: Tue, 7 May 2024 12:15:28 +0100 Subject: [PATCH 6/7] Update comment --- internal/releasesjson/downloader.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/releasesjson/downloader.go b/internal/releasesjson/downloader.go index 01ea7c8..490f8b0 100644 --- a/internal/releasesjson/downloader.go +++ b/internal/releasesjson/downloader.go @@ -63,8 +63,7 @@ func (d *Downloader) DownloadAndUnpack(ctx context.Context, pv *ProductVersion, archiveURL := pb.URL if d.BaseURL != "" { // If custom URL is set, use that instead of the one from the JSON. - // If using a custom URL, it may be because access to the HashiCorp Releases API is restricted. - // Also ensure that absolute download links from mocked responses are still pointing to the mock server if one is set. + // Also ensures that absolute download links from mocked responses are still pointing to the mock server if one is set. baseURL, err := url.Parse(d.BaseURL) if err != nil { return "", err From 65b06ce1eef364e642e7b1e07a7e624b11b6bca6 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 8 May 2024 08:53:47 +0100 Subject: [PATCH 7/7] Update internal/releasesjson/downloader.go --- internal/releasesjson/downloader.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/releasesjson/downloader.go b/internal/releasesjson/downloader.go index 490f8b0..ad86beb 100644 --- a/internal/releasesjson/downloader.go +++ b/internal/releasesjson/downloader.go @@ -63,7 +63,8 @@ func (d *Downloader) DownloadAndUnpack(ctx context.Context, pv *ProductVersion, archiveURL := pb.URL if d.BaseURL != "" { // If custom URL is set, use that instead of the one from the JSON. - // Also ensures that absolute download links from mocked responses are still pointing to the mock server if one is set. + // Also ensures that absolute download links from mocked responses + // are still pointing to the mock server if one is set. baseURL, err := url.Parse(d.BaseURL) if err != nil { return "", err