Allow overriding home dir and download url during install (#1542)

* Allow overriding home dir and download url during install

When installing, let the user override with PORTER_HOME and
PORTER_MIRROR the location where porter should be installed, or the
location from which it should be downloaded.

I have updated the porter CLI to understand the new configuration value
PORTER_MIRROR, which and also be specified as a flag or config file
setting.

This lets us test out alternate download infra from end-to-end, and also
allows a user to set up a mirror of Porter's assets.

I am also getting rid of the troubleshooting I had added for Azure's
storage and CDN now that we aren't using it anymore (e.g. dynamically
generated PORTER_TRACE user agents).

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Fix line ending

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Rename heading to url structure

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

* Review feedback

Signed-off-by: Carolyn Van Slyck <me@carolynvanslyck.com>

cmd/porter/mixins.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"fmt"
-
 	"get.porter.sh/porter/pkg/mixin"
 	"get.porter.sh/porter/pkg/pkgmgmt"
 	"get.porter.sh/porter/pkg/pkgmgmt/feed"
@@ -57,7 +55,9 @@ func buildMixinsSearchCommand(p *porter.Porter) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "search [QUERY]",
 		Short: "Search available mixins",
-		Long:  "Search available mixins. You can specify an optional mixin name query, where the results are filtered by mixins whose name contains the query term.",
+		Long: `Search available mixins. You can specify an optional mixin name query, where the results are filtered by mixins whose name contains the query term.
+
+By default the community mixin index at https://cdn.porter.sh/mixins/index.json is searched. To search from a mirror, set the environment variable PORTER_MIRROR, or mirror in the Porter config file, with the value to replace https://cdn.porter.sh with.`,
 		Example: `  porter mixin search
   porter mixin search helm
   porter mixin search -o json`,
@@ -69,8 +69,11 @@ func buildMixinsSearchCommand(p *porter.Porter) *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVarP(&opts.RawFormat, "output", "o", "table",
+	flags := cmd.Flags()
+	flags.StringVarP(&opts.RawFormat, "output", "o", "table",
 		"Output format, allowed values are: table, json, yaml")
+	flags.StringVar(&opts.Mirror, "mirror", pkgmgmt.DefaultPackageMirror,
+		"Mirror of official Porter assets")
 
 	return cmd
 }
@@ -80,6 +83,9 @@ func BuildMixinInstallCommand(p *porter.Porter) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "install NAME",
 		Short: "Install a mixin",
+		Long: `Install a mixin.
+
+By default mixins are downloaded from the official Porter mixin feed at https://cdn.porter.sh/mixins/atom.xml. To download from a mirror, set the environment variable PORTER_MIRROR, or mirror in the Porter config file, with the value to replace https://cdn.porter.sh with.`,
 		Example: `  porter mixin install helm --url https://cdn.porter.sh/mixins/helm
   porter mixin install helm --feed-url https://cdn.porter.sh/mixins/atom.xml
   porter mixin install azure --version v0.4.0-ralpha.1+dubonnet --url https://cdn.porter.sh/mixins/azure
@@ -92,12 +98,15 @@ func BuildMixinInstallCommand(p *porter.Porter) *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVarP(&opts.Version, "version", "v", "latest",
+	flags := cmd.Flags()
+	flags.StringVarP(&opts.Version, "version", "v", "latest",
 		"The mixin version. This can either be a version number, or a tagged release like 'latest' or 'canary'")
-	cmd.Flags().StringVar(&opts.URL, "url", "",
+	flags.StringVar(&opts.URL, "url", "",
 		"URL from where the mixin can be downloaded, for example https://github.com/org/proj/releases/downloads")
-	cmd.Flags().StringVar(&opts.FeedURL, "feed-url", "",
-		fmt.Sprintf(`URL of an atom feed where the mixin can be downloaded (default %s)`, mixin.DefaultFeedUrl))
+	flags.StringVar(&opts.FeedURL, "feed-url", "",
+		"URL of an atom feed where the mixin can be downloaded. Defaults to the official Porter mixin feed.")
+	flags.StringVar(&opts.Mirror, "mirror", pkgmgmt.DefaultPackageMirror,
+		"Mirror of official Porter assets")
 	return cmd
 }
 

cmd/porter/plugins.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"fmt"
-
 	"get.porter.sh/porter/pkg/pkgmgmt"
 	"get.porter.sh/porter/pkg/plugins"
 	"get.porter.sh/porter/pkg/porter"
@@ -57,7 +55,9 @@ func buildPluginSearchCommand(p *porter.Porter) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "search [QUERY]",
 		Short: "Search available plugins",
-		Long:  "Search available plugins. You can specify an optional plugin name query, where the results are filtered by plugins whose name contains the query term.",
+		Long: `Search available plugins. You can specify an optional plugin name query, where the results are filtered by plugins whose name contains the query term.
+
+By default the community plugin index at https://cdn.porter.sh/plugins/index.json is searched. To search from a mirror, set the environment variable PORTER_MIRROR, or mirror in the Porter config file, with the value to replace https://cdn.porter.sh with.`,
 		Example: `  porter plugin search
   porter plugin search azure
   porter plugin search -o json`,
@@ -69,8 +69,11 @@ func buildPluginSearchCommand(p *porter.Porter) *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVarP(&opts.RawFormat, "output", "o", "table",
+	flags := cmd.Flags()
+	flags.StringVarP(&opts.RawFormat, "output", "o", "table",
 		"Output format, allowed values are: table, json, yaml")
+	flags.StringVar(&opts.Mirror, "mirror", pkgmgmt.DefaultPackageMirror,
+		"Mirror of official Porter assets")
 
 	return cmd
 }
@@ -100,6 +103,9 @@ func BuildPluginInstallCommand(p *porter.Porter) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "install NAME",
 		Short: "Install a plugin",
+		Long: `Install a plugin.
+
+By default plugins are downloaded from the official Porter plugin feed at https://cdn.porter.sh/plugins/atom.xml. To download from a mirror, set the environment variable PORTER_MIRROR, or mirror in the Porter config file, with the value to replace https://cdn.porter.sh with.`,
 		Example: `  porter plugin install azure  
   porter plugin install azure --url https://cdn.porter.sh/plugins/azure
   porter plugin install azure --feed-url https://cdn.porter.sh/plugins/atom.xml
@@ -113,12 +119,16 @@ func BuildPluginInstallCommand(p *porter.Porter) *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVarP(&opts.Version, "version", "v", "latest",
+	flags := cmd.Flags()
+	flags.StringVarP(&opts.Version, "version", "v", "latest",
 		"The plugin version. This can either be a version number, or a tagged release like 'latest' or 'canary'")
-	cmd.Flags().StringVar(&opts.URL, "url", "",
+	flags.StringVar(&opts.URL, "url", "",
 		"URL from where the plugin can be downloaded, for example https://github.com/org/proj/releases/downloads")
-	cmd.Flags().StringVar(&opts.FeedURL, "feed-url", "",
-		fmt.Sprintf(`URL of an atom feed where the plugin can be downloaded (default %s)`, plugins.DefaultFeedUrl))
+	flags.StringVar(&opts.FeedURL, "feed-url", "",
+		"URL of an atom feed where the plugin can be downloaded. Defaults to the official Porter plugin feed.")
+	flags.StringVar(&opts.Mirror, "mirror", pkgmgmt.DefaultPackageMirror,
+		"Mirror of official Porter assets")
+
 	return cmd
 }
 

docs/content/install.md

@@ -13,9 +13,11 @@ We have a few release types available for you to use:
 * [Older Version](#older-version)
 
 You can also install and manage [mixins](#mixins) and [plugins](#plugins) using
-porter, and use the [Porter VS Code Extension][vscode-ext] for help authoring
+porter, and use the [Porter VS Code Extension][vscode-ext] to help author
 bundles.
 
+All the scripts for Porter v0.37.3+ support [customizing the installation through parameters](#install-script-parameters).
+
 [vscode-ext]: https://marketplace.visualstudio.com/items?itemName=ms-kubernetes-tools.porter-vscode
 [ps-link]: https://www.howtogeek.com/126469/how-to-create-a-powershell-profile/
 [mailing list]: https://groups.io/g/porter
@@ -129,3 +131,64 @@ All of the Porter-authored plugins are published to `https://cdn.porter.sh/plugi
 
 [releases]: https://github.com/getporter/porter/releases
 
+
+
+# Install Script Parameters
+
+The installation scripts provide the following parameters. Parameters can be specified with environment variables for the macOS and Linux scripts, and on Windows they are named parameters in the script.
+
+## PORTER_HOME
+
+Location where Porter is installed (defaults to ~/.porter).
+
+**Posix Shells**
+```bash
+PORTER_HOME=/alt/porter/home curl -L REPLACE_WITH_INSTALL_URL | bash
+```
+
+**PowerShell**
+```powershell
+iwr REPLACE_WITH_INSTALL_URL -OutFile install-porter.ps1 -UseBasicParsing
+.\install-porter.ps1 -PORTER_HOME C:\alt\porter\home
+```
+
+## PORTER_MIRROR
+
+Base URL where Porter assets, such as binaries and atom feeds, are downloaded.
+This lets you set up an internal mirror. Note that atom feeds and index files
+should be updated in the mirror to point to the mirrored location. Porter does
+not alter the contents of these files.
+
+**Posix Shells**
+```bash
+PORTER_MIRROR=https://example.com/porter curl -L REPLACE_WITH_INSTALL_URL | bash
+```
+
+**PowerShell**
+```powershell
+iwr REPLACE_WITH_INSTALL_URL -OutFile install-porter.ps1 -UseBasicParsing
+.\install-porter.ps1 -PORTER_MIRROR https://example.com/porter
+```
+
+### URL Structure
+
+Configuring a mirror of Porter's assets is out of scope of this document.
+Reach out on the Porter [mailing list] for assistance.
+
+Below is the general structure for Porter's asset URLs:
+
+```
+PERMALINK/
+  - install-linux.sh
+  - install-mac.sh
+  - install-windows.ps1
+  - porter-GOOS-GOARCH[FILE_EXT]
+mixins/
+  - atom.xml
+  - index.json
+  - MIXIN/PERMALINK/MIXIN-GOOS-GOARCH[FILE_EXT]
+plugins/
+  - atom.xml
+  - index.json
+  - PLUGIN/PERMALINK/PLUGIN-GOOS-GOARCH[FILE_EXT]
+```

examples/azure-terraform/porter.yaml

@@ -27,7 +27,7 @@ credentials:
   - name: client_secret
     env: AZURE_CLIENT_SECRET
 
-## This section defines what paramters are used by the bundle. These parameters are used by various 
+## This section defines what parameters are used by the bundle. These parameters are used by various 
 ## steps within the bundle
 parameters:
   - name: location

pkg/mixin/install.go

@@ -4,13 +4,11 @@ import (
 	"get.porter.sh/porter/pkg/pkgmgmt"
 )
 
-const DefaultFeedUrl = "https://cdn.porter.sh/mixins/atom.xml"
-
 type InstallOptions struct {
 	pkgmgmt.InstallOptions
 }
 
 func (o *InstallOptions) Validate(args []string) error {
-	o.DefaultFeedURL = DefaultFeedUrl
+	o.PackageType = "mixin"
 	return o.InstallOptions.Validate(args)
 }

pkg/mixin/install_test.go

@@ -12,5 +12,5 @@ func TestInstallOptions_Validate(t *testing.T) {
 	opts := InstallOptions{}
 	err := opts.Validate([]string{"pkg1"})
 	require.NoError(t, err, "Validate failed")
-	assert.Equal(t, DefaultFeedUrl, opts.FeedURL, "Feed URL was not defaulted to the mixin feed URL")
+	assert.NotEmpty(t, opts.FeedURL, "Feed URL was not defaulted")
 }

pkg/pkgmgmt/client/install.go

@@ -9,9 +9,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
-	"time"
 
-	"get.porter.sh/porter/pkg"
 	"get.porter.sh/porter/pkg/pkgmgmt"
 	"get.porter.sh/porter/pkg/pkgmgmt/feed"
 	"github.com/pkg/errors"
@@ -166,20 +164,12 @@ func (fs *FileSystem) downloadFile(url url.URL, destPath string, executable bool
 		return errors.Wrapf(err, "error creating web request to %s", url.String())
 	}
 
-	// Add debugging headers to our request
-	req.Header.Set("X-Azure-DebugInfo", "1")
-	userAgent := fmt.Sprintf("porter/%s porter_trace_%d %s", pkg.Version, time.Now().UnixNano(), req.UserAgent())
-	if fs.Debug {
-		fmt.Fprintln(fs.Err, "PORTER_TRACE:", userAgent)
-	}
-
-	req.Header.Set("User-Agent", userAgent)
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return errors.Wrapf(err, "error downloading %s\nPlease include the following information in any bug reports:\nPORTER_TRACE: %s", url.String(), userAgent)
+		return errors.Wrapf(err, "error downloading %s", url.String())
 	}
 	if resp.StatusCode != 200 {
-		return errors.Errorf("bad status returned when downloading %s (%d) %s\nPlease include the following information in any bug reports:\nPORTER_TRACE: %s\nHEADERS: %#v", url.String(), resp.StatusCode, resp.Status, userAgent, resp.Header)
+		return errors.Errorf("bad status returned when downloading %s (%d) %s", url.String(), resp.StatusCode, resp.Status)
 	}
 	defer resp.Body.Close()
 

pkg/pkgmgmt/client/install_test.go

@@ -19,29 +19,27 @@ import (
 func TestFileSystem_InstallFromUrl(t *testing.T) {
 	// serve out a fake package
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Header.Get("X-Azure-DebugInfo") == "" ||
-			!strings.Contains(r.Header.Get("User-Agent"), "porter_trace") {
-			w.WriteHeader(400)
-		}
-		fmt.Fprintf(w, "#!/usr/bin/env bash\necho i am a mixxin\n")
+		fmt.Fprintf(w, "#!/usr/bin/env bash\necho i am a random package\n")
 	}))
 	defer ts.Close()
 
 	c := config.NewTestConfig(t)
 	p := NewFileSystem(c.Config, "packages")
 
 	opts := pkgmgmt.InstallOptions{
-		Version: "latest",
-		URL:     ts.URL,
+		PackageType: "mixin",
+		Version:     "latest",
+		URL:         ts.URL,
 	}
-	opts.Validate([]string{"mixxin"})
+	err := opts.Validate([]string{"mypkg"})
+	require.NoError(t, err, "Validate failed")
 
-	err := p.Install(opts)
+	err = p.Install(opts)
 	require.NoError(t, err)
 
-	clientExists, _ := p.FileSystem.Exists("/root/.porter/packages/mixxin/mixxin")
+	clientExists, _ := p.FileSystem.Exists("/root/.porter/packages/mypkg/mypkg")
 	assert.True(t, clientExists)
-	runtimeExists, _ := p.FileSystem.Exists("/root/.porter/packages/mixxin/runtimes/mixxin-runtime")
+	runtimeExists, _ := p.FileSystem.Exists("/root/.porter/packages/mypkg/runtimes/mypkg-runtime")
 	assert.True(t, runtimeExists)
 }
 
@@ -68,10 +66,12 @@ func TestFileSystem_InstallFromFeedUrl(t *testing.T) {
 	p := NewFileSystem(c.Config, "packages")
 
 	opts := pkgmgmt.InstallOptions{
-		Version: "v1.2.4",
-		FeedURL: ts.URL + "/atom.xml",
+		PackageType: "plugin",
+		Version:     "v1.2.4",
+		FeedURL:     ts.URL + "/atom.xml",
 	}
-	opts.Validate([]string{"helm"})
+	err = opts.Validate([]string{"helm"})
+	require.NoError(t, err, "Validate failed")
 
 	err = p.Install(opts)
 	require.NoError(t, err)
@@ -88,7 +88,7 @@ func TestFileSystem_Install_RollbackMissingRuntime(t *testing.T) {
 		if strings.Contains(r.RequestURI, "linux-amd64") {
 			w.WriteHeader(400)
 		} else {
-			fmt.Fprintf(w, "#!/usr/bin/env bash\necho i am a client mixxin\n")
+			fmt.Fprintf(w, "#!/usr/bin/env bash\necho i am a client mypkg\n")
 		}
 	}))
 	defer ts.Close()
@@ -97,18 +97,19 @@ func TestFileSystem_Install_RollbackMissingRuntime(t *testing.T) {
 	p := NewFileSystem(c.Config, "packages")
 
 	parentDir, _ := p.GetPackagesDir()
-	pkgDir := path.Join(parentDir, "mixxin")
+	pkgDir := path.Join(parentDir, "mypkg")
 
 	opts := pkgmgmt.InstallOptions{
-		Version: "latest",
-		URL:     ts.URL,
+		PackageType: "mixin",
+		Version:     "latest",
+		URL:         ts.URL,
 	}
-	opts.Validate([]string{"mixxin"})
+	err := opts.Validate([]string{"mypkg"})
+	require.NoError(t, err, "Validate failed")
 
-	err := p.Install(opts)
+	err = p.Install(opts)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "bad status returned when downloading")
-	assert.Contains(t, err.Error(), "porter_trace_", "The error message should contain our special debug user agent")
 
 	// Make sure the package directory was removed
 	dirExists, _ := p.FileSystem.DirExists(pkgDir)
@@ -121,17 +122,19 @@ func TestFileSystem_Install_PackageInfoSavedWhenNoFileExists(t *testing.T) {
 
 	packageURL := "https://cdn.porter.sh/mixins/helm"
 	opts := pkgmgmt.InstallOptions{
-		Version: "v1.2.4",
-		URL:     packageURL,
+		PackageType: "plugin",
+		Version:     "v1.2.4",
+		URL:         packageURL,
 	}
 	name := "helm"
-	opts.Validate([]string{name})
+	err := opts.Validate([]string{name})
+	require.NoError(t, err, "Validate failed")
 
 	// ensure cache.json does not exist (yet)
 	cacheExists, _ := p.FileSystem.Exists("/root/.porter/packages/cache.json")
 	assert.False(t, cacheExists)
 
-	err := p.savePackageInfo(opts)
+	err = p.savePackageInfo(opts)
 	require.NoError(t, err)
 
 	// cache.json should have been created