Making some watch functionality usable as a stand-alone component

Gopkg.toml

@@ -76,7 +76,7 @@ ignored = [
 
 [[constraint]]
   name = "github.com/google/go-github"
-  version = "15.0.0"
+  version = "18.2.0"
 
 [[constraint]]
   name = "github.com/ghodss/yaml"

pkg/constants/filepaths.go

@@ -5,37 +5,60 @@ import "path"
 const (
 	// InstallerPrefixPath is the path prefix of installed assets
 	InstallerPrefixPath = "installer"
-	// ShipPathInternal is the default folder path of Ship configuration
-	ShipPathInternal = ".ship"
 	// KustomizeBasePath is the path to which assets to be kustomized are written
 	KustomizeBasePath = "base"
 	// GithubAssetSavePath is the path that github assets are initially fetched to
 	GithubAssetSavePath = "tmp-github-asset"
 )
 
+func init() {
+	SetShipRootDir(".ship")
+}
+
 var (
+	// ShipPathInternal is the default folder path of Ship configuration
+	ShipPathInternal string
 	// ShipPathInternalTmp is a temporary folder that will get cleaned up on exit
-	ShipPathInternalTmp = path.Join(ShipPathInternal, "tmp")
+	ShipPathInternalTmp string
 	// ShipPathInternalLog is a log file that will be preserved on failure for troubleshooting
-	ShipPathInternalLog = path.Join(ShipPathInternal, "debug.log")
+	ShipPathInternalLog string
 	// InternalTempHelmHome is the path to a helm home directory
-	InternalTempHelmHome = path.Join(ShipPathInternalTmp, ".helm")
+	InternalTempHelmHome string
 	// StatePath is the default state file path
-	StatePath = path.Join(ShipPathInternal, "state.json")
+	StatePath string
 	// ReleasePath is the default place to write a pulled release to the filesystem
-	ReleasePath = path.Join(ShipPathInternal, "release.yml")
+	ReleasePath string
 	// TempHelmValuesPath is the folder path used to store the updated values.yaml
-	TempHelmValuesPath = path.Join(HelmChartPath, "tmp")
+	TempHelmValuesPath string
 	// DefaultOverlaysPath is the folder path used for the default k8s patches removing helm and tiller labels
-	DefaultOverlaysPath = path.Join("overlays", "defaults")
+	DefaultOverlaysPath string
+	// TempApplyOverlayPath is the folder path used to apply patch
+	TempApplyOverlayPath string
 	// HelmChartPath is the path used to store Helm chart contents
-	HelmChartPath = path.Join(ShipPathInternalTmp, "chart")
+	HelmChartPath string
 	// HelmChartForkedPath is the path used to store Helm chart contents of the fork
-	HelmChartForkedPath = path.Join(ShipPathInternalTmp, "chart-forked")
+	HelmChartForkedPath string
 	// UnforkForkedBasePath is the path that unfork will save the forked in when unforking
-	UnforkForkedBasePath = path.Join(ShipPathInternalTmp, "fork", "base")
+	UnforkForkedBasePath string
 	// HelmLocalDependencyPath is the local temp path that local dependencies are initially saved to
 	HelmLocalDependencyPath = path.Join(ShipPathInternalTmp, "dependencies")
 	// Kustomize render path is the local path that kustomize steps will use to render yaml for display
-	KustomizeRenderPath = path.Join(ShipPathInternalTmp, "kustomize")
+	KustomizeRenderPath string
 )
+
+func SetShipRootDir(dir string) {
+	ShipPathInternal = dir
+	ShipPathInternalTmp = path.Join(ShipPathInternal, "tmp")
+	ShipPathInternalLog = path.Join(ShipPathInternal, "debug.log")
+	InternalTempHelmHome = path.Join(ShipPathInternalTmp, ".helm")
+	StatePath = path.Join(ShipPathInternal, "state.json")
+	ReleasePath = path.Join(ShipPathInternal, "release.yml")
+	TempHelmValuesPath = path.Join(HelmChartPath, "tmp")
+	DefaultOverlaysPath = path.Join("overlays", "defaults")
+	TempApplyOverlayPath = path.Join("overlays", "tmp-apply")
+	HelmChartPath = path.Join(ShipPathInternalTmp, "chart")
+	HelmChartForkedPath = path.Join(ShipPathInternalTmp, "chart-forked")
+	UnforkForkedBasePath = path.Join(ShipPathInternalTmp, "fork", "base")
+	HelmLocalDependencyPath = path.Join(ShipPathInternalTmp, "dependencies")
+	KustomizeRenderPath = path.Join(ShipPathInternalTmp, "kustomize")
+}

pkg/lifecycle/render/github/render.go

@@ -188,7 +188,7 @@ func (r *LocalRenderer) resolveNoProxyGithubAssets(asset api.GitHubAsset, builde
 
 	var fetcher apptype.FileFetcher
 	localFetchPath := filepath.Join(constants.InstallerPrefixPath, constants.GithubAssetSavePath)
-	fetcher = githubclient.NewGithubClient(r.Fs, r.Logger)
+	fetcher = githubclient.NewGithubClient(r.Logger)
 	if r.Viper.GetBool("prefer-git") {
 		var isSingleFile bool
 		var subdir string

pkg/lifecycle/render/helm/dependency.go

@@ -129,7 +129,7 @@ func (f *LocalTemplater) createDependencyUpstreamFromAsset(originalAsset api.Hel
 
 func (f *LocalTemplater) fetchLocalHelmDependency(upstream string, fetchPath string) (string, error) {
 	var fetcher apptype.FileFetcher
-	fetcher = githubclient.NewGithubClient(f.FS, f.Logger)
+	fetcher = githubclient.NewGithubClient(f.Logger)
 	if f.Viper.GetBool("prefer-git") {
 		var isSingleFile bool
 		var subdir string

pkg/ship/kustomize.go

@@ -96,7 +96,8 @@ func (s *Ship) Init(ctx context.Context) error {
 		return errors.New("No upstream provided")
 	}
 
-	maybeVersionedUpstream, err := s.Resolver.MaybeResolveVersionedUpstream(ctx, target, existingState)
+	p := s.Resolver.NewContentProcessor()
+	maybeVersionedUpstream, err := p.MaybeResolveVersionedUpstream(ctx, target, existingState)
 	if err != nil {
 		return errors.Wrap(err, "create versioned release")
 	}

pkg/ship/unfork.go

@@ -44,7 +44,8 @@ func (s *Ship) Unfork(ctx context.Context) error {
 		return errors.New("No upstream provided")
 	}
 
-	maybeVersionedUpstream, err := s.Resolver.MaybeResolveVersionedUpstream(ctx, upstream, existingState)
+	p := s.Resolver.NewContentProcessor()
+	maybeVersionedUpstream, err := p.MaybeResolveVersionedUpstream(ctx, upstream, existingState)
 	if err != nil {
 		return errors.Wrap(err, "create versioned upstream release")
 	}
@@ -54,7 +55,7 @@ func (s *Ship) Unfork(ctx context.Context) error {
 		return errors.New("No fork provided")
 	}
 
-	maybeVersionedFork, err := s.Resolver.MaybeResolveVersionedUpstream(ctx, fork, existingState)
+	maybeVersionedFork, err := p.MaybeResolveVersionedUpstream(ctx, fork, existingState)
 	if err != nil {
 		return errors.Wrap(err, "create versioned fork release")
 	}

pkg/ship/update.go

@@ -50,7 +50,8 @@ func (s *Ship) Update(ctx context.Context) error {
 		return errors.Errorf(`No upstream URL found at %s, please run "ship init"`, uiPrintableStatePath)
 	}
 
-	maybeVersionedUpstream, err := s.Resolver.MaybeResolveVersionedUpstream(ctx, upstreamURL, existingState)
+	p := s.Resolver.NewContentProcessor()
+	maybeVersionedUpstream, err := p.MaybeResolveVersionedUpstream(ctx, upstreamURL, existingState)
 	if err != nil {
 		return errors.New(`Unable to resolve versioned upstream ` + upstreamURL)
 	}

pkg/ship/watch.go

@@ -48,7 +48,8 @@ func (s *Ship) Watch(ctx context.Context) error {
 			return errors.New(`No current chart url found at ` + s.Viper.GetString("state-file") + `, please run "ship init"`)
 		}
 
-		maybeVersionedUpstream, err := s.Resolver.MaybeResolveVersionedUpstream(ctx, upstream, existingState)
+		p := s.Resolver.NewContentProcessor()
+		maybeVersionedUpstream, err := p.MaybeResolveVersionedUpstream(ctx, upstream, existingState)
 		if err != nil {
 			return errors.New(`Unable to resolve versioned upstream ` + upstream)
 		}
@@ -59,7 +60,7 @@ func (s *Ship) Watch(ctx context.Context) error {
 			return errors.New(`No current SHA found at ` + s.Viper.GetString("state-file") + `, please run "ship init"`)
 		}
 
-		contentSHA, err := s.Resolver.ReadContentSHAForWatch(ctx, maybeVersionedUpstream)
+		contentSHA, err := p.ReadContentSHAForWatch(ctx, maybeVersionedUpstream)
 		if err != nil {
 			return errors.Wrap(err, "read content SHA")
 		}

pkg/specs/apptype/determine_type.go

@@ -90,7 +90,7 @@ func (i *inspector) DetermineApplicationType(ctx context.Context, upstream strin
 	i.ui.Info(fmt.Sprintf("Attempting to retrieve upstream %s ...", upstream))
 	// use the integrated github client if the url is a github url and does not contain "//", unless perfer-git is set)
 	if i.viper.GetBool("prefer-git") == false && util.IsGithubURL(upstream) {
-		githubClient := githubclient.NewGithubClient(i.fs, i.logger)
+		githubClient := githubclient.NewGithubClient(i.logger)
 		return i.determineTypeFromContents(ctx, upstream, githubClient)
 	}
 
@@ -144,7 +144,7 @@ func (i *inspector) determineTypeFromContents(ctx context.Context, upstream stri
 		if _, ok := err.(errors2.FetchFilesError); ok {
 			i.ui.Info(fmt.Sprintf("Failed to retrieve upstream %s", upstream))
 
-			var retryError error
+			var retryError = err
 			retries := i.viper.GetInt("retries")
 			hasSucceeded := false
 			for idx := 1; idx <= retries && !hasSucceeded; idx++ {

pkg/specs/chart.go

@@ -16,6 +16,7 @@ import (
 	"github.com/replicatedhq/ship/pkg/api"
 	"github.com/replicatedhq/ship/pkg/constants"
 	"github.com/replicatedhq/ship/pkg/util"
+	"github.com/spf13/afero"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -265,7 +266,7 @@ For continuous notification and preparation of application updates via email, we
 func (r *Resolver) resolveMetadata(ctx context.Context, upstream, localPath string, applicationType string) (*api.ShipAppMetadata, error) {
 	debug := level.Debug(log.With(r.Logger, "method", "ResolveHelmMetadata"))
 
-	baseMetadata, err := r.ResolveBaseMetadata(upstream, localPath)
+	baseMetadata, err := r.NewContentProcessor().ResolveBaseMetadata(upstream, localPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "resolve base metadata")
 	}
@@ -333,45 +334,6 @@ func (r *Resolver) resolveMetadata(ctx context.Context, upstream, localPath stri
 	return baseMetadata, nil
 }
 
-// ResolveBaseMetadata resolves URL, ContentSHA, and Readme for the resource
-func (r *Resolver) ResolveBaseMetadata(upstream string, localPath string) (*api.ShipAppMetadata, error) {
-	debug := level.Debug(log.With(r.Logger, "method", "resolveBaseMetaData"))
-	var md api.ShipAppMetadata
-	md.URL = upstream
-	debug.Log("phase", "calculate-sha", "for", localPath)
-	contentSHA, err := r.shaSummer(r, localPath)
-	if err != nil {
-		return nil, errors.Wrapf(err, "calculate chart sha")
-	}
-	md.ContentSHA = contentSHA
-
-	localReadmePath := filepath.Join(localPath, "README.md")
-	debug.Log("phase", "read-readme", "from", localReadmePath)
-	readme, err := r.FS.ReadFile(localReadmePath)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return nil, errors.Wrapf(err, "read file from %s", localReadmePath)
-		}
-	}
-	if readme != nil {
-		md.Readme = string(readme)
-	} else {
-		// TODO default README better
-		md.Readme = fmt.Sprintf(`
-Deployment Generator
-===========================
-
-This is a deployment generator for
-
-    ship init %s
-
-Sources for your app have been generated at %s. This installer will walk you through
-customizing these resources and preparing them to deploy to your infrastructure.
-`, upstream, localPath)
-	}
-	return &md, nil
-}
-
 func (r *Resolver) maybeGetShipYAML(ctx context.Context, localPath string) (*api.Spec, error) {
 	localReleasePaths := []string{
 		filepath.Join(localPath, "ship.yaml"),
@@ -404,11 +366,11 @@ func (r *Resolver) maybeGetShipYAML(ctx context.Context, localPath string) (*api
 	return nil, nil
 }
 
-type shaSummer func(r *Resolver, localPath string) (string, error)
+type shaSummer func(fs afero.Afero, localPath string) (string, error)
 
-func (r *Resolver) calculateContentSHA(root string) (string, error) {
+func calculateContentSHA(fs afero.Afero, root string) (string, error) {
 	var contents []byte
-	err := r.FS.Walk(root, func(path string, info os.FileInfo, err error) error {
+	err := fs.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return errors.Wrapf(err, "fs walk")
 		}
@@ -430,7 +392,8 @@ func (r *Resolver) calculateContentSHA(root string) (string, error) {
 			return nil
 		}
 
-		fileContents, err := r.FS.ReadFile(path)
+		// TODO: Use checksum writer instead of loading ALL FILES in memory.
+		fileContents, err := fs.ReadFile(path)
 		if err != nil {
 			return errors.Wrapf(err, "read file")
 		}

pkg/specs/chart_test.go

@@ -48,10 +48,10 @@ func TestStableContentSha(t *testing.T) {
 		},
 	}
 
-	firstPass, err := r.calculateContentSHA("")
+	firstPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
-	secondPass, err := r.calculateContentSHA("")
+	secondPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	req.Equal(firstPass, secondPass)
@@ -75,13 +75,13 @@ func TestChangingContentSha(t *testing.T) {
 		},
 	}
 
-	firstPass, err := r.calculateContentSHA("")
+	firstPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	err = mockFs.WriteFile("newfile.txt", []byte("I AM A NEW FILE"), 0755)
 	req.NoError(err)
 
-	secondPass, err := r.calculateContentSHA("")
+	secondPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	req.NotEqual(firstPass, secondPass)
@@ -107,13 +107,13 @@ func TestStableGitContentSha(t *testing.T) {
 		},
 	}
 
-	firstPass, err := r.calculateContentSHA("")
+	firstPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	err = mockFs.WriteFile(".git/newfile.txt", []byte("I AM A NEW GIT FILE"), 0755)
 	req.NoError(err)
 
-	secondPass, err := r.calculateContentSHA("")
+	secondPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	req.Equal(firstPass, secondPass)
@@ -141,10 +141,10 @@ func TestRootIndependentContentSha(t *testing.T) {
 		},
 	}
 
-	firstPass, err := r.calculateContentSHA("path1")
+	firstPass, err := calculateContentSHA(r.FS, "path1")
 	req.NoError(err)
 
-	secondPass, err := r.calculateContentSHA("path2")
+	secondPass, err := calculateContentSHA(r.FS, "path2")
 	req.NoError(err)
 
 	req.Equal(firstPass, secondPass)
@@ -168,13 +168,13 @@ func TestMoveFileContentSha(t *testing.T) {
 		},
 	}
 
-	firstPass, err := r.calculateContentSHA("")
+	firstPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	err = mockFs.Rename("templates/README.md", "templated/README.md")
 	req.NoError(err)
 
-	secondPass, err := r.calculateContentSHA("")
+	secondPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	req.NotEqual(firstPass, secondPass)
@@ -198,15 +198,15 @@ func TestEditFileContentSha(t *testing.T) {
 		},
 	}
 
-	firstPass, err := r.calculateContentSHA("")
+	firstPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	err = mockFs.Remove("templates/README.md")
 	req.NoError(err)
 	err = mockFs.WriteFile("templates/README.md", []byte("not readme"), 0755)
 	req.NoError(err)
 
-	secondPass, err := r.calculateContentSHA("")
+	secondPass, err := calculateContentSHA(r.FS, "")
 	req.NoError(err)
 
 	req.NotEqual(firstPass, secondPass)

pkg/specs/constructor.go

@@ -50,12 +50,10 @@ func NewResolver(
 		Viper:            v,
 		ui:               ui,
 		appTypeInspector: determiner,
-		shaSummer: func(resolver *Resolver, s string) (string, error) {
-			return resolver.calculateContentSHA(s)
-		},
-		AppResolver:   appresolver,
-		GitHubFetcher: github,
-		NoOutro:       v.GetBool("no-outro"),
-		isEdit:        v.GetBool("isEdit"),
+		shaSummer:        calculateContentSHA,
+		AppResolver:      appresolver,
+		GitHubFetcher:    github,
+		NoOutro:          v.GetBool("no-outro"),
+		isEdit:           v.GetBool("isEdit"),
 	}
 }