render into a tempdir and do not apply changes to original dirs

also handle existing kustomization yaml

render to tempdir and display tempdir to user

in the background, build a sane kustomize flow including our default patches

cleanup temp kustomize files on exit

pkg/api/lifecycle.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/replicatedhq/ship/pkg/constants"
 )
@@ -127,7 +128,7 @@ func (k *Unfork) OverlayPath() string {
 func (u *Unfork) Shared() *StepShared { return &u.StepShared }
 func (k *Unfork) ShortName() string   { return "unfork" }
 
-// Kustomize is a lifeycle step to generate overlays for generated assets.
+// Kustomize is a lifecycle step to generate overlays for generated assets.
 // It does not take a kustomization.yml, rather it will generate one in the .ship/ folder
 type Kustomize struct {
 	StepShared `json:",inline" yaml:",inline" hcl:",inline"`
@@ -143,8 +144,9 @@ func (k *Kustomize) OverlayPath() string {
 	return k.Overlay
 }
 
-func (k *Kustomize) Shared() *StepShared { return &k.StepShared }
-func (k *Kustomize) ShortName() string   { return "kustomize" }
+func (k *Kustomize) Shared() *StepShared    { return &k.StepShared }
+func (k *Kustomize) ShortName() string      { return "kustomize" }
+func (k *Kustomize) TempRenderPath() string { return filepath.Join(constants.KustomizeRenderPath, k.ID) }
 
 // KustomizeIntro is a lifeycle step to display an informative intro page for kustomize
 type KustomizeIntro struct {

pkg/constants/filepaths.go

@@ -26,8 +26,8 @@ var (
 	ReleasePath = path.Join(ShipPathInternal, "release.yml")
 	// TempHelmValuesPath is the folder path used to store the updated values.yaml
 	TempHelmValuesPath = path.Join(HelmChartPath, "tmp")
-	// TempApplyOverlayPath is the folder path used to apply patch
-	TempApplyOverlayPath = path.Join("overlays", "tmp-apply")
+	// DefaultOverlaysPath is the folder path used for the default k8s patches removing helm and tiller labels
+	DefaultOverlaysPath = path.Join("overlays", "defaults")
 	// HelmChartPath is the path used to store Helm chart contents
 	HelmChartPath = path.Join(ShipPathInternalTmp, "chart")
 	// HelmChartForkedPath is the path used to store Helm chart contents of the fork
@@ -36,4 +36,6 @@ var (
 	UnforkForkedBasePath = path.Join(ShipPathInternalTmp, "fork", "base")
 	// HelmLocalDependencyPath is the local temp path that local dependencies are initially saved to
 	HelmLocalDependencyPath = path.Join(ShipPathInternalTmp, "dependencies")
+	// Kustomize render path is the local path that kustomize steps will use to render yaml for display
+	KustomizeRenderPath = path.Join(ShipPathInternalTmp, "kustomize")
 )

pkg/lifecycle/daemon/daemontypes/types.go

@@ -93,7 +93,7 @@ func NewStep(apiStep api.Step) Step {
 		}
 	} else if apiStep.Kustomize != nil {
 		step.Kustomize = &Kustomize{
-			BasePath: apiStep.Kustomize.Base,
+			BasePath: apiStep.Kustomize.TempRenderPath(),
 		}
 	} else if apiStep.KustomizeIntro != nil {
 		step.KustomizeIntro = &KustomizeIntro{}

pkg/lifecycle/daemon/routes_navcycle_getstep_test.go

@@ -601,7 +601,7 @@ func TestHydrateTemplatedKustomizeStep(t *testing.T) {
 		name    string
 		step    api.Step
 		state   state2.State
-		fs      map[string]string
+		fs      []string
 		release *api.Release
 		want    *daemontypes.StepResponse
 	}{
@@ -637,6 +637,9 @@ func TestHydrateTemplatedKustomizeStep(t *testing.T) {
 				},
 			},
 			state: state2.State{V1: &state2.V1{}},
+			fs: []string{
+				".ship/tmp/kustomize/bar",
+			},
 			want: &daemontypes.StepResponse{
 				CurrentStep: daemontypes.Step{
 					Source: api.Step{
@@ -650,7 +653,7 @@ func TestHydrateTemplatedKustomizeStep(t *testing.T) {
 						},
 					},
 					Kustomize: &daemontypes.Kustomize{
-						BasePath: "ABCDEF",
+						BasePath: ".ship/tmp/kustomize/bar",
 						Tree: filetree.Node{
 							Children: []filetree.Node{
 								filetree.Node{
@@ -702,6 +705,10 @@ func TestHydrateTemplatedKustomizeStep(t *testing.T) {
 				mockState.EXPECT().TryLoad().Return(test.state, nil)
 			}
 
+			for _, dir := range test.fs {
+				req.NoError(mockFs.MkdirAll(dir, 0644))
+			}
+
 			treeLoader := filetree.NewLoader(mockFs, testLogger, mockState)
 
 			v2 := &NavcycleRoutes{
@@ -726,6 +733,7 @@ func TestHydrateTemplatedKustomizeStep(t *testing.T) {
 
 			response, err := v2.hydrateStep(daemontypes.NewStep(builtStep))
 			req.NoError(err, "hydrate templated kustomize step")
+
 			req.Equal(test.want, response)
 		})
 	}

pkg/lifecycle/daemon/routes_navcycle_kustomize.go

@@ -136,7 +136,7 @@ func (d *NavcycleRoutes) kustomizeGetFile(c *gin.Context) {
 
 	var base []byte
 	if !isResource {
-		base, err = d.TreeLoader.LoadFile(step.Kustomize.Base, request.Path)
+		base, err = d.TreeLoader.LoadFile(step.Kustomize.TempRenderPath(), request.Path)
 		if err != nil {
 			level.Warn(d.Logger).Log("event", "load file failed", "err", err)
 			c.AbortWithError(500, err)
@@ -237,7 +237,7 @@ func (d *NavcycleRoutes) createOrMergePatch(c *gin.Context) {
 	}
 
 	debug.Log("event", "load.originalFile")
-	original, err := d.TreeLoader.LoadFile(step.Kustomize.Base, request.Original)
+	original, err := d.TreeLoader.LoadFile(step.Kustomize.TempRenderPath(), request.Original)
 	if err != nil {
 		level.Error(d.Logger).Log("event", "failed to read original file", "err", err)
 		c.AbortWithError(500, errors.New("internal_server_error"))

pkg/lifecycle/kustomize/daemonless.go

@@ -18,10 +18,11 @@ import (
 )
 
 type Kustomizer struct {
-	Logger  log.Logger
-	FS      afero.Afero
-	State   state.Manager
-	Patcher patch.ShipPatcher
+	Logger           log.Logger
+	FS               afero.Afero
+	State            state.Manager
+	Patcher          patch.ShipPatcher
+	renderedUpstream string
 }
 
 func NewDaemonlessKustomizer(

pkg/lifecycle/kustomize/kustomizer.go

@@ -174,9 +174,13 @@ func (l *Kustomizer) writeOverlay(
 	relativeResourcePaths []string,
 ) error {
 	// just always make a new kustomization.yaml for now
+	basePath, err := filepath.Rel(step.OverlayPath(), constants.DefaultOverlaysPath)
+	if err != nil {
+		return err
+	}
 	kustomization := ktypes.Kustomization{
 		Bases: []string{
-			filepath.Join("../../", step.Base),
+			basePath,
 		},
 		PatchesStrategicMerge: relativePatchPaths,
 		Resources:             relativeResourcePaths,
@@ -210,6 +214,16 @@ func (l *Kustomizer) writeBase(base string) error {
 	}
 	shipOverlay := currentKustomize.Ship()
 
+	existingKustomize, err := l.FS.Exists(filepath.Join(base, "kustomization.yaml"))
+	if err != nil {
+		return errors.Wrapf(err, "check for kustomization in %s", base)
+	}
+	if existingKustomize {
+		// no need to write base, kustomization already exists
+		// but we do need to remove excluded bases
+		return errors.Wrapf(util.ExcludeKubernetesResources(l.FS, base, shipOverlay.ExcludedBases), "write base %s", base)
+	}
+
 	baseKustomization := ktypes.Kustomization{}
 	if err := l.FS.Walk(
 		base,

pkg/lifecycle/kustomize/kustomizer_test.go

@@ -134,7 +134,7 @@ func Test_kustomizer_writeOverlay(t *testing.T) {
 			expectFile: `kind: ""
 apiversion: ""
 bases:
-- ../../base
+- ../defaults
 `,
 		},
 		{
@@ -147,7 +147,7 @@ patchesStrategicMerge:
 - b.yaml
 - c.yaml
 bases:
-- ../../base
+- ../defaults
 `,
 		},
 	}
@@ -381,7 +381,7 @@ func TestKustomizer(t *testing.T) {
 				"overlays/ship/kustomization.yaml": `kind: ""
 apiversion: ""
 bases:
-- ../../base
+- ../defaults
 `,
 				"base/kustomization.yaml": `kind: ""
 apiversion: ""
@@ -418,7 +418,7 @@ apiversion: ""
 patchesStrategicMerge:
 - deployment.yaml
 bases:
-- ../../base
+- ../defaults
 `,
 				"base/kustomization.yaml": `kind: ""
 apiversion: ""
@@ -469,7 +469,7 @@ apiversion: ""
 resources:
 - limitrange.yaml
 bases:
-- ../../base
+- ../defaults
 `,
 				"base/kustomization.yaml": `kind: ""
 apiversion: ""

pkg/lifecycle/kustomize/patch.go

@@ -51,26 +51,26 @@ var (
 func (l *Kustomizer) generateTillerPatches(step api.Kustomize) error {
 	debug := level.Debug(log.With(l.Logger, "struct", "kustomizer", "handler", "generateTillerPatches"))
 
-	debug.Log("event", "mkdir.tempApplyOverlayPath")
-	if err := l.FS.MkdirAll(constants.TempApplyOverlayPath, 0755); err != nil {
-		return errors.Wrap(err, "create temp apply overlay path")
+	debug.Log("event", "mkdir.DefaultOverlaysPath")
+	if err := l.FS.MkdirAll(constants.DefaultOverlaysPath, 0755); err != nil {
+		return errors.Wrapf(err, "create default overlays path at %s", constants.DefaultOverlaysPath)
 	}
 
 	defaultPatches := []patchOperation{removeChartPatch, removeHeritagePatch}
-	for _, defaultPatch := range defaultPatches {
+	for idx, defaultPatch := range defaultPatches {
 		defaultPatchAsSlice := []patchOperation{defaultPatch}
 
 		patchesB, err := json.Marshal(defaultPatchAsSlice)
 		if err != nil {
-			return errors.Wrap(err, "marshal heritage patch")
+			return errors.Wrapf(err, "marshal default patch idx %d", idx)
 		}
 
-		if err := l.FS.WriteFile(path.Join(constants.TempApplyOverlayPath, defaultPatch.writePath), patchesB, 0755); err != nil {
-			return errors.Wrap(err, "write heritage patch")
+		if err := l.FS.WriteFile(path.Join(constants.DefaultOverlaysPath, defaultPatch.writePath), patchesB, 0755); err != nil {
+			return errors.Wrapf(err, "write default patch idx %d", idx)
 		}
 	}
 
-	relativePathToBases, err := filepath.Rel(constants.TempApplyOverlayPath, step.Base)
+	relativePathToBases, err := filepath.Rel(constants.DefaultOverlaysPath, step.Base)
 	if err != nil {
 		return errors.Wrap(err, "relative path to bases")
 	}
@@ -93,16 +93,8 @@ func (l *Kustomizer) generateTillerPatches(step api.Kustomize) error {
 				return errors.Wrap(err, "walk path")
 			}
 
-			// ignore non-yaml
-			if filepath.Ext(targetPath) != ".yaml" && filepath.Ext(targetPath) != ".yml" {
-				return nil
-			}
-
-			if info.Mode().IsDir() {
-				return nil
-			}
-
-			if info.Name() == "kustomization.yaml" {
+			// this ignores non-k8s resources and things included in the list of excluded bases
+			if !l.shouldAddFileToBase(step.Base, excludedBases, targetPath) {
 				return nil
 			}
 
@@ -168,7 +160,7 @@ func (l *Kustomizer) generateTillerPatches(step api.Kustomize) error {
 	}
 
 	debug.Log("event", "writeFile.kustomization")
-	if err := l.FS.WriteFile(path.Join(constants.TempApplyOverlayPath, "kustomization.yaml"), kustomizationYamlB, 0755); err != nil {
+	if err := l.FS.WriteFile(path.Join(constants.DefaultOverlaysPath, "kustomization.yaml"), kustomizationYamlB, 0755); err != nil {
 		return errors.Wrap(err, "write temp kustomization")
 	}
 

pkg/lifecycle/kustomize/patch_test.go

@@ -45,6 +45,85 @@ metadata:
     heritage: Tiller
     chart: strawberry-1.0.0
   name: strawberry
+`,
+				},
+			},
+			expectKustomization: k8stypes.Kustomization{
+				Bases: []string{"../../strawberry"},
+				PatchesJson6902: []kustomizepatch.Json6902{
+					{
+						Path: "chart-patch.json",
+						Target: &kustomizepatch.Target{
+							Gvk:  gvk.Gvk{Group: "apps", Kind: "Deployment", Version: "v1beta2"},
+							Name: "strawberry",
+						},
+					},
+					{
+						Path: "heritage-patch.json",
+						Target: &kustomizepatch.Target{
+							Gvk:  gvk.Gvk{Group: "apps", Kind: "Deployment", Version: "v1beta2"},
+							Name: "strawberry",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "kustomization yaml",
+			step: api.Kustomize{
+				Base: "strawberry",
+			},
+			testFiles: []testFile{
+				{
+					path: "strawberry/kustomization.yaml",
+					contents: `apiVersion: apps/v1beta2
+bases:
+- ../../base
+patchesJson6902:
+- path: chart-patch.json
+  target:
+    group: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: cert-manager-cainjector
+    version: v1beta1
+`,
+				},
+			},
+			expectKustomization: k8stypes.Kustomization{
+				Bases:           []string{"../../strawberry"},
+				PatchesJson6902: nil,
+			},
+		},
+		{
+			name: "both kustomization and relevant yaml with heritage and chart labels",
+			step: api.Kustomize{
+				Base: "strawberry",
+			},
+			testFiles: []testFile{
+				{
+					path: "strawberry/deployment.yaml",
+					contents: `apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  labels:
+    app: strawberry
+    heritage: Tiller
+    chart: strawberry-1.0.0
+  name: strawberry
+`,
+				},
+				{
+					path: "strawberry/kustomization.yaml",
+					contents: `apiVersion: apps/v1beta2
+bases:
+- ../../base
+patchesJson6902:
+- path: chart-patch.json
+  target:
+    group: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: cert-manager-cainjector
+    version: v1beta1
 `,
 				},
 			},
@@ -185,7 +264,7 @@ metadata:
 			err = l.generateTillerPatches(tt.step)
 			req.NoError(err)
 
-			kustomizationB, err := mockFs.ReadFile(path.Join(constants.TempApplyOverlayPath, "kustomization.yaml"))
+			kustomizationB, err := mockFs.ReadFile(path.Join(constants.DefaultOverlaysPath, "kustomization.yaml"))
 			req.NoError(err)
 
 			kustomizationYaml := k8stypes.Kustomization{}