From f7d9ee90cdfaa7769dd35cae74a2d743f4d61359 Mon Sep 17 00:00:00 2001 From: Soule BA Date: Wed, 15 Dec 2021 16:23:54 +0100 Subject: [PATCH] Add e2e tests for build/diff kustomization Signed-off-by: Soule BA --- cmd/flux/build_kustomization.go | 2 +- cmd/flux/build_kustomization_test.go | 83 +++++ cmd/flux/diff_kustomization.go | 6 +- cmd/flux/diff_kustomization_test.go | 129 +++++++ cmd/flux/main_test.go | 40 ++- .../delete-service/deployment.yaml | 74 ++++ .../delete-service/hpa.yaml | 20 ++ .../delete-service/kustomization.yaml | 5 + .../podinfo-kustomization.yaml | 15 + .../build-kustomization/podinfo-result.yaml | 148 ++++++++ .../build-kustomization/podinfo-source.yaml | 16 + .../podinfo-without-service-result.yaml | 101 ++++++ .../podinfo/deployment.yaml | 74 ++++ .../build-kustomization/podinfo/hpa.yaml | 20 ++ .../podinfo/kustomization.yaml | 14 + .../build-kustomization/podinfo/service.yaml | 17 + .../podinfo/token.encrypted | 20 ++ .../diff-kustomization/deployment.yaml | 78 ++++ .../diff-with-deployment.golden | 4 + .../diff-with-drifted-key-sops-secret.golden | 10 + .../diff-with-drifted-secret.golden | 11 + .../diff-with-drifted-service.golden | 16 + ...diff-with-drifted-value-sops-secret.golden | 4 + .../diff-kustomization/key-sops-secret.yaml | 11 + .../diff-kustomization/kustomization.yaml | 11 + .../nothing-is-deployed.golden | 5 + .../testdata/diff-kustomization/secret.yaml | 12 + .../testdata/diff-kustomization/service.yaml | 21 ++ .../diff-kustomization/value-sops-secret.yaml | 11 + go.mod | 2 +- internal/kustomization/build.go | 113 +++--- internal/kustomization/diff.go | 63 ++-- internal/kustomization/kustomization.go | 60 ---- .../kustomization/kustomization_generator.go | 338 ++++++++++++++++-- .../kustomization/kustomization_varsub.go | 119 ++++-- 35 files changed, 1467 insertions(+), 206 deletions(-) create mode 100644 cmd/flux/build_kustomization_test.go create mode 100644 cmd/flux/diff_kustomization_test.go create mode 100644 cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml create mode 100644 cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml create mode 100644 cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-result.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-source.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/service.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/token.encrypted create mode 100644 cmd/flux/testdata/diff-kustomization/deployment.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden create mode 100644 cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/kustomization.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden create mode 100644 cmd/flux/testdata/diff-kustomization/secret.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/service.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml diff --git a/cmd/flux/build_kustomization.go b/cmd/flux/build_kustomization.go index 9262355cdd..0cebfd0dbe 100644 --- a/cmd/flux/build_kustomization.go +++ b/cmd/flux/build_kustomization.go @@ -64,7 +64,7 @@ func buildKsCmdRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid resource path %q", buildKsArgs.path) } - builder, err := kustomization.NewBuilder(rootArgs.kubeconfig, rootArgs.kubecontext, rootArgs.namespace, name, buildKsArgs.path, kustomization.WithTimeout(rootArgs.timeout)) + builder, err := kustomization.NewBuilder(kubeconfigArgs, name, buildKsArgs.path, kustomization.WithTimeout(rootArgs.timeout)) if err != nil { return err } diff --git a/cmd/flux/build_kustomization_test.go b/cmd/flux/build_kustomization_test.go new file mode 100644 index 0000000000..99ecb45c7a --- /dev/null +++ b/cmd/flux/build_kustomization_test.go @@ -0,0 +1,83 @@ +//go:build unit +// +build unit + +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func setup(t *testing.T, tmpl map[string]string) { + t.Helper() + testEnv.CreateObjectFile("./testdata/build-kustomization/podinfo-source.yaml", tmpl, t) + testEnv.CreateObjectFile("./testdata/build-kustomization/podinfo-kustomization.yaml", tmpl, t) +} + +func TestBuildKustomization(t *testing.T) { + tests := []struct { + name string + args string + resultFile string + assertFunc string + }{ + { + name: "no args", + args: "build kustomization podinfo", + resultFile: "invalid resource path \"\"", + assertFunc: "assertError", + }, + { + name: "build podinfo", + args: "build kustomization podinfo --path ./testdata/build-kustomization/podinfo", + resultFile: "./testdata/build-kustomization/podinfo-result.yaml", + assertFunc: "assertGoldenTemplateFile", + }, + { + name: "build podinfo without service", + args: "build kustomization podinfo --path ./testdata/build-kustomization/delete-service", + resultFile: "./testdata/build-kustomization/podinfo-without-service-result.yaml", + assertFunc: "assertGoldenTemplateFile", + }, + } + + tmpl := map[string]string{ + "fluxns": allocateNamespace("flux-system"), + } + setup(t, tmpl) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var assert assertFunc + + switch tt.assertFunc { + case "assertGoldenTemplateFile": + assert = assertGoldenTemplateFile(tt.resultFile, tmpl) + case "assertError": + assert = assertError(tt.resultFile) + } + + cmd := cmdTestCase{ + args: tt.args + " -n " + tmpl["fluxns"], + assert: assert, + } + + cmd.runTestCmd(t) + }) + } +} diff --git a/cmd/flux/diff_kustomization.go b/cmd/flux/diff_kustomization.go index bfc5a19538..4a2afed567 100644 --- a/cmd/flux/diff_kustomization.go +++ b/cmd/flux/diff_kustomization.go @@ -62,16 +62,18 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("invalid resource path %q", diffKsArgs.path) } - builder, err := kustomization.NewBuilder(rootArgs.kubeconfig, rootArgs.kubecontext, rootArgs.namespace, name, diffKsArgs.path, kustomization.WithTimeout(rootArgs.timeout)) + builder, err := kustomization.NewBuilder(kubeconfigArgs, name, diffKsArgs.path, kustomization.WithTimeout(rootArgs.timeout)) if err != nil { return err } - err = builder.Diff() + output, err := builder.Diff() if err != nil { return err } + cmd.Print(output) + return nil } diff --git a/cmd/flux/diff_kustomization_test.go b/cmd/flux/diff_kustomization_test.go new file mode 100644 index 0000000000..c7a11d3656 --- /dev/null +++ b/cmd/flux/diff_kustomization_test.go @@ -0,0 +1,129 @@ +//go:build unit +// +build unit + +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/fluxcd/flux2/internal/kustomization" + "github.com/fluxcd/pkg/ssa" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestDiffKustomization(t *testing.T) { + tests := []struct { + name string + args string + objectFile string + assert assertFunc + }{ + { + name: "no args", + args: "diff kustomization podinfo", + objectFile: "", + assert: assertError("invalid resource path \"\""), + }, + { + name: "diff nothing deployed", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "", + assert: assertGoldenFile("./testdata/diff-kustomization/nothing-is-deployed.golden"), + }, + { + name: "diff with a deployment object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/deployment.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-deployment.golden"), + }, + { + name: "diff with a drifted service object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/service.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-service.golden"), + }, + { + name: "diff with a drifted secret object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/secret.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-secret.golden"), + }, + { + name: "diff with a drifted key in sops secret object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/key-sops-secret.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden"), + }, + { + name: "diff with a drifted value in sops secret object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/value-sops-secret.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden"), + }, + } + + tmpl := map[string]string{ + "fluxns": allocateNamespace("flux-system"), + } + + b, _ := kustomization.NewBuilder(kubeconfigArgs, "podinfo", "") + + resourceManager, err := b.Manager() + if err != nil { + t.Fatal(err) + } + + setup(t, tmpl) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.objectFile != "" { + resourceManager.ApplyAll(context.Background(), createObjectFromFile(tt.objectFile, tmpl, t), ssa.DefaultApplyOptions()) + } + cmd := cmdTestCase{ + args: tt.args + " -n " + tmpl["fluxns"], + assert: tt.assert, + } + cmd.runTestCmd(t) + if tt.objectFile != "" { + testEnv.DeleteObjectFile(tt.objectFile, tmpl, t) + } + }) + } +} + +func createObjectFromFile(objectFile string, templateValues map[string]string, t *testing.T) []*unstructured.Unstructured { + buf, err := os.ReadFile(objectFile) + if err != nil { + t.Fatalf("Error reading file '%s': %v", objectFile, err) + } + content, err := executeTemplate(string(buf), templateValues) + if err != nil { + t.Fatalf("Error evaluating template file '%s': '%v'", objectFile, err) + } + clientObjects, err := readYamlObjects(strings.NewReader(content)) + if err != nil { + t.Fatalf("Error decoding yaml file '%s': %v", objectFile, err) + } + + return clientObjects +} diff --git a/cmd/flux/main_test.go b/cmd/flux/main_test.go index 10879e23d0..26b4e99e13 100644 --- a/cmd/flux/main_test.go +++ b/cmd/flux/main_test.go @@ -49,8 +49,8 @@ func allocateNamespace(prefix string) string { return fmt.Sprintf("%s-%d", prefix, id) } -func readYamlObjects(rdr io.Reader) ([]unstructured.Unstructured, error) { - objects := []unstructured.Unstructured{} +func readYamlObjects(rdr io.Reader) ([]*unstructured.Unstructured, error) { + objects := []*unstructured.Unstructured{} reader := k8syaml.NewYAMLReader(bufio.NewReader(rdr)) for { doc, err := reader.Read() @@ -65,7 +65,7 @@ func readYamlObjects(rdr io.Reader) ([]unstructured.Unstructured, error) { if err != nil { return nil, err } - objects = append(objects, *unstructuredObj) + objects = append(objects, unstructuredObj) } return objects, nil } @@ -96,7 +96,7 @@ func (m *testEnvKubeManager) CreateObjectFile(objectFile string, templateValues } } -func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstructured, t *testing.T) error { +func (m *testEnvKubeManager) CreateObjects(clientObjects []*unstructured.Unstructured, t *testing.T) error { for _, obj := range clientObjects { // First create the object then set its status if present in the // yaml file. Make a copy first since creating an object may overwrite @@ -107,7 +107,7 @@ func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstruct return err } obj.SetResourceVersion(createObj.GetResourceVersion()) - err = m.client.Status().Update(context.Background(), &obj) + err = m.client.Status().Update(context.Background(), obj) if err != nil { return err } @@ -115,6 +115,36 @@ func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstruct return nil } +func (m *testEnvKubeManager) DeleteObjectFile(objectFile string, templateValues map[string]string, t *testing.T) { + buf, err := os.ReadFile(objectFile) + if err != nil { + t.Fatalf("Error reading file '%s': %v", objectFile, err) + } + content, err := executeTemplate(string(buf), templateValues) + if err != nil { + t.Fatalf("Error evaluating template file '%s': '%v'", objectFile, err) + } + clientObjects, err := readYamlObjects(strings.NewReader(content)) + if err != nil { + t.Fatalf("Error decoding yaml file '%s': %v", objectFile, err) + } + err = m.DeleteObjects(clientObjects, t) + if err != nil { + t.Logf("Error deleting test objects: '%v'", err) + } +} + +func (m *testEnvKubeManager) DeleteObjects(clientObjects []*unstructured.Unstructured, t *testing.T) error { + for _, obj := range clientObjects { + err := m.client.Delete(context.Background(), obj) + if err != nil { + return err + } + } + + return nil +} + func (m *testEnvKubeManager) Stop() error { if m.testEnv == nil { return fmt.Errorf("do nothing because testEnv is nil") diff --git a/cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml b/cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml new file mode 100644 index 0000000000..33a65a3ac3 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo +spec: + minReadySeconds: 3 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: ghcr.io/stefanprodan/podinfo:6.0.3 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: "#34577c" + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml b/cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml new file mode 100644 index 0000000000..f8111598c0 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml @@ -0,0 +1,20 @@ +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + # scale up if usage is above + # 99% of the requested CPU (100m) + averageUtilization: 99 \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml b/cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml new file mode 100644 index 0000000000..1d0e99c5f6 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./deployment.yaml +- ./hpa.yaml diff --git a/cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml b/cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml new file mode 100644 index 0000000000..036185dc14 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: podinfo + namespace: {{ .fluxns }} +spec: + interval: 5m0s + path: ./kustomize + force: true + prune: true + sourceRef: + kind: GitRepository + name: podinfo + targetNamespace: default \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo-result.yaml b/cmd/flux/testdata/build-kustomization/podinfo-result.yaml new file mode 100644 index 0000000000..ce66ff0f58 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-result.yaml @@ -0,0 +1,148 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + minReadySeconds: 3 + progressDeadlineSeconds: 60 + revisionHistoryLimit: 5 + selector: + matchLabels: + app: podinfo + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9797" + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: '#34577c' + image: ghcr.io/stefanprodan/podinfo:6.0.10 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: podinfod + ports: + - containerPort: 9898 + name: http + protocol: TCP + - containerPort: 9797 + name: http-metrics + protocol: TCP + - containerPort: 9999 + name: grpc + protocol: TCP + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi +--- +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + maxReplicas: 4 + metrics: + - resource: + name: cpu + target: + averageUtilization: 99 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo +--- +apiVersion: v1 +kind: Service +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: http + - name: grpc + port: 9999 + protocol: TCP + targetPort: grpc + selector: + app: podinfo + type: ClusterIP +--- +apiVersion: v1 +data: + token: KipTT1BTKio= +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo-token-77t89m9b67 + namespace: default +type: Opaque +--- +apiVersion: v1 +data: + password: MWYyZDFlMmU2N2Rm + username: YWRtaW4= +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: db-user-pass-bkbd782d2c + namespace: default +type: Opaque diff --git a/cmd/flux/testdata/build-kustomization/podinfo-source.yaml b/cmd/flux/testdata/build-kustomization/podinfo-source.yaml new file mode 100644 index 0000000000..f1a33ecd15 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-source.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .fluxns }} +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: {{ .fluxns }} +spec: + interval: 30s + ref: + branch: master + url: https://github.com/stefanprodan/podinfo \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml b/cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml new file mode 100644 index 0000000000..943381c885 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + minReadySeconds: 3 + progressDeadlineSeconds: 60 + revisionHistoryLimit: 5 + selector: + matchLabels: + app: podinfo + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9797" + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: '#34577c' + image: ghcr.io/stefanprodan/podinfo:6.0.3 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: podinfod + ports: + - containerPort: 9898 + name: http + protocol: TCP + - containerPort: 9797 + name: http-metrics + protocol: TCP + - containerPort: 9999 + name: grpc + protocol: TCP + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi +--- +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + maxReplicas: 4 + metrics: + - resource: + name: cpu + target: + averageUtilization: 99 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo diff --git a/cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml b/cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml new file mode 100644 index 0000000000..1a3287bd04 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo +spec: + minReadySeconds: 3 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: ghcr.io/stefanprodan/podinfo:6.0.10 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: "#34577c" + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml b/cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml new file mode 100644 index 0000000000..f8111598c0 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml @@ -0,0 +1,20 @@ +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + # scale up if usage is above + # 99% of the requested CPU (100m) + averageUtilization: 99 \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml b/cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml new file mode 100644 index 0000000000..0ba076685d --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./deployment.yaml +- ./hpa.yaml +- ./service.yaml +secretGenerator: + - files: + - token=token.encrypted + name: podinfo-token + - literals: + - username=admin + - password=1f2d1e2e67df + name: db-user-pass diff --git a/cmd/flux/testdata/build-kustomization/podinfo/service.yaml b/cmd/flux/testdata/build-kustomization/podinfo/service.yaml new file mode 100644 index 0000000000..0d26eca389 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: podinfo +spec: + type: ClusterIP + selector: + app: podinfo + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: http + - port: 9999 + targetPort: grpc + protocol: TCP + name: grpc \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo/token.encrypted b/cmd/flux/testdata/build-kustomization/podinfo/token.encrypted new file mode 100644 index 0000000000..c88ac97278 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/token.encrypted @@ -0,0 +1,20 @@ + { + "data": "ENC[AES256_GCM,data:oBe5PlPmfQCUUc4sqKImjw==,iv:MLLEW15QC9kRdVVagJnzLCSk0xZGWIpAeTfHzyxT10g=,tag:K3GkBCGS+ut4Tpk6ndb0CA==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": [ + { + "recipient": "age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+ IFgyNTUxOSA1L2RpZHRrK1FSVmYrd1Va\nY0hxWFQzSDBsT1k3WjNtYmU1QmliaDJycXlNCnF1YjdNOThVbVNvMG9rNS9ZUXZw\nMnV0bnRUMGNtejFPbzM4U2UzWkszeVkKLS0tIGJ6UGhxMUV3YmVJTHlJSUJpRVRZ\nVjd0RVRadU8wekxXTHIrYUplYkN2aEEK0I/ MCEtXRk+b/N2G1JF3vHQT24dShWYD\nw+JIUSA3aLf2sv0zr2MdUEdVWBJoM8nT4D4xVbBORD+669W+9nDeSw==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2021-11-26T16:34:51Z", + "mac": "ENC[AES256_GCM,data:COGzf5YCHNNP6z4JaEKrjN3M8f5+Q1uKUKTMHwj388/ICmLyi2sSrTmj7PP+X7M9jTVwa8wVgYTpNLiVJx+LcxqvIXM0Tyo+/Cu1zrfao98aiACP8+TSEDiFQNtEus23H+d/X1hqMwRHDI3kQ+ 6scgEGnqY57r3RDSA3E8EhHr4=,iv:LxitVIYm8srZVqFueJh9loClA44Y2Z3XAVYmxesMmOg=,tag:Y8qFD8UGlDfwNSv7xlcn6A==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } + } \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/deployment.yaml b/cmd/flux/testdata/diff-kustomization/deployment.yaml new file mode 100644 index 0000000000..9b6b6e1bf0 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + minReadySeconds: 3 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: ghcr.io/stefanprodan/podinfo:6.0.10 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: "#34577c" + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden b/cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden new file mode 100644 index 0000000000..098497fccd --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden @@ -0,0 +1,4 @@ +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden new file mode 100644 index 0000000000..3826954479 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden @@ -0,0 +1,10 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 drifted + +data + - one map entry removed: + one map entry added: + drift-key: "*****" token: "*****" + +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden new file mode 100644 index 0000000000..ac76978f18 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden @@ -0,0 +1,11 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c drifted + +data.password + ± value change + - ****** + + ***** + diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden new file mode 100644 index 0000000000..d65e5968e4 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden @@ -0,0 +1,16 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo drifted + +spec.ports + ⇆ order changed + - http, grpc + + grpc, http + +spec.ports.http.port + ± value change + - 9899 + + 9898 + +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden new file mode 100644 index 0000000000..033db67e5d --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden @@ -0,0 +1,4 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml b/cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml new file mode 100644 index 0000000000..52f7cf46f0 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +data: + drift-key: bXktc2VjcmV0LXRva2VuCg== +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo-token-77t89m9b67 + namespace: default +type: Opaque diff --git a/cmd/flux/testdata/diff-kustomization/kustomization.yaml b/cmd/flux/testdata/diff-kustomization/kustomization.yaml new file mode 100644 index 0000000000..dfe99e3245 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./deployment.yaml +- ./hpa.yaml +- ./service.yaml +secretGenerator: + - literals: + - username=admin + - password=1f2d1e2e67df + name: secret-basic-auth \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden b/cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden new file mode 100644 index 0000000000..da1c23dae0 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden @@ -0,0 +1,5 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/secret.yaml b/cmd/flux/testdata/diff-kustomization/secret.yaml new file mode 100644 index 0000000000..3911cf0c16 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +data: + password: cGFzc3dvcmQK + username: YWRtaW4= +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: db-user-pass-bkbd782d2c + namespace: default +type: Opaque \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/service.yaml b/cmd/flux/testdata/diff-kustomization/service.yaml new file mode 100644 index 0000000000..640fbd2f59 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + type: ClusterIP + selector: + app: podinfo + ports: + - name: http + port: 9899 + protocol: TCP + targetPort: http + - port: 9999 + targetPort: grpc + protocol: TCP + name: grpc \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml b/cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml new file mode 100644 index 0000000000..1a469b2546 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +data: + token: ZHJpZnQtdmFsdWUK +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo-token-77t89m9b67 + namespace: default +type: Opaque \ No newline at end of file diff --git a/go.mod b/go.mod index 6ea59ef94f..e3cb91ad24 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/gonvenience/ytbx v1.4.2 github.com/google/go-cmp v0.5.6 github.com/google/go-containerregistry v0.2.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-retryablehttp v0.7.0 // indirect github.com/homeport/dyff v1.4.6 github.com/lucasb-eyer/go-colorful v1.2.0 @@ -100,7 +101,6 @@ require ( github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect diff --git a/internal/kustomization/build.go b/internal/kustomization/build.go index b2704c80dd..1b17ed4123 100644 --- a/internal/kustomization/build.go +++ b/internal/kustomization/build.go @@ -21,23 +21,26 @@ import ( "context" "encoding/base64" "fmt" - "os" - "path/filepath" "time" "github.com/fluxcd/flux2/internal/utils" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/cli-runtime/pkg/genericclioptions" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/kyaml/filesys" ) -const mask string = "**SOPS**" +const ( + controllerName = "kustomize-controller" + controllerGroup = "kustomize.toolkit.fluxcd.io" + mask string = "**SOPS**" +) var defaultTimeout = 80 * time.Second @@ -65,17 +68,13 @@ func WithTimeout(timeout time.Duration) BuilderOptionFunc { // NewBuilder returns a new Builder // to dp : create functional options -func NewBuilder(kubeconfig string, kubecontext string, namespace, name, resources string, opts ...BuilderOptionFunc) (*Builder, error) { - kubeClient, err := utils.KubeClient(kubeconfig, kubecontext) +func NewBuilder(rcg *genericclioptions.ConfigFlags, name, resources string, opts ...BuilderOptionFunc) (*Builder, error) { + kubeClient, err := utils.KubeClient(rcg) if err != nil { return nil, err } - cfg, err := utils.KubeConfig(kubeconfig, kubecontext) - if err != nil { - return nil, err - } - restMapper, err := apiutil.NewDynamicRESTMapper(cfg) + restMapper, err := rcg.ToRESTMapper() if err != nil { return nil, err } @@ -84,7 +83,7 @@ func NewBuilder(kubeconfig string, kubecontext string, namespace, name, resource client: kubeClient, restMapper: restMapper, name: name, - namespace: namespace, + namespace: *rcg.Namespace, resourcesPath: resources, } @@ -134,57 +133,70 @@ func (b *Builder) Build() ([]byte, error) { return resources, nil } -func (b *Builder) build() (resmap.ResMap, error) { +func (b *Builder) build() (m resmap.ResMap, err error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() // Get the kustomization object k, err := b.getKustomization(ctx) if err != nil { - return nil, err + return } + // store the kustomization object + b.kustomization = k + // generate kustomization.yaml if needed - saved, err := b.generate(*k, b.resourcesPath) - if err != nil { - return nil, fmt.Errorf("failed to generate kustomization.yaml: %w", err) + action, er := b.generate(*k, b.resourcesPath) + if er != nil { + errf := CleanDirectory(b.resourcesPath, action) + err = fmt.Errorf("failed to generate kustomization.yaml: %w", fmt.Errorf("%v %v", er, errf)) + return } + defer func() { + errf := CleanDirectory(b.resourcesPath, action) + if err == nil { + err = errf + } + }() + // build the kustomization - m, err := b.do(ctx, *k, b.resourcesPath) + m, err = b.do(ctx, *k, b.resourcesPath) if err != nil { - return nil, err + return } - // make sure secrets are masked for _, res := range m.Resources() { - err := trimSopsData(res) + // set owner labels + err = b.setOwnerLabels(res) if err != nil { - return nil, err + return } - } - // store the kustomization object - b.kustomization = k - - // overwrite the kustomization.yaml to make sure it's clean - err = overwrite(saved, b.resourcesPath) - if err != nil { - return nil, fmt.Errorf("failed to restore kustomization.yaml: %w", err) + // make sure secrets are masked + err = trimSopsData(res) + if err != nil { + return + } } - return m, nil + return } -func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath string) ([]byte, error) { - gen := NewGenerator(&kustomizeImpl{kustomization}) - return gen.WriteFile(dirPath) +func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath string) (action, error) { + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&kustomization) + if err != nil { + return "", err + } + gen := NewGenerator(unstructured.Unstructured{Object: data}) + return gen.WriteFile(dirPath, WithSaveOriginalKustomization()) } func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomization, dirPath string) (resmap.ResMap, error) { fs := filesys.MakeFsOnDisk() - m, err := buildKustomization(fs, dirPath) + m, err := BuildKustomization(fs, dirPath) if err != nil { return nil, fmt.Errorf("kustomize build failed: %w", err) } @@ -192,7 +204,11 @@ func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomizatio for _, res := range m.Resources() { // run variable substitutions if kustomization.Spec.PostBuild != nil { - outRes, err := substituteVariables(ctx, b.client, &kustomizeImpl{kustomization}, res) + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&kustomization) + if err != nil { + return nil, err + } + outRes, err := SubstituteVariables(ctx, b.client, unstructured.Unstructured{Object: data}, res) if err != nil { return nil, fmt.Errorf("var substitution failed for '%s': %w", res.GetName(), err) } @@ -209,6 +225,20 @@ func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomizatio return m, nil } +func (b *Builder) setOwnerLabels(res *resource.Resource) error { + labels := res.GetLabels() + + labels[controllerGroup+"/name"] = b.kustomization.GetName() + labels[controllerGroup+"/namespace"] = b.kustomization.GetNamespace() + + err := res.SetLabels(labels) + if err != nil { + return err + } + + return nil +} + func trimSopsData(res *resource.Resource) error { // sopsMess is the base64 encoded mask sopsMess := base64.StdEncoding.EncodeToString([]byte(mask)) @@ -233,12 +263,3 @@ func trimSopsData(res *resource.Resource) error { return nil } - -func overwrite(saved []byte, dirPath string) error { - kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName()) - err := os.WriteFile(kfile, saved, 0644) - if err != nil { - return fmt.Errorf("failed to overwrite kustomization.yaml: %w", err) - } - return nil -} diff --git a/internal/kustomization/diff.go b/internal/kustomization/diff.go index 4bf2f9820c..f084b56ba2 100644 --- a/internal/kustomization/diff.go +++ b/internal/kustomization/diff.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "fmt" + "io" "os" "path/filepath" "sort" @@ -17,6 +18,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/homeport/dyff/pkg/dyff" "github.com/lucasb-eyer/go-colorful" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/cli-utils/pkg/kstatus/polling" @@ -24,12 +26,7 @@ import ( "sigs.k8s.io/yaml" ) -const ( - controllerName = "kustomize-controller" - controllerGroup = "kustomize.toolkit.fluxcd.io" -) - -func (b *Builder) manager() (*ssa.ResourceManager, error) { +func (b *Builder) Manager() (*ssa.ResourceManager, error) { statusPoller := polling.NewStatusPoller(b.client, b.restMapper) owner := ssa.Owner{ Field: controllerName, @@ -39,20 +36,21 @@ func (b *Builder) manager() (*ssa.ResourceManager, error) { return ssa.NewResourceManager(b.client, statusPoller, owner), nil } -func (b *Builder) Diff() error { +func (b *Builder) Diff() (string, error) { + output := strings.Builder{} res, err := b.Build() if err != nil { - return err + return "", err } // convert the build result into Kubernetes unstructured objects objects, err := ssa.ReadObjects(bytes.NewReader(res)) if err != nil { - return err + return "", err } - resourceManager, err := b.manager() + resourceManager, err := b.Manager() if err != nil { - return err + return "", err } resourceManager.SetOwnerLabels(objects, b.kustomization.GetName(), b.kustomization.GetNamespace()) @@ -61,7 +59,7 @@ func (b *Builder) Diff() error { defer cancel() if err := ssa.SetNativeKindsDefaults(objects); err != nil { - return err + return "", err } // create an inventory of objects to be reconciled @@ -69,10 +67,10 @@ func (b *Builder) Diff() error { for _, obj := range objects { change, liveObject, mergedObject, err := resourceManager.Diff(ctx, obj) if err != nil { - if b.kustomization.Spec.Force && strings.Contains(err.Error(), "immutable") { - writeString(fmt.Sprintf("► %s created", obj.GetName()), bunt.Green) + if b.kustomization.Spec.Force && isImmutableError(err) { + output.WriteString(writeString(fmt.Sprintf("► %s created\n", obj.GetName()), bunt.Green)) } else { - writeString(fmt.Sprint(`✗`, err), bunt.Red) + output.WriteString(writeString(fmt.Sprint(`✗`, err), bunt.Red)) } continue } @@ -84,20 +82,20 @@ func (b *Builder) Diff() error { } if change.Action == string(ssa.CreatedAction) { - writeString(fmt.Sprintf("► %s created", change.Subject), bunt.Green) + output.WriteString(writeString(fmt.Sprintf("► %s created\n", change.Subject), bunt.Green)) } if change.Action == string(ssa.ConfiguredAction) { - writeString(fmt.Sprintf("► %s drifted", change.Subject), bunt.WhiteSmoke) + output.WriteString(writeString(fmt.Sprintf("► %s drifted\n", change.Subject), bunt.WhiteSmoke)) liveFile, mergedFile, tmpDir, err := writeYamls(liveObject, mergedObject) if err != nil { - return err + return "", err } defer cleanupDir(tmpDir) - err = diff(liveFile, mergedFile) + err = diff(liveFile, mergedFile, &output) if err != nil { - return err + return "", err } } @@ -109,15 +107,15 @@ func (b *Builder) Diff() error { if oldStatus.Inventory != nil { diffObjects, err := diffInventory(oldStatus.Inventory, newInventory) if err != nil { - return err + return "", err } for _, object := range diffObjects { - writeString(fmt.Sprintf("► %s deleted", ssa.FmtUnstructured(object)), bunt.OrangeRed) + output.WriteString(writeString(fmt.Sprintf("► %s deleted\n", ssa.FmtUnstructured(object)), bunt.OrangeRed)) } } } - return nil + return output.String(), nil } func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, string, string, error) { @@ -141,19 +139,19 @@ func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, st return liveFile, mergedFile, tmpDir, nil } -func writeString(t string, color colorful.Color) { - fmt.Println(bunt.Style( +func writeString(t string, color colorful.Color) string { + return bunt.Style( t, bunt.EachLine(), bunt.Foreground(color), - )) + ) } func cleanupDir(dir string) error { return os.RemoveAll(dir) } -func diff(liveFile, mergedFile string) error { +func diff(liveFile, mergedFile string, output io.Writer) error { from, to, err := ytbx.LoadFiles(liveFile, mergedFile) if err != nil { return fmt.Errorf("failed to load input files: %w", err) @@ -172,7 +170,7 @@ func diff(liveFile, mergedFile string) error { OmitHeader: true, } - if err := reportWriter.WriteReport(os.Stdout); err != nil { + if err := reportWriter.WriteReport(output); err != nil { return fmt.Errorf("failed to print report: %w", err) } @@ -285,3 +283,12 @@ func addObjectsToInventory(inv *kustomizev1.ResourceInventory, entry *ssa.Change return nil } + +func isImmutableError(err error) bool { + // Detect immutability like kubectl does + // https://github.com/kubernetes/kubectl/blob/8165f83007/pkg/cmd/apply/patcher.go#L201 + if errors.IsConflict(err) || errors.IsInvalid(err) { + return true + } + return false +} diff --git a/internal/kustomization/kustomization.go b/internal/kustomization/kustomization.go index 537d0011b4..c16afc996a 100644 --- a/internal/kustomization/kustomization.go +++ b/internal/kustomization/kustomization.go @@ -16,69 +16,9 @@ limitations under the License. package kustomization -import ( - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2" - "github.com/fluxcd/pkg/apis/kustomize" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Kustomize defines the methods to retrieve the kustomization information -// TO DO @souleb: move this to fluxcd/pkg along with generator and varsub -type Kustomize interface { - client.Object - GetTargetNamespace() string - GetPatches() []kustomize.Patch - GetPatchesStrategicMerge() []apiextensionsv1.JSON - GetPatchesJSON6902() []kustomize.JSON6902Patch - GetImages() []kustomize.Image - GetSubstituteFrom() []SubstituteReference - GetSubstitute() map[string]string -} - // SubstituteReference contains a reference to a resource containing // the variables name and value. type SubstituteReference struct { Kind string `json:"kind"` Name string `json:"name"` } - -// TO DO @souleb: this is a temporary hack to get the kustomize object -// from the kustomize controller. -// At some point we should remove this and have the kustomize controller implement -// the Kustomize interface. -type kustomizeImpl struct { - kustomizev1.Kustomization -} - -func (k *kustomizeImpl) GetTargetNamespace() string { - return k.Spec.TargetNamespace -} - -func (k *kustomizeImpl) GetPatches() []kustomize.Patch { - return k.Spec.Patches -} - -func (k *kustomizeImpl) GetPatchesStrategicMerge() []apiextensionsv1.JSON { - return k.Spec.PatchesStrategicMerge -} - -func (k *kustomizeImpl) GetPatchesJSON6902() []kustomize.JSON6902Patch { - return k.Spec.PatchesJSON6902 -} - -func (k *kustomizeImpl) GetImages() []kustomize.Image { - return k.Spec.Images -} - -func (k *kustomizeImpl) GetSubstituteFrom() []SubstituteReference { - refs := make([]SubstituteReference, 0, len(k.Spec.PostBuild.SubstituteFrom)) - for _, s := range k.Spec.PostBuild.SubstituteFrom { - refs = append(refs, SubstituteReference(s)) - } - return refs -} - -func (k *kustomizeImpl) GetSubstitute() map[string]string { - return k.Spec.PostBuild.Substitute -} diff --git a/internal/kustomization/kustomization_generator.go b/internal/kustomization/kustomization_generator.go index 829c3bdfb7..fc70fb8de4 100644 --- a/internal/kustomization/kustomization_generator.go +++ b/internal/kustomization/kustomization_generator.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Flux authors +Copyright 2022 The Flux authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,11 +19,15 @@ package kustomization import ( "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" "sync" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/krusty" "sigs.k8s.io/kustomize/api/provider" @@ -33,32 +37,74 @@ import ( "sigs.k8s.io/yaml" "github.com/fluxcd/pkg/apis/kustomize" + "github.com/hashicorp/go-multierror" +) + +const ( + specField = "spec" + targetNSField = "targetNamespace" + patchesField = "patches" + patchesSMField = "patchesStrategicMerge" + patchesJson6902Field = "patchesJson6902" + imagesField = "images" + originalKustomizationFile = "kustomization.yaml.original" +) + +type action string + +const ( + createdAction action = "created" + unchangedAction action = "unchanged" ) type KustomizeGenerator struct { - kustomization Kustomize + kustomization unstructured.Unstructured } -func NewGenerator(kustomization Kustomize) *KustomizeGenerator { +type SavingOptions func(dirPath, file string, action action) error + +func NewGenerator(kustomization unstructured.Unstructured) *KustomizeGenerator { return &KustomizeGenerator{ kustomization: kustomization, } } +func WithSaveOriginalKustomization() SavingOptions { + return func(dirPath, kfile string, action action) error { + // copy the original kustomization.yaml to the directory if we did not create it + if action != createdAction { + if err := copyFile(kfile, filepath.Join(dirPath, originalKustomizationFile)); err != nil { + errf := CleanDirectory(dirPath, action) + return fmt.Errorf("%v %v", err, errf) + } + } + return nil + } +} + // WriteFile generates a kustomization.yaml in the given directory if it does not exist. // It apply the flux kustomize resources to the kustomization.yaml and then write the // updated kustomization.yaml to the directory. -// It returns the original kustomization.yaml. -func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) { - if err := kg.generateKustomization(dirPath); err != nil { - return nil, err +// It returns an action that indicates if the kustomization.yaml was created or not. +// It is the caller responsability to clean up the directory by use the provided function CleanDirectory. +// example: +// err := CleanDirectory(dirPath, action) +// if err != nil { +// log.Fatal(err) +// } +func (kg *KustomizeGenerator) WriteFile(dirPath string, opts ...SavingOptions) (action, error) { + action, err := kg.generateKustomization(dirPath) + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("%v %v", err, errf) } kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName()) data, err := os.ReadFile(kfile) if err != nil { - return nil, err + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("%w %s", err, errf) } kus := kustypes.Kustomization{ @@ -69,36 +115,67 @@ func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) { } if err := yaml.Unmarshal(data, &kus); err != nil { - return nil, err + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("%v %v", err, errf) } - if kg.kustomization.GetTargetNamespace() != "" { - kus.Namespace = kg.kustomization.GetTargetNamespace() + tg, ok, err := kg.getNestedString(specField, targetNSField) + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("%v %v", err, errf) + } + if ok { + kus.Namespace = tg + } + + patches, err := kg.getPatches() + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("unable to get patches: %w", fmt.Errorf("%v %v", err, errf)) } - for _, m := range kg.kustomization.GetPatches() { + for _, p := range patches { kus.Patches = append(kus.Patches, kustypes.Patch{ - Patch: m.Patch, - Target: adaptSelector(&m.Target), + Patch: p.Patch, + Target: adaptSelector(&p.Target), }) } - for _, m := range kg.kustomization.GetPatchesStrategicMerge() { - kus.PatchesStrategicMerge = append(kus.PatchesStrategicMerge, kustypes.PatchStrategicMerge(m.Raw)) + patchesSM, err := kg.getPatchesStrategicMerge() + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("unable to get patchesStrategicMerge: %w", fmt.Errorf("%v %v", err, errf)) + } + + for _, p := range patchesSM { + kus.PatchesStrategicMerge = append(kus.PatchesStrategicMerge, kustypes.PatchStrategicMerge(p.Raw)) } - for _, m := range kg.kustomization.GetPatchesJSON6902() { - patch, err := json.Marshal(m.Patch) + patchesJSON, err := kg.getPatchesJson6902() + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("unable to get patchesJson6902: %w", fmt.Errorf("%v %v", err, errf)) + } + + for _, p := range patchesJSON { + patch, err := json.Marshal(p.Patch) if err != nil { - return nil, err + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("%v %v", err, errf) } kus.PatchesJson6902 = append(kus.PatchesJson6902, kustypes.Patch{ Patch: string(patch), - Target: adaptSelector(&m.Target), + Target: adaptSelector(&p.Target), }) } - for _, image := range kg.kustomization.GetImages() { + images, err := kg.getImages() + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("unable to get images: %w", fmt.Errorf("%v %v", err, errf)) + } + + for _, image := range images { newImage := kustypes.Image{ Name: image.Name, NewName: image.NewName, @@ -112,13 +189,141 @@ func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) { } manifest, err := yaml.Marshal(kus) + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("%v %v", err, errf) + } + + // copy the original kustomization.yaml to the directory if we did not create it + for _, opt := range opts { + if err := opt(dirPath, kfile, action); err != nil { + return action, fmt.Errorf("failed to save original kustomization.yaml: %w", err) + } + } + + err = os.WriteFile(kfile, manifest, os.ModePerm) + if err != nil { + errf := CleanDirectory(dirPath, action) + return action, fmt.Errorf("%v %v", err, errf) + } + + return action, nil +} + +func (kg *KustomizeGenerator) getPatches() ([]kustomize.Patch, error) { + patches, ok, err := kg.getNestedSlice(specField, patchesField) + if err != nil { + return nil, err + } + + var resultErr error + if ok { + res := make([]kustomize.Patch, 0, len(patches)) + for k, p := range patches { + patch, ok := p.(map[string]interface{}) + if !ok { + err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k) + resultErr = multierror.Append(resultErr, err) + } + var kpatch kustomize.Patch + err = runtime.DefaultUnstructuredConverter.FromUnstructured(patch, &kpatch) + if err != nil { + resultErr = multierror.Append(resultErr, err) + } + res = append(res, kpatch) + } + return res, resultErr + } + + return nil, resultErr + +} + +func (kg *KustomizeGenerator) getPatchesStrategicMerge() ([]apiextensionsv1.JSON, error) { + patches, ok, err := kg.getNestedSlice(specField, patchesSMField) + if err != nil { + return nil, err + } + + var resultErr error + if ok { + res := make([]apiextensionsv1.JSON, 0, len(patches)) + for k, p := range patches { + patch, ok := p.(map[string]interface{}) + if !ok { + err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k) + resultErr = multierror.Append(resultErr, err) + } + var kpatch apiextensionsv1.JSON + err = runtime.DefaultUnstructuredConverter.FromUnstructured(patch, &kpatch) + if err != nil { + resultErr = multierror.Append(resultErr, err) + } + res = append(res, kpatch) + } + return res, resultErr + } + + return nil, resultErr + +} + +func (kg *KustomizeGenerator) getPatchesJson6902() ([]kustomize.JSON6902Patch, error) { + patches, ok, err := kg.getNestedSlice(specField, patchesJson6902Field) + if err != nil { + return nil, err + } + + var resultErr error + if ok { + res := make([]kustomize.JSON6902Patch, 0, len(patches)) + for k, p := range patches { + patch, ok := p.(map[string]interface{}) + if !ok { + err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k) + resultErr = multierror.Append(resultErr, err) + } + var kpatch kustomize.JSON6902Patch + err = runtime.DefaultUnstructuredConverter.FromUnstructured(patch, &kpatch) + if err != nil { + resultErr = multierror.Append(resultErr, err) + } + res = append(res, kpatch) + } + return res, resultErr + } + + return nil, resultErr + +} + +func (kg *KustomizeGenerator) getImages() ([]kustomize.Image, error) { + img, ok, err := kg.getNestedSlice(specField, imagesField) if err != nil { return nil, err } - os.WriteFile(kfile, manifest, 0644) + var resultErr error + if ok { + res := make([]kustomize.Image, 0, len(img)) + for k, i := range img { + im, ok := i.(map[string]interface{}) + if !ok { + err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k) + resultErr = multierror.Append(resultErr, err) + } + var image kustomize.Image + err = runtime.DefaultUnstructuredConverter.FromUnstructured(im, &image) + if err != nil { + resultErr = multierror.Append(resultErr, err) + } + res = append(res, image) + } + return res, resultErr + } + + return nil, resultErr - return data, nil } func checkKustomizeImageExists(images []kustypes.Image, imageName string) (bool, int) { @@ -131,14 +336,32 @@ func checkKustomizeImageExists(images []kustypes.Image, imageName string) (bool, return false, -1 } -func (kg *KustomizeGenerator) generateKustomization(dirPath string) error { +func (kg *KustomizeGenerator) getNestedString(fields ...string) (string, bool, error) { + val, ok, err := unstructured.NestedString(kg.kustomization.Object, fields...) + if err != nil { + return "", ok, err + } + + return val, ok, nil +} + +func (kg *KustomizeGenerator) getNestedSlice(fields ...string) ([]interface{}, bool, error) { + val, ok, err := unstructured.NestedSlice(kg.kustomization.Object, fields...) + if err != nil { + return nil, ok, err + } + + return val, ok, nil +} + +func (kg *KustomizeGenerator) generateKustomization(dirPath string) (action, error) { fs := filesys.MakeFsOnDisk() // Determine if there already is a Kustomization file at the root, // as this means we do not have to generate one. for _, kfilename := range konfig.RecognizedKustomizationFileNames() { if kpath := filepath.Join(dirPath, kfilename); fs.Exists(kpath) && !fs.IsDir(kpath) { - return nil + return unchangedAction, nil } } @@ -186,18 +409,18 @@ func (kg *KustomizeGenerator) generateKustomization(dirPath string) error { abs, err := filepath.Abs(dirPath) if err != nil { - return err + return unchangedAction, err } files, err := scan(abs) if err != nil { - return err + return unchangedAction, err } kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName()) f, err := fs.Create(kfile) if err != nil { - return err + return unchangedAction, err } f.Close() @@ -216,10 +439,12 @@ func (kg *KustomizeGenerator) generateKustomization(dirPath string) error { kus.Resources = resources kd, err := yaml.Marshal(kus) if err != nil { - return err + // delete the kustomization file + errf := CleanDirectory(dirPath, createdAction) + return unchangedAction, fmt.Errorf("%v %v", err, errf) } - return os.WriteFile(kfile, kd, os.ModePerm) + return createdAction, os.WriteFile(kfile, kd, os.ModePerm) } func adaptSelector(selector *kustomize.Selector) (output *kustypes.Selector) { @@ -239,10 +464,10 @@ func adaptSelector(selector *kustomize.Selector) (output *kustypes.Selector) { // TODO: remove mutex when kustomize fixes the concurrent map read/write panic var kustomizeBuildMutex sync.Mutex -// buildKustomization wraps krusty.MakeKustomizer with the following settings: +// BuildKustomization wraps krusty.MakeKustomizer with the following settings: // - load files from outside the kustomization.yaml root // - disable plugins except for the builtin ones -func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, error) { +func BuildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, error) { // temporary workaround for concurrent map read and map write bug // https://github.com/kubernetes-sigs/kustomize/issues/3659 kustomizeBuildMutex.Lock() @@ -256,3 +481,52 @@ func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, e k := krusty.MakeKustomizer(buildOptions) return k.Run(fs, dirPath) } + +// CleanDirectory removes the kustomization.yaml file from the given directory. +func CleanDirectory(dirPath string, action action) error { + kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName()) + originalFile := filepath.Join(dirPath, originalKustomizationFile) + + // restore old file if it exists + if _, err := os.Stat(originalFile); err == nil { + err := os.Rename(originalFile, kfile) + if err != nil { + return fmt.Errorf("failed to cleanup repository: %w", err) + } + } + + if action == createdAction { + return os.Remove(kfile) + } + + return nil +} + +// copyFile copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist or else trucnated. +func copyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + + defer func() { + errf := out.Close() + if err == nil { + err = errf + } + }() + + if _, err = io.Copy(out, in); err != nil { + return + } + + return +} diff --git a/internal/kustomization/kustomization_varsub.go b/internal/kustomization/kustomization_varsub.go index 5a893eb9eb..f89e327316 100644 --- a/internal/kustomization/kustomization_varsub.go +++ b/internal/kustomization/kustomization_varsub.go @@ -23,7 +23,10 @@ import ( "strings" "github.com/drone/envsubst" + "github.com/hashicorp/go-multierror" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kustomize/api/resource" @@ -37,13 +40,13 @@ const ( DisabledValue = "disabled" ) -// substituteVariables replaces the vars with their values in the specified resource. +// SubstituteVariables replaces the vars with their values in the specified resource. // If a resource is labeled or annotated with // 'kustomize.toolkit.fluxcd.io/substitute: disabled' the substitution is skipped. -func substituteVariables( +func SubstituteVariables( ctx context.Context, kubeClient client.Client, - kustomization Kustomize, + kustomization unstructured.Unstructured, res *resource.Resource) (*resource.Resource, error) { resData, err := res.AsYAML() if err != nil { @@ -56,10 +59,46 @@ func substituteVariables( return nil, nil } + // load vars from ConfigMaps and Secrets data keys + vars, err := loadVars(ctx, kubeClient, kustomization) + if err != nil { + return nil, err + } + + // load in-line vars (overrides the ones from resources) + substitute, ok, err := unstructured.NestedStringMap(kustomization.Object, "spec", "postBuild", "substitute") + if err != nil { + return nil, err + } + if ok { + for k, v := range substitute { + vars[k] = strings.Replace(v, "\n", "", -1) + } + } + + // run bash variable substitutions + if len(vars) > 0 { + jsonData, err := varSubstitution(resData, vars) + if err != nil { + return nil, fmt.Errorf("YAMLToJSON: %w", err) + } + err = res.UnmarshalJSON(jsonData) + if err != nil { + return nil, fmt.Errorf("UnmarshalJSON: %w", err) + } + } + + return res, nil +} + +func loadVars(ctx context.Context, kubeClient client.Client, kustomization unstructured.Unstructured) (map[string]string, error) { vars := make(map[string]string) + substituteFrom, err := getSubstituteFrom(kustomization) + if err != nil { + return nil, fmt.Errorf("unable to get subsituteFrom: %w", err) + } - // load vars from ConfigMaps and Secrets data keys - for _, reference := range kustomization.GetSubstituteFrom() { + for _, reference := range substituteFrom { namespacedName := types.NamespacedName{Namespace: kustomization.GetNamespace(), Name: reference.Name} switch reference.Kind { case "ConfigMap": @@ -81,39 +120,57 @@ func substituteVariables( } } - // load in-line vars (overrides the ones from resources) - if kustomization.GetSubstitute() != nil { - for k, v := range kustomization.GetSubstitute() { - vars[k] = strings.Replace(v, "\n", "", -1) + return vars, nil +} + +func varSubstitution(data []byte, vars map[string]string) ([]byte, error) { + r, _ := regexp.Compile(varsubRegex) + for v := range vars { + if !r.MatchString(v) { + return nil, fmt.Errorf("'%s' var name is invalid, must match '%s'", v, varsubRegex) } } - // run bash variable substitutions - if len(vars) > 0 { - r, _ := regexp.Compile(varsubRegex) - for v := range vars { - if !r.MatchString(v) { - return nil, fmt.Errorf("'%s' var name is invalid, must match '%s'", v, varsubRegex) - } - } + output, err := envsubst.Eval(string(data), func(s string) string { + return vars[s] + }) + if err != nil { + return nil, fmt.Errorf("variable substitution failed: %w", err) + } - output, err := envsubst.Eval(string(resData), func(s string) string { - return vars[s] - }) - if err != nil { - return nil, fmt.Errorf("variable substitution failed: %w", err) - } + jsonData, err := yaml.YAMLToJSON([]byte(output)) + if err != nil { + return nil, fmt.Errorf("YAMLToJSON: %w", err) + } - jsonData, err := yaml.YAMLToJSON([]byte(output)) - if err != nil { - return nil, fmt.Errorf("YAMLToJSON: %w", err) - } + return jsonData, nil +} - err = res.UnmarshalJSON(jsonData) - if err != nil { - return nil, fmt.Errorf("UnmarshalJSON: %w", err) +func getSubstituteFrom(kustomization unstructured.Unstructured) ([]SubstituteReference, error) { + substituteFrom, ok, err := unstructured.NestedSlice(kustomization.Object, "spec", "postBuild", "substituteFrom") + if err != nil { + return nil, err + } + + var resultErr error + if ok { + res := make([]SubstituteReference, 0, len(substituteFrom)) + for k, s := range substituteFrom { + sub, ok := s.(map[string]interface{}) + if !ok { + err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k) + resultErr = multierror.Append(resultErr, err) + } + var substitute SubstituteReference + err = runtime.DefaultUnstructuredConverter.FromUnstructured(sub, &substitute) + if err != nil { + resultErr = multierror.Append(resultErr, err) + } + res = append(res, substitute) } + return res, nil } - return res, nil + return nil, resultErr + }