From 5b8996314c33c9208b73bda56979bd1bf4e277cc Mon Sep 17 00:00:00 2001 From: Soule BA Date: Wed, 15 Dec 2021 16:23:54 +0100 Subject: [PATCH] Add e2e tests for build kustomization Signed-off-by: Soule BA --- cmd/flux/build_kustomization_test.go | 68 +++++++++ cmd/flux/diff_kustomization.go | 4 +- cmd/flux/diff_kustomization_test.go | 127 +++++++++++++++++ cmd/flux/main_test.go | 40 +++++- .../delete-service/deployment.yaml | 74 ++++++++++ .../delete-service/hpa.yaml | 20 +++ .../delete-service/kustomization.yaml | 5 + .../podinfo-kustomization.yaml | 15 ++ .../build-kustomization/podinfo-result.yaml | 133 ++++++++++++++++++ .../build-kustomization/podinfo-source.yaml | 16 +++ .../podinfo-without-service-result.yaml | 95 +++++++++++++ .../podinfo/deployment.yaml | 74 ++++++++++ .../build-kustomization/podinfo/hpa.yaml | 20 +++ .../podinfo/kustomization.yaml | 14 ++ .../build-kustomization/podinfo/service.yaml | 17 +++ .../podinfo/token.encrypted | 20 +++ .../diff-kustomization/deployment.yaml | 78 ++++++++++ .../diff-with-deployment.golden | 4 + .../diff-with-drifted-key-sops-secret.golden | 10 ++ .../diff-with-drifted-secret.golden | 16 +++ .../diff-with-drifted-service.golden | 16 +++ ...diff-with-drifted-value-sops-secret.golden | 4 + .../diff-kustomization/key-sops-secret.yaml | 11 ++ .../diff-kustomization/kustomization.yaml | 11 ++ .../nothing-is-deployed.golden | 5 + .../testdata/diff-kustomization/secret.yaml | 12 ++ .../testdata/diff-kustomization/service.yaml | 21 +++ .../diff-kustomization/value-sops-secret.yaml | 11 ++ go.mod | 2 +- internal/kustomization/build.go | 16 ++- internal/kustomization/diff.go | 58 +++++--- internal/kustomization/kustomization.go | 60 -------- .../kustomization/kustomization_generator.go | 127 +++++++++++++---- .../kustomization/kustomization_varsub.go | 125 ++++++++++------ 34 files changed, 1164 insertions(+), 165 deletions(-) create mode 100644 cmd/flux/build_kustomization_test.go create mode 100644 cmd/flux/diff_kustomization_test.go create mode 100644 cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml create mode 100644 cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml create mode 100644 cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-result.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-source.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/service.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo/token.encrypted create mode 100644 cmd/flux/testdata/diff-kustomization/deployment.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden create mode 100644 cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden create mode 100644 cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/kustomization.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden create mode 100644 cmd/flux/testdata/diff-kustomization/secret.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/service.yaml create mode 100644 cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml diff --git a/cmd/flux/build_kustomization_test.go b/cmd/flux/build_kustomization_test.go new file mode 100644 index 0000000000..de0a5492a7 --- /dev/null +++ b/cmd/flux/build_kustomization_test.go @@ -0,0 +1,68 @@ +// +build unit + +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func setup(t *testing.T, tmpl map[string]string) { + t.Helper() + testEnv.CreateObjectFile("./testdata/build-kustomization/podinfo-source.yaml", tmpl, t) + testEnv.CreateObjectFile("./testdata/build-kustomization/podinfo-kustomization.yaml", tmpl, t) +} + +func TestBuildKustomization(t *testing.T) { + tests := []struct { + name string + args string + assert assertFunc + }{ + { + name: "no args", + args: "build kustomization podinfo", + assert: assertError("invalid resource path \"\""), + }, + { + name: "build podinfo", + args: "build kustomization podinfo --path ./testdata/build-kustomization/podinfo", + assert: assertGoldenFile("./testdata/build-kustomization/podinfo-result.yaml"), + }, + { + name: "build podinfo without service", + args: "build kustomization podinfo --path ./testdata/build-kustomization/delete-service", + assert: assertGoldenFile("./testdata/build-kustomization/podinfo-without-service-result.yaml"), + }, + } + + tmpl := map[string]string{ + "fluxns": allocateNamespace("flux-system"), + } + setup(t, tmpl) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := cmdTestCase{ + args: tt.args + " --namespace=" + tmpl["fluxns"], + assert: tt.assert, + } + cmd.runTestCmd(t) + }) + } +} diff --git a/cmd/flux/diff_kustomization.go b/cmd/flux/diff_kustomization.go index bfc5a19538..16579c3931 100644 --- a/cmd/flux/diff_kustomization.go +++ b/cmd/flux/diff_kustomization.go @@ -67,11 +67,13 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error { return err } - err = builder.Diff() + output, err := builder.Diff() if err != nil { return err } + cmd.Print(output) + return nil } diff --git a/cmd/flux/diff_kustomization_test.go b/cmd/flux/diff_kustomization_test.go new file mode 100644 index 0000000000..70222c4bc7 --- /dev/null +++ b/cmd/flux/diff_kustomization_test.go @@ -0,0 +1,127 @@ +// +build unit + +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "github.com/fluxcd/flux2/internal/kustomization" + "github.com/fluxcd/pkg/ssa" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "os" + "strings" + "testing" +) + +func TestDiffKustomization(t *testing.T) { + tests := []struct { + name string + args string + objectFile string + assert assertFunc + }{ + { + name: "no args", + args: "diff kustomization podinfo", + objectFile: "", + assert: assertError("invalid resource path \"\""), + }, + { + name: "diff nothing deployed", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "", + assert: assertGoldenFile("./testdata/diff-kustomization/nothing-is-deployed.golden"), + }, + { + name: "diff with a deployment object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/deployment.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-deployment.golden"), + }, + { + name: "diff with a drifted service object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/service.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-service.golden"), + }, + { + name: "diff with a drifted secret object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/secret.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-secret.golden"), + }, + { + name: "diff with a drifted key in sops secret object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/key-sops-secret.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden"), + }, + { + name: "diff with a drifted value in sops secret object", + args: "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo", + objectFile: "./testdata/diff-kustomization/value-sops-secret.yaml", + assert: assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden"), + }, + } + + tmpl := map[string]string{ + "fluxns": allocateNamespace("flux-system"), + } + + b, _ := kustomization.NewBuilder(rootArgs.kubeconfig, rootArgs.kubecontext, tmpl["fluxns"], "podinfo", "") + + resourceManager, err := b.Manager() + if err != nil { + t.Fatal(err) + } + + setup(t, tmpl) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.objectFile != "" { + resourceManager.ApplyAll(context.Background(), createObjectFromFile(tt.objectFile, tmpl, t), ssa.DefaultApplyOptions()) + } + cmd := cmdTestCase{ + args: tt.args + " --namespace=" + tmpl["fluxns"], + assert: tt.assert, + } + cmd.runTestCmd(t) + if tt.objectFile != "" { + testEnv.DeleteObjectFile(tt.objectFile, tmpl, t) + } + }) + } +} + +func createObjectFromFile(objectFile string, templateValues map[string]string, t *testing.T) []*unstructured.Unstructured { + buf, err := os.ReadFile(objectFile) + if err != nil { + t.Fatalf("Error reading file '%s': %v", objectFile, err) + } + content, err := executeTemplate(string(buf), templateValues) + if err != nil { + t.Fatalf("Error evaluating template file '%s': '%v'", objectFile, err) + } + clientObjects, err := readYamlObjects(strings.NewReader(content)) + if err != nil { + t.Fatalf("Error decoding yaml file '%s': %v", objectFile, err) + } + + return clientObjects +} diff --git a/cmd/flux/main_test.go b/cmd/flux/main_test.go index 10879e23d0..26b4e99e13 100644 --- a/cmd/flux/main_test.go +++ b/cmd/flux/main_test.go @@ -49,8 +49,8 @@ func allocateNamespace(prefix string) string { return fmt.Sprintf("%s-%d", prefix, id) } -func readYamlObjects(rdr io.Reader) ([]unstructured.Unstructured, error) { - objects := []unstructured.Unstructured{} +func readYamlObjects(rdr io.Reader) ([]*unstructured.Unstructured, error) { + objects := []*unstructured.Unstructured{} reader := k8syaml.NewYAMLReader(bufio.NewReader(rdr)) for { doc, err := reader.Read() @@ -65,7 +65,7 @@ func readYamlObjects(rdr io.Reader) ([]unstructured.Unstructured, error) { if err != nil { return nil, err } - objects = append(objects, *unstructuredObj) + objects = append(objects, unstructuredObj) } return objects, nil } @@ -96,7 +96,7 @@ func (m *testEnvKubeManager) CreateObjectFile(objectFile string, templateValues } } -func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstructured, t *testing.T) error { +func (m *testEnvKubeManager) CreateObjects(clientObjects []*unstructured.Unstructured, t *testing.T) error { for _, obj := range clientObjects { // First create the object then set its status if present in the // yaml file. Make a copy first since creating an object may overwrite @@ -107,7 +107,7 @@ func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstruct return err } obj.SetResourceVersion(createObj.GetResourceVersion()) - err = m.client.Status().Update(context.Background(), &obj) + err = m.client.Status().Update(context.Background(), obj) if err != nil { return err } @@ -115,6 +115,36 @@ func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstruct return nil } +func (m *testEnvKubeManager) DeleteObjectFile(objectFile string, templateValues map[string]string, t *testing.T) { + buf, err := os.ReadFile(objectFile) + if err != nil { + t.Fatalf("Error reading file '%s': %v", objectFile, err) + } + content, err := executeTemplate(string(buf), templateValues) + if err != nil { + t.Fatalf("Error evaluating template file '%s': '%v'", objectFile, err) + } + clientObjects, err := readYamlObjects(strings.NewReader(content)) + if err != nil { + t.Fatalf("Error decoding yaml file '%s': %v", objectFile, err) + } + err = m.DeleteObjects(clientObjects, t) + if err != nil { + t.Logf("Error deleting test objects: '%v'", err) + } +} + +func (m *testEnvKubeManager) DeleteObjects(clientObjects []*unstructured.Unstructured, t *testing.T) error { + for _, obj := range clientObjects { + err := m.client.Delete(context.Background(), obj) + if err != nil { + return err + } + } + + return nil +} + func (m *testEnvKubeManager) Stop() error { if m.testEnv == nil { return fmt.Errorf("do nothing because testEnv is nil") diff --git a/cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml b/cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml new file mode 100644 index 0000000000..33a65a3ac3 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo +spec: + minReadySeconds: 3 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: ghcr.io/stefanprodan/podinfo:6.0.3 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: "#34577c" + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml b/cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml new file mode 100644 index 0000000000..f8111598c0 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml @@ -0,0 +1,20 @@ +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + # scale up if usage is above + # 99% of the requested CPU (100m) + averageUtilization: 99 \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml b/cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml new file mode 100644 index 0000000000..1d0e99c5f6 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./deployment.yaml +- ./hpa.yaml diff --git a/cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml b/cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml new file mode 100644 index 0000000000..036185dc14 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: podinfo + namespace: {{ .fluxns }} +spec: + interval: 5m0s + path: ./kustomize + force: true + prune: true + sourceRef: + kind: GitRepository + name: podinfo + targetNamespace: default \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo-result.yaml b/cmd/flux/testdata/build-kustomization/podinfo-result.yaml new file mode 100644 index 0000000000..cd4009bba5 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-result.yaml @@ -0,0 +1,133 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: default +spec: + minReadySeconds: 3 + progressDeadlineSeconds: 60 + revisionHistoryLimit: 5 + selector: + matchLabels: + app: podinfo + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9797" + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: '#34577c' + image: ghcr.io/stefanprodan/podinfo:6.0.10 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: podinfod + ports: + - containerPort: 9898 + name: http + protocol: TCP + - containerPort: 9797 + name: http-metrics + protocol: TCP + - containerPort: 9999 + name: grpc + protocol: TCP + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi +--- +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo + namespace: default +spec: + maxReplicas: 4 + metrics: + - resource: + name: cpu + target: + averageUtilization: 99 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo +--- +apiVersion: v1 +kind: Service +metadata: + name: podinfo + namespace: default +spec: + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: http + - name: grpc + port: 9999 + protocol: TCP + targetPort: grpc + selector: + app: podinfo + type: ClusterIP +--- +apiVersion: v1 +data: + token: KipTT1BTKio= +kind: Secret +metadata: + name: podinfo-token-77t89m9b67 + namespace: default +type: Opaque +--- +apiVersion: v1 +data: + password: MWYyZDFlMmU2N2Rm + username: YWRtaW4= +kind: Secret +metadata: + name: db-user-pass-bkbd782d2c + namespace: default +type: Opaque diff --git a/cmd/flux/testdata/build-kustomization/podinfo-source.yaml b/cmd/flux/testdata/build-kustomization/podinfo-source.yaml new file mode 100644 index 0000000000..f1a33ecd15 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-source.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .fluxns }} +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: {{ .fluxns }} +spec: + interval: 30s + ref: + branch: master + url: https://github.com/stefanprodan/podinfo \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml b/cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml new file mode 100644 index 0000000000..22a3f81e64 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: default +spec: + minReadySeconds: 3 + progressDeadlineSeconds: 60 + revisionHistoryLimit: 5 + selector: + matchLabels: + app: podinfo + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9797" + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: '#34577c' + image: ghcr.io/stefanprodan/podinfo:6.0.3 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: podinfod + ports: + - containerPort: 9898 + name: http + protocol: TCP + - containerPort: 9797 + name: http-metrics + protocol: TCP + - containerPort: 9999 + name: grpc + protocol: TCP + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi +--- +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo + namespace: default +spec: + maxReplicas: 4 + metrics: + - resource: + name: cpu + target: + averageUtilization: 99 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo diff --git a/cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml b/cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml new file mode 100644 index 0000000000..1a3287bd04 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo +spec: + minReadySeconds: 3 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: ghcr.io/stefanprodan/podinfo:6.0.10 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: "#34577c" + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml b/cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml new file mode 100644 index 0000000000..f8111598c0 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml @@ -0,0 +1,20 @@ +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + # scale up if usage is above + # 99% of the requested CPU (100m) + averageUtilization: 99 \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml b/cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml new file mode 100644 index 0000000000..0ba076685d --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./deployment.yaml +- ./hpa.yaml +- ./service.yaml +secretGenerator: + - files: + - token=token.encrypted + name: podinfo-token + - literals: + - username=admin + - password=1f2d1e2e67df + name: db-user-pass diff --git a/cmd/flux/testdata/build-kustomization/podinfo/service.yaml b/cmd/flux/testdata/build-kustomization/podinfo/service.yaml new file mode 100644 index 0000000000..0d26eca389 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: podinfo +spec: + type: ClusterIP + selector: + app: podinfo + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: http + - port: 9999 + targetPort: grpc + protocol: TCP + name: grpc \ No newline at end of file diff --git a/cmd/flux/testdata/build-kustomization/podinfo/token.encrypted b/cmd/flux/testdata/build-kustomization/podinfo/token.encrypted new file mode 100644 index 0000000000..c88ac97278 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo/token.encrypted @@ -0,0 +1,20 @@ + { + "data": "ENC[AES256_GCM,data:oBe5PlPmfQCUUc4sqKImjw==,iv:MLLEW15QC9kRdVVagJnzLCSk0xZGWIpAeTfHzyxT10g=,tag:K3GkBCGS+ut4Tpk6ndb0CA==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": [ + { + "recipient": "age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+ IFgyNTUxOSA1L2RpZHRrK1FSVmYrd1Va\nY0hxWFQzSDBsT1k3WjNtYmU1QmliaDJycXlNCnF1YjdNOThVbVNvMG9rNS9ZUXZw\nMnV0bnRUMGNtejFPbzM4U2UzWkszeVkKLS0tIGJ6UGhxMUV3YmVJTHlJSUJpRVRZ\nVjd0RVRadU8wekxXTHIrYUplYkN2aEEK0I/ MCEtXRk+b/N2G1JF3vHQT24dShWYD\nw+JIUSA3aLf2sv0zr2MdUEdVWBJoM8nT4D4xVbBORD+669W+9nDeSw==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2021-11-26T16:34:51Z", + "mac": "ENC[AES256_GCM,data:COGzf5YCHNNP6z4JaEKrjN3M8f5+Q1uKUKTMHwj388/ICmLyi2sSrTmj7PP+X7M9jTVwa8wVgYTpNLiVJx+LcxqvIXM0Tyo+/Cu1zrfao98aiACP8+TSEDiFQNtEus23H+d/X1hqMwRHDI3kQ+ 6scgEGnqY57r3RDSA3E8EhHr4=,iv:LxitVIYm8srZVqFueJh9loClA44Y2Z3XAVYmxesMmOg=,tag:Y8qFD8UGlDfwNSv7xlcn6A==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } + } \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/deployment.yaml b/cmd/flux/testdata/diff-kustomization/deployment.yaml new file mode 100644 index 0000000000..9b6b6e1bf0 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + minReadySeconds: 3 + revisionHistoryLimit: 5 + progressDeadlineSeconds: 60 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9797" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: ghcr.io/stefanprodan/podinfo:6.0.10 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9898 + protocol: TCP + - name: http-metrics + containerPort: 9797 + protocol: TCP + - name: grpc + containerPort: 9999 + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --port-metrics=9797 + - --grpc-port=9999 + - --grpc-service-name=podinfo + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: "#34577c" + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + initialDelaySeconds: 5 + timeoutSeconds: 5 + resources: + limits: + cpu: 2000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden b/cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden new file mode 100644 index 0000000000..098497fccd --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden @@ -0,0 +1,4 @@ +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden new file mode 100644 index 0000000000..fe65956683 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden @@ -0,0 +1,10 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 drifted + +data + - one map entry removed: + one map entry added: + drift-key: "*****" token: "******" + +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden new file mode 100644 index 0000000000..8ba49c48d9 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden @@ -0,0 +1,16 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c drifted + +data.password + ± value change + - ***** + + ****** + +data.username + ± value change + - ***** + + ****** + diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden new file mode 100644 index 0000000000..d65e5968e4 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden @@ -0,0 +1,16 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo drifted + +spec.ports + ⇆ order changed + - http, grpc + + grpc, http + +spec.ports.http.port + ± value change + - 9899 + + 9898 + +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden new file mode 100644 index 0000000000..033db67e5d --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden @@ -0,0 +1,4 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml b/cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml new file mode 100644 index 0000000000..52f7cf46f0 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +data: + drift-key: bXktc2VjcmV0LXRva2VuCg== +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo-token-77t89m9b67 + namespace: default +type: Opaque diff --git a/cmd/flux/testdata/diff-kustomization/kustomization.yaml b/cmd/flux/testdata/diff-kustomization/kustomization.yaml new file mode 100644 index 0000000000..dfe99e3245 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ./deployment.yaml +- ./hpa.yaml +- ./service.yaml +secretGenerator: + - literals: + - username=admin + - password=1f2d1e2e67df + name: secret-basic-auth \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden b/cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden new file mode 100644 index 0000000000..da1c23dae0 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden @@ -0,0 +1,5 @@ +► Deployment/default/podinfo created +► HorizontalPodAutoscaler/default/podinfo created +► Service/default/podinfo created +► Secret/default/podinfo-token-77t89m9b67 created +► Secret/default/db-user-pass-bkbd782d2c created diff --git a/cmd/flux/testdata/diff-kustomization/secret.yaml b/cmd/flux/testdata/diff-kustomization/secret.yaml new file mode 100644 index 0000000000..3911cf0c16 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +data: + password: cGFzc3dvcmQK + username: YWRtaW4= +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: db-user-pass-bkbd782d2c + namespace: default +type: Opaque \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/service.yaml b/cmd/flux/testdata/diff-kustomization/service.yaml new file mode 100644 index 0000000000..640fbd2f59 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo + namespace: default +spec: + type: ClusterIP + selector: + app: podinfo + ports: + - name: http + port: 9899 + protocol: TCP + targetPort: http + - port: 9999 + targetPort: grpc + protocol: TCP + name: grpc \ No newline at end of file diff --git a/cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml b/cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml new file mode 100644 index 0000000000..1a469b2546 --- /dev/null +++ b/cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +data: + token: ZHJpZnQtdmFsdWUK +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: podinfo-token-77t89m9b67 + namespace: default +type: Opaque \ No newline at end of file diff --git a/go.mod b/go.mod index f60598f026..6da4aa792c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/fluxcd/image-reflector-controller/api v0.14.0 github.com/fluxcd/kustomize-controller/api v0.18.2 github.com/fluxcd/notification-controller/api v0.19.0 - github.com/fluxcd/pkg/apis/kustomize v0.2.0 + github.com/fluxcd/pkg/apis/kustomize v0.3.0 github.com/fluxcd/pkg/apis/meta v0.10.1 github.com/fluxcd/pkg/runtime v0.12.2 github.com/fluxcd/pkg/ssa v0.5.0 diff --git a/internal/kustomization/build.go b/internal/kustomization/build.go index b2704c80dd..0613c0cb35 100644 --- a/internal/kustomization/build.go +++ b/internal/kustomization/build.go @@ -28,6 +28,8 @@ import ( "github.com/fluxcd/flux2/internal/utils" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -178,13 +180,17 @@ func (b *Builder) build() (resmap.ResMap, error) { } func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath string) ([]byte, error) { - gen := NewGenerator(&kustomizeImpl{kustomization}) + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&kustomization) + if err != nil { + return nil, err + } + gen := NewGenerator(unstructured.Unstructured{Object: data}) return gen.WriteFile(dirPath) } func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomization, dirPath string) (resmap.ResMap, error) { fs := filesys.MakeFsOnDisk() - m, err := buildKustomization(fs, dirPath) + m, err := BuildKustomization(fs, dirPath) if err != nil { return nil, fmt.Errorf("kustomize build failed: %w", err) } @@ -192,7 +198,11 @@ func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomizatio for _, res := range m.Resources() { // run variable substitutions if kustomization.Spec.PostBuild != nil { - outRes, err := substituteVariables(ctx, b.client, &kustomizeImpl{kustomization}, res) + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&kustomization) + if err != nil { + return nil, err + } + outRes, err := SubstituteVariables(ctx, b.client, unstructured.Unstructured{Object: data}, res) if err != nil { return nil, fmt.Errorf("var substitution failed for '%s': %w", res.GetName(), err) } diff --git a/internal/kustomization/diff.go b/internal/kustomization/diff.go index 4bf2f9820c..f2f4db5619 100644 --- a/internal/kustomization/diff.go +++ b/internal/kustomization/diff.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "fmt" + "io" "os" "path/filepath" "sort" @@ -17,6 +18,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/homeport/dyff/pkg/dyff" "github.com/lucasb-eyer/go-colorful" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/cli-utils/pkg/kstatus/polling" @@ -29,7 +31,7 @@ const ( controllerGroup = "kustomize.toolkit.fluxcd.io" ) -func (b *Builder) manager() (*ssa.ResourceManager, error) { +func (b *Builder) Manager() (*ssa.ResourceManager, error) { statusPoller := polling.NewStatusPoller(b.client, b.restMapper) owner := ssa.Owner{ Field: controllerName, @@ -39,20 +41,21 @@ func (b *Builder) manager() (*ssa.ResourceManager, error) { return ssa.NewResourceManager(b.client, statusPoller, owner), nil } -func (b *Builder) Diff() error { +func (b *Builder) Diff() (string, error) { + output := strings.Builder{} res, err := b.Build() if err != nil { - return err + return "", err } // convert the build result into Kubernetes unstructured objects objects, err := ssa.ReadObjects(bytes.NewReader(res)) if err != nil { - return err + return "", err } - resourceManager, err := b.manager() + resourceManager, err := b.Manager() if err != nil { - return err + return "", err } resourceManager.SetOwnerLabels(objects, b.kustomization.GetName(), b.kustomization.GetNamespace()) @@ -61,7 +64,7 @@ func (b *Builder) Diff() error { defer cancel() if err := ssa.SetNativeKindsDefaults(objects); err != nil { - return err + return "", err } // create an inventory of objects to be reconciled @@ -69,10 +72,10 @@ func (b *Builder) Diff() error { for _, obj := range objects { change, liveObject, mergedObject, err := resourceManager.Diff(ctx, obj) if err != nil { - if b.kustomization.Spec.Force && strings.Contains(err.Error(), "immutable") { - writeString(fmt.Sprintf("► %s created", obj.GetName()), bunt.Green) + if b.kustomization.Spec.Force && isImmutableError(err) { + output.WriteString(writeString(fmt.Sprintf("► %s created\n", obj.GetName()), bunt.Green)) } else { - writeString(fmt.Sprint(`✗`, err), bunt.Red) + output.WriteString(writeString(fmt.Sprint(`✗`, err), bunt.Red)) } continue } @@ -84,20 +87,20 @@ func (b *Builder) Diff() error { } if change.Action == string(ssa.CreatedAction) { - writeString(fmt.Sprintf("► %s created", change.Subject), bunt.Green) + output.WriteString(writeString(fmt.Sprintf("► %s created\n", change.Subject), bunt.Green)) } if change.Action == string(ssa.ConfiguredAction) { - writeString(fmt.Sprintf("► %s drifted", change.Subject), bunt.WhiteSmoke) + output.WriteString(writeString(fmt.Sprintf("► %s drifted\n", change.Subject), bunt.WhiteSmoke)) liveFile, mergedFile, tmpDir, err := writeYamls(liveObject, mergedObject) if err != nil { - return err + return "", err } defer cleanupDir(tmpDir) - err = diff(liveFile, mergedFile) + err = diff(liveFile, mergedFile, &output) if err != nil { - return err + return "", err } } @@ -109,15 +112,15 @@ func (b *Builder) Diff() error { if oldStatus.Inventory != nil { diffObjects, err := diffInventory(oldStatus.Inventory, newInventory) if err != nil { - return err + return "", err } for _, object := range diffObjects { - writeString(fmt.Sprintf("► %s deleted", ssa.FmtUnstructured(object)), bunt.OrangeRed) + output.WriteString(writeString(fmt.Sprintf("► %s deleted\n", ssa.FmtUnstructured(object)), bunt.OrangeRed)) } } } - return nil + return output.String(), nil } func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, string, string, error) { @@ -141,19 +144,19 @@ func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, st return liveFile, mergedFile, tmpDir, nil } -func writeString(t string, color colorful.Color) { - fmt.Println(bunt.Style( +func writeString(t string, color colorful.Color) string { + return bunt.Style( t, bunt.EachLine(), bunt.Foreground(color), - )) + ) } func cleanupDir(dir string) error { return os.RemoveAll(dir) } -func diff(liveFile, mergedFile string) error { +func diff(liveFile, mergedFile string, output io.Writer) error { from, to, err := ytbx.LoadFiles(liveFile, mergedFile) if err != nil { return fmt.Errorf("failed to load input files: %w", err) @@ -172,7 +175,7 @@ func diff(liveFile, mergedFile string) error { OmitHeader: true, } - if err := reportWriter.WriteReport(os.Stdout); err != nil { + if err := reportWriter.WriteReport(output); err != nil { return fmt.Errorf("failed to print report: %w", err) } @@ -285,3 +288,12 @@ func addObjectsToInventory(inv *kustomizev1.ResourceInventory, entry *ssa.Change return nil } + +func isImmutableError(err error) bool { + // Detect immutability like kubectl does + // https://github.com/kubernetes/kubectl/blob/8165f83007/pkg/cmd/apply/patcher.go#L201 + if errors.IsConflict(err) || errors.IsInvalid(err) { + return true + } + return false +} diff --git a/internal/kustomization/kustomization.go b/internal/kustomization/kustomization.go index 537d0011b4..c16afc996a 100644 --- a/internal/kustomization/kustomization.go +++ b/internal/kustomization/kustomization.go @@ -16,69 +16,9 @@ limitations under the License. package kustomization -import ( - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2" - "github.com/fluxcd/pkg/apis/kustomize" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Kustomize defines the methods to retrieve the kustomization information -// TO DO @souleb: move this to fluxcd/pkg along with generator and varsub -type Kustomize interface { - client.Object - GetTargetNamespace() string - GetPatches() []kustomize.Patch - GetPatchesStrategicMerge() []apiextensionsv1.JSON - GetPatchesJSON6902() []kustomize.JSON6902Patch - GetImages() []kustomize.Image - GetSubstituteFrom() []SubstituteReference - GetSubstitute() map[string]string -} - // SubstituteReference contains a reference to a resource containing // the variables name and value. type SubstituteReference struct { Kind string `json:"kind"` Name string `json:"name"` } - -// TO DO @souleb: this is a temporary hack to get the kustomize object -// from the kustomize controller. -// At some point we should remove this and have the kustomize controller implement -// the Kustomize interface. -type kustomizeImpl struct { - kustomizev1.Kustomization -} - -func (k *kustomizeImpl) GetTargetNamespace() string { - return k.Spec.TargetNamespace -} - -func (k *kustomizeImpl) GetPatches() []kustomize.Patch { - return k.Spec.Patches -} - -func (k *kustomizeImpl) GetPatchesStrategicMerge() []apiextensionsv1.JSON { - return k.Spec.PatchesStrategicMerge -} - -func (k *kustomizeImpl) GetPatchesJSON6902() []kustomize.JSON6902Patch { - return k.Spec.PatchesJSON6902 -} - -func (k *kustomizeImpl) GetImages() []kustomize.Image { - return k.Spec.Images -} - -func (k *kustomizeImpl) GetSubstituteFrom() []SubstituteReference { - refs := make([]SubstituteReference, 0, len(k.Spec.PostBuild.SubstituteFrom)) - for _, s := range k.Spec.PostBuild.SubstituteFrom { - refs = append(refs, SubstituteReference(s)) - } - return refs -} - -func (k *kustomizeImpl) GetSubstitute() map[string]string { - return k.Spec.PostBuild.Substitute -} diff --git a/internal/kustomization/kustomization_generator.go b/internal/kustomization/kustomization_generator.go index 829c3bdfb7..1f8e3166d2 100644 --- a/internal/kustomization/kustomization_generator.go +++ b/internal/kustomization/kustomization_generator.go @@ -24,6 +24,8 @@ import ( "strings" "sync" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/krusty" "sigs.k8s.io/kustomize/api/provider" @@ -36,10 +38,10 @@ import ( ) type KustomizeGenerator struct { - kustomization Kustomize + kustomization unstructured.Unstructured } -func NewGenerator(kustomization Kustomize) *KustomizeGenerator { +func NewGenerator(kustomization unstructured.Unstructured) *KustomizeGenerator { return &KustomizeGenerator{ kustomization: kustomization, } @@ -72,42 +74,109 @@ func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) { return nil, err } - if kg.kustomization.GetTargetNamespace() != "" { - kus.Namespace = kg.kustomization.GetTargetNamespace() + tg, ok, err := unstructured.NestedString(kg.kustomization.Object, "spec", "targetNamespace") + if err != nil { + return nil, err + } + if ok { + kus.Namespace = tg } - for _, m := range kg.kustomization.GetPatches() { - kus.Patches = append(kus.Patches, kustypes.Patch{ - Patch: m.Patch, - Target: adaptSelector(&m.Target), - }) + patches, ok, err := unstructured.NestedSlice(kg.kustomization.Object, "spec", "patches") + if err != nil { + return nil, err + } + + if ok { + jsonStr, err := json.Marshal(patches) + if err != nil { + return nil, err + } + var patches []kustomize.Patch + if err := json.Unmarshal(jsonStr, &patches); err != nil { + return nil, err + } + + for _, p := range patches { + kus.Patches = append(kus.Patches, kustypes.Patch{ + Patch: p.Patch, + Target: adaptSelector(&p.Target), + }) + } } - for _, m := range kg.kustomization.GetPatchesStrategicMerge() { - kus.PatchesStrategicMerge = append(kus.PatchesStrategicMerge, kustypes.PatchStrategicMerge(m.Raw)) + patchesSM, ok, err := unstructured.NestedSlice(kg.kustomization.Object, "spec", "patchesStrategicMerge") + if err != nil { + return nil, err } - for _, m := range kg.kustomization.GetPatchesJSON6902() { - patch, err := json.Marshal(m.Patch) + if ok { + jsonStr, err := json.Marshal(patchesSM) if err != nil { return nil, err } - kus.PatchesJson6902 = append(kus.PatchesJson6902, kustypes.Patch{ - Patch: string(patch), - Target: adaptSelector(&m.Target), - }) + var patches []apiextensionsv1.JSON + if err := json.Unmarshal(jsonStr, &patches); err != nil { + return nil, err + } + + for _, p := range patches { + kus.PatchesStrategicMerge = append(kus.PatchesStrategicMerge, kustypes.PatchStrategicMerge(p.Raw)) + } } - for _, image := range kg.kustomization.GetImages() { - newImage := kustypes.Image{ - Name: image.Name, - NewName: image.NewName, - NewTag: image.NewTag, + patchesJSON, ok, err := unstructured.NestedSlice(kg.kustomization.Object, "spec", "patchesJson6902") + if err != nil { + return nil, err + } + + if ok { + jsonStr, err := json.Marshal(patchesJSON) + if err != nil { + return nil, err + } + var patches []kustomize.JSON6902Patch + if err := json.Unmarshal(jsonStr, &patches); err != nil { + return nil, err } - if exists, index := checkKustomizeImageExists(kus.Images, image.Name); exists { - kus.Images[index] = newImage - } else { - kus.Images = append(kus.Images, newImage) + + for _, p := range patches { + patch, err := json.Marshal(p.Patch) + if err != nil { + return nil, err + } + kus.PatchesJson6902 = append(kus.PatchesJson6902, kustypes.Patch{ + Patch: string(patch), + Target: adaptSelector(&p.Target), + }) + } + } + + img, ok, err := unstructured.NestedSlice(kg.kustomization.Object, "spec", "images") + if err != nil { + return nil, err + } + + if ok { + jsonStr, err := json.Marshal(img) + if err != nil { + return nil, err + } + var images []kustomize.Image + if err := json.Unmarshal(jsonStr, &images); err != nil { + return nil, err + } + for _, image := range images { + newImage := kustypes.Image{ + Name: image.Name, + NewName: image.NewName, + NewTag: image.NewTag, + } + if exists, index := checkKustomizeImageExists(kus.Images, image.Name); exists { + kus.Images[index] = newImage + } else { + kus.Images = append(kus.Images, newImage) + } } } @@ -116,7 +185,7 @@ func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) { return nil, err } - os.WriteFile(kfile, manifest, 0644) + os.WriteFile(kfile, manifest, os.ModePerm) return data, nil } @@ -239,10 +308,10 @@ func adaptSelector(selector *kustomize.Selector) (output *kustypes.Selector) { // TODO: remove mutex when kustomize fixes the concurrent map read/write panic var kustomizeBuildMutex sync.Mutex -// buildKustomization wraps krusty.MakeKustomizer with the following settings: +// BuildKustomization wraps krusty.MakeKustomizer with the following settings: // - load files from outside the kustomization.yaml root // - disable plugins except for the builtin ones -func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, error) { +func BuildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, error) { // temporary workaround for concurrent map read and map write bug // https://github.com/kubernetes-sigs/kustomize/issues/3659 kustomizeBuildMutex.Lock() diff --git a/internal/kustomization/kustomization_varsub.go b/internal/kustomization/kustomization_varsub.go index 5a893eb9eb..f61f69c0d0 100644 --- a/internal/kustomization/kustomization_varsub.go +++ b/internal/kustomization/kustomization_varsub.go @@ -18,12 +18,14 @@ package kustomization import ( "context" + "encoding/json" "fmt" "regexp" "strings" "github.com/drone/envsubst" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kustomize/api/resource" @@ -37,13 +39,13 @@ const ( DisabledValue = "disabled" ) -// substituteVariables replaces the vars with their values in the specified resource. +// SubstituteVariables replaces the vars with their values in the specified resource. // If a resource is labeled or annotated with // 'kustomize.toolkit.fluxcd.io/substitute: disabled' the substitution is skipped. -func substituteVariables( +func SubstituteVariables( ctx context.Context, kubeClient client.Client, - kustomization Kustomize, + kustomization unstructured.Unstructured, res *resource.Resource) (*resource.Resource, error) { resData, err := res.AsYAML() if err != nil { @@ -56,59 +58,29 @@ func substituteVariables( return nil, nil } - vars := make(map[string]string) - // load vars from ConfigMaps and Secrets data keys - for _, reference := range kustomization.GetSubstituteFrom() { - namespacedName := types.NamespacedName{Namespace: kustomization.GetNamespace(), Name: reference.Name} - switch reference.Kind { - case "ConfigMap": - resource := &corev1.ConfigMap{} - if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { - return nil, fmt.Errorf("substitute from 'ConfigMap/%s' error: %w", reference.Name, err) - } - for k, v := range resource.Data { - vars[k] = strings.Replace(v, "\n", "", -1) - } - case "Secret": - resource := &corev1.Secret{} - if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { - return nil, fmt.Errorf("substitute from 'Secret/%s' error: %w", reference.Name, err) - } - for k, v := range resource.Data { - vars[k] = strings.Replace(string(v), "\n", "", -1) - } - } + vars, err := loadVars(ctx, kubeClient, kustomization) + if err != nil { + return nil, err } // load in-line vars (overrides the ones from resources) - if kustomization.GetSubstitute() != nil { - for k, v := range kustomization.GetSubstitute() { + substitute, ok, err := unstructured.NestedStringMap(kustomization.Object, "spec", "postBuild", "substitute") + if err != nil { + return nil, err + } + if ok { + for k, v := range substitute { vars[k] = strings.Replace(v, "\n", "", -1) } } // run bash variable substitutions if len(vars) > 0 { - r, _ := regexp.Compile(varsubRegex) - for v := range vars { - if !r.MatchString(v) { - return nil, fmt.Errorf("'%s' var name is invalid, must match '%s'", v, varsubRegex) - } - } - - output, err := envsubst.Eval(string(resData), func(s string) string { - return vars[s] - }) - if err != nil { - return nil, fmt.Errorf("variable substitution failed: %w", err) - } - - jsonData, err := yaml.YAMLToJSON([]byte(output)) + jsonData, err := varSubstitution(resData, vars) if err != nil { return nil, fmt.Errorf("YAMLToJSON: %w", err) } - err = res.UnmarshalJSON(jsonData) if err != nil { return nil, fmt.Errorf("UnmarshalJSON: %w", err) @@ -117,3 +89,70 @@ func substituteVariables( return res, nil } + +func loadVars(ctx context.Context, kubeClient client.Client, kustomization unstructured.Unstructured) (map[string]string, error) { + vars := make(map[string]string) + substituteFrom, ok, err := unstructured.NestedSlice(kustomization.Object, "spec", "postBuild", "substituteFrom") + if err != nil { + return nil, err + } + + if ok { + // Convert map to json string + jsonStr, err := json.Marshal(substituteFrom) + if err != nil { + return nil, err + } + var refs []SubstituteReference + if err := json.Unmarshal(jsonStr, &refs); err != nil { + return nil, err + } + + for _, reference := range refs { + namespacedName := types.NamespacedName{Namespace: kustomization.GetNamespace(), Name: reference.Name} + switch reference.Kind { + case "ConfigMap": + resource := &corev1.ConfigMap{} + if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { + return nil, fmt.Errorf("substitute from 'ConfigMap/%s' error: %w", reference.Name, err) + } + for k, v := range resource.Data { + vars[k] = strings.Replace(v, "\n", "", -1) + } + case "Secret": + resource := &corev1.Secret{} + if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { + return nil, fmt.Errorf("substitute from 'Secret/%s' error: %w", reference.Name, err) + } + for k, v := range resource.Data { + vars[k] = strings.Replace(string(v), "\n", "", -1) + } + } + } + } + + return vars, nil +} + +func varSubstitution(data []byte, vars map[string]string) ([]byte, error) { + r, _ := regexp.Compile(varsubRegex) + for v := range vars { + if !r.MatchString(v) { + return nil, fmt.Errorf("'%s' var name is invalid, must match '%s'", v, varsubRegex) + } + } + + output, err := envsubst.Eval(string(data), func(s string) string { + return vars[s] + }) + if err != nil { + return nil, fmt.Errorf("variable substitution failed: %w", err) + } + + jsonData, err := yaml.YAMLToJSON([]byte(output)) + if err != nil { + return nil, fmt.Errorf("YAMLToJSON: %w", err) + } + + return jsonData, nil +}