SOPS: Decrypt dotenv files used in kustomize secret generator

controllers/kustomization_controller.go

@@ -563,6 +563,12 @@ func (r *KustomizationReconciler) build(ctx context.Context, kustomization kusto
 	}
 
 	fs := filesys.MakeFsOnDisk()
+	// decrypt .env files before building kustomization
+	if kustomization.Spec.Decryption != nil {
+		if err = dec.decryptDotEnvFiles(dirPath); err != nil {
+			return nil, fmt.Errorf("error decrypting .env file: %w", err)
+		}
+	}
 	m, err := buildKustomization(fs, dirPath)
 	if err != nil {
 		return nil, fmt.Errorf("kustomize build failed: %w", err)

controllers/kustomization_decryptor.go

@@ -35,7 +35,9 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/kustomize/api/resource"
+	kustypes "sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/yaml"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
@@ -77,35 +79,10 @@ func (kd *KustomizeDecryptor) Decrypt(res *resource.Resource) (*resource.Resourc
 	}
 
 	if kd.kustomization.Spec.Decryption != nil && kd.kustomization.Spec.Decryption.Provider == DecryptionProviderSOPS {
-
 		if bytes.Contains(out, []byte("sops:")) && bytes.Contains(out, []byte("mac: ENC[")) {
-			store := common.StoreForFormat(formats.Yaml)
-
-			tree, err := store.LoadEncryptedFile(out)
-			if err != nil {
-				return nil, fmt.Errorf("LoadEncryptedFile: %w", err)
-			}
-
-			key, err := tree.Metadata.GetDataKeyWithKeyServices(
-				[]keyservice.KeyServiceClient{
-					intkeyservice.NewLocalClient(intkeyservice.NewServer(false, kd.homeDir, kd.ageIdentities)),
-				},
-			)
-			if err != nil {
-				if userErr, ok := err.(sops.UserError); ok {
-					err = fmt.Errorf(userErr.UserError())
-				}
-				return nil, fmt.Errorf("GetDataKey: %w", err)
-			}
-
-			cipher := aes.NewCipher()
-			if _, err := tree.Decrypt(key, cipher); err != nil {
-				return nil, fmt.Errorf("AES decrypt: %w", err)
-			}
-
-			data, err := store.EmitPlainFile(tree.Branches)
+			data, err := kd.DataWithFormat(out, formats.Yaml, formats.Yaml)
 			if err != nil {
-				return nil, fmt.Errorf("EmitPlainFile: %w", err)
+				return nil, fmt.Errorf("DataWithFormat: %w", err)
 			}
 
 			jsonData, err := yaml.YAMLToJSON(data)
@@ -131,37 +108,10 @@ func (kd *KustomizeDecryptor) Decrypt(res *resource.Resource) (*resource.Resourc
 				}
 
 				if bytes.Contains(data, []byte("sops")) && bytes.Contains(data, []byte("ENC[")) {
-
-					store := common.StoreForFormat(formats.Yaml)
-
-					tree, err := store.LoadEncryptedFile(data)
-					if err != nil {
-						return nil, fmt.Errorf("LoadEncryptedFile: %w", err)
-					}
-
-					metadataKey, err := tree.Metadata.GetDataKeyWithKeyServices(
-						[]keyservice.KeyServiceClient{
-							intkeyservice.NewLocalClient(intkeyservice.NewServer(false, kd.homeDir, kd.ageIdentities)),
-						},
-					)
-
-					if err != nil {
-						if userErr, ok := err.(sops.UserError); ok {
-							err = fmt.Errorf(userErr.UserError())
-						}
-						return nil, fmt.Errorf("GetDataKey: %w", err)
-					}
-
-					cipher := aes.NewCipher()
-					if _, err := tree.Decrypt(metadataKey, cipher); err != nil {
-						return nil, fmt.Errorf("AES decrypt: %w", err)
-					}
-
-					outputStore := common.DefaultStoreForPath(key)
-
-					out, err := outputStore.EmitPlainFile(tree.Branches)
+					outputFormat := formats.FormatForPath(key)
+					out, err := kd.DataWithFormat(data, formats.Yaml, outputFormat)
 					if err != nil {
-						return nil, fmt.Errorf("EmitPlainFile: %w", err)
+						return nil, fmt.Errorf("DataWithFormat: %w", err)
 					}
 
 					dataMap[key] = base64.StdEncoding.EncodeToString(out)
@@ -231,3 +181,104 @@ func (kd *KustomizeDecryptor) gpgImport(path string) error {
 	}
 	return nil
 }
+
+func (kd *KustomizeDecryptor) decryptDotEnvFiles(dirpath string) error {
+	kustomizePath := filepath.Join(dirpath, konfig.DefaultKustomizationFileName())
+	ksData, err := ioutil.ReadFile(kustomizePath)
+	if err != nil {
+		return nil
+	}
+
+	kus := kustypes.Kustomization{
+		TypeMeta: kustypes.TypeMeta{
+			APIVersion: kustypes.KustomizationVersion,
+			Kind:       kustypes.KustomizationKind,
+		},
+	}
+
+	if err := yaml.Unmarshal(ksData, &kus); err != nil {
+		return err
+	}
+
+	// recursively decrypt .env files in directories in
+	for _, rsrc := range kus.Resources {
+		rsrcPath := filepath.Join(dirpath, rsrc)
+		isDir, err := isDir(rsrcPath)
+		if err == nil && isDir {
+			err := kd.decryptDotEnvFiles(rsrcPath)
+			if err != nil {
+				return fmt.Errorf("error decrypting .env files in dir '%s': %w",
+					rsrcPath, err)
+			}
+		}
+	}
+
+	secretGens := kus.SecretGenerator
+	for _, gen := range secretGens {
+		for _, envFile := range gen.EnvSources {
+			filepath := filepath.Join(dirpath, envFile)
+			data, err := ioutil.ReadFile(filepath)
+			if err != nil {
+				return err
+			}
+
+			if bytes.Contains(data, []byte("sops_mac=ENC[")) {
+				out, err := kd.DataWithFormat(data, formats.Dotenv, formats.Dotenv)
+				if err != nil {
+					return nil
+				}
+
+				err = ioutil.WriteFile(filepath, out, 0644)
+				if err != nil {
+					return fmt.Errorf("error writing to file: %w", err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (kd KustomizeDecryptor) DataWithFormat(data []byte, inputFormat, outputFormat formats.Format) ([]byte, error) {
+	store := common.StoreForFormat(inputFormat)
+
+	tree, err := store.LoadEncryptedFile(data)
+	if err != nil {
+		return nil, fmt.Errorf("LoadEncryptedFile: %w", err)
+	}
+
+	metadataKey, err := tree.Metadata.GetDataKeyWithKeyServices(
+		[]keyservice.KeyServiceClient{
+			intkeyservice.NewLocalClient(intkeyservice.NewServer(false, kd.homeDir, kd.ageIdentities)),
+		},
+	)
+	if err != nil {
+		if userErr, ok := err.(sops.UserError); ok {
+			err = fmt.Errorf(userErr.UserError())
+		}
+		return nil, fmt.Errorf("GetDataKey: %w", err)
+	}
+
+	cipher := aes.NewCipher()
+	if _, err := tree.Decrypt(metadataKey, cipher); err != nil {
+		return nil, fmt.Errorf("AES decrypt: %w", err)
+	}
+
+	outputStore := common.StoreForFormat(outputFormat)
+
+	out, err := outputStore.EmitPlainFile(tree.Branches)
+	if err != nil {
+		return nil, fmt.Errorf("EmitPlainFile: %w", err)
+	}
+
+	return out, err
+}
+
+func isDir(path string) (bool, error) {
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		return false, err
+	}
+
+	return fileInfo.IsDir(), nil
+}

controllers/kustomization_decryptor_test.go

@@ -49,14 +49,28 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
 	artifactURL, err := testServer.URLForFile(artifactFile)
 	g.Expect(err).ToNot(HaveOccurred())
 
+	overlayArtifactFile := "sops-" + randStringRunes(5)
+	overlayChecksum, err := createArtifact(testServer, "testdata/test-dotenv", overlayArtifactFile)
+	g.Expect(err).ToNot(HaveOccurred())
+	overlayArtifactUrl, err := testServer.URLForFile(overlayArtifactFile)
+	g.Expect(err).ToNot(HaveOccurred())
+
 	repositoryName := types.NamespacedName{
 		Name:      fmt.Sprintf("sops-%s", randStringRunes(5)),
 		Namespace: id,
 	}
 
+	overlayRepositoryName := types.NamespacedName{
+		Name:      fmt.Sprintf("sops-%s", randStringRunes(5)),
+		Namespace: id,
+	}
+
 	err = applyGitRepository(repositoryName, artifactURL, "main/"+artifactChecksum, artifactChecksum)
 	g.Expect(err).NotTo(HaveOccurred())
 
+	err = applyGitRepository(overlayRepositoryName, overlayArtifactUrl, "main/"+overlayChecksum, overlayChecksum)
+	g.Expect(err).NotTo(HaveOccurred())
+
 	pgpKey, err := ioutil.ReadFile("testdata/sops/pgp.asc")
 	g.Expect(err).ToNot(HaveOccurred())
 	ageKey, err := ioutil.ReadFile("testdata/sops/age.txt")
@@ -111,9 +125,18 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
 			TargetNamespace: id,
 		},
 	}
-
 	g.Expect(k8sClient.Create(context.TODO(), kustomization)).To(Succeed())
 
+	overlayKustomizationName := fmt.Sprintf("sops-%s", randStringRunes(5))
+	overlayKs := kustomization.DeepCopy()
+	overlayKs.ResourceVersion = ""
+	overlayKs.Name = overlayKustomizationName
+	overlayKs.Spec.SourceRef.Name = overlayRepositoryName.Name
+	overlayKs.Spec.SourceRef.Namespace = overlayRepositoryName.Namespace
+	overlayKs.Spec.Path = "./testdata/test-dotenv/overlays"
+
+	g.Expect(k8sClient.Create(context.TODO(), overlayKs)).To(Succeed())
+
 	g.Eventually(func() bool {
 		var obj kustomizev1.Kustomization
 		_ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), &obj)
@@ -133,6 +156,22 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
 		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "sops-day", Namespace: id}, &daySecret)).To(Succeed())
 		g.Expect(string(daySecret.Data["secret"])).To(Equal("day=Tuesday\n"))
 
+		var yearSecret corev1.Secret
+		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "sops-year", Namespace: id}, &yearSecret)).To(Succeed())
+		g.Expect(string(yearSecret.Data["year"])).To(Equal("2017"))
+
+		var unencryptedSecret corev1.Secret
+		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "unencrypted-sops-year", Namespace: id}, &unencryptedSecret)).To(Succeed())
+		g.Expect(string(unencryptedSecret.Data["year"])).To(Equal("2021"))
+
+		var year1Secret corev1.Secret
+		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "sops-year1", Namespace: id}, &year1Secret)).To(Succeed())
+		g.Expect(string(year1Secret.Data["year"])).To(Equal("year1"))
+
+		var year2Secret corev1.Secret
+		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "sops-year2", Namespace: id}, &year2Secret)).To(Succeed())
+		g.Expect(string(year2Secret.Data["year"])).To(Equal("year2"))
+
 		var encodedSecret corev1.Secret
 		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "sops-month", Namespace: id}, &encodedSecret)).To(Succeed())
 		g.Expect(string(encodedSecret.Data["month.yaml"])).To(Equal("month: May\n"))

controllers/testdata/sops/month/kustomization.yaml

@@ -4,5 +4,11 @@ secretGenerator:
 - name: sops-month
   files:
   - month.yaml
+- name: sops-year
+  envs:
+  - year.env
+- name: unencrypted-sops-year
+  envs:
+  - unencrypted-year.env
 generatorOptions:
   disableNameSuffixHash: true

controllers/testdata/sops/month/unencrypted-year.env

@@ -0,0 +1 @@
+year=2021

controllers/testdata/sops/month/year.env

@@ -0,0 +1,7 @@
+year=ENC[AES256_GCM,data:EfNnlA==,iv:pBaHDmjQ1d6JrA0Rk19giCQon7CP37hZ0dEQTkJEw1U=,tag:J29CEN9S6pSie8tsAD2REA==,type:str]
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3NHYyMHdNcXhMS2x6aXJq\nTHVhbUYrcW8waFduN1NoWUIyTWFuMmNEVGpzCjUxb05zUndSdnpiQng2VnZ2SkNF\nbnlzY0VmaVd1Z0xZR2FKdDRPQlhKSE0KLS0tIDlEaGgwT3VHcUg5QzFpenZNOTBk\nbUZ5QkRnY0kwMFpYanFLYTlvc0FXdXMKb32CnEO8yg91kkUMFXhBL5Sfz32dNOJT\ntNGdKcOGVBzOJVgU1RquB+5OcJdbuwdV7GCq8KvXqh5fypTI00hZeg==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_0__map_recipient=age1l44xcng8dqj32nlv6d930qvvrny05hglzcv9qpc7kxjc6902ma4qufys29
+sops_lastmodified=2021-10-14T15:35:45Z
+sops_mac=ENC[AES256_GCM,data:brSfy5j0wETn6YT7p8qoCSuI6bevGwrxBbtcqBSYRJ+GgLAr9a7rtwHK8/BnKCi1C1H/zGa1gEERqz2j6Zw0uS4V5lejvtDtfRn9DwYWQ2Aqo2zi4crfNhljerwQVa/Hy9pq2falIZyyhoDX30WOoLe+2eZWQXLtFlVkx4x7U1s=,iv:wr4szytKCN9j6dqccZZl0bkDUHsOtFSvDXjdpuZwTbA=,tag:N1uQ25uLS+E6yQPzXJRiNw==,type:str]
+sops_version=3.7.1
+sops_unencrypted_suffix=_unencrypted

controllers/testdata/test-dotenv/bases/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+secretGenerator:
+  - name: sops-year2
+    envs:
+      - ./secrets/year2.txt
+generatorOptions:
+  disableNameSuffixHash: true

controllers/testdata/test-dotenv/bases/secrets/year2.txt

@@ -0,0 +1,7 @@
+year=ENC[AES256_GCM,data:HoFRvaM=,iv:XNDFLkONNvKSKkbqErVx1/tnEtDuZIG3SficCd7NIaM=,tag:aC7SCerL01kYyXyXkWR2ag==,type:str]
+sops_unencrypted_suffix=_unencrypted
+sops_mac=ENC[AES256_GCM,data:s75x7NzSjmkovCOopnT1eIfXMAdwwsN8KoVdVbAYDTAsB856w/i/W/JshXAUdr5SnXHNbtwzEha/HSppnWEQw1nds18yZCeIW54QE7yxvBKw9Mhd3wxHWiZWziTY0awbYinbyQ45zpq1Iz97BueNjhwtZWMQzRKLQvwyqEljTHs=,iv:AuKqCzIgTYcogtyLrtM6VdgwKTlDE3uMxvVaWbpKBOA=,tag:Ija+U/97TxxWoXYDpG6+jg==,type:str]
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYV1FYTkdzV210SkswWmty\nZzZSVzlCUlRQcVNEOVpYSWNSSWtPd00rcDJrCjZKVVp6aFY2cHJQbm9oY2Q1Z2N3\nLzBWalF4ZHZYTU5kMlcwaGRvYkVKcFEKLS0tIG1QTjNuY0pRbFBqT3dFNFROQWU3\nTWQxNVlUNG8rblQyYmJoaCtKSGcrdE0KjUJ+hGiyCkzUG41mwT3rAb0BdwBF8303\nhBDRmW+DjP1ETrGTXviTS1Cq29IX1K2KdBRxixjtwewkXV/i87wHRA==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_0__map_recipient=age1l44xcng8dqj32nlv6d930qvvrny05hglzcv9qpc7kxjc6902ma4qufys29
+sops_lastmodified=2021-10-15T11:09:14Z
+sops_version=3.7.1

controllers/testdata/test-dotenv/overlays/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../bases
+secretGenerator:
+  - name: sops-year1
+    envs:
+      - year1.env
+generatorOptions:
+  disableNameSuffixHash: true

controllers/testdata/test-dotenv/overlays/year1.env

@@ -0,0 +1,7 @@
+year=ENC[AES256_GCM,data:tV/GLTE=,iv:AtEKKSUa4BiTnDzGMtpGrO78NuR0wMXzjKrQScbtX24=,tag:zAzcBzQ6ORO+NhcY3idHcA==,type:str]
+sops_lastmodified=2021-10-15T11:08:51Z
+sops_unencrypted_suffix=_unencrypted
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFU29oWEh2ckRjaCs3d1FJ\nYTkxN0dqY1lsc1dEUmZ4OGN0N1BHK0xxQld3CmpTL2Z2VDloQStCYnRmYnJ0SDFj\nVU9USmszbU44YUxzRi95Q0sxY2t0bkUKLS0tIC80Ulh1RWJPeUFqbUFNSjFOeGIy\nY001MzMwbnRsQXlsN1VVY2xLY20yazQKYhZQGZpay9J1cnGiHCKBY6DtYMCSIBo7\nAP41GiVukT6M4LT83TpWzWgbR/xNgreKdNpweYcw+Fp+wJHVeR3+fg==\n-----END AGE ENCRYPTED FILE-----\n
+sops_mac=ENC[AES256_GCM,data:rw8vAq+8nqa5/V8p/ICuVKXNQCeTIFExF33qy1YEbc8f4kePDhTlGqxluEytbWOhk+hzCd4POk+zY8bWBY2QSiq0lle2rCtE2WT3I04/+bHzX74yMBuadYLqiUFEhkra/58FXD404PPJBUrOy8mAPgWVczcqMexYhzz//tPdGMY=,iv:yk3CsyGigCSHonvMBTQvjg+kgNssf87KqlKeR6FE8sk=,tag:dCaOhh97ebJWNT5v35n6Iw==,type:str]
+sops_version=3.7.1
+sops_age__list_0__map_recipient=age1l44xcng8dqj32nlv6d930qvvrny05hglzcv9qpc7kxjc6902ma4qufys29