extend helm chart with extra environment and extra secrets (#631)

* add test for secret volume and mounts
* add mount of extra secrets
* add extra environment variable definition
* add ability to use secret values in environment variables

charts/logprep/Chart.yaml

@@ -6,7 +6,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "13.0.0"
+version: "13.1.0"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/logprep/templates/deployment.yaml

@@ -68,6 +68,9 @@ spec:
             - name: REQUESTS_CA_BUNDLE
               value: /home/logprep/certificates/{{ .Values.secrets.certificates.name }}
             {{- end }}
+            {{- if .Values.environment }}
+            {{- toYaml .Values.environment  | nindent 12 }}
+            {{- end }}
           volumeMounts:
             - name: logprep-temp
               mountPath: /tmp
@@ -97,13 +100,10 @@ spec:
             - name: output-config
               mountPath: /home/logprep/output-config.yaml
               subPath: output-config.yaml
-            {{- if .Values.secrets.certificates }}
-            - name: certificates
-              mountPath: /home/logprep/certificates/{{ .Values.secrets.certificates.name }}
-            {{- end }}
-            {{- if .Values.secrets.credentials }}
-            - name: credentials
-              mountPath: /home/logprep/credentials/{{ .Values.secrets.credentials.name }}
+            {{- range $key, $value := .Values.secrets }}
+            - name: {{ $key }}
+              mountPath: /home/logprep/{{ $key }}/{{ $value.name }}
+              subPath: {{ $value.name }}
             {{- end }}
           {{- if or .Values.exporter.enabled (eq .Values.input.type "http_input") }}
           {{- if eq .Values.input.type "http_input" }}
@@ -168,15 +168,10 @@ spec:
           configMap:
             name: {{ include "logprep.fullname" . }}-artifacts
         {{- end }}
-        {{- if .Values.secrets.certificates }}
-        - name: certificates
-          secret:
-            secretName: {{ .Values.secrets.certificates.name }}
-        {{- end }}
-        {{- if .Values.secrets.credentials }}
-        - name: credentials
+        {{- range $key, $value := .Values.secrets }}
+        - name: {{ $key }}
           secret:
-            secretName: {{ .Values.secrets.credentials.name }}
+            secretName: {{ $value.name }}
         {{- end }}
       {{- if .Values.affinity }}
       affinity:

charts/logprep/values.yaml

@@ -29,6 +29,8 @@ securityContext:
 # Optional secrets that will be mounted into the pod
 # Listed secrets are handled specially by the logprep deployment.
 # Additional secrets will be mounted as usual.
+# The key is the folder under /home/logprep and the value.name
+# (which is the name of the external secret) will be the name of the mounted file.
 # secrets:
 #   certificates:
 #     name: ca-cert # Name of the secret containing the ca certificate (or chain) in one data block
@@ -38,6 +40,22 @@ securityContext:
 #     name: logprep-image-pull-secret # Name of the secret containing the image pull secret
 secrets: {}
 
+# extra environment variables in format key: value
+# Example:
+#
+# environment:
+#   - name: MY_VAR
+#     value: "my value"
+#   - name: MY_OTHER_VAR
+#     value: "my other value"
+#   - name: SECRET_USERNAME
+#     valueFrom:
+#       secretKeyRef:
+#         name: backend-user
+#         key: backend-username
+#
+environment: []
+
 # Boolean to signal to use affinity to avoid deploying multiple instances of the
 # pod on the same node
 affinity: false

pyproject.toml

@@ -44,17 +44,17 @@ keywords = [
   "logdata",
 ]
 dependencies = [
-  "aiohttp>=3.9.2",         # CVE-2024-23334
+  "aiohttp>=3.9.2",           # CVE-2024-23334
   "attrs",
-  "certifi>=2023.7.22",     # CVE-2023-37920
-  "ciso8601",               # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11
+  "certifi>=2023.7.22",       # CVE-2023-37920
+  "ciso8601",                 # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11
   "colorama",
   "confluent-kafka>2",
   "geoip2",
   "hyperscan>=0.7.0",
   "jsonref",
   "luqum",
-  "mysql-connector-python",
+  "mysql-connector-python<9",
   "numpy>=1.26.0",
   "opensearch-py",
   "prometheus_client",
@@ -71,7 +71,7 @@ dependencies = [
   "schedule",
   "tldextract",
   "urlextract",
-  "urllib3>=1.26.17",       # CVE-2023-43804
+  "urllib3>=1.26.17",         # CVE-2023-43804
   "uvicorn",
   "wheel",
   "deepdiff",

tests/unit/charts/test_deployment.py

@@ -342,3 +342,51 @@ def test_artifacts_volume_not_populated_if_not_defined(self):
         volumes = self.deployment["spec.template.spec.volumes"]
         artifacts_volume = [volume for volume in volumes if volume["name"] == "artifacts"]
         assert len(artifacts_volume) == 0
+
+    def test_extra_secrets_volumes_are_populated(self):
+        logprep_values = {"secrets": {"mysecret": {"name": "external-secret"}}}
+        self.manifests = self.render_chart("logprep", logprep_values)
+        volumes = self.deployment["spec.template.spec.volumes"]
+        volume = [volume for volume in volumes if volume["name"] == "mysecret"]
+        assert volume
+
+    def test_extra_secrets_are_mounted(self):
+        logprep_values = {"secrets": {"mysecret": {"name": "external-secret"}}}
+        self.manifests = self.render_chart("logprep", logprep_values)
+        mounts = self.deployment["spec.template.spec.containers.0.volumeMounts"]
+        mount = [mount for mount in mounts if mount["name"] == "mysecret"]
+        assert mount
+
+    def test_environment_variables_are_populated(self):
+        logprep_values = {
+            "environment": [
+                {"name": "MY_VAR", "value": "my_value"},
+                {"name": "MY_OTHER_VAR", "value": "my_other_value"},
+            ]
+        }
+        self.manifests = self.render_chart("logprep", logprep_values)
+        env = self.deployment["spec.template.spec.containers.0.env"]
+        my_var = [variable for variable in env if variable["name"] == "MY_VAR"].pop()
+        assert my_var["value"] == "my_value"
+        my_var = [variable for variable in env if variable["name"] == "MY_OTHER_VAR"].pop()
+        assert my_var["value"] == "my_other_value"
+
+    def test_environment_variables_populated_from_secrets(self):
+        logprep_values = {
+            "environment": [
+                {
+                    "name": "MY_VAR",
+                    "value": "my_value",
+                },
+                {
+                    "name": "MY_OTHER_VAR",
+                    "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
+                },
+            ]
+        }
+        self.manifests = self.render_chart("logprep", logprep_values)
+        env = self.deployment["spec.template.spec.containers.0.env"]
+        my_var = [variable for variable in env if variable["name"] == "MY_VAR"].pop()
+        assert my_var["value"] == "my_value"
+        my_var = [variable for variable in env if variable["name"] == "MY_OTHER_VAR"].pop()
+        assert my_var["valueFrom"]["secretKeyRef"]["name"] == "my-secret"