Merge branch 'main' into 512-make-processorprocess-always-return-a-re…

…sult-object

.github/workflows/main.yml

@@ -55,3 +55,28 @@ jobs:
 
       - name: Upload coverage reports to Codecov with GitHub Action
         uses: codecov/codecov-action@v2
+
+  helm-charts:
+    # you have to update the `charts/logprep/Chart.yaml` versions to trigger a new release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: main
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.6.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/publish-release-to-pypi.yml

@@ -80,51 +80,3 @@ jobs:
             ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-${{ github.ref_name }}
             ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-stable
             ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-latest
-
-  helm-charts:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: main
-
-      - name: Configure Git
-        run: |
-          git config user.name "$GITHUB_ACTOR"
-          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-
-      - name: Install Helm
-        uses: azure/setup-helm@v4
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-
-      - name: template Chart.yaml
-        env:
-          LOGPREP_VERSION: ${{ github.ref_name }}
-        run: |
-          sed "s/{{ LOGPREP_VERSION }}/$LOGPREP_VERSION/g" charts/logprep/Chart.yaml.j2 > ./charts/logprep/Chart.yaml
-
-      - uses: stefanzweifel/git-auto-commit-action@v5
-        with:
-          commit_message: Update Logprep chart versions
-          branch: main
-          # Optional glob pattern of files which should be added to the commit
-          # Defaults to all (.)
-          # See the `pathspec`-documentation for git
-          # - https://git-scm.com/docs/git-add#Documentation/git-add.txt-ltpathspecgt82308203
-          # - https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec
-          file_pattern: "charts/logprep/Chart.yaml"
-
-      - name: update release tag
-        uses: rickstaa/action-create-tag@v1
-        id: "tag_update"
-        with:
-          tag: ${{ github.ref_name }}
-          force_push_tag: true
-
-      - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.6.0
-        env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

charts/logprep/Chart.yaml

@@ -6,10 +6,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "1.0.0"
+version: "13.1.0"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "12.0.0"
+appVersion: "13.0.0"

charts/logprep/templates/deployment.yaml

@@ -68,6 +68,9 @@ spec:
             - name: REQUESTS_CA_BUNDLE
               value: /home/logprep/certificates/{{ .Values.secrets.certificates.name }}
             {{- end }}
+            {{- if .Values.environment }}
+            {{- toYaml .Values.environment  | nindent 12 }}
+            {{- end }}
           volumeMounts:
             - name: logprep-temp
               mountPath: /tmp
@@ -97,13 +100,10 @@ spec:
             - name: output-config
               mountPath: /home/logprep/output-config.yaml
               subPath: output-config.yaml
-            {{- if .Values.secrets.certificates }}
-            - name: certificates
-              mountPath: /home/logprep/certificates/{{ .Values.secrets.certificates.name }}
-            {{- end }}
-            {{- if .Values.secrets.credentials }}
-            - name: credentials
-              mountPath: /home/logprep/credentials/{{ .Values.secrets.credentials.name }}
+            {{- range $key, $value := .Values.secrets }}
+            - name: {{ $key }}
+              mountPath: /home/logprep/{{ $key }}/{{ $value.name }}
+              subPath: {{ $value.name }}
             {{- end }}
           {{- if or .Values.exporter.enabled (eq .Values.input.type "http_input") }}
           {{- if eq .Values.input.type "http_input" }}
@@ -168,15 +168,10 @@ spec:
           configMap:
             name: {{ include "logprep.fullname" . }}-artifacts
         {{- end }}
-        {{- if .Values.secrets.certificates }}
-        - name: certificates
-          secret:
-            secretName: {{ .Values.secrets.certificates.name }}
-        {{- end }}
-        {{- if .Values.secrets.credentials }}
-        - name: credentials
+        {{- range $key, $value := .Values.secrets }}
+        - name: {{ $key }}
           secret:
-            secretName: {{ .Values.secrets.credentials.name }}
+            secretName: {{ $value.name }}
         {{- end }}
       {{- if .Values.affinity }}
       affinity:

charts/logprep/values.yaml

@@ -29,6 +29,8 @@ securityContext:
 # Optional secrets that will be mounted into the pod
 # Listed secrets are handled specially by the logprep deployment.
 # Additional secrets will be mounted as usual.
+# The key is the folder under /home/logprep and the value.name
+# (which is the name of the external secret) will be the name of the mounted file.
 # secrets:
 #   certificates:
 #     name: ca-cert # Name of the secret containing the ca certificate (or chain) in one data block
@@ -38,6 +40,22 @@ securityContext:
 #     name: logprep-image-pull-secret # Name of the secret containing the image pull secret
 secrets: {}
 
+# extra environment variables in format key: value
+# Example:
+#
+# environment:
+#   - name: MY_VAR
+#     value: "my value"
+#   - name: MY_OTHER_VAR
+#     value: "my other value"
+#   - name: SECRET_USERNAME
+#     valueFrom:
+#       secretKeyRef:
+#         name: backend-user
+#         key: backend-username
+#
+environment: []
+
 # Boolean to signal to use affinity to avoid deploying multiple instances of the
 # pod on the same node
 affinity: false

pyproject.toml

@@ -44,17 +44,17 @@ keywords = [
   "logdata",
 ]
 dependencies = [
-  "aiohttp>=3.9.2",         # CVE-2024-23334
+  "aiohttp>=3.9.2",           # CVE-2024-23334
   "attrs",
-  "certifi>=2023.7.22",     # CVE-2023-37920
-  "ciso8601",               # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11
+  "certifi>=2023.7.22",       # CVE-2023-37920
+  "ciso8601",                 # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11
   "colorama",
   "confluent-kafka>2",
   "geoip2",
   "hyperscan>=0.7.0",
   "jsonref",
   "luqum",
-  "mysql-connector-python==8.4.0",
+  "mysql-connector-python<9",
   "numpy>=1.26.0",
   "opensearch-py",
   "prometheus_client",
@@ -71,7 +71,7 @@ dependencies = [
   "schedule",
   "tldextract",
   "urlextract",
-  "urllib3>=1.26.17",       # CVE-2023-43804
+  "urllib3>=1.26.17",         # CVE-2023-43804
   "uvicorn",
   "wheel",
   "deepdiff",

tests/unit/charts/test_base.py

@@ -2,6 +2,7 @@
 # pylint: disable=attribute-defined-outside-init
 # pylint: disable=protected-access
 
+import re
 import subprocess
 from tempfile import NamedTemporaryFile
 from typing import Dict, Optional
@@ -68,7 +69,6 @@ def test_labels_are_set(self):
     @pytest.mark.parametrize(
         "label, label_value",
         [
-            ("app.kubernetes.io/version", "12.0.0"),
             ("app.kubernetes.io/name", "logprep-logprep"),
             ("app.kubernetes.io/application", "logprep"),
             ("app.kubernetes.io/managed-by", "Helm"),
@@ -78,3 +78,9 @@ def test_labels_are_set(self):
     def test_common_labels_are_set(self, label, label_value):
         for manifest in self.manifests:
             assert manifest["metadata.labels"][label] == label_value
+
+    def test_chart_version_is_set(self):
+        for manifest in self.manifests:
+            assert re.search(
+                r"\d+\.\d+\.\d+", manifest["metadata.labels"]["app.kubernetes.io/version"]
+            )

tests/unit/charts/test_deployment.py

@@ -342,3 +342,51 @@ def test_artifacts_volume_not_populated_if_not_defined(self):
         volumes = self.deployment["spec.template.spec.volumes"]
         artifacts_volume = [volume for volume in volumes if volume["name"] == "artifacts"]
         assert len(artifacts_volume) == 0
+
+    def test_extra_secrets_volumes_are_populated(self):
+        logprep_values = {"secrets": {"mysecret": {"name": "external-secret"}}}
+        self.manifests = self.render_chart("logprep", logprep_values)
+        volumes = self.deployment["spec.template.spec.volumes"]
+        volume = [volume for volume in volumes if volume["name"] == "mysecret"]
+        assert volume
+
+    def test_extra_secrets_are_mounted(self):
+        logprep_values = {"secrets": {"mysecret": {"name": "external-secret"}}}
+        self.manifests = self.render_chart("logprep", logprep_values)
+        mounts = self.deployment["spec.template.spec.containers.0.volumeMounts"]
+        mount = [mount for mount in mounts if mount["name"] == "mysecret"]
+        assert mount
+
+    def test_environment_variables_are_populated(self):
+        logprep_values = {
+            "environment": [
+                {"name": "MY_VAR", "value": "my_value"},
+                {"name": "MY_OTHER_VAR", "value": "my_other_value"},
+            ]
+        }
+        self.manifests = self.render_chart("logprep", logprep_values)
+        env = self.deployment["spec.template.spec.containers.0.env"]
+        my_var = [variable for variable in env if variable["name"] == "MY_VAR"].pop()
+        assert my_var["value"] == "my_value"
+        my_var = [variable for variable in env if variable["name"] == "MY_OTHER_VAR"].pop()
+        assert my_var["value"] == "my_other_value"
+
+    def test_environment_variables_populated_from_secrets(self):
+        logprep_values = {
+            "environment": [
+                {
+                    "name": "MY_VAR",
+                    "value": "my_value",
+                },
+                {
+                    "name": "MY_OTHER_VAR",
+                    "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}},
+                },
+            ]
+        }
+        self.manifests = self.render_chart("logprep", logprep_values)
+        env = self.deployment["spec.template.spec.containers.0.env"]
+        my_var = [variable for variable in env if variable["name"] == "MY_VAR"].pop()
+        assert my_var["value"] == "my_value"
+        my_var = [variable for variable in env if variable["name"] == "MY_OTHER_VAR"].pop()
+        assert my_var["valueFrom"]["secretKeyRef"]["name"] == "my-secret"