diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c971e7bf..90e144c09 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,3 +55,28 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v2 + + helm-charts: + # you have to update the `charts/logprep/Chart.yaml` versions to trigger a new release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: main + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v4 + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.6.0 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/publish-release-to-pypi.yml b/.github/workflows/publish-release-to-pypi.yml index 2c003cf69..e9cfe1815 100644 --- a/.github/workflows/publish-release-to-pypi.yml +++ b/.github/workflows/publish-release-to-pypi.yml @@ -80,51 +80,3 @@ jobs: ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-${{ github.ref_name }} ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-stable ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-latest - - helm-charts: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: main - - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - - name: Install Helm - uses: azure/setup-helm@v4 - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - - name: template Chart.yaml - env: - LOGPREP_VERSION: ${{ github.ref_name }} - run: | - sed "s/{{ LOGPREP_VERSION }}/$LOGPREP_VERSION/g" charts/logprep/Chart.yaml.j2 > ./charts/logprep/Chart.yaml - - - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: Update Logprep chart versions - branch: main - # Optional glob pattern of files which should be added to the commit - # Defaults to all (.) - # See the `pathspec`-documentation for git - # - https://git-scm.com/docs/git-add#Documentation/git-add.txt-ltpathspecgt82308203 - # - https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec - file_pattern: "charts/logprep/Chart.yaml" - - - name: update release tag - uses: rickstaa/action-create-tag@v1 - id: "tag_update" - with: - tag: ${{ github.ref_name }} - force_push_tag: true - - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.6.0 - env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/charts/logprep/Chart.yaml b/charts/logprep/Chart.yaml index a6dbb82fb..35fa2bcf0 100644 --- a/charts/logprep/Chart.yaml +++ b/charts/logprep/Chart.yaml @@ -6,10 +6,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "1.0.0" +version: "13.1.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "12.0.0" +appVersion: "13.0.0" diff --git a/charts/logprep/templates/deployment.yaml b/charts/logprep/templates/deployment.yaml index 5631e53d3..2547d49e5 100644 --- a/charts/logprep/templates/deployment.yaml +++ b/charts/logprep/templates/deployment.yaml @@ -68,6 +68,9 @@ spec: - name: REQUESTS_CA_BUNDLE value: /home/logprep/certificates/{{ .Values.secrets.certificates.name }} {{- end }} + {{- if .Values.environment }} + {{- toYaml .Values.environment | nindent 12 }} + {{- end }} volumeMounts: - name: logprep-temp mountPath: /tmp @@ -97,13 +100,10 @@ spec: - name: output-config mountPath: /home/logprep/output-config.yaml subPath: output-config.yaml - {{- if .Values.secrets.certificates }} - - name: certificates - mountPath: /home/logprep/certificates/{{ .Values.secrets.certificates.name }} - {{- end }} - {{- if .Values.secrets.credentials }} - - name: credentials - mountPath: /home/logprep/credentials/{{ .Values.secrets.credentials.name }} + {{- range $key, $value := .Values.secrets }} + - name: {{ $key }} + mountPath: /home/logprep/{{ $key }}/{{ $value.name }} + subPath: {{ $value.name }} {{- end }} {{- if or .Values.exporter.enabled (eq .Values.input.type "http_input") }} {{- if eq .Values.input.type "http_input" }} @@ -168,15 +168,10 @@ spec: configMap: name: {{ include "logprep.fullname" . }}-artifacts {{- end }} - {{- if .Values.secrets.certificates }} - - name: certificates - secret: - secretName: {{ .Values.secrets.certificates.name }} - {{- end }} - {{- if .Values.secrets.credentials }} - - name: credentials + {{- range $key, $value := .Values.secrets }} + - name: {{ $key }} secret: - secretName: {{ .Values.secrets.credentials.name }} + secretName: {{ $value.name }} {{- end }} {{- if .Values.affinity }} affinity: diff --git a/charts/logprep/values.yaml b/charts/logprep/values.yaml index f024602c2..1e455f886 100644 --- a/charts/logprep/values.yaml +++ b/charts/logprep/values.yaml @@ -29,6 +29,8 @@ securityContext: # Optional secrets that will be mounted into the pod # Listed secrets are handled specially by the logprep deployment. # Additional secrets will be mounted as usual. +# The key is the folder under /home/logprep and the value.name +# (which is the name of the external secret) will be the name of the mounted file. # secrets: # certificates: # name: ca-cert # Name of the secret containing the ca certificate (or chain) in one data block @@ -38,6 +40,22 @@ securityContext: # name: logprep-image-pull-secret # Name of the secret containing the image pull secret secrets: {} +# extra environment variables in format key: value +# Example: +# +# environment: +# - name: MY_VAR +# value: "my value" +# - name: MY_OTHER_VAR +# value: "my other value" +# - name: SECRET_USERNAME +# valueFrom: +# secretKeyRef: +# name: backend-user +# key: backend-username +# +environment: [] + # Boolean to signal to use affinity to avoid deploying multiple instances of the # pod on the same node affinity: false diff --git a/pyproject.toml b/pyproject.toml index 31c027943..3ea2abb8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,17 +44,17 @@ keywords = [ "logdata", ] dependencies = [ - "aiohttp>=3.9.2", # CVE-2024-23334 + "aiohttp>=3.9.2", # CVE-2024-23334 "attrs", - "certifi>=2023.7.22", # CVE-2023-37920 - "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 + "certifi>=2023.7.22", # CVE-2023-37920 + "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 "colorama", "confluent-kafka>2", "geoip2", "hyperscan>=0.7.0", "jsonref", "luqum", - "mysql-connector-python==8.4.0", + "mysql-connector-python<9", "numpy>=1.26.0", "opensearch-py", "prometheus_client", @@ -71,7 +71,7 @@ dependencies = [ "schedule", "tldextract", "urlextract", - "urllib3>=1.26.17", # CVE-2023-43804 + "urllib3>=1.26.17", # CVE-2023-43804 "uvicorn", "wheel", "deepdiff", diff --git a/tests/unit/charts/test_base.py b/tests/unit/charts/test_base.py index d1968dac0..a1fb3ca49 100644 --- a/tests/unit/charts/test_base.py +++ b/tests/unit/charts/test_base.py @@ -2,6 +2,7 @@ # pylint: disable=attribute-defined-outside-init # pylint: disable=protected-access +import re import subprocess from tempfile import NamedTemporaryFile from typing import Dict, Optional @@ -68,7 +69,6 @@ def test_labels_are_set(self): @pytest.mark.parametrize( "label, label_value", [ - ("app.kubernetes.io/version", "12.0.0"), ("app.kubernetes.io/name", "logprep-logprep"), ("app.kubernetes.io/application", "logprep"), ("app.kubernetes.io/managed-by", "Helm"), @@ -78,3 +78,9 @@ def test_labels_are_set(self): def test_common_labels_are_set(self, label, label_value): for manifest in self.manifests: assert manifest["metadata.labels"][label] == label_value + + def test_chart_version_is_set(self): + for manifest in self.manifests: + assert re.search( + r"\d+\.\d+\.\d+", manifest["metadata.labels"]["app.kubernetes.io/version"] + ) diff --git a/tests/unit/charts/test_deployment.py b/tests/unit/charts/test_deployment.py index dffa74c33..3ea3e01e3 100644 --- a/tests/unit/charts/test_deployment.py +++ b/tests/unit/charts/test_deployment.py @@ -342,3 +342,51 @@ def test_artifacts_volume_not_populated_if_not_defined(self): volumes = self.deployment["spec.template.spec.volumes"] artifacts_volume = [volume for volume in volumes if volume["name"] == "artifacts"] assert len(artifacts_volume) == 0 + + def test_extra_secrets_volumes_are_populated(self): + logprep_values = {"secrets": {"mysecret": {"name": "external-secret"}}} + self.manifests = self.render_chart("logprep", logprep_values) + volumes = self.deployment["spec.template.spec.volumes"] + volume = [volume for volume in volumes if volume["name"] == "mysecret"] + assert volume + + def test_extra_secrets_are_mounted(self): + logprep_values = {"secrets": {"mysecret": {"name": "external-secret"}}} + self.manifests = self.render_chart("logprep", logprep_values) + mounts = self.deployment["spec.template.spec.containers.0.volumeMounts"] + mount = [mount for mount in mounts if mount["name"] == "mysecret"] + assert mount + + def test_environment_variables_are_populated(self): + logprep_values = { + "environment": [ + {"name": "MY_VAR", "value": "my_value"}, + {"name": "MY_OTHER_VAR", "value": "my_other_value"}, + ] + } + self.manifests = self.render_chart("logprep", logprep_values) + env = self.deployment["spec.template.spec.containers.0.env"] + my_var = [variable for variable in env if variable["name"] == "MY_VAR"].pop() + assert my_var["value"] == "my_value" + my_var = [variable for variable in env if variable["name"] == "MY_OTHER_VAR"].pop() + assert my_var["value"] == "my_other_value" + + def test_environment_variables_populated_from_secrets(self): + logprep_values = { + "environment": [ + { + "name": "MY_VAR", + "value": "my_value", + }, + { + "name": "MY_OTHER_VAR", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + ] + } + self.manifests = self.render_chart("logprep", logprep_values) + env = self.deployment["spec.template.spec.containers.0.env"] + my_var = [variable for variable in env if variable["name"] == "MY_VAR"].pop() + assert my_var["value"] == "my_value" + my_var = [variable for variable in env if variable["name"] == "MY_OTHER_VAR"].pop() + assert my_var["valueFrom"]["secretKeyRef"]["name"] == "my-secret"