Feat/helm test (#114)

* feat: add FP handling * beautify * fix: use notifier image with mailx (#110) * fix: Dot not perform to lower * fix: Dot not perform to lower * mailx debugging * fix: downgrading defectdojo-client to last used known version * CIS icons for Slack & co.! * CIS icons for Slack & co.! * chore: change escaping * fix: to lower workflow name * fix: clamscan exit code * Update README.md * fix: Malware detection * feat: Enhance mail debugging * chore: Enhance debugging for mail enforcement * fix: Check of detectedMalwareCount * chore: Make Sanesecurity.Foxhole signature ignore more flexible * Update README.md * chor:e Update README.md * chore: Update cleanup-scandata.yml * Update kustomization.yml * chore: undo * fix: Image * chore: activate * feat: Use S3 API * Add s3 API * Use emptyDir instead of NFS (nodes need to have storage > ~50GB) * fix: Image source list without git content * fix: flatten images * fix: Some vars * feat: current status * feat: update * fix: git config * fix: git config * feat: Update for helm local * chore: enhance debugging * chore: enhance debugging * fix: Inspection * chore: move image-metadata to cluster-image-scanner * fix: sed * fix: Remove quotes for skopeo_config * feat: add variables.yaml feat: copy argo-main * feat: add combined messages * feat: add combined messages * fix: remove old creds * fix: version * feat: revert changes * feat: bundle notifications * chore: use pathSeparator * fix: return on empty dst * fix: return on empty src * feat: add smtp * fix: use statistic-client version 3 * Update values.yaml change name of SA --------- Co-authored-by: Roland Schilling <corrupt@users.noreply.github.com> Co-authored-by: corrupt <365169+corrupt@users.noreply.github.com>
SDA-SE · Mar 4, 2024 · 7822d37 · 7822d37
1 parent 54d91de
commit 7822d37
Show file tree

Hide file tree

Showing 132 changed files with 760 additions and 4,448 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,4 +1,4 @@
-name: Build Image and Test with Minikube
+name: Build Images
 
 on:
   pull_request:
@@ -105,26 +105,6 @@ jobs:
           registry-user: ${{ secrets.QUAY_IO_CLUSTERSCANNER_USERNAME }}
           registry-token: ${{ secrets.QUAY_IO_CLUSTERSCANNER_TOKEN }}
 
-  build_dependencycheck:
-    needs: [build_image_base]
-    runs-on: ubuntu-latest
-    # run on base image change
-    # do not execute for PRs that origin from forks due to security concerns and missing secrets
-    if: |
-      (github.event_name == 'schedule') ||
-      (always() &&
-      (needs.build_image_base.result == 'success' ||
-      (needs.build_image_base.result == 'skipped' && needs.changes.outputs.dependencycheck == 'true')) &&
-      ! (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork))
-    steps:
-      - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
-      - uses: ./.github/actions/build_image
-        with:
-          image-path: images/scan/dependency-check
-          image-name: cluster-image-scanner-scan-dependency-check
-          registry-user: ${{ secrets.QUAY_IO_CLUSTERSCANNER_USERNAME }}
-          registry-token: ${{ secrets.QUAY_IO_CLUSTERSCANNER_TOKEN }}
-
   build_distroless:
     needs: [build_image_base]
     runs-on: ubuntu-latest
@@ -265,48 +245,48 @@ jobs:
           registry-user: ${{ secrets.QUAY_IO_CLUSTERSCANNER_USERNAME }}
           registry-token: ${{ secrets.QUAY_IO_CLUSTERSCANNER_TOKEN }}
 
-  test_minikube:
-    needs:
-      [
-        build_image_base,
-        build_imagecollector,
-        build_imagesourcefetcher,
-        build_workflowrunner,
-        build_dependencycheck,
-        build_distroless,
-        build_lifetime,
-        build_runasroot,
-        build_new-version,
-        #build_malware, # takes too long to wait
-        build_notifier,
-        build_syft
-      ]
-    if: (github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch') || (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork) || (github.event_name == 'push')
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    steps:
-      - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
-      - name: start minikube
-        id: minikube
-        uses: medyagh/setup-minikube@master          
-      - name: setup, execute and run test
-        env:
-          DD_TOKEN_SECRET: "${{secrets.DD_TOKEN_SECRET}}"
-          DD_URL_PLACEHOLDER: "${{secrets.DD_URL_PLACEHOLDER}}"
-          DD_USER_PLACEHOLDER: "${{secrets.DD_USER_PLACEHOLDER}}"
-          SLACK_CLI_TOKEN_SECRET: "${{secrets.SLACK_CLI_TOKEN_SECRET}}"
-          DEPSCAN_DB_DRIVER_PLACEHOLDER: "${{secrets.DEPSCAN_DB_DRIVER_PLACEHOLDER}}"
-          DEPSCAN_DB_USERNAME_PLACEHOLDER: "${{secrets.DEPSCAN_DB_USERNAME_PLACEHOLDER}}"
-          DEPSCAN_DB_CONNECTSRING_PLACEHOLDER: "${{secrets.DEPSCAN_DB_CONNECTSRING_PLACEHOLDER}}"
-          DEPSCAN_DB_PASSWORD_PLACEHOLDER: "${{secrets.DEPSCAN_DB_PASSWORD_PLACEHOLDER}}"
-          GH_PRIVATE_KEY_BASE64: "${{secrets.GH_PRIVATE_KEY_BASE64}}"
-          GH_APP_LOGIN: "SDA-SE"
-          GH_APP_ID: "143160"
-          GH_INSTALLATION_ID: "19959424"
-          GIT_SOURCE_REPOSITORY: "https://raw.githubusercontent.com/SDA-SE/cluster-scan-test-images/master/test-all.json"
-          GIT_COLLECTOR_REPOSITORY: "github.com/SDA-SE/cluster-image-scanner-sda-internal-test-images.git"
-        run: |
-          pwd
-          ls
-          cd test_actions
-          ./setup.bash
+# to be executed in a private repository
+#  test_minikube:
+#    needs:
+#      [
+#        build_image_base,
+#        build_imagecollector,
+#        build_imagesourcefetcher,
+#        build_workflowrunner,
+#        build_distroless,
+#        build_lifetime,
+#        build_runasroot,
+#        build_new-version,
+#        #build_malware, # takes too long to wait
+#        build_notifier,
+#        build_syft
+#      ]
+#    if: (github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch') || (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork) || (github.event_name == 'push')
+#    runs-on: ubuntu-latest
+#    timeout-minutes: 120
+#    steps:
+#      - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+#      - name: start minikube
+#        id: minikube
+#        uses: medyagh/setup-minikube@master
+#      - name: setup, execute and run test
+#        env:
+#          DD_TOKEN_SECRET: "${{secrets.DD_TOKEN_SECRET}}"
+#          DD_URL_PLACEHOLDER: "${{secrets.DD_URL_PLACEHOLDER}}"
+#          DD_USER_PLACEHOLDER: "${{secrets.DD_USER_PLACEHOLDER}}"
+#          SLACK_CLI_TOKEN_SECRET: "${{secrets.SLACK_CLI_TOKEN_SECRET}}"
+#          DEPSCAN_DB_DRIVER_PLACEHOLDER: "${{secrets.DEPSCAN_DB_DRIVER_PLACEHOLDER}}"
+#          DEPSCAN_DB_USERNAME_PLACEHOLDER: "${{secrets.DEPSCAN_DB_USERNAME_PLACEHOLDER}}"
+#          DEPSCAN_DB_CONNECTSRING_PLACEHOLDER: "${{secrets.DEPSCAN_DB_CONNECTSRING_PLACEHOLDER}}"
+#          DEPSCAN_DB_PASSWORD_PLACEHOLDER: "${{secrets.DEPSCAN_DB_PASSWORD_PLACEHOLDER}}"
+#          GH_PRIVATE_KEY_BASE64: "${{secrets.GH_PRIVATE_KEY_BASE64}}"
+#          GH_APP_LOGIN: "SDA-SE"
+#          GH_APP_ID: "143160"
+#          GH_INSTALLATION_ID: "19959424"
+#          GIT_SOURCE_REPOSITORY: "https://raw.githubusercontent.com/SDA-SE/cluster-scan-test-images/master/test-all.json"
+#          GIT_COLLECTOR_REPOSITORY: "github.com/SDA-SE/cluster-image-scanner-sda-internal-test-images.git"
+#        run: |
+#          pwd
+#          ls
+#          cd test_actions
+#          ./setup.bash
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ The following steps are conducted.
 2. The Orchestrator (implemented via ArgoWorkflows) starts the workflow periodically (e.g. nightly)
 3. The images from the Collector can be pulled by the Image Fetcher 
 4. These files are kept in a separate directory and from there they are passed to the scanner
-5. This scanner - which then receives the libraries to be ignored via the suppressions file - then executes the scans described in the definitions of Dependency Check, Lifetime, Virus and further more.
+5. Multiple scanner are used, e.g. Dependency Track, Lifetime, Malware and further more.
 6. The vulnerability management system (in our case [OWASP DefectDojo](https://github.com/DefectDojo/django-DefectDojo)) then collects the results 
 7. Non responded to findings are made available to the developers via a communication channel (Slack/Email).
 
@@ -29,7 +29,6 @@ The following steps are conducted.
 # Images
 Images to be used by ArgoWorkflows are published in quay.io (2021-06-28):
 
-- `cluster-image-scanner-scan-dependency-check`
 - `cluster-image-scanner-scan-runasroot`
 - `cluster-image-scanner-scan-distroless`
 - `cluster-image-scanner-scan-lifetime`
@@ -52,6 +51,17 @@ We are looking forward to contributions. Take a look at our [Contribution Guidel
 # Responsible Disclosure and Security
 The [SECURITY.md](SECURITY.md) includes information on responsible disclosure and security related topics like security patches.
 
+# Deployment
+## Test
+```bash
+cd test_actions
+export IS_MINIKUBE=true # if minikube is used
+./setup.bash
+```
+
+## Production
+helm files are in `deployment/helm`.
+
 # Legal Notice
 The purpose of the ClusterImageScanner is not to replace the penetration testers or make them obsolete. We strongly recommend running extensive tests by experienced penetration testers on all your applications.
 The ClusterImageScanner is to be used only for testing purpose of your running applications/containers. You need a written agreement of the organization of the _environment under scan_ to scan components with the ClusterScanner.

diff --git a/argo-main.yml b/argo-main.yml
@@ -21,9 +21,9 @@ spec:
       - name: gitSecretName
         value: "github"
       - name: s3SecretName
-        value: "s3-api-credentials"
+        value: "api-credentials"
       - name: s3ConfigName
-        value: "s3-api-config"
+        value: "api-config"
       - name: imageSourceListConfigMapName
         value: "image-source-list"
       - name: registrySecretName
@@ -34,14 +34,10 @@ spec:
         value: "slacktoken"
       - name: emailSecretName
         value: "email"
-      - name: dependencyCheckDbConfigMapName
-        value: "dependency-check-db"
       - name: enforceSlackChannel
         value: "#security-notifications-test"
       - name: newVersionImageFilter
         value: "quay.io/sdase/|swaggerapi/petstore"
-      - name: dependencyCheckSuppressionsConfigMapName
-        value: "suppressions-sda"
       - name: allResultsGitTarget
         value: "" # "github.com/SDA-SE/cluster-image-scanner-test-results"
       - name: imageRegistryBase

diff --git a/...ge-metadata-orchestrator-base/.helmignore → ...age-scanner-orchestrator-base/.helmignore b/...ge-metadata-orchestrator-base/.helmignore → ...age-scanner-orchestrator-base/.helmignore
diff --git a/...age-metadata-orchestrator-base/Chart.yaml → ...mage-scanner-orchestrator-base/Chart.yaml b/...age-metadata-orchestrator-base/Chart.yaml → ...mage-scanner-orchestrator-base/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v2
-name: image-metadata-orchestrator-base
+name: cluster-image-scanner-orchestrator-base
 description: This chart contains base resources required by the orchestrator. It primarily contains argo workflow templates and their config.
 
 # A chart can be either an 'application' or a 'library' chart.

diff --git a/...ata-orchestrator-base/templates/NOTES.txt → ...ner-orchestrator-base/templates/NOTES.txt b/...ata-orchestrator-base/templates/NOTES.txt → ...ner-orchestrator-base/templates/NOTES.txt
diff --git a/...-orchestrator-base/templates/_helpers.tpl → ...-orchestrator-base/templates/_helpers.tpl b/...-orchestrator-base/templates/_helpers.tpl → ...-orchestrator-base/templates/_helpers.tpl
@@ -1,7 +1,7 @@
 {{/*
 Expand the name of the chart.
 */}}
-{{- define "image-metadata-orchestrator-base.name" -}}
+{{- define "cluster-image-scanner-orchestrator-base.name" -}}
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
@@ -10,7 +10,7 @@ Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 If release name contains chart name it will be used as a full name.
 */}}
-{{- define "image-metadata-orchestrator-base.fullname" -}}
+{{- define "cluster-image-scanner-orchestrator-base.fullname" -}}
 {{- if .Values.fullnameOverride }}
 {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
 {{- else }}
@@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name.
 {{/*
 Create chart name and version as used by the chart label.
 */}}
-{{- define "image-metadata-orchestrator-base.chart" -}}
+{{- define "cluster-image-scanner-orchestrator-base.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*
 Common labels
 */}}
-{{- define "image-metadata-orchestrator-base.labels" -}}
-helm.sh/chart: {{ include "image-metadata-orchestrator-base.chart" . }}
-{{ include "image-metadata-orchestrator-base.selectorLabels" . }}
+{{- define "cluster-image-scanner-orchestrator-base.labels" -}}
+helm.sh/chart: {{ include "cluster-image-scanner-orchestrator-base.chart" . }}
+{{ include "cluster-image-scanner-orchestrator-base.selectorLabels" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
@@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{/*
 Selector labels
 */}}
-{{- define "image-metadata-orchestrator-base.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "image-metadata-orchestrator-base.name" . }}
+{{- define "cluster-image-scanner-orchestrator-base.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "cluster-image-scanner-orchestrator-base.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
 {{/*
 Create the name of the service account to use
 */}}
-{{- define "image-metadata-orchestrator-base.serviceAccountName" -}}
+{{- define "cluster-image-scanner-orchestrator-base.serviceAccountName" -}}
 {{- if .Values.serviceAccount.create }}
-{{- default (include "image-metadata-orchestrator-base.fullname" .) .Values.serviceAccount.name }}
+{{- default (include "cluster-image-scanner-orchestrator-base.fullname" .) .Values.serviceAccount.name }}
 {{- else }}
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}

diff --git a/...rator-base/templates/cleanup-scandata.yml → ...rator-base/templates/cleanup-scandata.yml b/...rator-base/templates/cleanup-scandata.yml → ...rator-base/templates/cleanup-scandata.yml
diff --git a/...rator-base/templates/cleanup-unpacked.yml → ...rator-base/templates/cleanup-unpacked.yml b/...rator-base/templates/cleanup-unpacked.yml → ...rator-base/templates/cleanup-unpacked.yml
diff --git a/...mplates/delete-test-products-template.yml → ...mplates/delete-test-products-template.yml b/...mplates/delete-test-products-template.yml → ...mplates/delete-test-products-template.yml
diff --git a/deployment/helm/cluster-image-scanner-orchestrator-base/templates/email.cm.yml b/deployment/helm/cluster-image-scanner-orchestrator-base/templates/email.cm.yml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: email-parameter
+  namespace: {{ .Release.Namespace }}
+data:
+  smtp: "{{ .Values.smtp.smtp }}"
+  smtp-auth: "{{ .Values.smtp.auth }}"
+  smtp-auth-user: "{{ .Values.smtp.user }}"
+  SMTP_START_TLS: "{{ .Values.smtp.starttls }}"
+  SMTP_MAIL_PARAMETER: "{{ .Values.smtp.smtpMailParameter }}"
+  SMTP_ENFORCE_MAILTO: "{{ .Values.smtp.smtpEnforceMailTo }}"
diff --git a/deployment/helm/cluster-image-scanner-orchestrator-base/templates/email.secret.yml b/deployment/helm/cluster-image-scanner-orchestrator-base/templates/email.secret.yml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: email
+  namespace: {{ .Release.Namespace }}
+data:
+  smtp-auth-password: {{ .Values.smtp.password | b64enc }}
diff --git a/...hestrator-base/templates/exit-handler.yml → ...hestrator-base/templates/exit-handler.yml b/...hestrator-base/templates/exit-handler.yml → ...hestrator-base/templates/exit-handler.yml