Vault and External Secrets Operator integration

.dockerignore

@@ -10,7 +10,7 @@
 !fluxv2/
 !jenkins/
 !k8s-namespaces/
-!metrics/
+!system/
 !scm-manager/
 !scripts/jenkins/
 !scripts/scm-manager/

Dockerfile

@@ -4,9 +4,9 @@ ARG ENV=prod
 ARG JDK_VERSION='17'
 # Those are set by the micronaut BOM, see pom.xml
 ARG GROOVY_VERSION='3.0.13'
-ARG GRAAL_VERSION='22.2.0'
+ARG GRAAL_VERSION='22.3.0'
 
-FROM alpine:3.16.2 as alpine
+FROM alpine:3.17.1 as alpine
 
 # Keep in sync with the version in pom.xml
 FROM ghcr.io/graalvm/graalvm-ce:ol8-java${JDK_VERSION}-${GRAAL_VERSION} AS graal
@@ -17,11 +17,18 @@ WORKDIR /app
 COPY .mvn/ /app/.mvn/
 COPY mvnw /app/
 COPY pom.xml /app/
-RUN ./mvnw dependency:go-offline
+RUN ./mvnw dependency:resolve-plugins dependency:go-offline -B 
 
-FROM maven-cache as maven-build
-COPY src /app/src/
-COPY compiler.groovy .
+FROM graal as maven-build
+ENV MAVEN_OPTS='-Dmaven.repo.local=/mvn'
+COPY --from=maven-cache /mvn/ /mvn/
+COPY --from=maven-cache /app/ /app
+# Speed up build by not compiling tests
+COPY src/main /app/src/main
+COPY compiler.groovy /app
+
+WORKDIR /app
+# Build native image without micronaut
 RUN ./mvnw package -DskipTests
 # Use simple name for largest jar file -> Easier reuse in later stages
 RUN mv $(ls -S target/*.jar | head -n 1) /app/gitops-playground.jar
@@ -31,8 +38,8 @@ FROM alpine as downloader
 # When updating, 
 # * also update the checksum found at https://dl.k8s.io/release/v${K8S_VERSION}/bin/linux/amd64/kubectl.sha256
 # * also update k8s-related versions in vars.tf, init-cluster.sh and apply.sh
-ARG K8S_VERSION=1.21.2
-ARG KUBECTL_CHECKSUM=55b982527d76934c2f119e70bf0d69831d3af4985f72bb87cd4924b1c7d528da
+ARG K8S_VERSION=1.21.14
+ARG KUBECTL_CHECKSUM=0c1682493c2abd7bc5fe4ddcdb0b6e5d417aa7e067994ffeca964163a988c6ee
 # When updating, also update the checksum found at https://github.com/helm/helm/releases
 ARG HELM_VERSION=3.8.2
 ARG HELM_CHECKSUM=6cb9a48f72ab9ddfecab88d264c2f6508ab3cd42d9c09666be16a7bf006bed7b
@@ -61,7 +68,6 @@ RUN gpg --batch --verify helm.tar.gz.asc helm.tar.gz
 RUN mv linux-amd64/helm /dist/usr/local/bin
 
 # Kubectl
-RUN wget -q -O kubectl.sha256 https://dl.k8s.io/release/v${K8S_VERSION}/bin/linux/amd64/kubectl.sha256
 RUN wget -q -O kubectl https://dl.k8s.io/release/v${K8S_VERSION}/bin/linux/amd64/kubectl
 # kubectl binary download does not seem to offer signatures
 RUN echo "${KUBECTL_CHECKSUM}  kubectl" | sha256sum -c

Jenkinsfile

@@ -4,7 +4,8 @@ import com.cloudogu.ces.cesbuildlib.*
 
 String getDockerRegistryBaseUrl() { 'ghcr.io' }
 String getDockerImageName() { 'cloudogu/gitops-playground' }
-String getTrivyVersion() { '0.18.3' }
+// Note that from 0.30.x the resulting file will never be 0 kb in size, as checked in saveScanResultsOnVulnerabilities()
+String getTrivyVersion() { '0.29.2' }
 
 properties([
     // Dont keep builds forever to preserve space
@@ -88,7 +89,7 @@ node('docker') {
                                 docker.image(imageNames[0])
                                         .inside("-e KUBECONFIG=${env.WORKSPACE}/.kube/config " +
                                                 " --network=host --entrypoint=''" ) {
-                                            sh "/app/scripts/apply.sh --yes --trace --internal-registry-port=${registryPort} --argocd"
+                                            sh "/app/scripts/apply.sh --yes --trace --internal-registry-port=${registryPort} --argocd --monitoring --vault=dev"
                                         }
                             }
                         }
@@ -190,7 +191,10 @@ private void trivy(output, flags, imageName) {
             .mountJenkinsUser()
             .mountDockerSocket()
             .inside("-v ${env.WORKSPACE}/.trivy/.cache:/root/.cache/") {
-                sh "trivy image -o ${output} ${flags} ${imageName}"
+                // Scanning occasionally take longer than the default 5 min, increase timeout
+                // Avoid timouts with offline-scan. This does not affect updates of the trivy DB 
+                // https://github.com/aquasecurity/trivy/issues/3421
+                sh "trivy -d image --offline-scan --timeout 30m -o ${output} ${flags} ${imageName}"
             }
 }
 

README.md

@@ -5,16 +5,19 @@ Reproducible infrastructure to showcase GitOps workflows with Kubernetes.
 
 In fact, this rolls out a complete DevOps stack with different features including 
 * GitOps (with different controllers to choose from: Argo CD, Flux v1 and v2),
+* example applications and CI-pipelines (using Jenkins and our [GitOps library](https://github.com/cloudogu/gitops-build-lib)),
+* Notifications/Alerts (using Mailhog for Demo purposes)
 * Monitoring (using Prometheus and Grafana),
-* example applications and CI-pipelines (using Jenkins and our [GitOps library](https://github.com/cloudogu/gitops-build-lib)) and
-* soon Secrets management (using Vault).
+* Secrets management (using Vault).
 
 The gitops-playground is derived from our experiences in [consulting](https://cloudogu.com/en/consulting/?mtm_campaign=gitops-playground&mtm_kwd=consulting&mtm_source=github&mtm_medium=link) 
 and operating the [myCloudogu platform](https://my.cloudogu.com/).  
 For questions or suggestions you are welcome to join us at our myCloudogu [community forum](https://community.cloudogu.com/t/introducing-the-gitops-playground/107).
 
 [![Discuss it on myCloudogu](https://static.cloudogu.com/static/images/discuss-it.png)](https://community.cloudogu.com/t/introducing-the-gitops-playground/107)
 
+![Playground features](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/plantuml-src/gitops-playground-features.puml&fmt=svg) |
+
 # TLDR;
 
 You can run a local k8s cluster with the GitOps playground installed with only one command (on Linux)
@@ -53,6 +56,10 @@ We recommend running this command as an unprivileged user, that is inside the [d
   - [Jenkins](#jenkins)
   - [SCM-Manager](#scm-manager)
   - [Monitoring tools](#monitoring-tools)
+  - [Secrets Management Tools](#secrets-management-tools)
+    - [dev mode](#dev-mode)
+    - [prod mode](#prod-mode)
+    - [Example app](#example-app)
   - [Argo CD UI](#argo-cd-ui)
   - [Demo applications](#demo-applications)
     - [Flux V1](#flux-v1)
@@ -89,15 +96,25 @@ There a several options for running the GitOps playground
     (this can be run in production, e.g. with a [Cloudogu Ecosystem](https://cloudogu.com/en/ecosystem/?mtm_campaign=gitops-playground&mtm_kwd=ces&mtm_source=github&mtm_medium=link)) or  
   * to run everything inside the cluster (for demo only)  
 
-The diagrams below show an overview of the playground's architecture and three scenarios for running the playground. 
+The diagrams below show an overview of the playground's architecture and three scenarios for running the playground.
+For a simpler overview including all optional features such as monitoring and secrets management see intro at the very top.
 
 Note that running Jenkins inside the cluster is meant for demo purposes only. The third graphic shows our production 
 scenario with the Cloudogu EcoSystem (CES). Here better security and build performance is achieved using ephemeral 
 Jenkins build agents spawned in the cloud.
 
-| Demo on local machine | Demo on remote cluster | Production environment with CES |
-|--------------------|--------------------|--------------------|
-|![Playground on local machine](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/gitops-playground.puml&fmt=svg) | ![Playground on remote cluster](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/gitops-playground-remote.puml&fmt=svg)  | ![A possible production environment](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/production-setting.puml&fmt=svg) |
+| Demo on local machine                                                                                                                                                                                                                                                                                                                                    |
+|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ![Playground on local machine](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/plantuml-src/gitops-playground.puml&fmt=svg) |
+
+
+| Demo on remote cluster                                                                                                                                                                              | 
+|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|  ![Playground on remote cluster](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/plantuml-src/gitops-playground-remote.puml&fmt=svg) | 
+
+ | Production environment with CES                                                                                                                                                                   |
+|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ![A possible production environment](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/plantuml-src/production-setting.puml&fmt=svg) |
 
 ### Create Cluster
 
@@ -267,12 +284,18 @@ To override each image in all the applications you can use following parameters:
 If you are using a remote cluster you can set the `--argocd-url` parameter so that argocd-notification messages have a
 link to the corresponding application.
 
-##### Metrics
+##### Monitoring
 
-Set the parameter `--metrics` to enable deployment of monitoring and alerting tools like prometheus, grafana and mailhog.
+Set the parameter `--monitoring` to enable deployment of monitoring and alerting tools like prometheus, grafana and mailhog.
 
 See [Monitoring tools](#monitoring-tools) for details.
 
+##### Secrets Management
+
+Set the parameter `--vault=[dev|prod]` to enable deployment of secret management tools hashicorp vault and external 
+secrets operator.
+See [Secrets management tools](#secrets-managment-tools) for details.
+
 ### Remove playground
 
 For k3d, you can just `k3d cluster delete gitops-playground`. This will delete the whole cluster. If you just want to 
@@ -389,7 +412,7 @@ The user on the scm has to have privileges to:
 
 ### Monitoring tools
 
-Set the parameter `--metrics` so the [kube-prometheus-stack](https://github.com/prometheus-operator/kube-prometheus)
+Set the parameter `--monitoring` so the [kube-prometheus-stack](https://github.com/prometheus-operator/kube-prometheus)
 via its [helm-chart](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)
 is being deployed including Argo CD dashboards. 
 
@@ -410,6 +433,78 @@ the sync status failed, for example.
 
 **Note that this only works with Argo CD so far**
 
+### Secrets Management Tools
+
+Via the `vault` parameter, you can deploy Hashicorp Vault and the External Secrets Operator into your GitOps playground.
+
+With this, the whole flow from secret value in Vault to kubernetes `Secret` via External Secrets Operator can be seen in 
+action:
+
+![External Secret Operator <-> Vault - flow](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/plantuml-src/External-Secret-Operator-Flow.puml&fmt=svg)
+
+For this to work, the GitOps playground configures the whole chain in Kubernetes and vault (when [dev mode](#dev-mode) is used): 
+
+![External Secret Operator Custom Resources](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/gitops-playground/main/docs/plantuml-src/External-Secret-Operator-CRs.puml&fmt=svg)
+
+* In k8s `namespaces` `argocd-staging` and `argocd-production`:
+  * Creates `SecretStore` and `ServiceAccount` (used to authenticate with vault)
+  * Creates `ExternalSecrets`
+* In Vault:
+  * Create secrets for staging and prod
+  * Create a human user for changing the secrets
+  * Authorizes the service accounts on those secrets
+* Creates an [example app](#example-app) that uses the `secrets`
+
+#### dev mode 
+
+For testing you can set the parameter `--vault=dev` to deploy vault in development mode. This will lead to 
+* vault being transient, i.e. all changes during runtime are not persisted. Meaning a restart will reset to default.
+* Vault is initialized with some fixed secrets that are used in the example app, see bellow.
+* Vault authorization is initialized with service accounts used in example `SecretStore`s for external secrets operator
+* Vault is initialized with the usual `admin/admin` account (can be overriden with `--username` and `--password`)
+
+The secrets are then picked up by the `vault-backend` `SecretStore`s (connects External Secrets Operator with Vault) in 
+the namespace `argocd-staging` and `argocd-production` namespaces
+
+You can reach the vault UI on
+  * http://localhost:8200 (k3d)
+  * `scripts/get-remote-url vault-ui secrets` (remote k8s)
+  * You can log in vie the user account mentioned above.  
+    If necessary, the root token can be found on the log:
+    ```shell
+    kubectl logs -n secrets vault-0 | grep 'Root Token'
+    ```
+
+#### prod mode
+
+When using `vault=prod` you'll have to initialize vault manually but on the other hand it will persist changes.  
+
+If you want the example app to work, you'll have to manually 
+* set up vault, unseal it and 
+* authorize the `vault` service accounts in `argocd-production` and `argocd-staging` namspaces. See `SecretStore`s and 
+  [dev-post-start.sh](system/secrets/vault/dev-post-start.sh) for an example.
+
+
+#### Example app
+
+With vault in `dev` mode and ArgoCD enabled, the demo app `applications/nginx/argocd/helm-jenkins` will be deployed in 
+a way that exposes the vault secrets `secret/<stage>/nginx-secret` via HTTP on the URL `http://<host>/secret`, 
+for example `http://localhost:30024/secret`.
+
+While exposing secrets on the web is a very bad practice, it's very good for demoing auto reload of a secret changed in 
+vault.
+
+To demo this, you could
+* change the [staging secret](http://localhost:8200/ui/vault/secrets/secret/edit/staging/nginx-helm-jenkins)
+* Wait for the change to show on the web, e.g. like so
+```shell
+while ; do echo -n "$(date '+%Y-%m-%d %H:%M:%S'): " ; curl http://localhost:30024/secret/ ; echo; sleep 1; done
+```
+  This usually takes between a couple of seconds and 1-2 minutes.  
+  This time consists of `ExternalSecret`'s `refreshInterval`, as well as the [kubelet sync period](https://v1-25.docs.kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically)
+  (defaults to [1 Minute](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration)) 
+  + cache propagation delay
+
 ### Argo CD UI
 
 Argo CD's web UI is available at
@@ -423,7 +518,7 @@ Each GitOps operator comes with a couple of demo applications that allow for exp
 features.
 
 All applications are deployed via separated application and GitOps repos: 
-![](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/k8s-diagrams/cdd6bb77/diagrams/gitops-with-app-repo.puml&fmt=svg)
+![](https://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/cloudogu/k8s-diagrams/cdd6bb77/diagrams/gitops-with-app-repo.puml&fmt=png)
 
 * Separation of app repo and GitOps repo
 * Infrastructure as Code is maintained  in app repo,

applications/nginx/argocd/helm-dependency/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
-version: 10.0.1
+version: 13.2.21
 name: nginx
 dependencies:
   - name: nginx
-    version: 10.0.1
-    repository: https://charts.bitnami.com/bitnami
+    version: 13.2.21
+    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami

applications/nginx/argocd/helm-dependency/values.yaml

@@ -1,5 +1,6 @@
 nginx:
   service:
-    port: 80
+    ports:
+      http: 80
     nodePorts:
       http: 30026

applications/nginx/argocd/helm-jenkins/Jenkinsfile

@@ -1,16 +1,16 @@
 #!groovy
 
-String getApplication() { "nginx" }
+String getApplication() { "nginx-helm-jenkins" }
 String getScmManagerCredentials() { 'scmm-user' }
 String getConfigRepositoryPRBaseUrl() { env.SCMM_URL }
 String getConfigRepositoryPRRepo() { 'argocd/gitops' }
 String getCesBuildLibRepo() { "${env.SCMM_URL}/repo/common/ces-build-lib/" }
 String getCesBuildLibVersion() { '1.48.0' }
 String getGitOpsBuildLibRepo() { "${env.SCMM_URL}/repo/common/gitops-build-lib" }
 String getGitOpsBuildLibVersion() { '0.1.3'}
-String getHelmChartRepository() { "https://charts.bitnami.com/bitnami" }
+String getHelmChartRepository() { "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami" }
 String getHelmChartName() { "nginx" }
-String getHelmChartVersion() { "10.0.1" }
+String getHelmChartVersion() { "13.2.21" }
 String getMainBranch() { 'main' }
 
 cesBuildLib = library(identifier: "ces-build-lib@${cesBuildLibVersion}",

applications/nginx/argocd/helm-jenkins/k8s/production/external-secret.yaml

@@ -0,0 +1,17 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: nginx-helm-jenkins
+spec:
+  refreshInterval: "5s"
+  secretStoreRef:
+    name: vault-backend
+    kind: SecretStore
+  target:
+    # Name of secret to be created
+    name: nginx-helm-jenkins
+  data:
+    - secretKey: some-secret # Key within target secret
+      remoteRef:
+        key: secret/production/nginx-helm-jenkins
+        property: example

applications/nginx/argocd/helm-jenkins/k8s/staging/external-secret.yaml

@@ -0,0 +1,17 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: nginx-helm-jenkins
+spec:
+  refreshInterval: "5s"
+  secretStoreRef:
+    name: vault-backend
+    kind: SecretStore
+  target:
+    # Name of secret to be created
+    name: nginx-helm-jenkins
+  data:
+    - secretKey: some-secret # Key within target secret
+      remoteRef:
+        key: secret/staging/nginx-helm-jenkins
+        property: example

applications/nginx/argocd/helm-jenkins/k8s/values-production.yaml

@@ -1,5 +1,3 @@
-#namespace: argocd-production
 service:
-  port: 80
   nodePorts:
     http: 30025

applications/nginx/argocd/helm-jenkins/k8s/values-shared.yaml

@@ -1 +1,24 @@
-staticSiteConfigmap: index-nginx
+service:
+  ports:
+    http: 80
+# We don't use staticSiteConfigmap, so this is extensible for our secret example. See ArgoCD.groovy
+extraVolumes:
+  - name: index
+    configMap:
+      name: index-nginx
+      items:
+        - key: index.html
+          path: index.html
+  - name: secret
+    secret:
+      secretName: nginx-helm-jenkins
+      items:
+        - key: some-secret
+          path: index.html
+extraVolumeMounts: 
+  - name: index
+    mountPath: /app
+    readOnly: true
+  - name: secret
+    mountPath: /app/secret
+    readOnly: true

applications/nginx/argocd/helm-jenkins/k8s/values-staging.yaml

@@ -1,5 +1,3 @@
-#namespace: argocd-staging
 service:
-  port: 80
   nodePorts:
     http: 30024