feat!: allow redis password secret creation and GitLab PeerAuthentica…

…tion exceptions (#161)

## Description

This updates GL to be able to take in a Redis password and use Valkey in
its test bundle.

It also updates GL to have its own peer authentication exceptions if
using internal psql / valkey

> [!WARNING]
> **Breaking Change**: the default connection to `dev-redis` has been
changed to `valkey` - if you were using the `dev-redis` package you must
update these values.

## Related Issue

Relates to #150

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [X] Test, docs, adr added or updated as needed
- [X] [Contributor Guide
Steps](https://github.com/defenseunicorns/uds-package-gitlab/blob/main/CONTRIBUTING.md#developer-workflow)
followed
Release-As: v17.1.1-uds.1

.github/workflows/tag-and-release.yaml

@@ -23,7 +23,7 @@ jobs:
     needs: tag-new-version
     if: ${{ needs.tag-new-version.outputs.release_created == 'true' }}
     # TODO: (@WSTARR) since we don't run the tests on arm currently we don't need to use the 8 core
-    runs-on: ${{ matrix.architecture == 'arm64' && 'uds-ubuntu-arm64-4-core' || 'uds-ubuntu-big-boy-8-core' }}
+    runs-on: ${{ matrix.architecture == 'arm64' && 'uds-swf-ubuntu-arm64-4-core' || 'uds-swf-ubuntu-big-boy-8-core' }}
     name: Publish ${{ matrix.flavor }} ${{ matrix.architecture }}
     timeout-minutes: 40
     strategy:
@@ -52,21 +52,21 @@ jobs:
         uses: defenseunicorns/uds-common/.github/actions/playwright@772b3337950b7c8e0882c527263684306bba7ce4 # v0.7.1
 
       - name: Build Package
-        run: uds run -f tasks/publish.yaml build-package --set FLAVOR=${{ matrix.flavor }}
+        run: uds run -f tasks/publish.yaml build-package --set FLAVOR=${{ matrix.flavor }} --no-progress
 
       - name: Build Bundle
-        run: uds run -f tasks/publish.yaml build-test-bundle --set FLAVOR=${{ matrix.flavor }}
+        run: uds run -f tasks/publish.yaml build-test-bundle --set FLAVOR=${{ matrix.flavor }} --no-progress
 
       - name: Test Package
         if: ${{ runner.arch != 'ARM64' }}
-        run: uds run -f tasks/publish.yaml test-package --set FLAVOR=${{ matrix.flavor }}
+        run: uds run -f tasks/publish.yaml test-package --set FLAVOR=${{ matrix.flavor }} --no-progress
 
       - name: Publish Package
-        run: uds run -f tasks/publish.yaml publish-package --set FLAVOR=${{ matrix.flavor }}
+        run: uds run -f tasks/publish.yaml publish-package --set FLAVOR=${{ matrix.flavor }} --no-progress
 
       - name: Publish Bundle
         if: ${{ matrix.flavor == 'upstream' }}
-        run: uds run -f tasks/publish.yaml publish-test-bundle --set FLAVOR=${{ matrix.flavor }}
+        run: uds run -f tasks/publish.yaml publish-test-bundle --set FLAVOR=${{ matrix.flavor }} --no-progress
 
       - name: Debug Output
         if: ${{ always() }}

.github/workflows/test.yaml

@@ -34,7 +34,7 @@ permissions:
 jobs:
   run-test:
     name: ${{ matrix.type }} ${{ matrix.flavor }}
-    runs-on: uds-ubuntu-big-boy-8-core
+    runs-on: uds-swf-ubuntu-big-boy-8-core
     timeout-minutes: 25
     strategy:
       matrix:

bundle/uds-bundle.yaml

@@ -17,23 +17,56 @@ packages:
 
   - name: postgres-operator
     repository: ghcr.io/defenseunicorns/packages/uds/postgres-operator
-    ref: 1.10.1-uds.4-upstream
+    ref: 1.12.2-uds.1-upstream
     overrides:
       postgres-operator:
         uds-postgres-config:
-          variables:
-            - name: POSTGRESQL
-              description: "Configure postgres using CRs via the uds-postgres-config chart"
-              path: postgresql
+          values:
+            - path: postgresql
+              value:
+                enabled: true  # Set to false to not create the PostgreSQL resource
+                teamId: "uds"
+                volume:
+                  size: "10Gi"
+                numberOfInstances: 2
+                users:
+                  gitlab.gitlab: []  # database owner
+                databases:
+                  gitlabdb: gitlab.gitlab
+                version: "14"
+                ingress:
+                  - remoteNamespace: gitlab
 
-  - name: dev-redis
-    repository: ghcr.io/defenseunicorns/packages/uds/dev-redis
-    ref: 0.0.2
+  - name: valkey
+    repository: ghcr.io/defenseunicorns/packages/uds/valkey
+    ref: 7.2.5-uds.1-upstream
+    overrides:
+      valkey:
+        uds-valkey-config:
+          values:
+            - path: custom
+              value:
+                - direction: Ingress
+                  selector:
+                    app.kubernetes.io/name: valkey
+                  remoteNamespace: gitlab
+                  port: 6379
+                  description: "Ingress from GitLab"
+            - path: copyPassword
+              value:
+                enabled: true
+                namespace: gitlab
+                secretName: gitlab-redis
+                secretKey: password
 
   - name: dev-secrets
     path: ../
     ref: 0.1.0
 
+  - name: dev-peer-auth-shim
+    path: ../
+    ref: 0.1.0
+
   - name: gitlab
     path: ../
     # x-release-please-start-version
@@ -52,7 +85,23 @@ packages:
             - name: GITLAB_ADMIN_GROUPS
               description: "Array of group names that grant admin role gitlab when saml protocol is active."
               path: "sso.adminGroups"
+          values:
+            # TODO: (@WSTARR) The below two overrides will no longer be needed after the next release
+            - path: redis.namespace
+              value: valkey
+            - path: redis.selector
+              value:
+                app.kubernetes.io/name: valkey
         gitlab:
+          values:
+            - path: global.psql.host
+              value: pg-cluster.postgres.svc.cluster.local
+            - path: "global.psql.username"
+              value: "gitlab.gitlab"
+            - path: "global.psql.password.secret"
+              value: "gitlab.gitlab.pg-cluster.credentials.postgresql.acid.zalan.do"
+            - path: global.redis.host
+              value: valkey-master.valkey.svc.cluster.local
           variables:
             - name: GITLAB_SSO_ENABLED
               description: "Boolean to enable or disable sso things"
@@ -87,11 +136,3 @@ packages:
             - name: SHELL_REPLICAS
               description: "Gitlab Shell Min Replicas"
               path: "gitlab.gitlab-shell.minReplicas"
-            - name: POSTGRES_USERNAME
-              description: "Gitlab Postgres Username"
-              path: "global.psql.username"
-              default: "gitlab.gitlab"
-            - name: POSTGRES_SECRET_REF
-              description: "Gitlab Postgres Password Secret Ref"
-              path: "global.psql.password.secret"
-              default: "gitlab.gitlab.pg-cluster.credentials.postgresql.acid.zalan.do"

bundle/uds-config.yaml

@@ -13,23 +13,7 @@ variables:
       - name: uds-gitlab-uploads
       - name: uds-gitlab-registry
       - name: uds-gitlab-tmp
-  postgres-operator:
-    postgresql:
-      enabled: true  # Set to false to not create the PostgreSQL resource
-      teamId: "uds"
-      volume:
-        size: "10Gi"
-      numberOfInstances: 2
-      users:
-        gitlab.gitlab: []  # database owner
-      databases:
-        gitlabdb: gitlab.gitlab
-      version: "13"
-      ingress:
-        remoteGenerated: Anywhere
   gitlab:
-    gitlab_redis_endpoint: "redis-master.dev-redis.svc.cluster.local"
-    gitlab_db_endpoint: "pg-cluster.postgres.svc.cluster.local"
     DISABLE_REGISTRY_REDIRECT: "true"
     GITLAB_PAGES_ENABLED: true
     GITLAB_ADMIN_GROUPS: ["/GitLab Admin", "/UDS Core/Admin"]

chart/templates/postgres-peerauthentication.yaml

@@ -0,0 +1,17 @@
+{{- if and .Values.postgres.internal (.Capabilities.APIVersions.Has "security.istio.io/v1beta1") }}
+apiVersion: "security.istio.io/v1beta1"
+kind: PeerAuthentication
+metadata:
+  name: gitlab-postgres-peerauthentication-exception
+  namespace: {{ .Values.postgres.namespace }}
+spec:
+  mtls:
+    mode: STRICT
+  selector:
+    matchLabels:
+      {{ .Values.postgres.selector | toYaml }}
+  portLevelMtls:
+    # Postgres exception to support GitLab dependency init containers
+    {{ .Values.postgres.port }}:
+      mode: PERMISSIVE
+{{- end }}

chart/templates/redis-peerauthentication.yaml

@@ -0,0 +1,17 @@
+{{- if and .Values.redis.internal (.Capabilities.APIVersions.Has "security.istio.io/v1beta1") }}
+apiVersion: "security.istio.io/v1beta1"
+kind: PeerAuthentication
+metadata:
+  name: gitlab-redis-peerauthentication-exception
+  namespace: {{ .Values.redis.namespace }}
+spec:
+  mtls:
+    mode: STRICT
+  selector:
+    matchLabels:
+      {{ .Values.redis.selector | toYaml }}
+  portLevelMtls:
+    # Redis/Valkey exception to support GitLab dependency init containers
+    {{ .Values.redis.port }}:
+      mode: PERMISSIVE
+{{- end }}

chart/templates/redis-secret.yaml

@@ -0,0 +1,10 @@
+{{- if ne .Values.redis.password "" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gitlab-redis
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/opaque
+stringData:
+  password: {{ .Values.redis.password }}
+{{- end }}

chart/values.yaml

@@ -12,11 +12,13 @@ storage:
   namespace: dev-minio
   port: 9000
 redis:
+  password: ""
+
   # Set to false to use external redis
   internal: true
   selector:
-    app.kubernetes.io/name: redis
-  namespace: dev-redis
+    app.kubernetes.io/name: valkey
+  namespace: valkey
   port: 6379
 postgres:
   password: ""

docs/configuration.md

@@ -17,27 +17,55 @@ GitLab uses Postgres as its backing database service and supports the [common da
 
 ### Manual Database Connection
 
-If you are using the UDS Postgres Operator or another external database that uses usernames/passwords you can use the following Helm overrides to configure it:
+If you are using the [UDS Postgres Operator](https://github.com/defenseunicorns/uds-package-postgres-operator/) or another external database that uses usernames/passwords you can use the following Helm overrides to configure it:
 
 #### `uds-gitlab-config` chart:
 
 > [!IMPORTANT]
-> The `postgres.password` setting is not applicable when using the UDS Postgres Operator package or when supplying a secret manually.
+> The `postgres.password` setting is not applicable when using the UDS Postgres Operator package or when supplying a secret manually!
 
 - `postgres.password` - provides a password to generate a secret to pass to GitLab
 
 
 #### `gitlab` chart:
 
 > [!IMPORTANT]
-> The `global.psql.password.secret` setting is not applicable when providing a password to the `uds-gitlab-config` chart manually.
+> The `global.psql.password.secret` and `global.psql.password.key` settings are not applicable when providing a password to the `uds-gitlab-config` chart manually.
 
 - `global.psql.username` - provides the username to use when connecting to the database (i.e. `gitlab.gitlab`)
-- `global.psql.password.secret` - provides the secret that contains the database password (i.e. `gitlab.gitlab.pg-cluster.credentials.postgresql.acid.zalan.do`)
+- `global.psql.password.secret` - provides the secret that contains the database password (defaults to `gitlab-postgres`)
+- `global.psql.password.key` - provides the secret key that contains the database password (defaults to `password`)
 - `global.psql.host` - provides the endpoint to use to connect to the database (i.e. `pg-cluster.postgres.svc.cluster.local`)
+- `global.psql.port` - provides the port to use to connect to the database (defaults to `5432`)
 
 ### IAM Roles for Service Accounts
 
 The Software Factory team has not yet tested IRSA with AWS RDS - there is an open issue linked below with further linked issues to test this that could act as a starting point to implement:
 
 https://github.com/defenseunicorns/uds-software-factory/issues/45
+
+## Redis / Valkey
+
+GitLab uses Redis as a key value store for caching, job queueing and more and supports external providers (such as Elasticache) as well as the [UDS Valkey](https://github.com/defenseunicorns/uds-package-valkey/) package to provide the service.
+
+### Manual Database Connection
+
+You can use the following Helm overrides to configure a connection to Redis / Valkey:
+
+#### `uds-gitlab-config` chart:
+
+> [!IMPORTANT]
+> The `redis.password` setting is not applicable when using the UDS Valkey package or when supplying a secret manually!
+
+- `redis.password` - provides a password to generate a secret to pass to GitLab
+
+#### `gitlab` chart:
+
+> [!IMPORTANT]
+> The `global.redis.auth.secret` and `global.redis.auth.key` settings are not applicable when providing a password to the `uds-gitlab-config` chart manually.
+
+- `global.redis.auth.secret` - provides the secret that contains the key value store password (defaults to `gitlab-redis`)
+- `global.redis.auth.key` - provides the key within the secret that contains the key value store password (defaults to `password`)
+- `global.redis.scheme` - provides the scheme to use to connect to the key value store (i.e. `redis` or `rediss`)
+- `global.redis.host` - provides the endpoint to use to connect to the key value store (i.e. `pg-cluster.postgres.svc.cluster.local`)
+- `global.redis.port` - provides the port to use to connect to the key value store (defaults to `6379`)

src/dev-secrets/zarf.yaml

@@ -27,20 +27,3 @@ components:
       - name: gitlab-minio
         files:
           - "minio-secret.yaml"
-  - name: redis-password
-    required: true
-    actions:
-      onDeploy:
-        before:
-          - cmd: ./zarf tools kubectl get secret -n dev-redis redis --template='{{ index .data "redis-password" }}' | base64 -d
-            mute: true
-            setVariables:
-              - name: REDIS_PASSWORD
-                sensitive: true
-  - name: gitlab-redis
-    required: true
-    manifests:
-      - name: gitlab-redis
-        namespace: gitlab
-        files:
-          - redis-secret.yaml

src/peer-auth-shim/postgres-peerauthentication.yaml

@@ -0,0 +1,15 @@
+apiVersion: "security.istio.io/v1beta1"
+kind: PeerAuthentication
+metadata:
+  name: gitlab-postgres-peerauthentication-shim-exception
+  namespace: postgres
+spec:
+  mtls:
+    mode: STRICT
+  selector:
+    matchLabels:
+      cluster-name: pg-cluster
+  portLevelMtls:
+    # Postgres exception to support GitLab dependency init containers
+    5432:
+      mode: PERMISSIVE

src/peer-auth-shim/redis-peerauthentication.yaml

@@ -0,0 +1,15 @@
+apiVersion: "security.istio.io/v1beta1"
+kind: PeerAuthentication
+metadata:
+  name: gitlab-redis-peerauthentication-shim-exception
+  namespace: valkey
+spec:
+  mtls:
+    mode: STRICT
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: valkey
+  portLevelMtls:
+    # Redis/Valkey exception to support GitLab dependency init containers
+    6379:
+      mode: PERMISSIVE

src/peer-auth-shim/zarf.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/zarf/main/zarf.schema.json
+# TODO: (@WSTARR) This will no longer be needed after the next release of GitLab
+kind: ZarfPackageConfig
+metadata:
+  name: dev-peer-auth-shim
+  version: "0.1.0"
+
+components:
+  - name: gitlab-peer-auth-shim
+    required: true
+    manifests:
+      - name: gitlab-peer-auth-shim
+        files:
+          - "postgres-peerauthentication.yaml"
+          - "redis-peerauthentication.yaml"

tasks/dependencies.yaml

@@ -10,3 +10,4 @@ tasks:
     actions:
       - cmd: ./uds zarf package create src/dev-secrets/ --confirm --no-progress --architecture=${{ .inputs.architecture }} --skip-sbom ${{ .inputs.options }}
       - cmd: ./uds zarf package create src/namespace/ --confirm --no-progress --architecture=${{ .inputs.architecture }} --skip-sbom ${{ .inputs.options }}
+      - cmd: ./uds zarf package create src/peer-auth-shim/ --confirm --no-progress --architecture=${{ .inputs.architecture }} --skip-sbom ${{ .inputs.options }}

zarf.yaml

@@ -15,7 +15,7 @@ variables:
   - name: BUCKET_SUFFIX
     default: ""
   - name: GITLAB_REDIS_ENDPOINT
-    default: "redis-master"
+    default: ""
   - name: GITLAB_REDIS_SCHEME
     default: "redis"
   - name: GITLAB_DB_NAME