ROX-13709: Introduce the external secrets operator on the Data Plane (#…

…1078)
stackrox · Nov 2, 2023 · a71d1d6 · a71d1d6
1 parent 3418a46
commit a71d1d6
Show file tree

Hide file tree

Showing 13 changed files with 8,211 additions and 62 deletions.
diff --git a/Makefile b/Makefile
@@ -104,7 +104,7 @@ $(TOOLS_VENV_DIR): $(TOOLS_DIR)/requirements.txt
 	trap "rm -rf $(TOOLS_VENV_DIR)" ERR; \
 	python3 -m venv $(TOOLS_VENV_DIR); \
 	. $(TOOLS_VENV_DIR)/bin/activate; \
-	pip install --upgrade pip==22.3.1; \
+	pip install --upgrade pip==23.3.1; \
 	pip install -r $(TOOLS_DIR)/requirements.txt; \
 	touch $(TOOLS_VENV_DIR) # update directory modification timestamp even if no changes were made by pip. This will allow to skip this target if the directory is up-to-date
 

diff --git a/dp-terraform/helm/rhacs-terraform/Chart.lock b/dp-terraform/helm/rhacs-terraform/Chart.lock
@@ -14,5 +14,8 @@ dependencies:
 - name: secured-cluster
   repository: ""
   version: 0.1.0
-digest: sha256:4b3301d2cdd6907207fb21ad741b6fa1e5302aaff1ce6fe5315cab8519908d61
-generated: "2023-07-06T21:15:28.778426+02:00"
+- name: external-secrets
+  repository: https://charts.external-secrets.io/
+  version: 0.9.5
+digest: sha256:4d1257d43daeda9d4f956f141edaba7f708838cbd2de86048f37261e9627f9cc
+generated: "2023-10-30T11:48:03.686258+01:00"
diff --git a/dp-terraform/helm/rhacs-terraform/Chart.yaml b/dp-terraform/helm/rhacs-terraform/Chart.yaml
@@ -36,8 +36,10 @@ dependencies:
     condition: logging.enabled
   - name: audit-logs
     version: "0.1.0"
-    repository: ""
     condition: audit-logs.enabled
   - name: secured-cluster
     version: "0.1.0"
     condition: secured-cluster.enabled
+  - name: external-secrets
+    version: "0.9.5"
+    repository: https://charts.external-secrets.io/
diff --git a/dp-terraform/helm/rhacs-terraform/crds/external-secrets.yaml b/dp-terraform/helm/rhacs-terraform/crds/external-secrets.yaml
diff --git a/dp-terraform/helm/rhacs-terraform/templates/_helpers.tpl b/dp-terraform/helm/rhacs-terraform/templates/_helpers.tpl
diff --git a/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync-secret.yaml b/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync-secret.yaml
@@ -1,15 +1,52 @@
-apiVersion: v1
-kind: Secret
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
 metadata:
-  name: fleetshard-sync
+  name: fleetshard-sync-ext-secret
   namespace: {{ .Release.Namespace }}
-  labels:
-    app: fleetshard-sync
-stringData:
-  rhsso-service-account-client-id: {{ .Values.fleetshardSync.redHatSSO.clientId | quote }}
-  rhsso-service-account-client-secret: {{ .Values.fleetshardSync.redHatSSO.clientSecret | quote }}
-  image-pull.dockerconfigjson: {{ template "imagePullSecret" . }}
-  {{- if eq .Values.fleetshardSync.aws.enableTokenAuth false }}
-  aws-access-key-id: {{ required "fleetshardSync.aws.accessKeyId is required when fleetshardSync.aws.enableTokenAuth = false" .Values.fleetshardSync.aws.accessKeyId | quote }}
-  aws-secret-access-key: {{ required "fleetshardSync.aws.secretAccessKey is required when fleetshardSync.aws.enableTokenAuth = false" .Values.fleetshardSync.aws.secretAccessKey | quote }}
-  {{- end }}
+spec:
+  secretStoreRef:
+    name: {{ .Values.secretStore.aws.secretsManagerSecretStoreName }}
+    kind: ClusterSecretStore
+  target:
+    name: fleetshard-sync
+    creationPolicy: Owner
+  data:
+    - secretKey: rhsso-service-account-client-id # pragma: allowlist secret
+      remoteRef:
+        key: "fleetshard-sync"
+        property: "rhsso_service_account_client_id"
+    - secretKey: rhsso-service-account-client-secret # pragma: allowlist secret
+      remoteRef:
+        key: "fleetshard-sync"
+        property: "rhsso_service_account_client_secret"
+    - secretKey: telemetry-storage-key # pragma: allowlist secret
+      remoteRef:
+        key: "fleetshard-sync"
+        property: "telemetry_storage_key"
+    {{- if not .Values.fleetshardSync.aws.enableTokenAuth }}
+    - secretKey: aws-access-key-id # pragma: allowlist secret
+      remoteRef:
+        key: "fleetshard-sync"
+        property: "aws_access_key_id"
+    - secretKey: aws-secret-access-key # pragma: allowlist secret
+      remoteRef:
+        key: "fleetshard-sync"
+        property: "aws_secret_access_key"
+    {{- end }}
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: fleetshard-sync-ext-parameters
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretStoreRef:
+    name: {{ .Values.secretStore.aws.parameterStoreSecretStoreName }}
+    kind: ClusterSecretStore
+  target:
+    name: fleetshard-sync-parameters
+    creationPolicy: Owner
+  data:
+    - secretKey: aws-role-arn # pragma: allowlist secret
+      remoteRef:
+        key: "/fleetshard-sync/aws_role_arn"
diff --git a/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml b/dp-terraform/helm/rhacs-terraform/templates/fleetshard-sync.yaml
@@ -83,11 +83,17 @@ spec:
         - name: AWS_REGION
           value: {{ .Values.fleetshardSync.aws.region }}
         - name: AWS_ROLE_ARN
-          value: {{ .Values.fleetshardSync.aws.roleARN }}
+          valueFrom:
+            secretKeyRef:
+              name: fleetshard-sync-parameters
+              key: "aws-role-arn"
         - name: TELEMETRY_STORAGE_ENDPOINT
           value: {{ .Values.fleetshardSync.telemetry.storage.endpoint | quote }}
         - name: TELEMETRY_STORAGE_KEY
-          value: {{ .Values.fleetshardSync.telemetry.storage.key | quote }}
+          valueFrom:
+            secretKeyRef:
+              name: fleetshard-sync
+              key: "telemetry-storage-key"
         {{- if .Values.fleetshardSync.aws.enableTokenAuth }}
         - name: AWS_WEB_IDENTITY_TOKEN_FILE
           value: "/var/run/secrets/tokens/aws-token"

diff --git a/dp-terraform/helm/rhacs-terraform/templates/secret-store.yaml b/dp-terraform/helm/rhacs-terraform/templates/secret-store.yaml
@@ -0,0 +1,54 @@
+{{- define "aws.auth" -}}
+auth:
+  {{- if .Values.secretStore.aws.enableTokenAuth }}
+  jwt:
+    serviceAccountRef:
+      name: {{ index .Values "external-secrets" "fullnameOverride" }}
+      namespace: {{ .Release.Namespace }}
+  {{- else }}
+  secretRef:
+    accessKeyIDSecretRef:
+      name: {{ .Values.secretStore.aws.secretName }}
+      key: access-key-id
+    secretAccessKeySecretRef:
+      name: {{ .Values.secretStore.aws.secretName }}
+      key: secret-access-key
+  {{- end }}
+{{- end -}}
+
+{{- if not .Values.secretStore.aws.enableTokenAuth }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.secretStore.aws.secretName }}
+  namespace: {{ .Release.Namespace }}
+stringData:
+  access-key-id: {{ .Values.secretStore.aws.accessKeyId | quote }}
+  secret-access-key: {{ .Values.secretStore.aws.secretAccessKey | quote }}
+type: Opaque
+{{- end }}
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ClusterSecretStore
+metadata:
+  name: {{ .Values.secretStore.aws.secretsManagerSecretStoreName }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  provider:
+    aws:
+      service: SecretsManager
+      region: {{ .Values.secretStore.aws.region }}
+{{ include "aws.auth" . | indent 6 }}
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ClusterSecretStore
+metadata:
+  name: {{ .Values.secretStore.aws.parameterStoreSecretStoreName }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  provider:
+    aws:
+      service: ParameterStore
+      region: {{ .Values.secretStore.aws.region }}
+{{ include "aws.auth" . | indent 6 }}
diff --git a/dp-terraform/helm/rhacs-terraform/terraform_cluster.sh b/dp-terraform/helm/rhacs-terraform/terraform_cluster.sh
@@ -22,12 +22,12 @@ export AWS_AUTH_HELPER="${AWS_AUTH_HELPER:-aws-saml}"
 
 init_chamber
 
-load_external_config fleetshard-sync FLEETSHARD_SYNC_
 load_external_config cloudwatch-exporter CLOUDWATCH_EXPORTER_
 load_external_config logging LOGGING_
 load_external_config observability OBSERVABILITY_
 load_external_config secured-cluster SECURED_CLUSTER_
-load_external_config quay/rhacs-eng QUAY_
+
+AWS_ACCOUNT_ID="${AWS_ACCOUNT_ID:-$(aws sts get-caller-identity --query "Account" --output text)}"
 
 case $ENVIRONMENT in
   dev)
@@ -137,6 +137,7 @@ fi
 OPERATOR_SOURCE="redhat-operators"
 OPERATOR_USE_UPSTREAM="${OPERATOR_USE_UPSTREAM:-false}"
 if [[ "${OPERATOR_USE_UPSTREAM}" == "true" ]]; then
+    load_external_config quay/rhacs-eng QUAY_
     quay_basic_auth="${QUAY_READ_ONLY_USERNAME}:${QUAY_READ_ONLY_PASSWORD}"
     pull_secret_json="$(mktemp)"
     trap 'rm -f "${pull_secret_json}"' EXIT
@@ -163,25 +164,17 @@ invoke_helm "${SCRIPT_DIR}" rhacs-terraform \
   --set fleetshardSync.clusterName="${CLUSTER_NAME}" \
   --set fleetshardSync.environment="${ENVIRONMENT}" \
   --set fleetshardSync.fleetManagerEndpoint="${FM_ENDPOINT}" \
-  --set fleetshardSync.redHatSSO.clientId="${FLEETSHARD_SYNC_RHSSO_SERVICE_ACCOUNT_CLIENT_ID}" \
-  --set fleetshardSync.redHatSSO.clientSecret="${FLEETSHARD_SYNC_RHSSO_SERVICE_ACCOUNT_CLIENT_SECRET}" \
   --set fleetshardSync.managedDB.enabled=true \
   --set fleetshardSync.managedDB.subnetGroup="${CLUSTER_MANAGED_DB_SUBNET_GROUP}" \
   --set fleetshardSync.managedDB.securityGroup="${CLUSTER_MANAGED_DB_SECURITY_GROUP}" \
   --set fleetshardSync.managedDB.performanceInsights=true \
   --set fleetshardSync.aws.region="${CLUSTER_REGION}" \
-  --set fleetshardSync.aws.roleARN="${FLEETSHARD_SYNC_AWS_ROLE_ARN}" \
   --set fleetshardSync.gitops.enabled="${RHACS_GITOPS_ENABLED:-}" \
   --set fleetshardSync.targetedOperatorUpgrades.enabled="${RHACS_TARGETED_OPERATOR_UPGRADES:-}" \
-  --set fleetshardSync.telemetry.storage.endpoint="${FLEETSHARD_SYNC_TELEMETRY_STORAGE_ENDPOINT:-}" \
-  --set fleetshardSync.telemetry.storage.key="${FLEETSHARD_SYNC_TELEMETRY_STORAGE_KEY:-}" \
   --set fleetshardSync.resources.requests.cpu="${FLEETSHARD_SYNC_CPU_REQUEST}" \
   --set fleetshardSync.resources.requests.memory="${FLEETSHARD_SYNC_MEMORY_REQUEST}" \
   --set fleetshardSync.resources.limits.cpu="${FLEETSHARD_SYNC_CPU_LIMIT}" \
   --set fleetshardSync.resources.limits.memory="${FLEETSHARD_SYNC_MEMORY_LIMIT}" \
-  --set fleetshardSync.imageCredentials.registry="quay.io" \
-  --set fleetshardSync.imageCredentials.username="${QUAY_READ_ONLY_USERNAME}" \
-  --set fleetshardSync.imageCredentials.password="${QUAY_READ_ONLY_PASSWORD}" \
   --set fleetshardSync.secretEncryption.type="kms" \
   --set fleetshardSync.secretEncryption.keyID="${CLUSTER_SECRET_ENCRYPTION_KEY_ID}" \
   --set cloudwatch.aws.accessKeyId="${CLOUDWATCH_EXPORTER_AWS_ACCESS_KEY_ID:-}" \
@@ -215,8 +208,8 @@ invoke_helm "${SCRIPT_DIR}" rhacs-terraform \
   --set secured-cluster.collector.serviceTLS.cert="${SECURED_CLUSTER_COLLECTOR_CERT}" \
   --set secured-cluster.collector.serviceTLS.key="${SECURED_CLUSTER_COLLECTOR_KEY}" \
   --set secured-cluster.sensor.serviceTLS.cert="${SECURED_CLUSTER_SENSOR_CERT}" \
-  --set secured-cluster.sensor.serviceTLS.key="${SECURED_CLUSTER_SENSOR_KEY}"
-
+  --set secured-cluster.sensor.serviceTLS.key="${SECURED_CLUSTER_SENSOR_KEY}" \
+  --set external-secrets.serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn="arn:aws:iam::${AWS_ACCOUNT_ID}:role/ExternalSecretsServiceRole"
 # To uninstall an existing release:
 # helm uninstall rhacs-terraform --namespace rhacs
 #

diff --git a/dp-terraform/helm/rhacs-terraform/values.yaml b/dp-terraform/helm/rhacs-terraform/values.yaml
@@ -21,8 +21,6 @@ fleetshardSync:
   # Red Hat SSO secrets, only required in combination with authType=RHSSO. The client credentials can be found within
   # Bitwarden (ACS RH SSO Fleet* serviceaccount).
   redHatSSO:
-    clientId: ""
-    clientSecret: ""
     endpoint: "https://sso.redhat.com"
     realm: "redhat-external"
   egressProxy:
@@ -36,31 +34,21 @@ fleetshardSync:
     securityGroup: ""
     performanceInsights: true
   secretEncryption:
-    type: "" # local or kms
+    type: kms # local or kms
     keyID: ""
   aws:
     region: "us-east-1" # TODO(2023-05-01): Remove the default value here as we now set it explicitly
-    roleARN: ""
     enableTokenAuth: true
-    accessKeyId: ""
-    secretAccessKey: ""
   telemetry:
     storage:
       endpoint: ""
-      # API key to push telemetry data to a remote backend. Leaving it empty disables telemetry.
-      key: ""
   resources:
     requests:
       cpu: "500m"
       memory: "512Mi"
     limits:
       cpu: "500m"
       memory: "512Mi"
-  imageCredentials:
-    registry: quay.io
-    username: ""
-    password: ""
-    email: "quayuser@example.com"
   gitops:
     enabled: false
   targetedOperatorUpgrades:
@@ -179,3 +167,35 @@ secured-cluster:
     serviceTLS:
       cert: ""
       key: ""
+
+external-secrets:
+  fullnameOverride: rhacs-external-secrets
+  installCRDs: false
+  image:
+    repository: quay.io/app-sre/external-secrets
+    tag: v0.9.5
+  securityContext:
+    runAsUser: 1001130000
+  webhook:
+    securityContext:
+      runAsUser: 1001130001
+    image:
+      repository: quay.io/app-sre/external-secrets
+      tag: v0.9.5
+  certController:
+    securityContext:
+      runAsUser: 1001130002
+    image:
+      repository: quay.io/app-sre/external-secrets
+      tag: v0.9.5
+
+secretStore:
+  aws:
+    region: "us-east-1"
+    enableTokenAuth: true
+    # used only when enableTokenAuth == false
+    secretName: aws-access-keys # pragma: allowlist secret
+    accessKeyId: ""
+    secretAccessKey: ""
+    secretsManagerSecretStoreName: secrets-manager-secret-store # pragma: allowlist secret
+    parameterStoreSecretStoreName: parameter-store-secret-store # pragma: allowlist secret
diff --git a/dp-terraform/ocm/install_addon.sh b/dp-terraform/ocm/install_addon.sh
@@ -6,7 +6,6 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 # shellcheck source=scripts/lib/external_config.sh
 source "$SCRIPT_DIR/../../scripts/lib/external_config.sh"
 
-
 if [[ $# -ne 2 ]]; then
     echo "Usage: $0 [environment] [cluster]" >&2
     echo "Known environments: dev integration stage prod"
@@ -21,7 +20,6 @@ export AWS_AUTH_HELPER="${AWS_AUTH_HELPER:-aws-saml}"
 
 init_chamber
 
-load_external_config fleetshard-sync FLEETSHARD_SYNC_
 load_external_config cloudwatch-exporter CLOUDWATCH_EXPORTER_
 load_external_config logging LOGGING_
 load_external_config observability OBSERVABILITY_
@@ -114,6 +112,9 @@ escape_linebreaks() {
 # Allows to load an external cluster config (e.g. acs-dev-dp-01) and apply it to a different cluster with override
 OCM_CLUSTER_ID="${OVERRIDE_CLUSTER_ID:-${CLUSTER_ID}}"
 
+OCM_SUBSCRIPTION_ID=$(ocm get cluster "$OCM_CLUSTER_ID" | jq -r '.subscription.id')
+AWS_ACCOUNT_ID=$(ocm get "/api/accounts_mgmt/v1/subscriptions/${OCM_SUBSCRIPTION_ID}" | jq -r '.cloud_account_id')
+
 OCM_COMMAND="patch"
 OCM_ENDPOINT="/api/clusters_mgmt/v1/clusters/${OCM_CLUSTER_ID}/addons/acs-fleetshard"
 OCM_PAYLOAD=$(cat << EOF
@@ -131,17 +132,11 @@ OCM_PAYLOAD=$(cat << EOF
             { "id": "fleetshardSyncAuthType", "value": "RHSSO" },
             { "id": "fleetshardSyncImageTag", "value": "quay.io/${FLEETSHARD_SYNC_ORG}/${FLEETSHARD_SYNC_IMAGE}:${FLEETSHARD_SYNC_TAG}" },
             { "id": "fleetshardSyncAwsRegion", "value": "${CLUSTER_REGION}" },
-            { "id": "fleetshardSyncAwsRoleArn", "value": "${FLEETSHARD_SYNC_AWS_ROLE_ARN}" },
             { "id": "fleetshardSyncFleetManagerEndpoint", "value": "${FM_ENDPOINT}" },
-            { "id": "fleetshardSyncImageCredentialsPassword", "value": "${QUAY_READ_ONLY_PASSWORD}" },
-            { "id": "fleetshardSyncImageCredentialsRegistry", "value": "quay.io" },
-            { "id": "fleetshardSyncImageCredentialsUsername", "value": "${QUAY_READ_ONLY_USERNAME}" },
             { "id": "fleetshardSyncManagedDbEnabled", "value": "true" },
             { "id": "fleetshardSyncManagedDbPerformanceInsights", "value": "true" },
             { "id": "fleetshardSyncManagedDbSecurityGroup", "value": "${CLUSTER_MANAGED_DB_SECURITY_GROUP}" },
             { "id": "fleetshardSyncManagedDbSubnetGroup", "value": "${CLUSTER_MANAGED_DB_SUBNET_GROUP}" },
-            { "id": "fleetshardSyncRedHatSsoClientId", "value": "${FLEETSHARD_SYNC_RHSSO_SERVICE_ACCOUNT_CLIENT_ID}" },
-            { "id": "fleetshardSyncRedHatSsoClientSecret", "value": "${FLEETSHARD_SYNC_RHSSO_SERVICE_ACCOUNT_CLIENT_SECRET}" },
             { "id": "fleetshardSyncRedHatSsoEndpoint", "value": "https://sso.redhat.com" },
             { "id": "fleetshardSyncRedHatSsoRealm", "value": "redhat-external" },
             { "id": "fleetshardSyncResourcesLimitsCpu", "value": "${FLEETSHARD_SYNC_CPU_LIMIT}" },
@@ -150,8 +145,6 @@ OCM_PAYLOAD=$(cat << EOF
             { "id": "fleetshardSyncResourcesRequestsMemory", "value": "${FLEETSHARD_SYNC_MEMORY_REQUEST}" },
             { "id": "fleetshardSyncSecretEncryptionKeyId", "value": "${CLUSTER_SECRET_ENCRYPTION_KEY_ID}" },
             { "id": "fleetshardSyncSecretEncryptionType", "value": "kms" },
-            { "id": "fleetshardSyncTelemetryStorageEndpoint", "value": "${FLEETSHARD_SYNC_TELEMETRY_STORAGE_ENDPOINT:-}" },
-            { "id": "fleetshardSyncTelemetryStorageKey", "value": "${FLEETSHARD_SYNC_TELEMETRY_STORAGE_KEY:-}" },
             { "id": "loggingAwsAccessKeyId", "value": "${LOGGING_AWS_ACCESS_KEY_ID}" },
             { "id": "loggingAwsRegion", "value": "us-east-1" },
             { "id": "loggingAwsSecretAccessKey", "value": "${LOGGING_AWS_SECRET_ACCESS_KEY}" },
@@ -176,7 +169,8 @@ OCM_PAYLOAD=$(cat << EOF
             { "id": "securedClusterCollectorServiceTlsKey", "value": "$(escape_linebreaks "${SECURED_CLUSTER_COLLECTOR_KEY}")" },
             { "id": "securedClusterEnabled", "value": "${SECURED_CLUSTER_ENABLED}" },
             { "id": "securedClusterSensorServiceTlsCert", "value": "$(escape_linebreaks "${SECURED_CLUSTER_SENSOR_CERT}")" },
-            { "id": "securedClusterSensorServiceTlsKey", "value": "$(escape_linebreaks "${SECURED_CLUSTER_SENSOR_KEY}")" }
+            { "id": "securedClusterSensorServiceTlsKey", "value": "$(escape_linebreaks "${SECURED_CLUSTER_SENSOR_KEY}")" },
+            { "id": "externalSecretsAwsRoleArn", "value": "arn:aws:iam::${AWS_ACCOUNT_ID}:role/ExternalSecretsServiceRole" }
         ]
     }
 }

diff --git a/scripts/lib/helm.sh b/scripts/lib/helm.sh
@@ -6,9 +6,9 @@ function invoke_helm() {
     local -r release="${1}"
     shift
 
-    helm repo add vector "https://helm.vector.dev"
+    helm repo add external-secrets "https://charts.external-secrets.io/"
 
-    # Build the external dependencies like the vector helm chart bundle.
+    # Build the external dependencies like the external-secrets helm chart bundle.
     helm dependencies build "${dir}"
 
     if [[ "${ENVIRONMENT}" == "dev" ]]; then

diff --git a/tools/requirements.txt b/tools/requirements.txt
@@ -1 +1,2 @@
 git+https://gitlab.corp.redhat.com/compute/aws-automation.git@dc044fef
+awscli==1.29.74