feat: Add custom resource health for flux resources (#19082)

* feat: Add custom resource health for flux resources

Signed-off-by: Adrian Berger <adrian.berger@bedag.ch>

* chore: Update upgrade docs

Signed-off-by: Adrian Berger <adrian.berger@bedag.ch>

---------

Signed-off-by: Adrian Berger <adrian.berger@bedag.ch>
Signed-off-by: Dan Garfield <dan@codefresh.io>
Co-authored-by: Dan Garfield <dan@codefresh.io>

docs/operator-manual/upgrading/2.12-2.13.md

@@ -22,9 +22,26 @@ The following actions are now available:
 
 If you want to use these actions do not forget to update the permissions (RBAC) for your Argo CD instance.
 
+## Custom Resource Health for Flux Resources
+
+[`Custom Resource Health`](../health.md#custom-health-checks) has been added for Flux Resources.  
+The following Flux resources now support health checks:
+- HelmRelease
+- ImagePolicy
+- ImageRepository
+- ImageUpdateAutomation
+- Kustomization
+- Receiver
+- Bucket
+- GitRepository
+- HelmChart
+- HelmRepository
+- OCIRepository
+
 ## Upgraded Dex Version
 
 Dex [v2.39.0](https://github.com/dexidp/dex/releases/tag/v2.39.0) included a breaking change for the LDAP connector:
 
 > The validation of username and password in the LDAP connector is much more strict now.
 > As of today, Dex uses the EscapeFilter function to check for special characters in credentials and prevent injections by denying such requests.
+

resource_customizations/helm.toolkit.fluxcd.io/HelmRelease/health.lua

@@ -0,0 +1,40 @@
+local hs = {}
+if obj.status ~= nil then
+  if obj.status.conditions ~= nil then
+    local numProgressing = 0
+    local numSucceeded = 0
+    local message = ""
+    for _, condition in ipairs(obj.status.conditions) do
+      if condition.type == "Ready" then
+        if condition.status == "True" then
+          numSucceeded = numSucceeded + 1
+        elseif condition.status == "Unknown" then
+          numProgressing = numProgressing + 1
+        end
+        message = condition.reason
+      elseif condition.type == "Reconciling" and condition.status == "True" then
+        numProgressing = numProgressing + 1
+      elseif condition.type == "Released" or condition.type == "TestSuccess" then
+        if condition.status == "True" then
+          numSucceeded = numSucceeded + 1
+        end
+      end
+    end
+    if(numProgressing == 2) then
+      hs.message = message
+      hs.status = "Progressing"
+      return hs
+    elseif(numSucceeded == 2) then
+      hs.message = message
+      hs.status = "Healthy"
+      return hs
+    else
+      hs.message = message
+      hs.status = "Degraded"
+      return hs
+    end
+  end
+end
+hs.message = "Status unknown"
+hs.status = "Progressing"
+return hs

resource_customizations/helm.toolkit.fluxcd.io/HelmRelease/health_test.yaml

@@ -0,0 +1,13 @@
+tests:
+  - healthStatus:
+      status: Progressing
+      message: Progressing
+    inputPath: testdata/progressing.yaml
+  - healthStatus:
+      status: Degraded
+      message: RollbackSucceeded
+    inputPath: testdata/degraded.yaml
+  - healthStatus:
+      status: Healthy
+      message: InstallSucceeded
+    inputPath: testdata/healthy.yaml

resource_customizations/helm.toolkit.fluxcd.io/HelmRelease/testdata/degraded.yaml

@@ -0,0 +1,70 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: podinfo
+  namespace: default
+spec:
+  interval: 10m
+  timeout: 5m
+  chart:
+    spec:
+      chart: podinfo
+      version: '6.5.*'
+      sourceRef:
+        kind: HelmRepository
+        name: podinfo
+      interval: 5m
+  releaseName: podinfo
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  test:
+    enable: true
+  driftDetection:
+    mode: enabled
+    ignore:
+    - paths: ["/spec/replicas"]
+      target:
+        kind: Deployment
+  values:
+    replicaCount: 2
+status:
+  conditions:
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Helm rollback to previous release default/podinfo.v24 with
+      chart podinfo@6.5.4 succeeded
+    observedGeneration: 5
+    reason: ProgressingWithRetry
+    status: "True"
+    type: Reconciling
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Helm rollback to previous release default/podinfo.v24 with
+      chart podinfo@6.5.4 succeeded
+    observedGeneration: 5
+    reason: RollbackSucceeded
+    status: "False"
+    type: Ready
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: "Helm upgrade failed for release default/podinfo with chart
+      podinfo@6.5.4: cannot patch \"podinfo\" with kind Deployment: admission webhook
+      \"validate.kyverno.svc-fail\" denied the request: \n\nresource Deployment/default/podinfo
+      was blocked due to the following policies \n\ndisallow-privilege-escalation:\n
+      \ autogen-privilege-escalation: 'validation error: Privilege escalation is disallowed.\n
+      \   The fields spec.containers[*].securityContext.allowPrivilegeEscalation,
+      spec.initContainers[*].securityContext.allowPrivilegeEscalation,\n    and spec.ephemeralContainers[*].securityContext.allowPrivilegeEscalation
+      must\n    be set to `false`. rule autogen-privilege-escalation failed at path
+      /spec/template/spec/containers/0/securityContext/allowPrivilegeEscalation/'"
+    observedGeneration: 5
+    reason: UpgradeFailed
+    status: "False"
+    type: Released
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Helm rollback to previous release default/podinfo.v24 with
+      chart podinfo@6.5.4 succeeded
+    observedGeneration: 5
+    reason: RollbackSucceeded
+    status: "True"
+    type: Remediated

resource_customizations/helm.toolkit.fluxcd.io/HelmRelease/testdata/healthy.yaml

@@ -0,0 +1,49 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: podinfo
+  namespace: default
+spec:
+  interval: 10m
+  timeout: 5m
+  chart:
+    spec:
+      chart: podinfo
+      version: '6.5.*'
+      sourceRef:
+        kind: HelmRepository
+        name: podinfo
+      interval: 5m
+  releaseName: podinfo
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  test:
+    enable: true
+  driftDetection:
+    mode: enabled
+    ignore:
+    - paths: ["/spec/replicas"]
+      target:
+        kind: Deployment
+  values:
+    replicaCount: 2
+status:
+  conditions:
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Helm install succeeded for release default/podinfo.v1 with
+      chart podinfo@6.5.4
+    observedGeneration: 2
+    reason: InstallSucceeded
+    status: "True"
+    type: Ready
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Helm install succeeded for release default/podinfo.v1 with
+      chart podinfo@6.5.4
+    observedGeneration: 1
+    reason: InstallSucceeded
+    status: "True"
+    type: Released

resource_customizations/helm.toolkit.fluxcd.io/HelmRelease/testdata/progressing.yaml

@@ -0,0 +1,54 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: podinfo
+  namespace: default
+spec:
+  interval: 10m
+  timeout: 5m
+  chart:
+    spec:
+      chart: podinfo
+      version: '6.5.*'
+      sourceRef:
+        kind: HelmRepository
+        name: podinfo
+      interval: 5m
+  releaseName: podinfo
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  test:
+    enable: true
+  driftDetection:
+    mode: enabled
+    ignore:
+    - paths: ["/spec/replicas"]
+      target:
+        kind: Deployment
+  values:
+    replicaCount: 2
+status:
+  conditions:
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Running 'upgrade' action with timeout of 5m0s
+    observedGeneration: 3
+    reason: Progressing
+    status: "True"
+    type: Reconciling
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Running 'upgrade' action with timeout of 5m0s
+    observedGeneration: 3
+    reason: Progressing
+    status: Unknown
+    type: Ready
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Helm install succeeded for release default/podinfo.v1 with
+      chart podinfo@6.5.4
+    observedGeneration: 1
+    reason: InstallSucceeded
+    status: "True"
+    type: Released

resource_customizations/image.toolkit.fluxcd.io/ImagePolicy/health.lua

@@ -0,0 +1,38 @@
+local hs = {}
+if obj.status ~= nil then
+  if obj.status.conditions ~= nil then
+    local numProgressing = 0
+    local numSucceeded = 0
+    local message = ""
+    for _, condition in ipairs(obj.status.conditions) do
+      if condition.type == "Ready" then
+        if condition.status == "True" then
+          numSucceeded = numSucceeded + 1
+        elseif condition.status == "False" then
+          numProgressing = numProgressing + 1
+        end
+        message = condition.reason
+      elseif condition.type == "Reconciling" and condition.status == "True" then
+        if condition.reason == "NewGeneration" or condition.reason == "AccessingRepository" or condition.reason == "ApplyingPolicy" then
+          numProgressing = numProgressing + 1
+        end
+      end
+    end
+    if(numProgressing == 2) then
+      hs.message = message
+      hs.status = "Progressing"
+      return hs
+    elseif(numSucceeded == 1) then
+      hs.message = message
+      hs.status = "Healthy"
+      return hs
+    else
+      hs.message = message
+      hs.status = "Degraded"
+      return hs
+    end
+  end
+end
+hs.message = "Status unknown"
+hs.status = "Progressing"
+return hs

resource_customizations/image.toolkit.fluxcd.io/ImagePolicy/health_test.yaml

@@ -0,0 +1,13 @@
+tests:
+  # - healthStatus:
+  #     status: Progressing
+  #     message: DependencyNotReady
+  #   inputPath: testdata/progressing.yaml
+  - healthStatus:
+      status: Degraded
+      message: DependencyNotReady
+    inputPath: testdata/degraded.yaml
+  - healthStatus:
+      status: Healthy
+      message: Succeeded
+    inputPath: testdata/healthy.yaml

resource_customizations/image.toolkit.fluxcd.io/ImagePolicy/testdata/degraded.yaml

@@ -0,0 +1,26 @@
+apiVersion: image.toolkit.fluxcd.io/v1beta2
+kind: ImagePolicy
+metadata:
+  name: podinfo
+  namespace: argocd
+spec:
+  imageRepositoryRef:
+    name: podinfo-faulty
+  policy:
+    semver:
+      range: x.x.x
+status:
+  conditions:
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: 'processing object: new generation 1 -> 2'
+    observedGeneration: 2
+    reason: ProgressingWithRetry
+    status: "True"
+    type: Reconciling
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: 'failed to get the referred ImageRepository: referenced ImageRepository
+      does not exist: ImageRepository.image.toolkit.fluxcd.io "podinfo-faulty" not found'
+    observedGeneration: 2
+    reason: DependencyNotReady
+    status: "False"
+    type: Ready

resource_customizations/image.toolkit.fluxcd.io/ImagePolicy/testdata/healthy.yaml

@@ -0,0 +1,19 @@
+apiVersion: image.toolkit.fluxcd.io/v1beta2
+kind: ImagePolicy
+metadata:
+  name: podinfo
+  namespace: argocd
+spec:
+  imageRepositoryRef:
+    name: podinfo
+  policy:
+    semver:
+      range: x.x.x
+status:
+  conditions:
+  - lastTransitionTime: "2024-07-16T12:00:00Z"
+    message: Latest image tag for 'stefanprodan/podinfo' resolved to 5.1.4
+    observedGeneration: 1
+    reason: Succeeded
+    status: "True"
+    type: Ready

resource_customizations/image.toolkit.fluxcd.io/ImagePolicy/testdata/progressing.yaml

@@ -0,0 +1,13 @@
+apiVersion: image.toolkit.fluxcd.io/v1beta2
+kind: ImagePolicy
+metadata:
+  name: podinfo
+  namespace: argocd
+spec:
+  imageRepositoryRef:
+    name: podinfo
+  policy:
+    semver:
+      range: x.x.x
+status:
+  conditions: []