feat(health): Add Health Checks for IngressController resources (argo…

…proj#12776) Signed-off-by: Jack Henschel <jack.henschel@cern.ch>
tesla59 · Dec 16, 2023 · f8ad772 · f8ad772
1 parent 1dd8841
commit f8ad772
Show file tree

Hide file tree

Showing 6 changed files with 381 additions and 0 deletions.
diff --git a/resource_customizations/operator.openshift.io/IngressController/health.lua b/resource_customizations/operator.openshift.io/IngressController/health.lua
@@ -0,0 +1,31 @@
+-- healthcheck for IngressController resources
+hs = {}
+if obj.status ~= nil then
+  if obj.status.conditions ~= nil then
+      -- if the status conditions are present, iterate over them and check their status
+    for _, condition in pairs(obj.status.conditions) do
+      if condition.type == "Degraded" and condition.status == "True" then
+        hs.status = "Degraded"
+        hs.message = condition.message
+        return hs
+      elseif condition.type == "DeploymentReplicasAllAvailable" and condition.status == "False" then
+        hs.status = "Progressing"
+        hs.message =  condition.message
+        return hs
+      elseif condition.type == "Progressing" and condition.status == "True" then
+        hs.status = "Progressing"
+        hs.message =  condition.reason
+        return hs
+      elseif condition.type == "Available" and condition.status == "True" then
+        hs.status = "Healthy"
+        hs.message = "IngressController is available"
+        return hs
+      end
+    end
+  end
+end
+
+-- default status when none of the previous condition matches
+hs.status = "Progressing"
+hs.message = "Status of IngressController is not known yet"
+return hs
diff --git a/resource_customizations/operator.openshift.io/IngressController/health_test.yaml b/resource_customizations/operator.openshift.io/IngressController/health_test.yaml
@@ -0,0 +1,17 @@
+tests:
+- healthStatus:
+    status: Progressing
+    message: "Status of IngressController is not known yet"
+  inputPath: testdata/progressing_initialization.yaml
+- healthStatus:
+    status: Progressing
+    message: "0/1 of replicas are available"
+  inputPath: testdata/progressing_pod_rollout.yaml
+- healthStatus:
+    status: Degraded
+    message: "One or more other status conditions indicate a degraded state."
+  inputPath: testdata/degraded.yaml
+- healthStatus:
+    status: Healthy
+    message: "IngressController is available"
+  inputPath: testdata/healthy.yaml
diff --git a/resource_customizations/operator.openshift.io/IngressController/testdata/degraded.yaml b/resource_customizations/operator.openshift.io/IngressController/testdata/degraded.yaml
@@ -0,0 +1,103 @@
+---
+apiVersion: operator.openshift.io/v1
+kind: IngressController
+metadata:
+  name: default
+  namespace: openshift-ingress-operator
+spec:
+  domain: openshift.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      statsPort: 1936
+    type: HostNetwork
+  nodePlacement:
+    nodeSelector:
+      matchLabels:
+        node-role.kubernetes.io/worker: ""
+  replicas: 1
+status:
+  availableReplicas: 0
+  conditions:
+  - lastTransitionTime: "2023-01-28T10:05:06Z"
+    reason: Valid
+    status: "True"
+    type: Admitted
+  - lastTransitionTime: "2023-01-28T10:09:15Z"
+    status: "True"
+    type: PodsScheduled
+  - lastTransitionTime: "2023-01-28T10:05:06Z"
+    message: The configured endpoint publishing strategy does not include a managed
+      load balancer
+    reason: EndpointPublishingStrategyExcludesManagedLoadBalancer
+    status: "False"
+    type: LoadBalancerManaged
+  - lastTransitionTime: "2023-01-28T10:05:06Z"
+    message: No DNS zones are defined in the cluster dns config.
+    reason: NoDNSZones
+    status: "False"
+    type: DNSManaged
+  - lastTransitionTime: "2023-01-28T10:05:06Z"
+    status: "False"
+    type: Progressing
+  - lastTransitionTime: "2023-01-28T10:13:55Z"
+    message: "One or more other status conditions indicate a degraded state."
+    # message: 'One or more other status conditions indicate a degraded state: CanaryChecksSucceeding=False
+    #   (CanaryChecksRepetitiveFailures: Canary route checks for the default ingress
+    #   controller are failing)'
+    reason: DegradedConditions
+    status: "True"
+    type: Degraded
+  - lastTransitionTime: "2023-01-28T10:05:06Z"
+    message: IngressController is upgradeable.
+    reason: Upgradeable
+    status: "True"
+    type: Upgradeable
+  - lastTransitionTime: "2023-01-28T10:12:55Z"
+    message: Canary route checks for the default ingress controller are failing
+    reason: CanaryChecksRepetitiveFailures
+    status: "False"
+    type: CanaryChecksSucceeding
+  domain: openshift.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      protocol: TCP
+      statsPort: 1936
+    type: HostNetwork
+  namespaceSelector: {}
+  observedGeneration: 2
+  routeSelector: {}
+  selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default
+  tlsProfile:
+    ciphers:
+    - ECDHE-ECDSA-CHACHA20-POLY1305
+    - ECDHE-RSA-CHACHA20-POLY1305
+    - ECDHE-ECDSA-AES128-GCM-SHA256
+    - ECDHE-RSA-AES128-GCM-SHA256
+    - ECDHE-ECDSA-AES256-GCM-SHA384
+    - ECDHE-RSA-AES256-GCM-SHA384
+    - DHE-RSA-AES128-GCM-SHA256
+    - DHE-RSA-AES256-GCM-SHA384
+    - ECDHE-ECDSA-AES128-SHA256
+    - ECDHE-RSA-AES128-SHA256
+    - ECDHE-ECDSA-AES128-SHA
+    - ECDHE-RSA-AES256-SHA384
+    - ECDHE-RSA-AES128-SHA
+    - ECDHE-ECDSA-AES256-SHA384
+    - ECDHE-ECDSA-AES256-SHA
+    - ECDHE-RSA-AES256-SHA
+    - DHE-RSA-AES128-SHA256
+    - DHE-RSA-AES128-SHA
+    - DHE-RSA-AES256-SHA256
+    - DHE-RSA-AES256-SHA
+    - AES128-GCM-SHA256
+    - AES256-GCM-SHA384
+    - AES128-SHA256
+    - AES256-SHA256
+    - AES128-SHA
+    - AES256-SHA
+    - '!DSS'
+    minTLSVersion: VersionTLS11
diff --git a/resource_customizations/operator.openshift.io/IngressController/testdata/healthy.yaml b/resource_customizations/operator.openshift.io/IngressController/testdata/healthy.yaml
@@ -0,0 +1,93 @@
+---
+apiVersion: operator.openshift.io/v1
+kind: IngressController
+metadata:
+  name: apps-shard-2
+  namespace: openshift-ingress-operator
+spec:
+  domain: openshift-apps-shard-2.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      statsPort: 1936
+    type: HostNetwork
+  nodePlacement:
+    nodeSelector:
+      matchLabels:
+        node-role.kubernetes.io/worker: ""
+  replicas: 1
+status:
+  availableReplicas: 1
+  conditions:
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    reason: Valid
+    status: "True"
+    type: Admitted
+  - lastTransitionTime: "2023-01-28T09:43:42Z"
+    status: "True"
+    type: PodsScheduled
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: The deployment has Available status condition set to True
+    reason: DeploymentAvailable
+    status: "True"
+    type: DeploymentAvailable
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: Minimum replicas requirement is met
+    reason: DeploymentMinimumReplicasMet
+    status: "True"
+    type: DeploymentReplicasMinAvailable
+  - lastTransitionTime: "2023-01-28T09:44:36Z"
+    message: All replicas are available
+    reason: DeploymentReplicasAvailable
+    status: "True"
+    type: DeploymentReplicasAllAvailable
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: The configured endpoint publishing strategy does not include a managed
+      load balancer
+    reason: EndpointPublishingStrategyExcludesManagedLoadBalancer
+    status: "False"
+    type: LoadBalancerManaged
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: No DNS zones are defined in the cluster dns config.
+    reason: NoDNSZones
+    status: "False"
+    type: DNSManaged
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    status: "True"
+    type: Available
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    status: "False"
+    type: Progressing
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    status: "False"
+    type: Degraded
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: IngressController is upgradeable.
+    reason: Upgradeable
+    status: "True"
+    type: Upgradeable
+  domain: openshift-apps-shard-2.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      protocol: TCP
+      statsPort: 1936
+    type: HostNetwork
+  observedGeneration: 5
+  selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller=apps-shard-2
+  tlsProfile:
+    ciphers:
+    - ECDHE-ECDSA-AES128-GCM-SHA256
+    - ECDHE-RSA-AES128-GCM-SHA256
+    - ECDHE-ECDSA-AES256-GCM-SHA384
+    - ECDHE-RSA-AES256-GCM-SHA384
+    - ECDHE-ECDSA-CHACHA20-POLY1305
+    - ECDHE-RSA-CHACHA20-POLY1305
+    - DHE-RSA-AES128-GCM-SHA256
+    - DHE-RSA-AES256-GCM-SHA384
+    - TLS_AES_128_GCM_SHA256
+    - TLS_AES_256_GCM_SHA384
+    - TLS_CHACHA20_POLY1305_SHA256
+    minTLSVersion: VersionTLS12
diff --git a/...izations/operator.openshift.io/IngressController/testdata/progressing_initialization.yaml b/...izations/operator.openshift.io/IngressController/testdata/progressing_initialization.yaml
@@ -0,0 +1,36 @@
+---
+apiVersion: operator.openshift.io/v1
+kind: IngressController
+metadata:
+  name: apps-shard-2
+  namespace: openshift-ingress-operator
+spec:
+  domain: openshift-apps-shard-2.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      statsPort: 1936
+    type: HostNetwork
+  nodePlacement:
+    nodeSelector:
+      matchLabels:
+        node-role.kubernetes.io/worker: ""
+  replicas: 1
+status:
+  availableReplicas: 0
+  conditions:
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    reason: Valid
+    status: "True"
+    type: Admitted
+  domain: openshift-apps-shard-2.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      protocol: TCP
+      statsPort: 1936
+    type: HostNetwork
+  observedGeneration: 1
+  selector: ""
diff --git a/...tomizations/operator.openshift.io/IngressController/testdata/progressing_pod_rollout.yaml b/...tomizations/operator.openshift.io/IngressController/testdata/progressing_pod_rollout.yaml
@@ -0,0 +1,101 @@
+---
+apiVersion: operator.openshift.io/v1
+kind: IngressController
+metadata:
+  name: apps-shard-2
+  namespace: openshift-ingress-operator
+spec:
+  domain: openshift-apps-shard-2.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      statsPort: 1936
+    type: HostNetwork
+  nodePlacement:
+    nodeSelector:
+      matchLabels:
+        node-role.kubernetes.io/worker: ""
+  replicas: 1
+status:
+  availableReplicas: 0
+  conditions:
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    reason: Valid
+    status: "True"
+    type: Admitted
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: 'Some pods are not scheduled: Pod "router-apps-shard-2-7b5cb5f98d-gk4hj"
+      cannot be scheduled: 0/6 nodes are available: 2 node(s) didn''t have free ports
+      for the requested pod ports, 3 node(s) had untolerated taint {node-role.kubernetes.io/master:
+      }, 5 node(s) didn''t match Pod''s node affinity/selector. preemption: 0/6 nodes
+      are available: 1 node(s) didn''t have free ports for the requested pod ports,
+      5 Preemption is not helpful for scheduling. Make sure you have sufficient worker
+      nodes.'
+    reason: PodsNotScheduled
+    status: "False"
+    type: PodsScheduled
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: The deployment has Available status condition set to True
+    reason: DeploymentAvailable
+    status: "True"
+    type: DeploymentAvailable
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: Minimum replicas requirement is met
+    reason: DeploymentMinimumReplicasMet
+    status: "True"
+    type: DeploymentReplicasMinAvailable
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: 0/1 of replicas are available
+    reason: DeploymentReplicasNotAvailable
+    status: "False"
+    type: DeploymentReplicasAllAvailable
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: The configured endpoint publishing strategy does not include a managed
+      load balancer
+    reason: EndpointPublishingStrategyExcludesManagedLoadBalancer
+    status: "False"
+    type: LoadBalancerManaged
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: No DNS zones are defined in the cluster dns config.
+    reason: NoDNSZones
+    status: "False"
+    type: DNSManaged
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    status: "True"
+    type: Available
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    status: "False"
+    type: Progressing
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    status: "False"
+    type: Degraded
+  - lastTransitionTime: "2023-01-28T09:34:36Z"
+    message: IngressController is upgradeable.
+    reason: Upgradeable
+    status: "True"
+    type: Upgradeable
+  domain: openshift-apps-shard-2.example.com
+  endpointPublishingStrategy:
+    hostNetwork:
+      httpPort: 80
+      httpsPort: 443
+      protocol: TCP
+      statsPort: 1936
+    type: HostNetwork
+  observedGeneration: 2
+  selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller=apps-shard-2
+  tlsProfile:
+    ciphers:
+    - ECDHE-ECDSA-AES128-GCM-SHA256
+    - ECDHE-RSA-AES128-GCM-SHA256
+    - ECDHE-ECDSA-AES256-GCM-SHA384
+    - ECDHE-RSA-AES256-GCM-SHA384
+    - ECDHE-ECDSA-CHACHA20-POLY1305
+    - ECDHE-RSA-CHACHA20-POLY1305
+    - DHE-RSA-AES128-GCM-SHA256
+    - DHE-RSA-AES256-GCM-SHA384
+    - TLS_AES_128_GCM_SHA256
+    - TLS_AES_256_GCM_SHA384
+    - TLS_CHACHA20_POLY1305_SHA256
+    minTLSVersion: VersionTLS12