Skip to content

Commit

Permalink
add analysis failure count
Browse files Browse the repository at this point in the history
Signed-off-by: JuHyung-Son <sonju0427@gmail.com>
  • Loading branch information
JuHyung-Son committed Jan 16, 2024
1 parent efc169f commit 49e1ebf
Showing 1 changed file with 15 additions and 0 deletions.
15 changes: 15 additions & 0 deletions controllers/k8sgpt_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ var (
Name: "k8sgpt_number_of_failed_backend_ai_calls",
Help: "The total number of failed backend AI calls",
}, []string{"backend", "deployment", "namespace"})
// analysisFailureCount is for the number of analysis failures
analysisFailureCount int
)

// K8sGPTReconciler reconciles a K8sGPT object
Expand Down Expand Up @@ -210,8 +212,21 @@ func (r *K8sGPTReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
"namespace": deployment.Namespace}).Inc()
}
k8sgptReconcileErrorCount.Inc()
analysisFailureCount++
if analysisFailureCount > 5 {
k8sgptConfig.Spec.AI.Enabled = false
err = r.Update(ctx, k8sgptConfig)
if err != nil {
k8sgptReconcileErrorCount.Inc()
return r.finishReconcile(err, false)
}
analysisFailureCount = 0
}
return r.finishReconcile(err, false)
}
// Reset analysisFailureCount
analysisFailureCount = 0

// Update metrics count
if k8sgptConfig.Spec.AI.Enabled && len(response.Results) > 0 {
k8sgptNumberOfBackendAICalls.With(prometheus.Labels{
Expand Down

0 comments on commit 49e1ebf

Please sign in to comment.