feat: have pre-req retry upon check fail

Signed-off-by: Humair Khan <HumairAK@users.noreply.github.com>
opendatahub-io · Feb 13, 2024 · c3221fc · c3221fc
1 parent 0df6ce3
commit c3221fc
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 6 deletions.
diff --git a/config/base/params.env b/config/base/params.env
@@ -25,4 +25,4 @@ ZAP_LOG_LEVEL=info
 MAX_CONCURRENT_RECONCILES=10
 DSPO_HEALTHCHECK_DATABASE_CONNECTIONTIMEOUT=15s
 DSPO_HEALTHCHECK_OBJECTSTORE_CONNECTIONTIMEOUT=15s
-DSPO_REQUEUE_TIME=2m
+DSPO_REQUEUE_TIME=20s
diff --git a/controllers/config/defaults.go b/controllers/config/defaults.go
@@ -150,7 +150,7 @@ const DefaultObjStoreConnectionTimeout = time.Second * 15
 
 const DefaultMaxConcurrentReconciles = 10
 
-const DefaultRequeueTime = 2 * time.Minute
+const DefaultRequeueTime = time.Second * 20
 
 func GetConfigRequiredFields() []string {
 	return requiredFields

diff --git a/controllers/database.go b/controllers/database.go
@@ -54,7 +54,10 @@ var ConnectAndQueryDatabase = func(host, port, username, password, dbname string
 
 	testStatement := "SELECT 1;"
 	_, err = db.QueryContext(ctx, testStatement)
-	return err == nil, nil
+	if err != nil {
+		return false, err
+	}
+	return true, nil
 }
 
 func (r *DSPAReconciler) isDatabaseAccessible(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication,
@@ -80,7 +83,7 @@ func (r *DSPAReconciler) isDatabaseAccessible(ctx context.Context, dsp *dspav1al
 	decodePass, _ := b64.StdEncoding.DecodeString(params.DBConnection.Password)
 	dbConnectionTimeout := config.GetDurationConfigWithDefault(config.DBConnectionTimeoutConfigName, config.DefaultDBConnectionTimeout)
 
-	log.V(1).Info(fmt.Sprintf("Database Heath Check connection timeout: %s", dbConnectionTimeout))
+	log.V(1).Info(fmt.Sprintf("Attempting Database Heath Check connection (with timeout: %s)", dbConnectionTimeout))
 
 	dbHealthCheckPassed, err := ConnectAndQueryDatabase(params.DBConnection.Host,
 		params.DBConnection.Port,
@@ -90,8 +93,11 @@ func (r *DSPAReconciler) isDatabaseAccessible(ctx context.Context, dsp *dspav1al
 		dbConnectionTimeout)
 
 	if err != nil {
-		log.Info("Unable to connect to Database")
-	} else {
+		log.Info(fmt.Sprintf("Unable to connect to Database: %v", err))
+		return false, err
+	}
+
+	if dbHealthCheckPassed {
 		log.Info("Database Health Check Successful")
 	}
 

diff --git a/controllers/dspipeline_controller.go b/controllers/dspipeline_controller.go
@@ -297,6 +297,9 @@ func (r *DSPAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 		if err != nil {
 			return ctrl.Result{}, err
 		}
+	} else {
+		log.Info(fmt.Sprintf("Health check for Database or Object Store failed, retrying in %d seconds.", int(requeueTime.Seconds())))
+		return ctrl.Result{Requeue: true, RequeueAfter: requeueTime}, nil
 	}
 
 	log.Info("Updating CR status")