Update removedChecker to use countInterval (#839)

* Update removedChecker to use countInterval

* gofmt

* Requested changes

* Revert using errgroup

Co-authored-by: Mason Liang <mason@0x.org>

zeroex/orderwatch/order_watcher.go

@@ -29,9 +29,14 @@ const (
 	// were not caught by the event watcher process.
 	minCleanupInterval = 1 * time.Hour
 
-	// minRemovedCheckInterval specifies the minimum amount of time between checks
-	// on whether to remove orders flaggged for removal from the DB
-	minRemovedCheckInterval = 5 * time.Minute
+	// maxDeleteInterval specifies the maximum amount of time between calls to permanentlyDeleteStaleRemovedOrders
+	maxDeleteInterval = 5 * time.Minute
+
+	// checkDatabaseUtilizationThresholdInterval specifies the amount of time between calls to db.CountOrders. If the count is higher than a threshold, then permanentlyDeleteStaleRemovedOrders will be called, so this is the minimum interval between calls to permanentlyDeleteStaleRemovedOrders.
+	checkDatabaseUtilizationThresholdInterval = 5 * time.Second
+
+	// If, after a call to db.CountOrders, the database utilization exceeds databaseUtilizationThreshold, then permanentlyDeleteStaleRemovedOrders will be called.
+	databaseUtilizationThreshold = 0.5
 
 	// defaultLastUpdatedBuffer specifies how long it must have been since an order was
 	// last updated in order to be re-validated by the cleanup worker
@@ -264,20 +269,28 @@ func (w *Watcher) cleanupLoop(ctx context.Context) error {
 }
 
 func (w *Watcher) removedCheckerLoop(ctx context.Context) error {
-	for {
-		start := time.Now()
-		if err := w.permanentlyDeleteStaleRemovedOrders(ctx); err != nil {
-			return err
-		}
+	if err := w.permanentlyDeleteStaleRemovedOrders(ctx); err != nil {
+		return err
+	}
+	lastDeleted := time.Now()
 
+	for {
 		select {
 		case <-ctx.Done():
 			return nil
-		// Wait minRemovedCheckInterval before calling permanentlyDeleteStaleRemovedOrders again. Since
-		// we only start waiting _after_ permanentlyDeleteStaleRemovedOrders completes, we will never
-		// have multiple calls to permanentlyDeleteStaleRemovedOrders running in parallel
-		case <-time.After(minRemovedCheckInterval - time.Since(start)):
-			continue
+		case <-time.After(checkDatabaseUtilizationThresholdInterval):
+			count, err := w.db.CountOrders(nil)
+			if err != nil {
+				return err
+			}
+			databaseUtilization := float64(count) / float64(w.maxOrders)
+
+			if time.Since(lastDeleted) > maxDeleteInterval || databaseUtilization > databaseUtilizationThreshold {
+				if err := w.permanentlyDeleteStaleRemovedOrders(ctx); err != nil {
+					return err
+				}
+				lastDeleted = time.Now()
+			}
 		}
 	}
 }