fix operator timeout metrics (#1541)

Signed-off-by: Ryan Leung <rleungx@gmail.com>
tikv · May 21, 2019 · acf3e28 · acf3e28
1 parent cdc23ae
commit acf3e28
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/server/cluster.go b/server/cluster.go
@@ -601,7 +601,7 @@ func (c *RaftCluster) checkOperators() {
 			log.Info("operator timeout",
 				zap.Uint64("region-id", op.RegionID()),
 				zap.Stringer("operator", op))
-			opController.RemoveOperator(op)
+			opController.RemoveTimeoutOperator(op)
 		}
 	}
 }

diff --git a/server/schedule/operator.go b/server/schedule/operator.go
@@ -370,7 +370,6 @@ func (o *Operator) IsTimeout() bool {
 		timeout = time.Since(o.createTime) > LeaderOperatorWaitTime
 	}
 	if timeout {
-		operatorCounter.WithLabelValues(o.Desc(), "timeout").Inc()
 		return true
 	}
 	return false

diff --git a/server/schedule/operator_controller.go b/server/schedule/operator_controller.go
@@ -93,7 +93,7 @@ func (oc *OperatorController) Dispatch(region *core.RegionInfo, source string) {
 			oc.RemoveOperator(op)
 		} else if timeout {
 			log.Info("operator timeout", zap.Uint64("region-id", region.GetID()), zap.Reflect("operator", op))
-			oc.RemoveOperator(op)
+			oc.RemoveTimeoutOperator(op)
 			oc.opRecords.Put(op, pdpb.OperatorStatus_TIMEOUT)
 		}
 	}
@@ -238,6 +238,14 @@ func (oc *OperatorController) RemoveOperator(op *Operator) {
 	oc.removeOperatorLocked(op)
 }
 
+// RemoveTimeoutOperator removes a operator which is timeout from the running operators.
+func (oc *OperatorController) RemoveTimeoutOperator(op *Operator) {
+	oc.Lock()
+	defer oc.Unlock()
+	operatorCounter.WithLabelValues(op.Desc(), "timeout").Inc()
+	oc.removeOperatorLocked(op)
+}
+
 // GetOperatorStatus gets the operator and its status with the specify id.
 func (oc *OperatorController) GetOperatorStatus(id uint64) *OperatorWithStatus {
 	oc.Lock()