add gauge metrics when no active zone is not found

common/metrics/defs.go

@@ -1025,6 +1025,8 @@ const (
 	ControllerNumOpenCGExtents
 	// ControllerNumConsumedCGExtents represents the count of consumed cg extents
 	ControllerNumConsumedCGExtents
+	// ControllerNoActiveZone indicates there's no active zone from dynamic config
+	ControllerNoActiveZone
 
 	// -- Replicator metrics -- //
 
@@ -1234,6 +1236,7 @@ var metricDefs = map[ServiceIdx]map[int]metricDefinition{
 		ControllerRetentionJobDuration:             {Timer, "controller.retentionmgr.jobduration"},
 		ControllerGetAddressLatency:                {Timer, "controller.retentionmgr.getaddresslatency"},
 		ControllerPurgeMessagesLatency:             {Timer, "controller.retentionmgr.purgemessageslatency"},
+		ControllerNoActiveZone:                     {Gauge, "controller.no-active-zone"},
 	},
 
 	// definitions for Replicator metrics

services/controllerhost/consumer.go

@@ -801,7 +801,7 @@ func refreshOutputHostsForConsGroup(context *Context,
 		}
 
 		// If we shouldn't consume in this zone(for a multi_zone cg), short circuit and return
-		if !shouldConsumeInZone(context, cgDesc, cfg) {
+		if !shouldConsumeInZone(context, m3Scope, cgDesc, cfg) {
 			consumeDisabled = true
 			writeToCache(int64(outputCacheTTL))
 			return outputAddrs, nil
@@ -863,7 +863,7 @@ func refreshOutputHostsForConsGroup(context *Context,
 // If failover mode is enabled in dynamic config, the active zone will be the one specified in dynamic config
 // Otherwise, use the per cg override if it's specified
 // Last, check the active zone in dynamic config. If specified, use it. Otherwise always return false
-func shouldConsumeInZone(context *Context, cgDesc *shared.ConsumerGroupDescription, dConfig ControllerDynamicConfig) bool {
+func shouldConsumeInZone(context *Context, m3Scope int, cgDesc *shared.ConsumerGroupDescription, dConfig ControllerDynamicConfig) bool {
 	if strings.EqualFold(dConfig.FailoverMode, `enabled`) {
 		return strings.EqualFold(context.localZone, dConfig.ActiveZone)
 	}
@@ -877,6 +877,7 @@ func shouldConsumeInZone(context *Context, cgDesc *shared.ConsumerGroupDescripti
 	}
 
 	context.log.Warn(`no active zone from dynamic config !`)
+	context.m3Client.UpdateGauge(m3Scope, metrics.ControllerNoActiveZone, 1)
 
 	return false
 }