Add resource exhausted metric with cause (#940)

temporalio · Oct 31, 2022 · 40f7da0 · 40f7da0
1 parent 3f3eca4
commit 40f7da0
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 6 deletions.
diff --git a/internal/common/metrics/constants.go b/internal/common/metrics/constants.go
@@ -63,12 +63,14 @@ const (
 	WorkerTaskSlotsAvailable = TemporalMetricsPrefix + "worker_task_slots_available"
 	PollerStartCounter       = TemporalMetricsPrefix + "poller_start"
 
-	TemporalRequest            = TemporalMetricsPrefix + "request"
-	TemporalRequestFailure     = TemporalRequest + "_failure"
-	TemporalRequestLatency     = TemporalRequest + "_latency"
-	TemporalLongRequest        = TemporalMetricsPrefix + "long_request"
-	TemporalLongRequestFailure = TemporalLongRequest + "_failure"
-	TemporalLongRequestLatency = TemporalLongRequest + "_latency"
+	TemporalRequest                      = TemporalMetricsPrefix + "request"
+	TemporalRequestFailure               = TemporalRequest + "_failure"
+	TemporalRequestLatency               = TemporalRequest + "_latency"
+	TemporalLongRequest                  = TemporalMetricsPrefix + "long_request"
+	TemporalLongRequestFailure           = TemporalLongRequest + "_failure"
+	TemporalLongRequestLatency           = TemporalLongRequest + "_latency"
+	TemporalRequestResourceExhausted     = TemporalRequest + "_resource_exhausted"
+	TemporalLongRequestResourceExhausted = TemporalLongRequest + "_resource_exhausted"
 
 	StickyCacheHit                 = TemporalMetricsPrefix + "sticky_cache_hit"
 	StickyCacheMiss                = TemporalMetricsPrefix + "sticky_cache_miss"
@@ -87,6 +89,7 @@ const (
 	ActivityTypeNameTagName = "activity_type"
 	TaskQueueTagName        = "task_queue"
 	OperationTagName        = "operation"
+	CauseTagName            = "cause"
 )
 
 // Metric tag values

diff --git a/internal/common/metrics/grpc.go b/internal/common/metrics/grpc.go
@@ -27,7 +27,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gogo/status"
+	enumspb "go.temporal.io/api/enums/v1"
+	"go.temporal.io/api/serviceerror"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 )
 
 // HandlerContextKey is the context key for a MetricHandler value.
@@ -105,5 +109,18 @@ func recordRequestEnd(handler Handler, longPoll bool, suffix string, start time.
 		}
 		failureMetric += suffix
 		handler.Counter(failureMetric).Inc(1)
+
+		// If it's a resource exhausted, extract cause if present and increment
+		if s := status.Convert(err); s.Code() == codes.ResourceExhausted {
+			resMetric := TemporalRequestResourceExhausted
+			if longPoll {
+				resMetric = TemporalLongRequestResourceExhausted
+			}
+			var cause enumspb.ResourceExhaustedCause
+			if resErr, _ := serviceerror.FromStatus(s).(*serviceerror.ResourceExhausted); resErr != nil {
+				cause = resErr.Cause
+			}
+			handler.WithTags(map[string]string{CauseTagName: cause.String()}).Counter(resMetric).Inc(1)
+		}
 	}
 }
diff --git a/internal/grpc_dialer_test.go b/internal/grpc_dialer_test.go
@@ -40,9 +40,11 @@ import (
 	"github.com/pborman/uuid"
 	"github.com/stretchr/testify/require"
 	"go.temporal.io/api/common/v1"
+	"go.temporal.io/api/enums/v1"
 	"go.temporal.io/api/errordetails/v1"
 	"go.temporal.io/api/serviceerror"
 	"go.temporal.io/api/workflowservice/v1"
+	"go.temporal.io/sdk/internal/common/metrics"
 	"go.temporal.io/sdk/internal/common/retry"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -401,6 +403,39 @@ func TestCustomResolver(t *testing.T) {
 	require.Equal(t, 4, s2.signalWorkflowInvokeCount())
 }
 
+func TestResourceExhaustedCause(t *testing.T) {
+	// Start gRPC server
+	srv, err := startTestGRPCServer()
+	require.NoError(t, err)
+	defer srv.Stop()
+	handler := metrics.NewCapturingHandler()
+
+	// Attempt dial with a resource exhausted cause
+	s, _ := status.New(codes.ResourceExhausted, "some resource exhausted").WithDetails(&errordetails.ResourceExhaustedFailure{
+		Cause: enums.RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT,
+	})
+	srv.getSystemInfoResponseError = s.Err()
+	_, err = DialClient(ClientOptions{HostPort: srv.addr, MetricsHandler: handler})
+	require.Error(t, err)
+
+	// Attempt dial with a cause-less resource exhausted
+	srv.getSystemInfoResponseError = status.New(codes.ResourceExhausted, "some resource exhausted").Err()
+	_, err = DialClient(ClientOptions{HostPort: srv.addr, MetricsHandler: handler})
+	require.Error(t, err)
+
+	// Make sure we have 1 metric with cause and 1 without
+	var foundWithCause, foundWithoutCause bool
+	for _, counter := range handler.Counters() {
+		if counter.Tags["operation"] == "GetSystemInfo" && counter.Tags["cause"] == "ConcurrentLimit" {
+			foundWithCause = true
+		} else if counter.Tags["operation"] == "GetSystemInfo" && counter.Tags["cause"] == "Unspecified" {
+			foundWithoutCause = true
+		}
+	}
+	require.True(t, foundWithCause)
+	require.True(t, foundWithoutCause)
+}
+
 type testGRPCServer struct {
 	workflowservice.UnimplementedWorkflowServiceServer
 	*grpc.Server