From d708ad2c85107f3d98cdb32a1f51b515c3d712b2 Mon Sep 17 00:00:00 2001 From: Ilya Ozherelyev Date: Tue, 20 Feb 2024 14:40:18 +0100 Subject: [PATCH 1/3] Added workflow_type to logs and added domain name in the log --- .../history/task/timer_active_task_executor.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/service/history/task/timer_active_task_executor.go b/service/history/task/timer_active_task_executor.go index 6f39fdfe559..af93b0373e0 100644 --- a/service/history/task/timer_active_task_executor.go +++ b/service/history/task/timer_active_task_executor.go @@ -435,6 +435,11 @@ func (t *timerActiveTaskExecutor) executeDecisionTimeoutTask( } defer func() { release(retError) }() + domainName, err := t.shard.GetDomainCache().GetDomainName(task.DomainID) + if err != nil { + return fmt.Errorf("unable to find domainID: %v, err: %v", task.DomainID, err) + } + mutableState, err := loadMutableStateForTimerTask(ctx, wfContext, task, t.metricsClient, t.logger) if err != nil { return err @@ -443,6 +448,11 @@ func (t *timerActiveTaskExecutor) executeDecisionTimeoutTask( return nil } + wfType := mutableState.GetWorkflowType() + if wfType == nil { + return fmt.Errorf("unable to find workflow type, task %s", task) + } + scheduleID := task.EventID decision, ok := mutableState.GetDecisionInfo(scheduleID) if !ok { @@ -464,13 +474,14 @@ func (t *timerActiveTaskExecutor) executeDecisionTimeoutTask( if isStickyDecision { decisionTypeTag = stickyDecisionTypeTag } + tags := []metrics.Tag{metrics.WorkflowTypeTag(wfType.GetName()), decisionTypeTag} switch execution.TimerTypeFromInternal(types.TimeoutType(task.TimeoutType)) { case execution.TimerTypeStartToClose: t.emitTimeoutMetricScopeWithDomainTag( mutableState.GetExecutionInfo().DomainID, metrics.TimerActiveTaskDecisionTimeoutScope, execution.TimerTypeStartToClose, - decisionTypeTag, + tags..., ) if _, err := mutableState.AddDecisionTaskTimedOutEvent( decision.ScheduleID, @@ -488,6 +499,7 @@ func (t *timerActiveTaskExecutor) executeDecisionTimeoutTask( if !isStickyDecision { t.logger.Warn("Potential lost normal decision task", + tag.WorkflowDomainName(domainName), tag.WorkflowDomainID(task.GetDomainID()), tag.WorkflowID(task.GetWorkflowID()), tag.WorkflowRunID(task.GetRunID()), @@ -501,7 +513,7 @@ func (t *timerActiveTaskExecutor) executeDecisionTimeoutTask( mutableState.GetExecutionInfo().DomainID, metrics.TimerActiveTaskDecisionTimeoutScope, execution.TimerTypeScheduleToStart, - decisionTypeTag, + tags..., ) _, err := mutableState.AddDecisionTaskScheduleToStartTimeoutEvent(scheduleID) if err != nil { From ddf546cac0854131eaf1de0bbab1b8a12a524325 Mon Sep 17 00:00:00 2001 From: Ilya Ozherelyev Date: Tue, 20 Feb 2024 14:51:34 +0100 Subject: [PATCH 2/3] add workflowtype dimenstion to activity schedule_to_start --- service/history/task/timer_active_task_executor.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/service/history/task/timer_active_task_executor.go b/service/history/task/timer_active_task_executor.go index af93b0373e0..0865e83b1e6 100644 --- a/service/history/task/timer_active_task_executor.go +++ b/service/history/task/timer_active_task_executor.go @@ -281,6 +281,11 @@ func (t *timerActiveTaskExecutor) executeActivityTimeoutTask( return nil } + wfType := mutableState.GetWorkflowType() + if wfType == nil { + return fmt.Errorf("unable to find workflow type, task %s", task) + } + timerSequence := execution.NewTimerSequence(mutableState) referenceTime := t.shard.GetTimeSource().Now() resurrectionCheckMinDelay := t.config.ResurrectionCheckMinDelay(mutableState.GetDomainEntry().GetInfo().Name) @@ -398,6 +403,7 @@ Loop: mutableState.GetExecutionInfo().DomainID, metrics.TimerActiveTaskActivityTimeoutScope, timerSequenceID.TimerType, + metrics.WorkflowTypeTag(wfType.GetName()), ) if _, err := mutableState.AddActivityTaskTimedOutEvent( activityInfo.ScheduleID, From a437891cfef0b8ffb8b477c5f4c7346c8ebfafb9 Mon Sep 17 00:00:00 2001 From: Ilya Ozherelyev Date: Wed, 21 Feb 2024 15:51:06 +0100 Subject: [PATCH 3/3] additional log for timed out activities --- .../history/task/timer_active_task_executor.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/service/history/task/timer_active_task_executor.go b/service/history/task/timer_active_task_executor.go index 0865e83b1e6..2352b9d5ad7 100644 --- a/service/history/task/timer_active_task_executor.go +++ b/service/history/task/timer_active_task_executor.go @@ -273,6 +273,11 @@ func (t *timerActiveTaskExecutor) executeActivityTimeoutTask( } defer func() { release(retError) }() + domainName, err := t.shard.GetDomainCache().GetDomainName(task.DomainID) + if err != nil { + return fmt.Errorf("unable to find domainID: %v, err: %v", task.DomainID, err) + } + mutableState, err := loadMutableStateForTimerTask(ctx, wfContext, task, t.metricsClient, t.logger) if err != nil { return err @@ -405,6 +410,16 @@ Loop: timerSequenceID.TimerType, metrics.WorkflowTypeTag(wfType.GetName()), ) + + t.logger.Info("Activity timed out", + tag.WorkflowDomainName(domainName), + tag.WorkflowDomainID(task.GetDomainID()), + tag.WorkflowID(task.GetWorkflowID()), + tag.WorkflowRunID(task.GetRunID()), + tag.ScheduleAttempt(task.ScheduleAttempt), + tag.FailoverVersion(task.GetVersion()), + ) + if _, err := mutableState.AddActivityTaskTimedOutEvent( activityInfo.ScheduleID, activityInfo.StartedID,