fix(executor): handle podlog in deadlineExceed termination. Fixes #7092

#7081 (#7093)

Signed-off-by: Tianchu Zhao <evantczhao@gmail.com>

Makefile

@@ -51,7 +51,7 @@ GOTEST                ?= go test -v
 PROFILE               ?= minimal
 PLUGINS               ?= $(shell [ $PROFILE = plugins ] && echo false || echo true)
 # by keeping this short we speed up the tests
-DEFAULT_REQUEUE_TIME  ?= 100ms
+DEFAULT_REQUEUE_TIME  ?= 1s
 # whether or not to start the Argo Service in TLS mode
 SECURE                := false
 AUTH_MODE             := hybrid

examples/timeouts-step.yaml

@@ -11,6 +11,7 @@ spec:
   templates:
   - name: sleep
     container:
-      image: debian:9.5-slim
-      command: [sleep, 1d]
+      image: argoproj/argosay:v2
+      command: [bash, -c]
+      args: ["echo 123; sleep 1d"]
     activeDeadlineSeconds: 10

test/e2e/artifacts_test.go

@@ -132,18 +132,35 @@ func (s *ArtifactsSuite) TestOutputResult() {
 }
 
 func (s *ArtifactsSuite) TestMainLog() {
-	s.Given().
-		Workflow("@testdata/basic-workflow.yaml").
-		When().
-		SubmitWorkflow().
-		WaitForWorkflow(fixtures.ToBeSucceeded).
-		Then().
-		ExpectWorkflow(func(t *testing.T, m *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
-			n := status.Nodes[m.Name]
-			if assert.NotNil(t, n) {
-				assert.Len(t, n.Outputs.Artifacts, 1)
-			}
-		})
+	s.Run("Basic", func() {
+		s.Given().
+			Workflow("@testdata/basic-workflow.yaml").
+			When().
+			SubmitWorkflow().
+			WaitForWorkflow(fixtures.ToBeSucceeded).
+			Then().
+			ExpectWorkflow(func(t *testing.T, m *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
+				n := status.Nodes[m.Name]
+				if assert.NotNil(t, n) {
+					assert.Len(t, n.Outputs.Artifacts, 1)
+				}
+			})
+	})
+	s.Need(fixtures.None(fixtures.Docker, fixtures.Kubelet))
+	s.Run("ActiveDeadlineSeconds", func() {
+		s.Given().
+			Workflow("@expectedfailures/timeouts-step.yaml").
+			When().
+			SubmitWorkflow().
+			WaitForWorkflow(fixtures.ToBeFailed).
+			Then().
+			ExpectWorkflow(func(t *testing.T, m *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
+				n := status.Nodes[m.Name]
+				if assert.NotNil(t, n) {
+					assert.Len(t, n.Outputs.Artifacts, 1)
+				}
+			})
+	})
 }
 
 func TestArtifactsSuite(t *testing.T) {

test/e2e/fixtures/needs.go

@@ -18,6 +18,7 @@ var (
 	Emissary = Executor("emissary")
 	K8SAPI   = Executor("k8sapi")
 	Kubelet  = Executor("kubelet")
+	PNS      = Executor("pns")
 )
 
 func Executor(e string) Need {

workflow/common/common.go

@@ -221,6 +221,9 @@ const (
 
 	// ArgoProgressPath defines the path to a file used for self reporting progress
 	ArgoProgressPath = "/var/run/argo/progress"
+
+	// ErrDeadlineExceeded is the pod status reason when exceed deadline
+	ErrDeadlineExceeded = "DeadlineExceeded"
 )
 
 // AnnotationKeyKillCmd specifies the command to use to kill to container, useful for injected sidecars

workflow/executor/k8sapi/k8sapi.go

@@ -11,7 +11,8 @@ import (
 	restclient "k8s.io/client-go/rest"
 
 	"github.com/argoproj/argo-workflows/v3/errors"
-	"github.com/argoproj/argo-workflows/v3/workflow/executor/common"
+	"github.com/argoproj/argo-workflows/v3/workflow/common"
+	execcommon "github.com/argoproj/argo-workflows/v3/workflow/executor/common"
 )
 
 type K8sAPIExecutor struct {
@@ -44,7 +45,7 @@ func (k *K8sAPIExecutor) GetOutputStream(ctx context.Context, containerName stri
 // Wait for the container to complete
 func (k *K8sAPIExecutor) Wait(ctx context.Context, containerNames []string) error {
 	return k.Until(ctx, func(pod *corev1.Pod) bool {
-		return common.AllTerminated(pod.Status.ContainerStatuses, containerNames)
+		return execcommon.AllTerminated(pod.Status.ContainerStatuses, containerNames) || pod.Status.Reason == common.ErrDeadlineExceeded
 	})
 }