engine: update container metadata on agent restart

agent/engine/docker_task_engine.go

@@ -37,6 +37,7 @@ import (
 	utilsync "github.com/aws/amazon-ecs-agent/agent/utils/sync"
 	"github.com/aws/amazon-ecs-agent/agent/utils/ttime"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/cihub/seelog"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -285,6 +286,33 @@ func (engine *DockerTaskEngine) synchronizeState() {
 	engine.saver.Save()
 }
 
+func updateContainerMetadata(metadata *DockerContainerMetadata, container *api.Container, task *api.Task) {
+	container.SetCreatedAt(metadata.CreatedAt)
+	container.SetStartedAt(metadata.StartedAt)
+	container.SetFinishedAt(metadata.FinishedAt)
+
+	// Set the labels if it's not set
+	if len(metadata.Labels) != 0 && len(container.GetLabels()) == 0 {
+		container.SetLabels(metadata.Labels)
+	}
+
+	// Update Volume
+	if metadata.Volumes != nil {
+		task.UpdateMountPoints(container, metadata.Volumes)
+	}
+
+	// Set Exitcode if it's not set
+	if metadata.ExitCode != nil && (container.GetKnownExitCode() == nil ||
+		aws.IntValue(metadata.ExitCode) != aws.IntValue(container.GetKnownExitCode())) {
+		container.SetKnownExitCode(metadata.ExitCode)
+	}
+
+	// Set port mappings
+	if len(metadata.PortBindings) != 0 && len(container.KnownPortBindings) == 0 {
+		container.KnownPortBindings = metadata.PortBindings
+	}
+}
+
 // synchronizeContainerStatus checks and updates the container status with docker
 func (engine *DockerTaskEngine) synchronizeContainerStatus(container *api.DockerContainer, task *api.Task) {
 	if container.DockerID == "" {
@@ -294,32 +322,37 @@ func (engine *DockerTaskEngine) synchronizeContainerStatus(container *api.Docker
 		if err != nil {
 			log.Warn("Could not find matching container for expected", "name", container.DockerName)
 		} else {
+			// update the container metadata in case the container status/metadata changed during agent restart
+			metadata := metadataFromContainer(describedContainer)
+			updateContainerMetadata(&metadata, container.Container, task)
 			container.DockerID = describedContainer.ID
+
 			container.Container.SetKnownStatus(dockerStateToState(describedContainer.State))
-			container.Container.SetCreatedAt(describedContainer.Created)
-			container.Container.SetStartedAt(describedContainer.State.StartedAt)
-			container.Container.SetFinishedAt(describedContainer.State.FinishedAt)
 			// update mappings that need dockerid
 			engine.state.AddContainer(container, task)
 			engine.imageManager.RecordContainerReference(container.Container)
-			container.Container.SetLabels(describedContainer.Config.Labels)
 		}
 		return
 	}
 
 	currentState, metadata := engine.client.DescribeContainer(container.DockerID)
 	if metadata.Error != nil {
 		currentState = api.ContainerStopped
-		if !container.Container.KnownTerminal() {
-			container.Container.ApplyingError = api.NewNamedError(&ContainerVanishedError{})
+		// If this is a Docker API error
+		if metadata.Error.ErrorName() == "CannotDescribeContainerError" {
 			log.Warn("Could not describe previously known container; assuming dead", "err", metadata.Error, "id", container.DockerID, "name", container.DockerName)
-			engine.imageManager.RemoveContainerReferenceFromImageState(container.Container)
+			if !container.Container.KnownTerminal() {
+				container.Container.ApplyingError = api.NewNamedError(&ContainerVanishedError{})
+				engine.imageManager.RemoveContainerReferenceFromImageState(container.Container)
+			}
+		} else {
+			// If this is a container state error
+			updateContainerMetadata(&metadata, container.Container, task)
+			container.Container.ApplyingError = api.NewNamedError(metadata.Error)
 		}
 	} else {
-		container.Container.SetLabels(metadata.Labels)
-		container.Container.SetCreatedAt(metadata.CreatedAt)
-		container.Container.SetStartedAt(metadata.StartedAt)
-		container.Container.SetFinishedAt(metadata.FinishedAt)
+		// update the container metadata in case the container status/metadata changed during agent restart
+		updateContainerMetadata(&metadata, container.Container, task)
 		engine.imageManager.RecordContainerReference(container.Container)
 		if engine.cfg.ContainerMetadataEnabled && !container.Container.IsMetadataFileUpdated() {
 			go engine.updateMetadataFile(task, container)
@@ -329,6 +362,16 @@ func (engine *DockerTaskEngine) synchronizeContainerStatus(container *api.Docker
 		// update the container known status
 		container.Container.SetKnownStatus(currentState)
 	}
+	// Update task ExecutionStoppedAt timestamp
+	if container.Container.GetKnownStatus() == api.ContainerStopped && container.Container.IsEssential() {
+		now := engine.time().Now()
+		ok := task.SetExecutionStoppedAt(now)
+		if ok {
+			seelog.Infof("Recording execution stopped time for a task, essential container in task stopped, task %s, time: %s",
+				task.String(), now.String())
+		}
+	}
+
 }
 
 // CheckTaskState inspects the state of all containers within a task and writes

agent/engine/task_manager.go

@@ -29,6 +29,7 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/statemanager"
 	utilsync "github.com/aws/amazon-ecs-agent/agent/utils/sync"
 	"github.com/aws/amazon-ecs-agent/agent/utils/ttime"
+
 	"github.com/cihub/seelog"
 )
 
@@ -342,10 +343,7 @@ func (mtask *managedTask) handleContainerChange(containerChange dockerContainerC
 	currentKnownStatus := containerKnownStatus
 	container.SetKnownStatus(event.Status)
 
-	container.SetCreatedAt(event.CreatedAt)
-	container.SetStartedAt(event.StartedAt)
-	container.SetFinishedAt(event.FinishedAt)
-
+	updateContainerMetadata(&event.DockerContainerMetadata, container, mtask.Task)
 	if event.Error != nil {
 		proceedAnyway := mtask.handleEventError(containerChange, currentKnownStatus)
 		if !proceedAnyway {
@@ -370,17 +368,7 @@ func (mtask *managedTask) handleContainerChange(containerChange dockerContainerC
 		seelog.Warnf("Failed to write container change event to event stream, err %v", err)
 	}
 
-	if event.ExitCode != nil && event.ExitCode != container.GetKnownExitCode() {
-		container.SetKnownExitCode(event.ExitCode)
-	}
-	if event.PortBindings != nil {
-		container.KnownPortBindings = event.PortBindings
-	}
-	if event.Volumes != nil {
-		mtask.UpdateMountPoints(container, event.Volumes)
-	}
-
-	mtask.emitContainerEvent(mtask.Task, container, "")
+	mtask.engine.emitContainerEvent(mtask.Task, container, "")
 	if mtask.UpdateStatus() {
 		llog.Debug("Container change also resulted in task change")
 		// If knownStatus changed, let it be known