diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index 42ca7c772f6..b5f91ce5f8f 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -37,6 +37,7 @@ import ( utilsync "github.com/aws/amazon-ecs-agent/agent/utils/sync" "github.com/aws/amazon-ecs-agent/agent/utils/ttime" + "github.com/aws/aws-sdk-go/aws" "github.com/cihub/seelog" "github.com/pkg/errors" "golang.org/x/net/context" @@ -285,6 +286,33 @@ func (engine *DockerTaskEngine) synchronizeState() { engine.saver.Save() } +func updateContainerMetadata(metadata *DockerContainerMetadata, container *api.Container, task *api.Task) { + container.SetCreatedAt(metadata.CreatedAt) + container.SetStartedAt(metadata.StartedAt) + container.SetFinishedAt(metadata.FinishedAt) + + // Set the labels if it's not set + if len(metadata.Labels) != 0 && len(container.GetLabels()) == 0 { + container.SetLabels(metadata.Labels) + } + + // Update Volume + if metadata.Volumes != nil { + task.UpdateMountPoints(container, metadata.Volumes) + } + + // Set Exitcode if it's not set + if metadata.ExitCode != nil && (container.GetKnownExitCode() == nil || + aws.IntValue(metadata.ExitCode) != aws.IntValue(container.GetKnownExitCode())) { + container.SetKnownExitCode(metadata.ExitCode) + } + + // Set port mappings + if len(metadata.PortBindings) != 0 && len(container.KnownPortBindings) == 0 { + container.KnownPortBindings = metadata.PortBindings + } +} + // synchronizeContainerStatus checks and updates the container status with docker func (engine *DockerTaskEngine) synchronizeContainerStatus(container *api.DockerContainer, task *api.Task) { if container.DockerID == "" { @@ -294,15 +322,15 @@ func (engine *DockerTaskEngine) synchronizeContainerStatus(container *api.Docker if err != nil { log.Warn("Could not find matching container for expected", "name", container.DockerName) } else { + // update the container metadata in case the container status/metadata changed during agent restart + metadata := metadataFromContainer(describedContainer) + updateContainerMetadata(&metadata, container.Container, task) container.DockerID = describedContainer.ID + container.Container.SetKnownStatus(dockerStateToState(describedContainer.State)) - container.Container.SetCreatedAt(describedContainer.Created) - container.Container.SetStartedAt(describedContainer.State.StartedAt) - container.Container.SetFinishedAt(describedContainer.State.FinishedAt) // update mappings that need dockerid engine.state.AddContainer(container, task) engine.imageManager.RecordContainerReference(container.Container) - container.Container.SetLabels(describedContainer.Config.Labels) } return } @@ -310,16 +338,21 @@ func (engine *DockerTaskEngine) synchronizeContainerStatus(container *api.Docker currentState, metadata := engine.client.DescribeContainer(container.DockerID) if metadata.Error != nil { currentState = api.ContainerStopped - if !container.Container.KnownTerminal() { - container.Container.ApplyingError = api.NewNamedError(&ContainerVanishedError{}) + // If this is a Docker API error + if metadata.Error.ErrorName() == "CannotDescribeContainerError" { log.Warn("Could not describe previously known container; assuming dead", "err", metadata.Error, "id", container.DockerID, "name", container.DockerName) - engine.imageManager.RemoveContainerReferenceFromImageState(container.Container) + if !container.Container.KnownTerminal() { + container.Container.ApplyingError = api.NewNamedError(&ContainerVanishedError{}) + engine.imageManager.RemoveContainerReferenceFromImageState(container.Container) + } + } else { + // If this is a container state error + updateContainerMetadata(&metadata, container.Container, task) + container.Container.ApplyingError = api.NewNamedError(metadata.Error) } } else { - container.Container.SetLabels(metadata.Labels) - container.Container.SetCreatedAt(metadata.CreatedAt) - container.Container.SetStartedAt(metadata.StartedAt) - container.Container.SetFinishedAt(metadata.FinishedAt) + // update the container metadata in case the container status/metadata changed during agent restart + updateContainerMetadata(&metadata, container.Container, task) engine.imageManager.RecordContainerReference(container.Container) if engine.cfg.ContainerMetadataEnabled && !container.Container.IsMetadataFileUpdated() { go engine.updateMetadataFile(task, container) @@ -329,6 +362,16 @@ func (engine *DockerTaskEngine) synchronizeContainerStatus(container *api.Docker // update the container known status container.Container.SetKnownStatus(currentState) } + // Update task ExecutionStoppedAt timestamp + if container.Container.GetKnownStatus() == api.ContainerStopped && container.Container.IsEssential() { + now := engine.time().Now() + ok := task.SetExecutionStoppedAt(now) + if ok { + seelog.Infof("Recording execution stopped time for a task, essential container in task stopped, task %s, time: %s", + task.String(), now.String()) + } + } + } // CheckTaskState inspects the state of all containers within a task and writes diff --git a/agent/engine/task_manager.go b/agent/engine/task_manager.go index be232d5e4aa..9094aab66d5 100644 --- a/agent/engine/task_manager.go +++ b/agent/engine/task_manager.go @@ -29,6 +29,7 @@ import ( "github.com/aws/amazon-ecs-agent/agent/statemanager" utilsync "github.com/aws/amazon-ecs-agent/agent/utils/sync" "github.com/aws/amazon-ecs-agent/agent/utils/ttime" + "github.com/cihub/seelog" ) @@ -342,10 +343,7 @@ func (mtask *managedTask) handleContainerChange(containerChange dockerContainerC currentKnownStatus := containerKnownStatus container.SetKnownStatus(event.Status) - container.SetCreatedAt(event.CreatedAt) - container.SetStartedAt(event.StartedAt) - container.SetFinishedAt(event.FinishedAt) - + updateContainerMetadata(&event.DockerContainerMetadata, container, mtask.Task) if event.Error != nil { proceedAnyway := mtask.handleEventError(containerChange, currentKnownStatus) if !proceedAnyway { @@ -370,17 +368,7 @@ func (mtask *managedTask) handleContainerChange(containerChange dockerContainerC seelog.Warnf("Failed to write container change event to event stream, err %v", err) } - if event.ExitCode != nil && event.ExitCode != container.GetKnownExitCode() { - container.SetKnownExitCode(event.ExitCode) - } - if event.PortBindings != nil { - container.KnownPortBindings = event.PortBindings - } - if event.Volumes != nil { - mtask.UpdateMountPoints(container, event.Volumes) - } - - mtask.emitContainerEvent(mtask.Task, container, "") + mtask.engine.emitContainerEvent(mtask.Task, container, "") if mtask.UpdateStatus() { llog.Debug("Container change also resulted in task change") // If knownStatus changed, let it be known diff --git a/amazon-ecs-cni-plugins b/amazon-ecs-cni-plugins index 23927539cec..397bc51f0db 160000 --- a/amazon-ecs-cni-plugins +++ b/amazon-ecs-cni-plugins @@ -1 +1 @@ -Subproject commit 23927539cec8307f67b441d6003af3f36741e0b4 +Subproject commit 397bc51f0db390b3902524781744591bcf34219c