Merge branch 'dev' into boltdb

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 1.42.0
+* Feature - Support for sub second precision in FluentD [#2538](https://github.com/aws/amazon-ecs-agent/pull/2538).
+* Bug - Fixed a bug that caused configured values for ImageCleanupExclusionList
+to be ignored in some situations [#2513](https://github.com/aws/amazon-ecs-agent/pull/2513)
+
 ## 1.41.1
 * Bug - Fixed a bug [#2476](https://github.com/aws/amazon-ecs-agent/issues/2476) where HostPort is not present in ECS Task Metadata Endpoint response with bridge network type [#2495](https://github.com/aws/amazon-ecs-agent/pull/2495)
 

VERSION

@@ -1 +1 @@
-1.41.1
+1.42.0

agent/Gopkg.toml

@@ -80,7 +80,6 @@ required = ["github.com/golang/mock/mockgen/model"]
 [[constraint]]
   name = "github.com/vishvananda/netlink"
   revision ="fe3b5664d23a11b52ba59bece4ff29c52772a56b"
-
 [[constraint]]
   name = "github.com/didip/tollbooth"
   version = "3.0.2"
@@ -96,3 +95,7 @@ required = ["github.com/golang/mock/mockgen/model"]
 [[constraint]]
   branch = "master"
   name = "github.com/awslabs/go-config-generator-for-fluentd-and-fluentbit"
+
+[[constraint]]
+  name = "github.com/containernetworking/plugins"
+  version = "0.8.6"

agent/acs/handler/task_manifest_handler.go

@@ -52,7 +52,6 @@ func newTaskManifestHandler(ctx context.Context,
 
 	// Create a cancelable context from the parent context
 	derivedContext, cancel := context.WithCancel(ctx)
-
 	return taskManifestHandler{
 		messageBufferTaskManifest:                make(chan *ecsacs.TaskManifestMessage),
 		messageBufferTaskManifestAck:             make(chan string),
@@ -187,7 +186,7 @@ func (taskManifestHandler *taskManifestHandler) sendTaskStopVerificationMessage(
 
 // compares the list of tasks received in the task manifest message and tasks running on the the instance
 // It returns all the task that are running on the instance but not present in task manifest message task list
-func compareTasks(receivedTaskList []*ecsacs.TaskIdentifier, runningTaskList []*apitask.Task) []*ecsacs.TaskIdentifier {
+func compareTasks(receivedTaskList []*ecsacs.TaskIdentifier, runningTaskList []*apitask.Task, clusterARN string) []*ecsacs.TaskIdentifier {
 	tasksToBeKilled := make([]*ecsacs.TaskIdentifier, 0)
 	for _, runningTask := range runningTaskList {
 		// For every task running on the instance check if the task is present in receivedTaskList with the DesiredState
@@ -204,8 +203,9 @@ func compareTasks(receivedTaskList []*ecsacs.TaskIdentifier, runningTaskList []*
 			}
 			if !taskPresent {
 				tasksToBeKilled = append(tasksToBeKilled, &ecsacs.TaskIdentifier{
-					DesiredStatus: aws.String(apitaskstatus.TaskStoppedString),
-					TaskArn:       aws.String(runningTask.Arn),
+					DesiredStatus:  aws.String(apitaskstatus.TaskStoppedString),
+					TaskArn:        aws.String(runningTask.Arn),
+					TaskClusterArn: aws.String(clusterARN),
 				})
 			}
 		}
@@ -240,6 +240,7 @@ func (taskManifestHandler *taskManifestHandler) handleTaskManifestSingleMessage(
 	message *ecsacs.TaskManifestMessage) error {
 	taskListManifestHandler := message.Tasks
 	seqNumberFromMessage := *message.Timeline
+	clusterARN := *message.ClusterArn
 	agentLatestSequenceNumber := *taskManifestHandler.latestSeqNumberTaskManifest
 
 	// Check if the sequence number of message received is more than the one stored in Agent
@@ -255,7 +256,7 @@ func (taskManifestHandler *taskManifestHandler) handleTaskManifestSingleMessage(
 			return err
 		}
 
-		tasksToKill := compareTasks(taskListManifestHandler, runningTasksOnInstance)
+		tasksToKill := compareTasks(taskListManifestHandler, runningTasksOnInstance, clusterARN)
 
 		// Update messageId so that it can be compared to the messageId in TaskStopVerificationAck message
 		taskManifestHandler.setMessageId(*message.MessageId)

agent/acs/handler/task_manifest_handler_test.go

@@ -68,8 +68,8 @@ func TestManifestHandlerKillAllTasks(t *testing.T) {
 
 	//Task that needs to be stopped, sent back by agent
 	taskIdentifierFinal := []*ecsacs.TaskIdentifier{
-		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn1")},
-		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2")},
+		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn1"), TaskClusterArn: aws.String(cluster)},
+		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2"), TaskClusterArn: aws.String(cluster)},
 	}
 
 	taskStopVerificationMessage := &ecsacs.TaskStopVerificationMessage{
@@ -107,7 +107,7 @@ func TestManifestHandlerKillAllTasks(t *testing.T) {
 		ClusterArn:           aws.String(cluster),
 		ContainerInstanceArn: aws.String(containerInstanceArn),
 		Tasks: []*ecsacs.TaskIdentifier{
-			{DesiredStatus: aws.String("STOPPED"), TaskArn: aws.String("arn-long")},
+			{DesiredStatus: aws.String("STOPPED"), TaskArn: aws.String("arn-long"), TaskClusterArn: aws.String(cluster)},
 		},
 		Timeline: aws.Int64(testSeqNum),
 	}
@@ -164,8 +164,8 @@ func TestManifestHandlerKillFewTasks(t *testing.T) {
 
 	//Task that needs to be stopped, sent back by agent
 	taskIdentifierFinal := []*ecsacs.TaskIdentifier{
-		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2")},
-		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn3")},
+		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2"), TaskClusterArn: aws.String(cluster)},
+		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn3"), TaskClusterArn: aws.String(cluster)},
 	}
 
 	taskStopVerificationMessage := &ecsacs.TaskStopVerificationMessage{
@@ -202,12 +202,14 @@ func TestManifestHandlerKillFewTasks(t *testing.T) {
 		ContainerInstanceArn: aws.String(containerInstanceArn),
 		Tasks: []*ecsacs.TaskIdentifier{
 			{
-				DesiredStatus: aws.String(apitaskstatus.TaskRunningString),
-				TaskArn:       aws.String("arn1"),
+				DesiredStatus:  aws.String(apitaskstatus.TaskRunningString),
+				TaskArn:        aws.String("arn1"),
+				TaskClusterArn: aws.String(cluster),
 			},
 			{
-				DesiredStatus: aws.String(apitaskstatus.TaskStoppedString),
-				TaskArn:       aws.String("arn2"),
+				DesiredStatus:  aws.String(apitaskstatus.TaskStoppedString),
+				TaskArn:        aws.String("arn2"),
+				TaskClusterArn: aws.String(cluster),
 			},
 		},
 		Timeline: aws.Int64(testSeqNum),
@@ -355,20 +357,21 @@ func TestManifestHandlerDifferentTaskLists(t *testing.T) {
 
 	// tasks that suppose to be running
 	taskIdentifierInitial := ecsacs.TaskIdentifier{
-		DesiredStatus: aws.String(apitaskstatus.TaskStoppedString),
-		TaskArn:       aws.String("arn1"),
+		DesiredStatus:  aws.String(apitaskstatus.TaskStoppedString),
+		TaskArn:        aws.String("arn1"),
+		TaskClusterArn: aws.String(cluster),
 	}
 
 	//Task that needs to be stopped, sent back by agent
 	taskIdentifierAckFinal := []*ecsacs.TaskIdentifier{
-		{DesiredStatus: aws.String(apitaskstatus.TaskRunningString), TaskArn: aws.String("arn1")},
-		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2")},
+		{DesiredStatus: aws.String(apitaskstatus.TaskRunningString), TaskArn: aws.String("arn1"), TaskClusterArn: aws.String(cluster)},
+		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2"), TaskClusterArn: aws.String(cluster)},
 	}
 
 	//Task that needs to be stopped, sent back by agent
 	taskIdentifierMessage := []*ecsacs.TaskIdentifier{
-		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn1")},
-		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2")},
+		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn1"), TaskClusterArn: aws.String(cluster)},
+		{DesiredStatus: aws.String(apitaskstatus.TaskStoppedString), TaskArn: aws.String("arn2"), TaskClusterArn: aws.String(cluster)},
 	}
 
 	taskStopVerificationMessage := &ecsacs.TaskStopVerificationMessage{
@@ -518,7 +521,7 @@ func TestCompareTasksDifferentTasks(t *testing.T) {
 		{Arn: "arn1", DesiredStatusUnsafe: apitaskstatus.TaskRunning},
 	}
 
-	compareTaskList := compareTasks(receivedTaskList, taskList)
+	compareTaskList := compareTasks(receivedTaskList, taskList, "test-cluster-arn")
 
 	assert.Equal(t, 2, len(compareTaskList))
 }
@@ -540,7 +543,7 @@ func TestCompareTasksSameTasks(t *testing.T) {
 		{Arn: "arn1", DesiredStatusUnsafe: apitaskstatus.TaskRunning},
 	}
 
-	compareTaskList := compareTasks(receivedTaskList, taskList)
+	compareTaskList := compareTasks(receivedTaskList, taskList, "test-cluster-arn")
 
 	assert.Equal(t, 0, len(compareTaskList))
 }

agent/acs/update_handler/updater.go

@@ -106,7 +106,7 @@ func (u *updater) stageUpdateHandler() func(req *ecsacs.StageUpdateMessage) {
 			u.reset()
 		}
 
-		if !u.config.UpdatesEnabled {
+		if !u.config.UpdatesEnabled.Enabled() {
 			nack("Updates are disabled")
 			return
 		}
@@ -225,7 +225,7 @@ func (u *updater) performUpdateHandler(state dockerstate.TaskEngineState, dataCl
 
 		seelog.Debug("Got perform update request")
 
-		if !u.config.UpdatesEnabled {
+		if !u.config.UpdatesEnabled.Enabled() {
 			reason := "Updates are disabled"
 			seelog.Errorf("Nacking PerformUpdate; reason: %s", reason)
 			u.acs.MakeRequest(&ecsacs.NackRequest{

agent/acs/update_handler/updater_test.go

@@ -50,7 +50,7 @@ func ptr(i interface{}) interface{} {
 func mocks(t *testing.T, cfg *config.Config) (*updater, *gomock.Controller, *config.Config, *mock_client.MockClientServer, *mock_http.MockRoundTripper) {
 	if cfg == nil {
 		cfg = &config.Config{
-			UpdatesEnabled:    true,
+			UpdatesEnabled:    config.BooleanDefaultFalse{Value: config.ExplicitlyEnabled},
 			UpdateDownloadDir: filepath.Clean("/tmp/test/"),
 		}
 	}
@@ -91,7 +91,7 @@ func mockOS() func() {
 }
 func TestStageUpdateWithUpdatesDisabled(t *testing.T) {
 	u, ctrl, _, mockacs, _ := mocks(t, &config.Config{
-		UpdatesEnabled: false,
+		UpdatesEnabled: config.BooleanDefaultFalse{Value: config.ExplicitlyDisabled},
 	})
 	defer ctrl.Finish()
 
@@ -116,7 +116,7 @@ func TestStageUpdateWithUpdatesDisabled(t *testing.T) {
 
 func TestPerformUpdateWithUpdatesDisabled(t *testing.T) {
 	u, ctrl, cfg, mockacs, _ := mocks(t, &config.Config{
-		UpdatesEnabled: false,
+		UpdatesEnabled: config.BooleanDefaultFalse{Value: config.ExplicitlyDisabled},
 	})
 	defer ctrl.Finish()
 

agent/api/task/task.go

@@ -298,7 +298,7 @@ func (task *Task) initializeVolumes(cfg *config.Config, dockerClient dockerapi.D
 	if err != nil {
 		return apierrors.NewResourceInitError(task.Arn, err)
 	}
-	err = task.initializeDockerVolumes(cfg.SharedVolumeMatchFullConfig, dockerClient, ctx)
+	err = task.initializeDockerVolumes(cfg.SharedVolumeMatchFullConfig.Enabled(), dockerClient, ctx)
 	if err != nil {
 		return apierrors.NewResourceInitError(task.Arn, err)
 	}

agent/api/task/task_linux_test.go

@@ -568,7 +568,7 @@ func TestPostUnmarshalWithCPULimitsFail(t *testing.T) {
 		ResourcesMapUnsafe: make(map[string][]taskresource.TaskResource),
 	}
 	cfg := config.Config{
-		TaskCPUMemLimit: config.ExplicitlyEnabled,
+		TaskCPUMemLimit: config.BooleanDefaultTrue{Value: config.ExplicitlyEnabled},
 	}
 	assert.Error(t, task.PostUnmarshalTask(&cfg, nil, nil, nil, nil))
 	assert.Equal(t, 0, len(task.GetResources()))

agent/api/task/task_windows.go

@@ -47,10 +47,10 @@ const (
 type PlatformFields struct {
 	// CpuUnbounded determines whether a mix of unbounded and bounded CPU tasks
 	// are allowed to run in the instance
-	CpuUnbounded bool `json:"cpuUnbounded"`
+	CpuUnbounded config.BooleanDefaultFalse `json:"cpuUnbounded"`
 	// MemoryUnbounded determines whether a mix of unbounded and bounded Memory tasks
 	// are allowed to run in the instance
-	MemoryUnbounded bool `json:"memoryUnbounded"`
+	MemoryUnbounded config.BooleanDefaultFalse `json:"memoryUnbounded"`
 }
 
 var cpuShareScaleFactor = runtime.NumCPU() * cpuSharesPerCore
@@ -130,7 +130,7 @@ func (task *Task) platformHostConfigOverride(hostConfig *dockercontainer.HostCon
 	}
 	hostConfig.CPUShares = 0
 
-	if hostConfig.Memory <= 0 && task.PlatformFields.MemoryUnbounded {
+	if hostConfig.Memory <= 0 && task.PlatformFields.MemoryUnbounded.Enabled() {
 		// As of version  17.06.2-ee-6 of docker. MemoryReservation is not supported on windows. This ensures that
 		// this parameter is not passed, allowing to launch a container without a hard limit.
 		hostConfig.MemoryReservation = 0
@@ -144,7 +144,7 @@ func (task *Task) platformHostConfigOverride(hostConfig *dockercontainer.HostCon
 // want.  Instead, we convert 0 to 2 to be closer to expected behavior. The
 // reason for 2 over 1 is that 1 is an invalid value (Linux's choice, not Docker's).
 func (task *Task) dockerCPUShares(containerCPU uint) int64 {
-	if containerCPU <= 1 && !task.PlatformFields.CpuUnbounded {
+	if containerCPU <= 1 && !task.PlatformFields.CpuUnbounded.Enabled() {
 		seelog.Debugf(
 			"Converting CPU shares to allowed minimum of 2 for task arn: [%s] and cpu shares: %d",
 			task.Arn, containerCPU)

agent/api/task/task_windows_test.go

@@ -110,7 +110,7 @@ func TestPostUnmarshalWindowsCanonicalPaths(t *testing.T) {
 	seqNum := int64(42)
 	task, err := TaskFromACS(&taskFromAcs, &ecsacs.PayloadMessage{SeqNum: &seqNum})
 	assert.Nil(t, err, "Should be able to handle acs task")
-	cfg := config.Config{TaskCPUMemLimit: config.ExplicitlyDisabled}
+	cfg := config.Config{TaskCPUMemLimit: config.BooleanDefaultTrue{Value: config.ExplicitlyDisabled}}
 	task.PostUnmarshalTask(&cfg, nil, nil, nil, nil)
 
 	for _, container := range task.Containers { // remove v3 endpoint from each container because it's randomly generated
@@ -211,43 +211,43 @@ func TestCPUPercentBasedOnUnboundedEnabled(t *testing.T) {
 	cpuShareScaleFactor := runtime.NumCPU() * cpuSharesPerCore
 	testcases := []struct {
 		cpu          int64
-		cpuUnbounded bool
+		cpuUnbounded config.BooleanDefaultFalse
 		cpuPercent   int64
 	}{
 		{
 			cpu:          0,
-			cpuUnbounded: true,
+			cpuUnbounded: config.BooleanDefaultFalse{Value: config.ExplicitlyEnabled},
 			cpuPercent:   0,
 		},
 		{
 			cpu:          1,
-			cpuUnbounded: true,
+			cpuUnbounded: config.BooleanDefaultFalse{Value: config.ExplicitlyEnabled},
 			cpuPercent:   1,
 		},
 		{
 			cpu:          0,
-			cpuUnbounded: false,
+			cpuUnbounded: config.BooleanDefaultFalse{Value: config.ExplicitlyDisabled},
 			cpuPercent:   1,
 		},
 		{
 			cpu:          1,
-			cpuUnbounded: false,
+			cpuUnbounded: config.BooleanDefaultFalse{Value: config.ExplicitlyDisabled},
 			cpuPercent:   1,
 		},
 		{
 			cpu:          100,
-			cpuUnbounded: true,
+			cpuUnbounded: config.BooleanDefaultFalse{Value: config.ExplicitlyEnabled},
 			cpuPercent:   100 * percentageFactor / int64(cpuShareScaleFactor),
 		},
 		{
 			cpu:          100,
-			cpuUnbounded: false,
+			cpuUnbounded: config.BooleanDefaultFalse{Value: config.ExplicitlyDisabled},
 			cpuPercent:   100 * percentageFactor / int64(cpuShareScaleFactor),
 		},
 	}
 	for _, tc := range testcases {
 		t.Run(fmt.Sprintf("container cpu-%d,cpu unbounded tasks enabled- %t,expected cpu percent-%d",
-			tc.cpu, tc.cpuUnbounded, tc.cpuPercent), func(t *testing.T) {
+			tc.cpu, tc.cpuUnbounded.Enabled(), tc.cpuPercent), func(t *testing.T) {
 			testTask := &Task{
 				Containers: []*apicontainer.Container{
 					{
@@ -295,7 +295,7 @@ func TestWindowsMemoryReservationOption(t *testing.T) {
 			},
 		},
 		PlatformFields: PlatformFields{
-			MemoryUnbounded: false,
+			MemoryUnbounded: config.BooleanDefaultFalse{Value: config.ExplicitlyDisabled},
 		},
 	}
 
@@ -307,7 +307,7 @@ func TestWindowsMemoryReservationOption(t *testing.T) {
 	assert.EqualValues(t, nonZeroMemoryReservationValue, cfg.MemoryReservation)
 
 	// With MemoryUnbounded set to true, tasks with no memory hard limit will have their memory reservation set to zero
-	testTask.PlatformFields.MemoryUnbounded = true
+	testTask.PlatformFields.MemoryUnbounded = config.BooleanDefaultFalse{Value: config.ExplicitlyEnabled}
 	cfg, configErr = testTask.DockerHostConfig(testTask.Containers[0], dockerMap(testTask),
 		defaultDockerClientAPIVersion, &config.Config{})