From 0501dd3a5cf27bbf2a514ccdbe0b1cc6c9e82152 Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Thu, 3 Jun 2021 13:29:46 -0700 Subject: [PATCH 01/14] agent_capability: Decoupled Linux/Windows Execcmd checks --- agent/app/agent_capability.go | 6 ------ agent/app/agent_capability_unix.go | 19 ++++++++++++++----- agent/app/agent_capability_unspecified.go | 8 ++++++++ agent/app/agent_capability_windows.go | 12 ++++++++++++ agent/config/config_windows.go | 5 +++++ 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/agent/app/agent_capability.go b/agent/app/agent_capability.go index 991b3cbcfdf..7f120a149c1 100644 --- a/agent/app/agent_capability.go +++ b/agent/app/agent_capability.go @@ -68,7 +68,6 @@ const ( capabilityEnvFilesS3 = "env-files.s3" capabilityFSxWindowsFileServer = "fsxWindowsFileServer" capabilityExec = "execute-command" - capabilityDepsRootDir = "/managed-agents" capabilityExecBinRelativePath = "bin" capabilityExecConfigRelativePath = "config" capabilityExecCertsRelativePath = "certs" @@ -97,11 +96,6 @@ var ( // ecs agent version 1.39.0 supports bulk loading env vars through environmentFiles in S3 capabilityEnvFilesS3, } - capabilityExecRequiredBinaries = []string{ - "amazon-ssm-agent", - "ssm-agent-worker", - "ssm-session-worker", - } capabilityExecRequiredCerts = []string{ "tls-ca-bundle.pem", } diff --git a/agent/app/agent_capability_unix.go b/agent/app/agent_capability_unix.go index 7d5a663aa8a..e8f15e6b275 100644 --- a/agent/app/agent_capability_unix.go +++ b/agent/app/agent_capability_unix.go @@ -30,11 +30,20 @@ import ( ) const ( - AVX = "avx" - AVX2 = "avx2" - SSE41 = "sse4_1" - SSE42 = "sse4_2" - CpuInfoPath = "/proc/cpuinfo" + AVX = "avx" + AVX2 = "avx2" + SSE41 = "sse4_1" + SSE42 = "sse4_2" + CpuInfoPath = "/proc/cpuinfo" + capabilityDepsRootDir = "/managed-agents" +) + +var ( + capabilityExecRequiredBinaries = []string{ + "amazon-ssm-agent", + "ssm-agent-worker", + "ssm-session-worker", + } ) func (agent *ecsAgent) appendVolumeDriverCapabilities(capabilities []*ecs.Attribute) []*ecs.Attribute { diff --git a/agent/app/agent_capability_unspecified.go b/agent/app/agent_capability_unspecified.go index dc091113888..62824b71e2f 100644 --- a/agent/app/agent_capability_unspecified.go +++ b/agent/app/agent_capability_unspecified.go @@ -27,6 +27,14 @@ import ( "github.com/cihub/seelog" ) +const ( + capabilityDepsRootDir = "" +) + +var ( + capabilityExecRequiredBinaries = []string{} +) + func (agent *ecsAgent) appendVolumeDriverCapabilities(capabilities []*ecs.Attribute) []*ecs.Attribute { // "local" is default docker driver capabilities = appendNameOnlyAttribute(capabilities, attributePrefix+capabilityDockerPluginInfix+volume.DockerLocalVolumeDriver) diff --git a/agent/app/agent_capability_windows.go b/agent/app/agent_capability_windows.go index 1606f7446e4..a7c3c46f231 100644 --- a/agent/app/agent_capability_windows.go +++ b/agent/app/agent_capability_windows.go @@ -16,6 +16,9 @@ package app import ( + "path/filepath" + + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs" "github.com/aws/amazon-ecs-agent/agent/ecscni" "github.com/aws/amazon-ecs-agent/agent/taskresource/volume" @@ -23,6 +26,15 @@ import ( "github.com/cihub/seelog" ) +var ( + capabilityDepsRootDir = filepath.Join(config.AmazonECSProgramFiles, "managed-agents") + capabilityExecRequiredBinaries = []string{ + "amazon-ssm-agent.exe", + "ssm-agent-worker.exe", + "ssm-session-worker.exe", + } +) + func (agent *ecsAgent) appendVolumeDriverCapabilities(capabilities []*ecs.Attribute) []*ecs.Attribute { // "local" is default docker driver return appendNameOnlyAttribute(capabilities, attributePrefix+capabilityDockerPluginInfix+volume.DockerLocalVolumeDriver) diff --git a/agent/config/config_windows.go b/agent/config/config_windows.go index a5162f46eb2..3ea3ac8bf18 100644 --- a/agent/config/config_windows.go +++ b/agent/config/config_windows.go @@ -72,6 +72,11 @@ const ( defaultCNIPluginDirName = "cni" ) +var ( + envProgramFiles = utils.DefaultIfBlank(os.Getenv("ProgramFiles"), `C:\Program Files`) + AmazonECSProgramFiles = filepath.Join(envProgramFiles, "Amazon", "ECS") +) + // DefaultConfig returns the default configuration for Windows func DefaultConfig() Config { programData := utils.DefaultIfBlank(os.Getenv("ProgramData"), `C:\ProgramData`) From cf7cc7f99442466669252182f59e9af6cd148cd0 Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Tue, 15 Jun 2021 09:57:47 -0700 Subject: [PATCH 02/14] agent_capability: added checks for SSM plugin and removed cert checks on Windows --- agent/app/agent_capability.go | 18 ++++-------------- agent/app/agent_capability_unix.go | 12 ++++++++++++ agent/app/agent_capability_unspecified.go | 1 + agent/app/agent_capability_windows.go | 17 ++++++++++++++++- agent/config/config_windows.go | 1 + 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/agent/app/agent_capability.go b/agent/app/agent_capability.go index 7f120a149c1..d0164f34ce9 100644 --- a/agent/app/agent_capability.go +++ b/agent/app/agent_capability.go @@ -96,9 +96,6 @@ var ( // ecs agent version 1.39.0 supports bulk loading env vars through environmentFiles in S3 capabilityEnvFilesS3, } - capabilityExecRequiredCerts = []string{ - "tls-ca-bundle.pem", - } // use empty struct as value type to simulate set capabilityExecInvalidSsmVersions = map[string]struct{}{} @@ -121,6 +118,10 @@ var ( externalSpecificCapabilities = []string{ attributePrefix + capabilityExternal, } + + capabilityExecRootDir = filepath.Join(capabilityDepsRootDir, capabilityExec) + binDir = filepath.Join(capabilityExecRootDir, capabilityExecBinRelativePath) + configDir = filepath.Join(capabilityExecRootDir, capabilityExecConfigRelativePath) ) // capabilities returns the supported capabilities of this agent / docker-client pair. @@ -381,17 +382,6 @@ func (agent *ecsAgent) appendExecCapabilities(capabilities []*ecs.Attribute) ([] // for an instance to be exec-enabled, it needs resources needed by SSM (binaries, configuration files and certs) // the following bind mounts are defined in ecs-init and added to the ecs-agent container - capabilityExecRootDir := filepath.Join(capabilityDepsRootDir, capabilityExec) - binDir := filepath.Join(capabilityExecRootDir, capabilityExecBinRelativePath) - configDir := filepath.Join(capabilityExecRootDir, capabilityExecConfigRelativePath) - certsDir := filepath.Join(capabilityExecRootDir, capabilityExecCertsRelativePath) - - // top-level folders, /bin, /config, /certs - dependencies := map[string][]string{ - binDir: []string{}, - configDir: []string{}, - certsDir: capabilityExecRequiredCerts, - } if exists, err := dependenciesExist(dependencies); err != nil || !exists { return capabilities, err } diff --git a/agent/app/agent_capability_unix.go b/agent/app/agent_capability_unix.go index e8f15e6b275..eff6534a01e 100644 --- a/agent/app/agent_capability_unix.go +++ b/agent/app/agent_capability_unix.go @@ -16,6 +16,7 @@ package app import ( + "path/filepath" "strings" "github.com/aws/amazon-ecs-agent/agent/config" @@ -39,11 +40,22 @@ const ( ) var ( + certsDir = filepath.Join(capabilityExecRootDir, capabilityExecCertsRelativePath) + capabilityExecRequiredCerts = []string{ + "tls-ca-bundle.pem", + } capabilityExecRequiredBinaries = []string{ "amazon-ssm-agent", "ssm-agent-worker", "ssm-session-worker", } + + // top-level folders, /bin, /config, /certs + dependencies = map[string][]string{ + binDir: []string{}, + configDir: []string{}, + certsDir: capabilityExecRequiredCerts, + } ) func (agent *ecsAgent) appendVolumeDriverCapabilities(capabilities []*ecs.Attribute) []*ecs.Attribute { diff --git a/agent/app/agent_capability_unspecified.go b/agent/app/agent_capability_unspecified.go index 62824b71e2f..0a62f9e810f 100644 --- a/agent/app/agent_capability_unspecified.go +++ b/agent/app/agent_capability_unspecified.go @@ -33,6 +33,7 @@ const ( var ( capabilityExecRequiredBinaries = []string{} + dependencies = map[string][]string{} ) func (agent *ecsAgent) appendVolumeDriverCapabilities(capabilities []*ecs.Attribute) []*ecs.Attribute { diff --git a/agent/app/agent_capability_windows.go b/agent/app/agent_capability_windows.go index a7c3c46f231..25a8a7e33c4 100644 --- a/agent/app/agent_capability_windows.go +++ b/agent/app/agent_capability_windows.go @@ -27,12 +27,27 @@ import ( ) var ( - capabilityDepsRootDir = filepath.Join(config.AmazonECSProgramFiles, "managed-agents") + capabilityDepsRootDir = filepath.Join(config.AmazonECSProgramFiles, "managed-agents") + ssmPluginDir = filepath.Join(config.AmazonProgramFiles, "SSM", "Plugins") + sessionManagerShellDir = filepath.Join(ssmPluginDir, "SessionManagerShell") + awsCloudWatchDir = filepath.Join(ssmPluginDir, "awsCloudWatch") + awsDomainJoin = filepath.Join(ssmPluginDir, "awsDomainJoin") + capabilityExecRequiredBinaries = []string{ "amazon-ssm-agent.exe", "ssm-agent-worker.exe", "ssm-session-worker.exe", } + + // top-level folders, /bin, /config, /plugins + dependencies = map[string][]string{ + binDir: []string{}, + configDir: []string{}, + ssmPluginDir: []string{}, + sessionManagerShellDir: []string{}, + awsCloudWatchDir: []string{}, + awsDomainJoin: []string{}, + } ) func (agent *ecsAgent) appendVolumeDriverCapabilities(capabilities []*ecs.Attribute) []*ecs.Attribute { diff --git a/agent/config/config_windows.go b/agent/config/config_windows.go index 3ea3ac8bf18..bf10cb4085e 100644 --- a/agent/config/config_windows.go +++ b/agent/config/config_windows.go @@ -74,6 +74,7 @@ const ( var ( envProgramFiles = utils.DefaultIfBlank(os.Getenv("ProgramFiles"), `C:\Program Files`) + AmazonProgramFiles = filepath.Join(envProgramFiles, "Amazon") AmazonECSProgramFiles = filepath.Join(envProgramFiles, "Amazon", "ECS") ) From 157ddfe60d53da671a03aae4f50cad646454992d Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Wed, 16 Jun 2021 14:05:08 -0700 Subject: [PATCH 03/14] execcmd manager init/start task: Decoupled Linux and Windows code files --- agent/engine/execcmd/manager.go | 3 -- .../execcmd/manager_init_task_windows.go | 33 +++++++++++++++++++ agent/engine/execcmd/manager_linux.go | 21 ++++++++++++ agent/engine/execcmd/manager_start_windows.go | 19 +++++++++++ agent/engine/execcmd/manager_unsupported.go | 9 ++--- agent/engine/execcmd/manager_windows.go | 27 +++++++++++++++ 6 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 agent/engine/execcmd/manager_init_task_windows.go create mode 100644 agent/engine/execcmd/manager_linux.go create mode 100644 agent/engine/execcmd/manager_start_windows.go create mode 100644 agent/engine/execcmd/manager_windows.go diff --git a/agent/engine/execcmd/manager.go b/agent/engine/execcmd/manager.go index af5822333a5..05523713555 100644 --- a/agent/engine/execcmd/manager.go +++ b/agent/engine/execcmd/manager.go @@ -27,9 +27,6 @@ import ( ) const ( - hostExecDepsDir = "/var/lib/ecs/deps/execute-command" - HostBinDir = hostExecDepsDir + "/bin" - ExecuteCommandAgentName = ecs.ManagedAgentNameExecuteCommandAgent defaultStartRetryTimeout = time.Minute * 10 defaultRetryMinDelay = time.Second * 1 diff --git a/agent/engine/execcmd/manager_init_task_windows.go b/agent/engine/execcmd/manager_init_task_windows.go new file mode 100644 index 00000000000..95e9841b5a7 --- /dev/null +++ b/agent/engine/execcmd/manager_init_task_windows.go @@ -0,0 +1,33 @@ +// +build windows + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +import ( + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + dockercontainer "github.com/docker/docker/api/types/container" +) + +const ( + // ECSAgentExecLogDir here is used used while cleaning up exec logs when task exits. + // When this path is empty, nothing is cleaned up for unsupported platforms. + ECSAgentExecLogDir = "" +) + +// InitializeContainer adds the necessary bind mounts in order for the ExecCommandAgent to run properly in the container +// Note: exec cmd agent is a linux feature, thus implemented here as a no-op. +func (m *manager) InitializeContainer(taskId string, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) error { + return nil +} diff --git a/agent/engine/execcmd/manager_linux.go b/agent/engine/execcmd/manager_linux.go new file mode 100644 index 00000000000..5984eceb97c --- /dev/null +++ b/agent/engine/execcmd/manager_linux.go @@ -0,0 +1,21 @@ +// +build linux + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +const ( + hostExecDepsDir = "/var/lib/ecs/deps/execute-command" + HostBinDir = hostExecDepsDir + "/bin" +) diff --git a/agent/engine/execcmd/manager_start_windows.go b/agent/engine/execcmd/manager_start_windows.go new file mode 100644 index 00000000000..641951096e9 --- /dev/null +++ b/agent/engine/execcmd/manager_start_windows.go @@ -0,0 +1,19 @@ +package execcmd + +import ( + "context" + + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + apitask "github.com/aws/amazon-ecs-agent/agent/api/task" + "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi" +) + +// Note: exec cmd agent is a linux/windows feature, thus implemented here as a no-op. +func (m *manager) RestartAgentIfStopped(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) (RestartStatus, error) { + return NotRestarted, nil +} + +// Note: exec cmd agent is a linux/windows feature, thus implemented here as a no-op. +func (m *manager) StartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) error { + return nil +} diff --git a/agent/engine/execcmd/manager_unsupported.go b/agent/engine/execcmd/manager_unsupported.go index f6a41ff6d40..4a8124c3b2b 100644 --- a/agent/engine/execcmd/manager_unsupported.go +++ b/agent/engine/execcmd/manager_unsupported.go @@ -1,4 +1,4 @@ -//go:build !linux +//go:build !linux && !windows // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // @@ -27,20 +27,21 @@ const ( // ECSAgentExecLogDir here is used used while cleaning up exec logs when task exits. // When this path is empty, nothing is cleaned up for unsupported platforms. ECSAgentExecLogDir = "" + HostBinDir = "" ) -// Note: exec cmd agent is a linux-only feature, thus implemented here as a no-op. +// Note: exec cmd agent is a linux/windows feature, thus implemented here as a no-op. func (m *manager) RestartAgentIfStopped(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) (RestartStatus, error) { return NotRestarted, nil } -// Note: exec cmd agent is a linux-only feature, thus implemented here as a no-op. +// Note: exec cmd agent is a linux/windows feature, thus implemented here as a no-op. func (m *manager) StartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) error { return nil } // InitializeContainer adds the necessary bind mounts in order for the ExecCommandAgent to run properly in the container -// Note: exec cmd agent is a linux-only feature, thus implemented here as a no-op. +// Note: exec cmd agent is a linux/windows feature, thus implemented here as a no-op. func (m *manager) InitializeContainer(taskId string, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) error { return nil } diff --git a/agent/engine/execcmd/manager_windows.go b/agent/engine/execcmd/manager_windows.go new file mode 100644 index 00000000000..085b52bbe26 --- /dev/null +++ b/agent/engine/execcmd/manager_windows.go @@ -0,0 +1,27 @@ +// +build windows + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +import ( + "path/filepath" + + "github.com/aws/amazon-ecs-agent/agent/config" +) + +var ( + hostExecDepsDir = filepath.Join(config.AmazonECSProgramFiles, "managed-agents", "execute-command") + HostBinDir = filepath.Join(hostExecDepsDir, "bin") +) From 59654b7b52d59df01d39a2cd6ed554a19076f756 Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Fri, 18 Jun 2021 14:20:23 -0700 Subject: [PATCH 04/14] execcmd manager init task: Windows functionality added --- agent/config/config_windows.go | 9 +- agent/engine/execcmd/manager_init_task.go | 208 ++++++++++++++ .../engine/execcmd/manager_init_task_linux.go | 271 ++++-------------- .../execcmd/manager_init_task_linux_test.go | 34 --- .../engine/execcmd/manager_init_task_test.go | 57 ++++ .../execcmd/manager_init_task_windows.go | 169 ++++++++++- .../execcmd/manager_init_task_windows_test.go | 260 +++++++++++++++++ 7 files changed, 748 insertions(+), 260 deletions(-) create mode 100644 agent/engine/execcmd/manager_init_task.go create mode 100644 agent/engine/execcmd/manager_init_task_test.go create mode 100644 agent/engine/execcmd/manager_init_task_windows_test.go diff --git a/agent/config/config_windows.go b/agent/config/config_windows.go index bf10cb4085e..181c9545af9 100644 --- a/agent/config/config_windows.go +++ b/agent/config/config_windows.go @@ -73,9 +73,14 @@ const ( ) var ( - envProgramFiles = utils.DefaultIfBlank(os.Getenv("ProgramFiles"), `C:\Program Files`) - AmazonProgramFiles = filepath.Join(envProgramFiles, "Amazon") + envProgramFiles = utils.DefaultIfBlank(os.Getenv("ProgramFiles"), `C:\Program Files`) + envProgramData = utils.DefaultIfBlank(os.Getenv("ProgramData"), `C:\ProgramData`) + + AmazonProgramFiles = filepath.Join(envProgramFiles, "Amazon") + AmazonProgramData = filepath.Join(envProgramData, "Amazon") + AmazonECSProgramFiles = filepath.Join(envProgramFiles, "Amazon", "ECS") + AmazonECSProgramData = filepath.Join(AmazonProgramData, "ECS") ) // DefaultConfig returns the default configuration for Windows diff --git a/agent/engine/execcmd/manager_init_task.go b/agent/engine/execcmd/manager_init_task.go new file mode 100644 index 00000000000..24808342a0d --- /dev/null +++ b/agent/engine/execcmd/manager_init_task.go @@ -0,0 +1,208 @@ +// +build linux windows + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + apicontainerstatus "github.com/aws/amazon-ecs-agent/agent/api/container/status" + dockercontainer "github.com/docker/docker/api/types/container" + + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + "github.com/pborman/uuid" +) + +const ( + namelessContainerPrefix = "nameless-container-" + + // filePerm is the permission for the exec agent config file. + filePerm = 0644 + defaultSessionLimit = 2 + + containerConfigFileName = "amazon-ssm-agent.json" + ContainerConfigDirName = "config" + + ExecAgentLogConfigFileName = "seelog.xml" +) + +var ( + execAgentConfigTemplate = `{ + "Mgs": { + "Region": "", + "Endpoint": "", + "StopTimeoutMillis": 20000, + "SessionWorkersLimit": %d + }, + "Agent": { + "Region": "", + "OrchestrationRootDir": "", + "ContainerMode": true + } +}` + + errExecCommandManagedAgentNotFound = fmt.Errorf("managed agent not found (%s)", ExecuteCommandAgentName) +) + +// InitializeContainer adds the necessary bind mounts in order for the ExecCommandAgent to run properly in the container +// TODO: [ecs-exec] Should we validate the ssm agent binaries & certs are valid and fail here if they're not? (bind mount will succeed even if files don't exist in host) +func (m *manager) InitializeContainer(taskId string, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) (rErr error) { + defer func() { + if rErr != nil { + container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ + InitFailed: true, + Status: apicontainerstatus.ManagedAgentStopped, + Reason: rErr.Error(), + }) + } + }() + ma, ok := container.GetManagedAgentByName(ExecuteCommandAgentName) + if !ok { + return errExecCommandManagedAgentNotFound + } + sessionWorkersLimit := getSessionWorkersLimit(ma) + cn := fileSystemSafeContainerName(container) + uuid := newUUID() + + latestBinVersionDir, rErr := m.getLatestVersionedHostBinDir() + if rErr != nil { + return rErr + } + + rErr = addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid, sessionWorkersLimit, hostConfig) + if rErr != nil { + return rErr + } + + container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ + ID: uuid, + }) + + return nil +} + +func (m *manager) getLatestVersionedHostBinDir() (string, error) { + versions, err := retrieveAgentVersions(ecsAgentDepsBinDir) + if err != nil { + return "", err + } + sort.Sort(sort.Reverse(byAgentVersion(versions))) + + var latest string + for _, v := range versions { + vStr := v.String() + ecsAgentDepsVersionedBinDir := filepath.Join(ecsAgentDepsBinDir, vStr) + if !fileExists(filepath.Join(ecsAgentDepsVersionedBinDir, SSMAgentBinName)) { + continue // try falling back to the previous version + } + // TODO: [ecs-exec] This requirement will be removed for SSM agent V2 + if !fileExists(filepath.Join(ecsAgentDepsVersionedBinDir, SSMAgentWorkerBinName)) { + continue // try falling back to the previous version + } + if !fileExists(filepath.Join(ecsAgentDepsVersionedBinDir, SessionWorkerBinName)) { + continue // try falling back to the previous version + } + latest = filepath.Join(m.hostBinDir, vStr) + break + } + if latest == "" { + return "", fmt.Errorf("no valid versions were found in %s", m.hostBinDir) + } + return latest, nil +} + +func getReadOnlyBindMountMapping(hostDir, containerDir string) string { + return getBindMountMapping(hostDir, containerDir) + ":ro" +} + +func getBindMountMapping(hostDir, containerDir string) string { + return hostDir + ":" + containerDir +} + +var newUUID = uuid.New + +func fileSystemSafeContainerName(c *apicontainer.Container) string { + // Trim leading hyphens since they're not valid directory names + cn := strings.TrimLeft(c.Name, "-") + if cn == "" { + // Fallback name in the extreme case that we end up with an empty string after trimming all leading hyphens. + return namelessContainerPrefix + newUUID() + } + return cn +} + +func getSessionWorkersLimit(ma apicontainer.ManagedAgent) int { + // TODO [ecs-exec] : verify that returning the default session limit (2) is ok in case of any errors, misconfiguration + limit := defaultSessionLimit + if ma.Properties == nil { // This means ACS didn't send the limit + return limit + } + limitStr, ok := ma.Properties["sessionLimit"] + if !ok { // This also means ACS didn't send the limit + return limit + } + limit, err := strconv.Atoi(limitStr) + if err != nil { // This means ACS send a limit that can't be converted to an int + return limit + } + if limit <= 0 { + limit = defaultSessionLimit + } + return limit +} + +var removeAll = os.RemoveAll + +var getFileContent = readFileContent + +func readFileContent(filePath string) ([]byte, error) { + return ioutil.ReadFile(filePath) +} + +func getExecAgentConfigHash(config string) string { + hash := sha256.New() + hash.Write([]byte(config)) + return base64.URLEncoding.EncodeToString(hash.Sum(nil)) +} + +var osStat = os.Stat + +func fileExists(path string) bool { + if fi, err := osStat(path); err == nil { + return !fi.IsDir() + } + return false +} + +func isDir(path string) bool { + if fi, err := osStat(path); err == nil { + return fi.IsDir() + } + return false +} + +var createNewExecAgentConfigFile = createNewConfigFile + +func createNewConfigFile(config, configFilePath string) error { + return ioutil.WriteFile(configFilePath, []byte(config), filePerm) +} diff --git a/agent/engine/execcmd/manager_init_task_linux.go b/agent/engine/execcmd/manager_init_task_linux.go index d811a5c3a3b..6398bf9bb1a 100644 --- a/agent/engine/execcmd/manager_init_task_linux.go +++ b/agent/engine/execcmd/manager_init_task_linux.go @@ -15,26 +15,14 @@ package execcmd import ( - "crypto/sha256" - "encoding/base64" "fmt" - "io/ioutil" - "os" "path/filepath" - "sort" - "strconv" "strings" dockercontainer "github.com/docker/docker/api/types/container" - "github.com/pborman/uuid" - - apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" - apicontainerstatus "github.com/aws/amazon-ecs-agent/agent/api/container/status" ) const ( - namelessContainerPrefix = "nameless-container-" - ecsAgentExecDepsDir = "/managed-agents/execute-command" // ecsAgentDepsBinDir is the directory where ECS Agent will read versions of SSM agent @@ -43,10 +31,6 @@ const ( ContainerDepsDirPrefix = "/ecs-execute-command-" - // filePerm is the permission for the exec agent config file. - filePerm = 0644 - defaultSessionLimit = 2 - SSMAgentBinName = "amazon-ssm-agent" SSMAgentWorkerBinName = "ssm-agent-worker" SessionWorkerBinName = "ssm-session-worker" @@ -58,32 +42,16 @@ const ( HostCertFile = "/var/lib/ecs/deps/execute-command/certs/tls-ca-bundle.pem" ContainerCertFileSuffix = "certs/amazon-ssm-agent.crt" - containerConfigFileName = "amazon-ssm-agent.json" - ContainerConfigDirName = "config" ContainerConfigFileSuffix = "configuration/" + containerConfigFileName // ECSAgentExecConfigDir is the directory where ECS Agent will write the ExecAgent config files to ECSAgentExecConfigDir = ecsAgentExecDepsDir + "/" + ContainerConfigDirName // HostExecConfigDir is the dir where ExecAgents Config files will live - HostExecConfigDir = hostExecDepsDir + "/" + ContainerConfigDirName - ExecAgentLogConfigFileName = "seelog.xml" - ContainerLogConfigFile = "configuration/" + ExecAgentLogConfigFileName + HostExecConfigDir = hostExecDepsDir + "/" + ContainerConfigDirName + ContainerLogConfigFile = "configuration/" + ExecAgentLogConfigFileName ) var ( - execAgentConfigTemplate = `{ - "Mgs": { - "Region": "", - "Endpoint": "", - "StopTimeoutMillis": 20000, - "SessionWorkersLimit": %d - }, - "Agent": { - "Region": "", - "OrchestrationRootDir": "", - "ContainerMode": true - } -}` execAgentLogConfigTemplate = ` @@ -107,161 +75,10 @@ var ( ` // TODO: [ecs-exec] seelog config needs to be implemented following a similar approach to ss, config - execAgentConfigFileNameTemplate = `amazon-ssm-agent-%s.json` - logConfigFileNameTemplate = `seelog-%s.xml` - errExecCommandManagedAgentNotFound = fmt.Errorf("managed agent not found (%s)", ExecuteCommandAgentName) + execAgentConfigFileNameTemplate = `amazon-ssm-agent-%s.json` + logConfigFileNameTemplate = `seelog-%s.xml` ) -// InitializeContainer adds the necessary bind mounts in order for the ExecCommandAgent to run properly in the container -// TODO: [ecs-exec] Should we validate the ssm agent binaries & certs are valid and fail here if they're not? (bind mount will succeed even if files don't exist in host) -func (m *manager) InitializeContainer(taskId string, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) (rErr error) { - defer func() { - if rErr != nil { - container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ - InitFailed: true, - Status: apicontainerstatus.ManagedAgentStopped, - Reason: rErr.Error(), - }) - } - }() - ma, ok := container.GetManagedAgentByName(ExecuteCommandAgentName) - if !ok { - return errExecCommandManagedAgentNotFound - } - configFile, rErr := GetExecAgentConfigFileName(getSessionWorkersLimit(ma)) - if rErr != nil { - rErr = fmt.Errorf("could not generate ExecAgent Config File: %v", rErr) - return rErr - } - logConfigFile, rErr := GetExecAgentLogConfigFile() - if rErr != nil { - rErr = fmt.Errorf("could not generate ExecAgent LogConfig file: %v", rErr) - return rErr - } - uuid := newUUID() - containerDepsFolder := ContainerDepsDirPrefix + uuid - - latestBinVersionDir, rErr := m.getLatestVersionedHostBinDir() - if rErr != nil { - return rErr - } - - if !certsExist() { - rErr = fmt.Errorf("could not find certs") - return rErr - } - - // Add ssm binary mounts - hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( - filepath.Join(latestBinVersionDir, SSMAgentBinName), - filepath.Join(containerDepsFolder, SSMAgentBinName))) - - hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( - filepath.Join(latestBinVersionDir, SSMAgentWorkerBinName), - filepath.Join(containerDepsFolder, SSMAgentWorkerBinName))) - - hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( - filepath.Join(latestBinVersionDir, SessionWorkerBinName), - filepath.Join(containerDepsFolder, SessionWorkerBinName))) - - // Add exec agent config file mount - hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( - filepath.Join(HostExecConfigDir, configFile), - filepath.Join(containerDepsFolder, ContainerConfigFileSuffix))) - - // Add exec agent log config file mount - hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( - filepath.Join(HostExecConfigDir, logConfigFile), - filepath.Join(containerDepsFolder, ContainerLogConfigFile))) - - // Append TLS cert mount - hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( - HostCertFile, - filepath.Join(containerDepsFolder, ContainerCertFileSuffix))) - - // Add ssm log bind mount - cn := fileSystemSafeContainerName(container) - hostConfig.Binds = append(hostConfig.Binds, getBindMountMapping( - filepath.Join(HostLogDir, taskId, cn), - ContainerLogDir)) - - container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ - ID: uuid, - }) - - return nil -} - -func (m *manager) getLatestVersionedHostBinDir() (string, error) { - versions, err := retrieveAgentVersions(ecsAgentDepsBinDir) - if err != nil { - return "", err - } - sort.Sort(sort.Reverse(byAgentVersion(versions))) - - var latest string - for _, v := range versions { - vStr := v.String() - ecsAgentDepsVersionedBinDir := filepath.Join(ecsAgentDepsBinDir, vStr) - if !fileExists(filepath.Join(ecsAgentDepsVersionedBinDir, SSMAgentBinName)) { - continue // try falling back to the previous version - } - // TODO: [ecs-exec] This requirement will be removed for SSM agent V2 - if !fileExists(filepath.Join(ecsAgentDepsVersionedBinDir, SSMAgentWorkerBinName)) { - continue // try falling back to the previous version - } - if !fileExists(filepath.Join(ecsAgentDepsVersionedBinDir, SessionWorkerBinName)) { - continue // try falling back to the previous version - } - latest = filepath.Join(m.hostBinDir, vStr) - break - } - if latest == "" { - return "", fmt.Errorf("no valid versions were found in %s", m.hostBinDir) - } - return latest, nil -} - -func getReadOnlyBindMountMapping(hostDir, containerDir string) string { - return getBindMountMapping(hostDir, containerDir) + ":ro" -} - -func getBindMountMapping(hostDir, containerDir string) string { - return hostDir + ":" + containerDir -} - -var newUUID = uuid.New - -func fileSystemSafeContainerName(c *apicontainer.Container) string { - // Trim leading hyphens since they're not valid directory names - cn := strings.TrimLeft(c.Name, "-") - if cn == "" { - // Fallback name in the extreme case that we end up with an empty string after trimming all leading hyphens. - return namelessContainerPrefix + newUUID() - } - return cn -} - -func getSessionWorkersLimit(ma apicontainer.ManagedAgent) int { - // TODO [ecs-exec] : verify that returning the default session limit (2) is ok in case of any errors, misconfiguration - limit := defaultSessionLimit - if ma.Properties == nil { // This means ACS didn't send the limit - return limit - } - limitStr, ok := ma.Properties["sessionLimit"] - if !ok { // This also means ACS didn't send the limit - return limit - } - limit, err := strconv.Atoi(limitStr) - if err != nil { // This means ACS send a limit that can't be converted to an int - return limit - } - if limit <= 0 { - limit = defaultSessionLimit - } - return limit -} - var GetExecAgentLogConfigFile = getAgentLogConfigFile func getAgentLogConfigFile() (string, error) { @@ -285,8 +102,6 @@ func getAgentLogConfigFile() (string, error) { return logConfigFileName, nil } -var removeAll = os.RemoveAll - func validConfigExists(configFilePath, expectedHash string) bool { config, err := getFileContent(configFilePath) if err != nil { @@ -296,12 +111,6 @@ func validConfigExists(configFilePath, expectedHash string) bool { return strings.Compare(expectedHash, hash) == 0 } -var getFileContent = readFileContent - -func readFileContent(filePath string) ([]byte, error) { - return ioutil.ReadFile(filePath) -} - var GetExecAgentConfigFileName = getAgentConfigFileName func getAgentConfigFileName(sessionLimit int) (string, error) { @@ -326,34 +135,62 @@ func getAgentConfigFileName(sessionLimit int) (string, error) { return configFileName, nil } -func getExecAgentConfigHash(config string) string { - hash := sha256.New() - hash.Write([]byte(config)) - return base64.URLEncoding.EncodeToString(hash.Sum(nil)) +func certsExist() bool { + return fileExists(filepath.Join(ecsAgentDepsCertsDir, "tls-ca-bundle.pem")) } -var osStat = os.Stat - -func fileExists(path string) bool { - if fi, err := osStat(path); err == nil { - return !fi.IsDir() +// This function creates any necessary config directories/files and ensures that +// the ssm-agent binaries, configs, logs, and plugin is bind mounted +func addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid string, sessionWorkersLimit int, hostConfig *dockercontainer.HostConfig) error { + configFile, rErr := GetExecAgentConfigFileName(sessionWorkersLimit) + if rErr != nil { + rErr = fmt.Errorf("could not generate ExecAgent Config File: %v", rErr) + return rErr + } + logConfigFile, rErr := GetExecAgentLogConfigFile() + if rErr != nil { + rErr = fmt.Errorf("could not generate ExecAgent LogConfig file: %v", rErr) + return rErr } - return false -} -func isDir(path string) bool { - if fi, err := osStat(path); err == nil { - return fi.IsDir() + containerDepsFolder := ContainerDepsDirPrefix + uuid + + if !certsExist() { + rErr = fmt.Errorf("could not find certs") + return rErr } - return false -} -var createNewExecAgentConfigFile = createNewConfigFile + // Add ssm binary mounts + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + filepath.Join(latestBinVersionDir, SSMAgentBinName), + filepath.Join(containerDepsFolder, SSMAgentBinName))) -func createNewConfigFile(config, configFilePath string) error { - return ioutil.WriteFile(configFilePath, []byte(config), filePerm) -} + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + filepath.Join(latestBinVersionDir, SSMAgentWorkerBinName), + filepath.Join(containerDepsFolder, SSMAgentWorkerBinName))) -func certsExist() bool { - return fileExists(filepath.Join(ecsAgentDepsCertsDir, "tls-ca-bundle.pem")) + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + filepath.Join(latestBinVersionDir, SessionWorkerBinName), + filepath.Join(containerDepsFolder, SessionWorkerBinName))) + + // Add exec agent config file mount + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + filepath.Join(HostExecConfigDir, configFile), + filepath.Join(containerDepsFolder, ContainerConfigFileSuffix))) + + // Add exec agent log config file mount + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + filepath.Join(HostExecConfigDir, logConfigFile), + filepath.Join(containerDepsFolder, ContainerLogConfigFile))) + + // Append TLS cert mount + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + HostCertFile, + filepath.Join(containerDepsFolder, ContainerCertFileSuffix))) + + // Add ssm log bind mount + hostConfig.Binds = append(hostConfig.Binds, getBindMountMapping( + filepath.Join(HostLogDir, taskId, cn), + ContainerLogDir)) + return nil } diff --git a/agent/engine/execcmd/manager_init_task_linux_test.go b/agent/engine/execcmd/manager_init_task_linux_test.go index 59e0c0c1d35..cb242d995b2 100644 --- a/agent/engine/execcmd/manager_init_task_linux_test.go +++ b/agent/engine/execcmd/manager_init_task_linux_test.go @@ -19,7 +19,6 @@ import ( "fmt" "io/ioutil" "os" - "strconv" "testing" dockercontainer "github.com/docker/docker/api/types/container" @@ -309,39 +308,6 @@ func TestGetExecAgentConfigFileName(t *testing.T) { } } -func TestGetSessionWorkersLimit(t *testing.T) { - var tests = []struct { - sessionLimit int - expectedLimit int - }{ - { - sessionLimit: -2, - expectedLimit: 2, - }, - { - sessionLimit: 0, - expectedLimit: 2, - }, - { - sessionLimit: 1, - expectedLimit: 1, - }, - { - sessionLimit: 2, - expectedLimit: 2, - }, - } - for _, tc := range tests { - ma := apicontainer.ManagedAgent{ - Properties: map[string]string{ - "sessionLimit": strconv.Itoa(tc.sessionLimit), - }, - } - limit := getSessionWorkersLimit(ma) - assert.Equal(t, tc.expectedLimit, limit) - } -} - func TestGetExecAgentLogConfigFile(t *testing.T) { hash := getExecAgentConfigHash(execAgentLogConfigTemplate) var tests = []struct { diff --git a/agent/engine/execcmd/manager_init_task_test.go b/agent/engine/execcmd/manager_init_task_test.go new file mode 100644 index 00000000000..7c4548a16bd --- /dev/null +++ b/agent/engine/execcmd/manager_init_task_test.go @@ -0,0 +1,57 @@ +// +build unit + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +import ( + "strconv" + "testing" + + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + "github.com/stretchr/testify/assert" +) + +func TestGetSessionWorkersLimit(t *testing.T) { + var tests = []struct { + sessionLimit int + expectedLimit int + }{ + { + sessionLimit: -2, + expectedLimit: 2, + }, + { + sessionLimit: 0, + expectedLimit: 2, + }, + { + sessionLimit: 1, + expectedLimit: 1, + }, + { + sessionLimit: 2, + expectedLimit: 2, + }, + } + for _, tc := range tests { + ma := apicontainer.ManagedAgent{ + Properties: map[string]string{ + "sessionLimit": strconv.Itoa(tc.sessionLimit), + }, + } + limit := getSessionWorkersLimit(ma) + assert.Equal(t, tc.expectedLimit, limit) + } +} diff --git a/agent/engine/execcmd/manager_init_task_windows.go b/agent/engine/execcmd/manager_init_task_windows.go index 95e9841b5a7..5c7766431b0 100644 --- a/agent/engine/execcmd/manager_init_task_windows.go +++ b/agent/engine/execcmd/manager_init_task_windows.go @@ -16,18 +16,173 @@ package execcmd import ( - apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/aws/amazon-ecs-agent/agent/config" dockercontainer "github.com/docker/docker/api/types/container" ) const ( - // ECSAgentExecLogDir here is used used while cleaning up exec logs when task exits. - // When this path is empty, nothing is cleaned up for unsupported platforms. - ECSAgentExecLogDir = "" + folderPerm = 0755 +) + +var ( + ecsAgentExecDepsDir = config.AmazonECSProgramFiles + "\\managed-agents\\execute-command" + + // ecsAgentDepsBinDir is the directory where ECS Agent will read versions of SSM agent + ecsAgentDepsBinDir = ecsAgentExecDepsDir + "\\bin" + + containerDepsFolder = "C:\\Program Files\\Amazon\\SSM" + + SSMAgentBinName = "amazon-ssm-agent.exe" + SSMAgentWorkerBinName = "ssm-agent-worker.exe" + SessionWorkerBinName = "ssm-session-worker.exe" + + HostLogDir = config.AmazonECSProgramData + "\\exec" + ContainerLogDir = config.AmazonProgramData + "\\SSM" + + SSMPluginDir = "C:\\Program Files\\Amazon\\SSM\\Plugins" + // since ecs agent windows is not running in a container, the agent and host log dirs are the same + ECSAgentExecLogDir = config.AmazonECSProgramData + "\\exec" + + // ECSAgentExecConfigDir is the directory where ECS Agent will write the ExecAgent config files to + ECSAgentExecConfigDir = ecsAgentExecDepsDir + "\\" + ContainerConfigDirName + // HostExecConfigDir is the dir where ExecAgents Config files will live + HostExecConfigDir = hostExecDepsDir + "\\" + ContainerConfigDirName + + configFiles = []string{ + "amazon-ssm-agent.json", + "seelog.xml", + } + + execAgentLogConfigTemplate = ` + + + + + + + + + + + + + + + + + + +` ) -// InitializeContainer adds the necessary bind mounts in order for the ExecCommandAgent to run properly in the container -// Note: exec cmd agent is a linux feature, thus implemented here as a no-op. -func (m *manager) InitializeContainer(taskId string, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) error { +var GetExecAgentConfigDir = getAgentConfigDir + +// Retrieves cached config dir, creates new one if needed +func getAgentConfigDir(sessionLimit int) (string, error) { + agentConfig := fmt.Sprintf(execAgentConfigTemplate, sessionLimit) + hash := getExecAgentConfigHash(agentConfig + execAgentLogConfigTemplate) + // check if cached config dir exists already + configDirPath := filepath.Join(ECSAgentExecConfigDir, hash) + if isDir(configDirPath) && validConfigDirExists(configDirPath, hash) { + return hash, nil + } + // check if config dir is a file; if true, remove it + if fileExists(configDirPath) { + if err := removeAll(configDirPath); err != nil { + return "", err + } + } + // create new config dir + if err := createNewExecAgentConfigDir(agentConfig, configDirPath); err != nil { + return "", err + } + return hash, nil +} + +var createNewExecAgentConfigDir = createNewConfigDir + +var mkdirAll = os.MkdirAll + +func createNewConfigDir(agentConfig, configDirPath string) error { + // make top level config directory + err := mkdirAll(configDirPath, folderPerm) + if err != nil { + return err + } + + // make individual config files + agentConfigFilePath := filepath.Join(configDirPath, containerConfigFileName) + err = createNewExecAgentConfigFile(agentConfigFilePath, agentConfig) + if err != nil { + return err + } + + logConfigFilePath := filepath.Join(configDirPath, ExecAgentLogConfigFileName) + err = createNewExecAgentConfigFile(logConfigFilePath, execAgentLogConfigTemplate) + if err != nil { + return err + } + + return nil +} + +func validConfigDirExists(configDirPath, expectedHash string) bool { + // check that each config file exists and create a hash of their summed contents + contentSum := "" + for _, file := range configFiles { + if isDir(filepath.Join(configDirPath, file)) { + return false + } + config, err := getFileContent(filepath.Join(configDirPath, file)) + if err != nil { + return false + } + contentSum += string(config) + } + + hash := getExecAgentConfigHash(contentSum) + return strings.Compare(expectedHash, hash) == 0 +} + +// This function creates any necessary config directories/files and ensures that +// the ssm-agent binaries, configs, logs, and plugin is bind mounted +func addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid string, sessionWorkersLimit int, hostConfig *dockercontainer.HostConfig) error { + // In windows host mounts are not created automatically, so need to create + rErr := os.MkdirAll(filepath.Join(HostLogDir, taskId, cn), folderPerm) + if rErr != nil { + return rErr + } + + configDirHash, rErr := GetExecAgentConfigDir(sessionWorkersLimit) + if rErr != nil { + rErr = fmt.Errorf("could not generate ExecAgent Config dir: %v", rErr) + return rErr + } + + // Add ssm binary mount + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + latestBinVersionDir, + containerDepsFolder)) + + // Add ssm configuration dir mount + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( + filepath.Join(ECSAgentExecConfigDir, configDirHash), + filepath.Join(containerDepsFolder, "configuration"))) + + // Add ssm log bind mount + hostConfig.Binds = append(hostConfig.Binds, getBindMountMapping( + filepath.Join(HostLogDir, taskId, cn), + ContainerLogDir)) + + // add ssm plugin bind mount (needed for execcmd windows) + hostConfig.Binds = append(hostConfig.Binds, getBindMountMapping( + SSMPluginDir, + SSMPluginDir)) + return nil } diff --git a/agent/engine/execcmd/manager_init_task_windows_test.go b/agent/engine/execcmd/manager_init_task_windows_test.go new file mode 100644 index 00000000000..0bc6f08a9dc --- /dev/null +++ b/agent/engine/execcmd/manager_init_task_windows_test.go @@ -0,0 +1,260 @@ +// +build windows,unit + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInitializeContainer(t *testing.T) {} + +func TestGetExecAgentConfigDir(t *testing.T) { + hash := getExecAgentConfigHash(fmt.Sprintf(execAgentConfigTemplate, 2) + execAgentLogConfigTemplate) + + var tests = []struct { + expectedDir string + expectedError error + execAgentConfigDirExist bool + configDirIsFile bool + removeFileErr error + existingLogConfigReadErr error + existingLogConfig string + logConfigIsDir bool + existingAgentConfigReadErr error + existingAgentConfig string + agentConfigIsDir bool + createNewConfigDirError error + createNewConfigFileError error + }{ + { + expectedDir: hash, + expectedError: nil, + execAgentConfigDirExist: true, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + expectedDir: hash, + expectedError: nil, + execAgentConfigDirExist: false, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + expectedDir: hash, + expectedError: nil, + execAgentConfigDirExist: true, + existingLogConfig: "junk", + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + expectedDir: hash, + expectedError: nil, + execAgentConfigDirExist: true, + existingLogConfigReadErr: errors.New("read file error"), + existingLogConfig: "", + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + expectedDir: hash, + expectedError: nil, + execAgentConfigDirExist: true, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfigReadErr: errors.New("read file error"), + existingAgentConfig: "", + }, + { + expectedDir: hash, + expectedError: nil, + execAgentConfigDirExist: true, + configDirIsFile: true, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + expectedDir: "", + expectedError: errors.New("create file error"), + execAgentConfigDirExist: false, + createNewConfigFileError: errors.New("create file error"), + }, + { + expectedDir: "", + expectedError: errors.New("create dir error"), + execAgentConfigDirExist: false, + createNewConfigDirError: errors.New("create dir error"), + }, + // todo: need to fix + { + expectedDir: "", + expectedError: errors.New("remove file error"), + execAgentConfigDirExist: true, + configDirIsFile: true, + removeFileErr: errors.New("remove file error"), + }, + + // todo: add the is_dir flags + } + defer func() { + osStat = os.Stat + getFileContent = readFileContent + createNewExecAgentConfigFile = createNewConfigFile + removeAll = os.RemoveAll + mkdirAll = os.MkdirAll + }() + + for _, tc := range tests { + osStat = func(name string) (os.FileInfo, error) { + if filepath.Base(name) == "amazon-ssm-agent.json" { + return &mockFileInfo{name: "", isDir: tc.agentConfigIsDir}, nil + } else if filepath.Base(name) == "seelog.xml" { + return &mockFileInfo{name: "", isDir: tc.logConfigIsDir}, nil + } else { // this is the case for the top level \config\hash folder + if tc.execAgentConfigDirExist { + return &mockFileInfo{name: "", isDir: !tc.configDirIsFile}, nil + } + return &mockFileInfo{}, errors.New("no such file") + } + + } + removeAll = func(name string) error { + return tc.removeFileErr + } + getFileContent = func(path string) ([]byte, error) { + if filepath.Base(path) == "amazon-ssm-agent.json" { + return []byte(tc.existingAgentConfig), tc.existingAgentConfigReadErr + } + + if filepath.Base(path) == "seelog.xml" { + return []byte(tc.existingLogConfig), tc.existingLogConfigReadErr + } + return nil, nil + } + createNewExecAgentConfigFile = func(c, f string) error { + return tc.createNewConfigFileError + } + + mkdirAll = func(path string, perm os.FileMode) error { + return tc.createNewConfigDirError + } + configDir, err := GetExecAgentConfigDir(2) + assert.Equal(t, tc.expectedDir, configDir) + assert.Equal(t, tc.expectedError, err) + } + +} + +func TestGetValidConfigDirExists(t *testing.T) { + var tests = []struct { + isValid bool + existingLogConfigReadErr error + existingLogConfig string + existingLogConfigIsDir bool + existingAgentConfigReadErr error + existingAgentConfig string + existingAgentConfigIsDir bool + }{ + { + isValid: true, + existingLogConfigReadErr: nil, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfigReadErr: nil, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + isValid: false, + existingLogConfigReadErr: nil, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfigReadErr: nil, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 3), + }, + { + isValid: false, + existingLogConfigReadErr: nil, + existingLogConfig: "junk", + existingAgentConfigReadErr: nil, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + isValid: false, + existingLogConfigReadErr: errors.New("read file error"), + existingLogConfig: "", + existingAgentConfigReadErr: nil, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + isValid: false, + existingLogConfigReadErr: nil, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfigReadErr: errors.New("read file error"), + existingAgentConfig: "", + }, + { + isValid: false, + existingLogConfigReadErr: nil, + existingLogConfig: execAgentLogConfigTemplate, + existingLogConfigIsDir: true, + existingAgentConfigReadErr: nil, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + }, + { + isValid: false, + existingLogConfigReadErr: nil, + existingLogConfig: execAgentLogConfigTemplate, + existingAgentConfigReadErr: nil, + existingAgentConfig: fmt.Sprintf(execAgentConfigTemplate, 2), + existingAgentConfigIsDir: true, + }, + } + defer func() { + getFileContent = readFileContent + osStat = os.Stat + }() + configDirPath := "C:\\configpath" + for _, tc := range tests { + getFileContent = func(path string) ([]byte, error) { + // amazon-ssm-agent.json + if path == filepath.Join(configDirPath, containerConfigFileName) { + return []byte(tc.existingAgentConfig), tc.existingAgentConfigReadErr + } + + // seelog.xml + if path == filepath.Join(configDirPath, ExecAgentLogConfigFileName) { + return []byte(tc.existingLogConfig), tc.existingLogConfigReadErr + } + + return nil, nil + } + + osStat = func(path string) (os.FileInfo, error) { + if filepath.Base(path) == "amazon-ssm-agent.json" { + return &mockFileInfo{name: "", isDir: tc.existingAgentConfigIsDir}, nil + } + + if filepath.Base(path) == "seelog.xml" { + return &mockFileInfo{name: "", isDir: tc.existingLogConfigIsDir}, nil + } + + return &mockFileInfo{}, errors.New("no such file") + } + assert.Equal(t, tc.isValid, validConfigDirExists(configDirPath, getExecAgentConfigHash(fmt.Sprintf(execAgentConfigTemplate, 2)+execAgentLogConfigTemplate))) + } +} From 21d34ba8b1a0056f146b646e833b65f01c32c05a Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Tue, 15 Jun 2021 00:34:39 -0700 Subject: [PATCH 05/14] execcmd: manager start task Windows/Linux code decoupling + implementation --- agent/engine/execcmd/manager_start.go | 192 +++++++ agent/engine/execcmd/manager_start_linux.go | 176 +------ .../execcmd/manager_start_linux_test.go | 462 +---------------- agent/engine/execcmd/manager_start_test.go | 476 ++++++++++++++++++ agent/engine/execcmd/manager_start_windows.go | 33 +- .../execcmd/manager_start_windows_test.go | 21 + 6 files changed, 717 insertions(+), 643 deletions(-) create mode 100644 agent/engine/execcmd/manager_start.go create mode 100644 agent/engine/execcmd/manager_start_test.go create mode 100644 agent/engine/execcmd/manager_start_windows_test.go diff --git a/agent/engine/execcmd/manager_start.go b/agent/engine/execcmd/manager_start.go new file mode 100644 index 00000000000..57890c491f6 --- /dev/null +++ b/agent/engine/execcmd/manager_start.go @@ -0,0 +1,192 @@ +// +build linux windows + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strconv" + "time" + + "github.com/cihub/seelog" + "github.com/docker/docker/api/types" + + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + "github.com/aws/amazon-ecs-agent/agent/api/container/status" + apitask "github.com/aws/amazon-ecs-agent/agent/api/task" + "github.com/aws/amazon-ecs-agent/agent/dockerclient" + "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi" + "github.com/aws/amazon-ecs-agent/agent/utils/retry" +) + +// RestartAgentIfStopped restarts the ExecCommandAgent in the container passed as parameter, only for ExecCommandAgent-enabled containers. +// The status of the ExecCommandAgent in the container is retrieved using a docker exec inspect call, using the dockerExecID +// stored in the AgentMetadata.DockerExecID. +// +// If the ExecCommandAgent is still running (or has never started), no action is taken. +// To actually restart the ExecCommandAgent, this function invokes this instance's StartAgent method. +func (m *manager) RestartAgentIfStopped(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) (RestartStatus, error) { + if !IsExecEnabledContainer(container) { + seelog.Warnf("Task engine [%s]: an attempt to restart ExecCommandAgent for a non ExecCommandAgent-enabled container was made; container %s", task.Arn, containerId) + return NotRestarted, nil + } + ma, _ := container.GetManagedAgentByName(ExecuteCommandAgentName) + metadata := MapToAgentMetadata(ma.Metadata) + if !m.isAgentStarted(ma) { + return NotRestarted, nil + } + res, err := m.inspectExecAgentProcess(ctx, client, metadata) + if err != nil || res == nil { + // We don't want to report InspectContainerExec errors, just that we don't know what the status of the agent is + return Unknown, nil + } + if res.Running { // agent still running, nothing to do + return NotRestarted, nil + } + // Restart if not running + //TODO: [ecs-exec] retry only for certain exit codes? + seelog.Warnf("Task engine [%s]: ExecCommandAgent Process stopped (exitCode=%d) for container %s, restarting...", task.Arn, res.ExitCode, containerId) + container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ + ID: ma.ID, + }) + err = m.StartAgent(ctx, client, task, container, containerId) + if err != nil { + return NotRestarted, err + } + return Restarted, nil +} + +func (m *manager) inspectExecAgentProcess(ctx context.Context, client dockerapi.DockerClient, metadata AgentMetadata) (*types.ContainerExecInspect, error) { + backoff := retry.NewExponentialBackoff(m.retryMinDelay, m.retryMaxDelay, retryJitterMultiplier, retryDelayMultiplier) + ctx, cancel := context.WithTimeout(ctx, m.inspectRetryTimeout) + defer cancel() + var ( + inspectRes *types.ContainerExecInspect + inspectErr error + ) + retry.RetryNWithBackoffCtx(ctx, backoff, maxRetries, func() error { + inspectRes, inspectErr = client.InspectContainerExec(ctx, metadata.DockerExecID, dockerclient.ContainerExecInspectTimeout) + if inspectErr != nil { + retryable := true + if _, ok := inspectErr.(*dockerapi.DockerTimeoutError); ok { + retryable = false + } + return StartError{ + error: inspectErr, + retryable: retryable, + } + } + return nil + }) + return inspectRes, inspectErr +} + +// StartAgent idempotently starts the ExecCommandAgent in the container passed as parameter, only for ExecCommandAgent-enabled containers. +// If no error is returned, it can be assumed the ExecCommandAgent is started. +func (m *manager) StartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) error { + if !IsExecEnabledContainer(container) { + seelog.Warnf("Task engine [%s]: an attempt to start ExecCommandAgent for a non ExecCommandAgent-enabled container was made; container %s", task.Arn, containerId) + return nil + } + ma, _ := container.GetManagedAgentByName(ExecuteCommandAgentName) + existingMetadata := MapToAgentMetadata(ma.Metadata) + if ma.ID == "" { + return errors.New("container has not been initialized: missing UUID") + } + // Guarantee idempotency if the container already has been started + if m.isAgentStarted(ma) { + res, err := m.inspectExecAgentProcess(ctx, client, existingMetadata) + if err != nil { + seelog.Warnf("Task engine [%s]: could not verify if the ExecCommandAgent was already running for container %s: %v", task.Arn, containerId, err) + } else if res.Running { // agent is already running, nothing to do + seelog.Warnf("Task engine [%s]: an attempt was made to start the ExecCommandAgent but it was already running (%s)", task.Arn, containerId) + return nil + } + } + + backoff := retry.NewExponentialBackoff(m.retryMinDelay, m.retryMaxDelay, retryJitterMultiplier, retryDelayMultiplier) + ctx, cancel := context.WithTimeout(ctx, m.startRetryTimeout) + defer cancel() + var startErr error + + var execMD *AgentMetadata + retry.RetryNWithBackoffCtx(ctx, backoff, maxRetries, func() error { + execMD, startErr = m.doStartAgent(ctx, client, task, ma, containerId) + if startErr != nil { + seelog.Warnf("Task engine [%s]: exec command agent failed to start for container %s: %v. Retrying...", task.Arn, containerId, startErr) + } + return startErr + }) + if startErr != nil { + container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ + ID: ma.ID, + Status: status.ManagedAgentStopped, + Reason: startErr.Error(), + }) + return startErr + } + container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ + ID: ma.ID, + Status: status.ManagedAgentRunning, + LastStartedAt: time.Now(), + Metadata: execMD.ToMap(), + }) + return nil +} + +func (m *manager) doStartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, ma apicontainer.ManagedAgent, containerId string) (*AgentMetadata, error) { + execAgentCmdBinDir := getExecAgentCmdBinDir(&ma) + execAgentCmd := filepath.Join(execAgentCmdBinDir, SSMAgentBinName) + execCfg := types.ExecConfig{ + User: execAgentCmdUser, + Detach: true, + Cmd: []string{execAgentCmd}, + } + newMD := &AgentMetadata{} + execRes, err := client.CreateContainerExec(ctx, containerId, execCfg, dockerclient.ContainerExecCreateTimeout) + if err != nil { + return newMD, StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [create]: %v", err), retryable: true} + } + + seelog.Debugf("Task engine [%s]: created ExecCommandAgent for container: %s -> docker exec id: %s", task.Arn, containerId, execRes.ID) + + err = client.StartContainerExec(ctx, execRes.ID, types.ExecStartCheck{Detach: true, Tty: false}, dockerclient.ContainerExecStartTimeout) + if err != nil { + return newMD, StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [pre-start]: %v", err), retryable: true} + } + seelog.Debugf("Task engine [%s]: sent ExecCommandAgent start signal for container: %s -> docker exec id: %s", task.Arn, containerId, execRes.ID) + + inspect, err := client.InspectContainerExec(ctx, execRes.ID, dockerclient.ContainerExecInspectTimeout) + if err != nil { + return newMD, StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [inspect]: %v", err), retryable: true} + } + seelog.Debugf("Task engine [%s]: inspect ExecCommandAgent for container: %s -> pid: %d, exitCode: %d, running:%v, err:%v", + task.Arn, containerId, inspect.Pid, inspect.ExitCode, inspect.Running, err) + + if !inspect.Running { //TODO: [ecs-exec] retry only for certain exit codes? + return newMD, StartError{ + error: fmt.Errorf("ExecuteCommandAgent process exited with exit code: %d", inspect.ExitCode), + retryable: true, + } + } + seelog.Infof("Task engine [%s]: started ExecCommandAgent for container: %s -> docker exec id: %s", task.Arn, containerId, execRes.ID) + newMD.PID = strconv.Itoa(inspect.Pid) + newMD.DockerExecID = execRes.ID + newMD.CMD = execAgentCmd + return newMD, nil +} diff --git a/agent/engine/execcmd/manager_start_linux.go b/agent/engine/execcmd/manager_start_linux.go index 2b2568b9d09..b98634c856d 100644 --- a/agent/engine/execcmd/manager_start_linux.go +++ b/agent/engine/execcmd/manager_start_linux.go @@ -14,178 +14,12 @@ // permissions and limitations under the License. package execcmd -import ( - "context" - "errors" - "fmt" - "path/filepath" - "strconv" - "time" +import apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" - "github.com/cihub/seelog" - "github.com/docker/docker/api/types" - - apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" - "github.com/aws/amazon-ecs-agent/agent/api/container/status" - apitask "github.com/aws/amazon-ecs-agent/agent/api/task" - "github.com/aws/amazon-ecs-agent/agent/dockerclient" - "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi" - "github.com/aws/amazon-ecs-agent/agent/utils/retry" +const ( + execAgentCmdUser = "0" ) -// RestartAgentIfStopped restarts the ExecCommandAgent in the container passed as parameter, only for ExecCommandAgent-enabled containers. -// The status of the ExecCommandAgent in the container is retrieved using a docker exec inspect call, using the dockerExecID -// stored in the AgentMetadata.DockerExecID. -// -// If the ExecCommandAgent is still running (or has never started), no action is taken. -// To actually restart the ExecCommandAgent, this function invokes this instance's StartAgent method. -func (m *manager) RestartAgentIfStopped(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) (RestartStatus, error) { - if !IsExecEnabledContainer(container) { - seelog.Warnf("Task engine [%s]: an attempt to restart ExecCommandAgent for a non ExecCommandAgent-enabled container was made; container %s", task.Arn, containerId) - return NotRestarted, nil - } - ma, _ := container.GetManagedAgentByName(ExecuteCommandAgentName) - metadata := MapToAgentMetadata(ma.Metadata) - if !m.isAgentStarted(ma) { - return NotRestarted, nil - } - res, err := m.inspectExecAgentProcess(ctx, client, metadata) - if err != nil || res == nil { - // We don't want to report InspectContainerExec errors, just that we don't know what the status of the agent is - return Unknown, nil - } - if res.Running { // agent still running, nothing to do - return NotRestarted, nil - } - // Restart if not running - //TODO: [ecs-exec] retry only for certain exit codes? - seelog.Warnf("Task engine [%s]: ExecCommandAgent Process stopped (exitCode=%d) for container %s, restarting...", task.Arn, res.ExitCode, containerId) - container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ - ID: ma.ID, - }) - err = m.StartAgent(ctx, client, task, container, containerId) - if err != nil { - return NotRestarted, err - } - return Restarted, nil -} - -func (m *manager) inspectExecAgentProcess(ctx context.Context, client dockerapi.DockerClient, metadata AgentMetadata) (*types.ContainerExecInspect, error) { - backoff := retry.NewExponentialBackoff(m.retryMinDelay, m.retryMaxDelay, retryJitterMultiplier, retryDelayMultiplier) - ctx, cancel := context.WithTimeout(ctx, m.inspectRetryTimeout) - defer cancel() - var ( - inspectRes *types.ContainerExecInspect - inspectErr error - ) - retry.RetryNWithBackoffCtx(ctx, backoff, maxRetries, func() error { - inspectRes, inspectErr = client.InspectContainerExec(ctx, metadata.DockerExecID, dockerclient.ContainerExecInspectTimeout) - if inspectErr != nil { - retryable := true - if _, ok := inspectErr.(*dockerapi.DockerTimeoutError); ok { - retryable = false - } - return StartError{ - error: inspectErr, - retryable: retryable, - } - } - return nil - }) - return inspectRes, inspectErr -} - -// StartAgent idempotently starts the ExecCommandAgent in the container passed as parameter, only for ExecCommandAgent-enabled containers. -// If no error is returned, it can be assumed the ExecCommandAgent is started. -func (m *manager) StartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) error { - if !IsExecEnabledContainer(container) { - seelog.Warnf("Task engine [%s]: an attempt to start ExecCommandAgent for a non ExecCommandAgent-enabled container was made; container %s", task.Arn, containerId) - return nil - } - ma, _ := container.GetManagedAgentByName(ExecuteCommandAgentName) - existingMetadata := MapToAgentMetadata(ma.Metadata) - if ma.ID == "" { - return errors.New("container has not been initialized: missing UUID") - } - // Guarantee idempotency if the container already has been started - if m.isAgentStarted(ma) { - res, err := m.inspectExecAgentProcess(ctx, client, existingMetadata) - if err != nil { - seelog.Warnf("Task engine [%s]: could not verify if the ExecCommandAgent was already running for container %s: %v", task.Arn, containerId, err) - } else if res.Running { // agent is already running, nothing to do - seelog.Warnf("Task engine [%s]: an attempt was made to start the ExecCommandAgent but it was already running (%s)", task.Arn, containerId) - return nil - } - } - - backoff := retry.NewExponentialBackoff(m.retryMinDelay, m.retryMaxDelay, retryJitterMultiplier, retryDelayMultiplier) - ctx, cancel := context.WithTimeout(ctx, m.startRetryTimeout) - defer cancel() - var startErr error - - var execMD *AgentMetadata - retry.RetryNWithBackoffCtx(ctx, backoff, maxRetries, func() error { - execMD, startErr = m.doStartAgent(ctx, client, task, ma, containerId) - if startErr != nil { - seelog.Warnf("Task engine [%s]: exec command agent failed to start for container %s: %v. Retrying...", task.Arn, containerId, startErr) - } - return startErr - }) - if startErr != nil { - container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ - ID: ma.ID, - Status: status.ManagedAgentStopped, - Reason: startErr.Error(), - }) - return startErr - } - container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{ - ID: ma.ID, - Status: status.ManagedAgentRunning, - LastStartedAt: time.Now(), - Metadata: execMD.ToMap(), - }) - return nil -} - -func (m *manager) doStartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, ma apicontainer.ManagedAgent, containerId string) (*AgentMetadata, error) { - execAgentCmdBinDir := ContainerDepsDirPrefix + ma.ID - execAgentCmd := filepath.Join(execAgentCmdBinDir, SSMAgentBinName) - execCfg := types.ExecConfig{ - User: "0", - Detach: true, - Cmd: []string{execAgentCmd}, - } - newMD := &AgentMetadata{} - execRes, err := client.CreateContainerExec(ctx, containerId, execCfg, dockerclient.ContainerExecCreateTimeout) - if err != nil { - return newMD, StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [create]: %v", err), retryable: true} - } - - seelog.Debugf("Task engine [%s]: created ExecCommandAgent for container: %s -> docker exec id: %s", task.Arn, containerId, execRes.ID) - - err = client.StartContainerExec(ctx, execRes.ID, types.ExecStartCheck{Detach: true, Tty: false}, dockerclient.ContainerExecStartTimeout) - if err != nil { - return newMD, StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [pre-start]: %v", err), retryable: true} - } - seelog.Debugf("Task engine [%s]: sent ExecCommandAgent start signal for container: %s -> docker exec id: %s", task.Arn, containerId, execRes.ID) - - inspect, err := client.InspectContainerExec(ctx, execRes.ID, dockerclient.ContainerExecInspectTimeout) - if err != nil { - return newMD, StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [inspect]: %v", err), retryable: true} - } - seelog.Debugf("Task engine [%s]: inspect ExecCommandAgent for container: %s -> pid: %d, exitCode: %d, running:%v, err:%v", - task.Arn, containerId, inspect.Pid, inspect.ExitCode, inspect.Running, err) - - if !inspect.Running { //TODO: [ecs-exec] retry only for certain exit codes? - return newMD, StartError{ - error: fmt.Errorf("ExecuteCommandAgent process exited with exit code: %d", inspect.ExitCode), - retryable: true, - } - } - seelog.Infof("Task engine [%s]: started ExecCommandAgent for container: %s -> docker exec id: %s", task.Arn, containerId, execRes.ID) - newMD.PID = strconv.Itoa(inspect.Pid) - newMD.DockerExecID = execRes.ID - newMD.CMD = execAgentCmd - return newMD, nil +func getExecAgentCmdBinDir(ma *apicontainer.ManagedAgent) string { + return ContainerDepsDirPrefix + ma.ID } diff --git a/agent/engine/execcmd/manager_start_linux_test.go b/agent/engine/execcmd/manager_start_linux_test.go index ec08b6a5ad2..2613f9ab2cd 100644 --- a/agent/engine/execcmd/manager_start_linux_test.go +++ b/agent/engine/execcmd/manager_start_linux_test.go @@ -14,463 +14,7 @@ // permissions and limitations under the License. package execcmd -import ( - "context" - "errors" - "fmt" - "strconv" - "testing" - "time" - - "github.com/pborman/uuid" - - "github.com/aws/amazon-ecs-agent/agent/api/container" - apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" - apicontainerstatus "github.com/aws/amazon-ecs-agent/agent/api/container/status" - errors2 "github.com/aws/amazon-ecs-agent/agent/api/errors" - apitask "github.com/aws/amazon-ecs-agent/agent/api/task" - "github.com/aws/amazon-ecs-agent/agent/dockerclient" - "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi" - mock_dockerapi "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi/mocks" - - "github.com/docker/docker/api/types" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" +const ( + specTestCmd = "/ecs-execute-command-test-uid/amazon-ssm-agent" + specUser = "0" ) - -func getAgentMetadata(container *container.Container) AgentMetadata { - ma, _ := container.GetManagedAgentByName(ExecuteCommandAgentName) - return MapToAgentMetadata(ma.Metadata) -} - -func TestStartAgent(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - client := mock_dockerapi.NewMockDockerClient(ctrl) - const ( - testPid1 = 9876 - testContainerRuntimeId = "123abc" - testDockerExecId = "mockDockerExecID" - ) - nowTime := time.Now() - zeroTime := time.Time{} - var ( - mockError = errors.New("mock error") - testContainers = []*apicontainer.Container{{ - RuntimeID: testContainerRuntimeId, - }} - ) - - tt := []struct { - name string - execEnabled bool - containers []*apicontainer.Container - expectCreateContainerExec bool - createContainerExecRes *types.IDResponse - createContainerExecErr error - expectStartContainerExec bool - startContainerExecErr error - expectInspectContainerExec bool - inspectContainerExecRes *types.ContainerExecInspect - inspectContainerExecErr error - expectedError error - expectedStatus apicontainerstatus.ManagedAgentStatus - expectedStartTime time.Time - }{ - { - name: "test exec disabled", - execEnabled: false, - containers: testContainers, - }, - { - name: "test with create container error", - execEnabled: true, - containers: testContainers, - expectCreateContainerExec: true, - expectedStatus: apicontainerstatus.ManagedAgentStopped, - createContainerExecErr: mockError, - expectedError: StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [create]: %v", mockError), retryable: true}, - expectedStartTime: zeroTime, - }, - { - name: "test with start container error", - execEnabled: true, - containers: testContainers, - expectCreateContainerExec: true, - createContainerExecRes: &types.IDResponse{ - ID: testDockerExecId, - }, - expectStartContainerExec: true, - expectedStatus: apicontainerstatus.ManagedAgentStopped, - startContainerExecErr: mockError, - expectedError: StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [pre-start]: %v", mockError), retryable: true}, - expectedStartTime: zeroTime, - }, - { - name: "test with inspect container error", - execEnabled: true, - containers: testContainers, - expectCreateContainerExec: true, - createContainerExecRes: &types.IDResponse{ - ID: testDockerExecId, - }, - expectStartContainerExec: true, - startContainerExecErr: nil, // Simulate StartContainerExec succeeds - expectInspectContainerExec: true, - expectedStatus: apicontainerstatus.ManagedAgentStopped, - inspectContainerExecErr: mockError, - expectedError: StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [inspect]: %v", mockError), retryable: true}, - expectedStartTime: zeroTime, - }, - { - name: "test happy path", - execEnabled: true, - containers: testContainers, - expectCreateContainerExec: true, - createContainerExecRes: &types.IDResponse{ - ID: testDockerExecId, - }, - expectStartContainerExec: true, - startContainerExecErr: nil, // Simulate StartContainerExec succeeds - expectInspectContainerExec: true, - expectedStatus: apicontainerstatus.ManagedAgentRunning, - expectedStartTime: nowTime, - inspectContainerExecRes: &types.ContainerExecInspect{ - ExecID: testDockerExecId, - Pid: testPid1, - Running: true, - }, - }, - } - testUUID := "test-uid" - defer func() { - newUUID = uuid.New - }() - newUUID = func() string { - return testUUID - } - for _, test := range tt { - t.Run(test.name, func(t *testing.T) { - - testTask := &apitask.Task{ - Arn: "taskArn:aws:ecs:region:account-id:task/test-task-taskArn", - Containers: test.containers, - } - if test.execEnabled { - for _, c := range testTask.Containers { - c.ManagedAgentsUnsafe = []apicontainer.ManagedAgent{ - { - Name: ExecuteCommandAgentName, - ManagedAgentState: apicontainer.ManagedAgentState{ - ID: testUUID, - }, - }, - } - } - } - - times := maxRetries - retryableErr, isRetryable := test.expectedError.(errors2.RetriableError) - if test.expectedError == nil || (isRetryable && !retryableErr.Retry()) { - times = 1 - } - if test.expectCreateContainerExec { - execCfg := types.ExecConfig{ - User: "0", - Detach: true, - Cmd: []string{"/ecs-execute-command-test-uid/amazon-ssm-agent"}, - } - client.EXPECT().CreateContainerExec(gomock.Any(), testTask.Containers[0].RuntimeID, execCfg, dockerclient.ContainerExecCreateTimeout). - Return(test.createContainerExecRes, test.createContainerExecErr). - Times(times) - } - - if test.expectStartContainerExec { - client.EXPECT().StartContainerExec(gomock.Any(), testDockerExecId, gomock.Any(), dockerclient.ContainerExecStartTimeout). - Return(test.startContainerExecErr). - Times(times) - } - - if test.expectInspectContainerExec { - client.EXPECT().InspectContainerExec(gomock.Any(), testDockerExecId, dockerclient.ContainerExecInspectTimeout). - Return(test.inspectContainerExecRes, test.inspectContainerExecErr). - Times(times) - } - - mgr := newTestManager() - prevMetadata := getAgentMetadata(test.containers[0]) - err := mgr.StartAgent(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) - if test.expectedError != nil { - assert.Equal(t, test.expectedError, err, "Wrong error returned") - // When there's an error, ExecCommandAgentMetadata should not be modified - newMetadata := getAgentMetadata(test.containers[0]) - assert.Equal(t, prevMetadata, newMetadata) - } else { // No error case - assert.NoError(t, err, "No error was expected") - if !test.execEnabled { - _, ok := test.containers[0].GetManagedAgentByName(ExecuteCommandAgentName) - assert.False(t, ok) - } else { - ma, _ := test.containers[0].GetManagedAgentByName(ExecuteCommandAgentName) - execMD := getAgentMetadata(test.containers[0]) - assert.Equal(t, strconv.Itoa(testPid1), execMD.PID, "PID not equal") - assert.Equal(t, testDockerExecId, execMD.DockerExecID, "DockerExecId not equal") - assert.Equal(t, "/ecs-execute-command-test-uid/amazon-ssm-agent", execMD.CMD) - assert.Equal(t, test.expectedStatus, ma.Status, "Exec status not equal") - assert.WithinDuration(t, test.expectedStartTime, ma.LastStartedAt, 5*time.Second, "StartedAt not equal") - } - } - }) - } -} - -func TestIdempotentStartAgent(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - client := mock_dockerapi.NewMockDockerClient(ctrl) - const ( - testDockerExecId = "abc" - testPid = 111 - ) - var ( - testPidStr = strconv.Itoa(testPid) - testCmd = "/ecs-execute-command-test-uid/amazon-ssm-agent" - ) - testUUID := "test-uid" - defer func() { - newUUID = uuid.New - }() - newUUID = func() string { - return testUUID - } - - testTask := &apitask.Task{ - Arn: "taskArn:aws:ecs:region:account-id:task/test-task-taskArn", - Containers: []*apicontainer.Container{{ - RuntimeID: "123", - ManagedAgentsUnsafe: []apicontainer.ManagedAgent{ - { - Name: ExecuteCommandAgentName, - ManagedAgentState: apicontainer.ManagedAgentState{ - ID: testUUID, - }, - }, - }, - }}, - } - - execCfg := types.ExecConfig{ - User: "0", - Detach: true, - Cmd: []string{"/ecs-execute-command-test-uid/amazon-ssm-agent"}, - } - client.EXPECT().CreateContainerExec(gomock.Any(), testTask.Containers[0].RuntimeID, execCfg, dockerclient.ContainerExecCreateTimeout). - Return(&types.IDResponse{ID: testDockerExecId}, nil). - Times(1) - - client.EXPECT().StartContainerExec(gomock.Any(), testDockerExecId, gomock.Any(), dockerclient.ContainerExecStartTimeout). - Return(nil). - Times(1) - - client.EXPECT().InspectContainerExec(gomock.Any(), testDockerExecId, dockerclient.ContainerExecInspectTimeout). - Return(&types.ContainerExecInspect{ - ExecID: testDockerExecId, - Pid: testPid, - Running: true, - }, nil). - Times(2) - - mgr := newTestManager() - err := mgr.StartAgent(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) - assert.NoError(t, err) - - ma, _ := testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) - execMD := getAgentMetadata(testTask.Containers[0]) - firstStart := ma.LastStartedAt - assert.NotNil(t, ma.LastStartedAt) - assert.Equal(t, testPidStr, execMD.PID) - assert.Equal(t, testDockerExecId, execMD.DockerExecID) - assert.Equal(t, testCmd, execMD.CMD) - assert.Equal(t, apicontainerstatus.ManagedAgentRunning, ma.Status) - - // Second call to start. The mock's expected call times is 1 (except for inspect); the absence of "too many calls" - // along with unchanged metadata guarantees idempotency - err = mgr.StartAgent(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) - assert.NoError(t, err) - - ma, _ = testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) - execMD = getAgentMetadata(testTask.Containers[0]) - // check StartedAt is not Zero and hasn't been updated - assert.NotNil(t, ma.LastStartedAt) - assert.False(t, ma.LastStartedAt.IsZero()) - assert.Equal(t, ma.LastStartedAt, firstStart) - assert.Equal(t, testPidStr, execMD.PID) - assert.Equal(t, testDockerExecId, execMD.DockerExecID) - assert.Equal(t, testCmd, execMD.CMD) - assert.Equal(t, apicontainerstatus.ManagedAgentRunning, ma.Status) -} - -func TestRestartAgentIfStopped(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - client := mock_dockerapi.NewMockDockerClient(ctrl) - nowTime := time.Now() - - const ( - testContainerId = "123" - testNewDockerExecID = "newDockerExecId" - testNewPID = 111 - ) - testUUID := "test-uid" - defer func() { - newUUID = uuid.New - }() - newUUID = func() string { - return testUUID - } - var ( - mockError = errors.New("mock error") - dockerTimeoutErr = &dockerapi.DockerTimeoutError{} - testExecAgentState = apicontainer.ManagedAgentState{ - ID: testUUID, - LastStartedAt: nowTime, - Metadata: map[string]interface{}{ - "PID": "456", - "DockerExecID": "789", - "CMD": "amazon-ssm-agent", - }, - } - ) - tt := []struct { - name string - execEnabled bool - expectedRestartStatus RestartStatus - execAgentState apicontainer.ManagedAgentState - containerExecInspectRes *types.ContainerExecInspect - expectedInspectErr error - expectedRestartErr error - expectedExecAgentStatus apicontainerstatus.ManagedAgentStatus - }{ - { - name: "test with exec agent disabled", - execEnabled: false, - expectedRestartStatus: NotRestarted, - }, - { - name: "test with inspect timeout error", - execEnabled: true, - execAgentState: testExecAgentState, - expectedInspectErr: dockerTimeoutErr, - expectedRestartErr: nil, - expectedRestartStatus: Unknown, - expectedExecAgentStatus: apicontainerstatus.ManagedAgentStopped, - }, - { - name: "test with other inspect error", - execEnabled: true, - execAgentState: testExecAgentState, - expectedInspectErr: mockError, - expectedRestartErr: nil, - expectedRestartStatus: Unknown, - expectedExecAgentStatus: apicontainerstatus.ManagedAgentStopped, - }, - { - name: "test with exec command still running", - execEnabled: true, - execAgentState: testExecAgentState, - containerExecInspectRes: &types.ContainerExecInspect{ - Running: true, - }, - expectedRestartStatus: NotRestarted, - expectedExecAgentStatus: apicontainerstatus.ManagedAgentRunning, - }, - { - name: "test with exec command stopped", - execEnabled: true, - execAgentState: testExecAgentState, - containerExecInspectRes: &types.ContainerExecInspect{ - Running: false, - }, - expectedRestartStatus: Restarted, - expectedExecAgentStatus: apicontainerstatus.ManagedAgentRunning, - }, - } - - for _, test := range tt { - t.Run(test.name, func(t *testing.T) { - testTask := &apitask.Task{ - Arn: "taskArn:aws:ecs:region:account-id:task/test-task-taskArn", - Containers: []*apicontainer.Container{{ - RuntimeID: testContainerId, - }}, - } - - if test.execEnabled { - testTask.Containers[0].ManagedAgentsUnsafe = []apicontainer.ManagedAgent{ - { - Name: ExecuteCommandAgentName, - ManagedAgentState: test.execAgentState, - }, - } - times := 1 - if test.expectedInspectErr == mockError { - times = maxRetries - } - execMD := getAgentMetadata(testTask.Containers[0]) - client.EXPECT().InspectContainerExec(gomock.Any(), execMD.DockerExecID, dockerclient.ContainerExecInspectTimeout). - Return(test.containerExecInspectRes, test.expectedInspectErr).Times(times) - } - - // Expect calls made by Start() - if test.containerExecInspectRes != nil && !test.containerExecInspectRes.Running { - client.EXPECT().CreateContainerExec(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(&types.IDResponse{ID: testNewDockerExecID}, nil). - Times(1) - - client.EXPECT().StartContainerExec(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(nil). - Times(1) - - client.EXPECT().InspectContainerExec(gomock.Any(), gomock.Any(), gomock.Any()). - Return(&types.ContainerExecInspect{ - ExecID: testNewDockerExecID, - Pid: testNewPID, - Running: true, - }, nil). - Times(1) - } - - mgr := newTestManager() - restarted, err := mgr.RestartAgentIfStopped(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) - assert.Equal(t, test.expectedRestartErr, err) - assert.Equal(t, test.expectedRestartStatus, restarted, "expected: %s, actual: %s", test.expectedRestartStatus, restarted) - - if test.expectedRestartStatus != Restarted { - ma, _ := testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) - actualState := ma.ManagedAgentState - assert.Equal(t, test.execAgentState, actualState, "ExecCommandAgentMetadata was incorrectly modified") - } else { - ma, _ := testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) - actualState := ma.ManagedAgentState - execMD := getAgentMetadata(testTask.Containers[0]) - assert.Equal(t, strconv.Itoa(testNewPID), execMD.PID, - "ExecCommandAgentMetadata.PID is not the newest after restart") - assert.Equal(t, testNewDockerExecID, execMD.DockerExecID, - "ExecCommandAgentMetadata.ExecID is not the newest after restart") - assert.Equal(t, "/ecs-execute-command-test-uid/amazon-ssm-agent", execMD.CMD, - "ExecCommandAgentMetadata.CMD is not the newest after restart") - assert.Equal(t, test.expectedExecAgentStatus, actualState.Status, - "ExecCommandAgentStatus is not correct after restart") - } - }) - } -} - -func newTestManager() *manager { - m := NewManager() - m.retryMaxDelay = time.Millisecond * 30 - m.retryMinDelay = time.Millisecond * 1 - m.startRetryTimeout = time.Second * 2 - m.inspectRetryTimeout = time.Second - return m -} diff --git a/agent/engine/execcmd/manager_start_test.go b/agent/engine/execcmd/manager_start_test.go new file mode 100644 index 00000000000..3810ed25646 --- /dev/null +++ b/agent/engine/execcmd/manager_start_test.go @@ -0,0 +1,476 @@ +// +build unit + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +package execcmd + +import ( + "context" + "errors" + "fmt" + "strconv" + "testing" + "time" + + "github.com/pborman/uuid" + + "github.com/aws/amazon-ecs-agent/agent/api/container" + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + apicontainerstatus "github.com/aws/amazon-ecs-agent/agent/api/container/status" + errors2 "github.com/aws/amazon-ecs-agent/agent/api/errors" + apitask "github.com/aws/amazon-ecs-agent/agent/api/task" + "github.com/aws/amazon-ecs-agent/agent/dockerclient" + "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi" + mock_dockerapi "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi/mocks" + + "github.com/docker/docker/api/types" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func getAgentMetadata(container *container.Container) AgentMetadata { + ma, _ := container.GetManagedAgentByName(ExecuteCommandAgentName) + return MapToAgentMetadata(ma.Metadata) +} + +func TestStartAgent(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + client := mock_dockerapi.NewMockDockerClient(ctrl) + const ( + testPid1 = 9876 + testContainerRuntimeId = "123abc" + testDockerExecId = "mockDockerExecID" + ) + nowTime := time.Now() + zeroTime := time.Time{} + var ( + mockError = errors.New("mock error") + testContainers = []*apicontainer.Container{{ + RuntimeID: testContainerRuntimeId, + }} + ) + + tt := []struct { + name string + execEnabled bool + containers []*apicontainer.Container + expectCreateContainerExec bool + createContainerExecRes *types.IDResponse + createContainerExecErr error + expectStartContainerExec bool + startContainerExecErr error + expectInspectContainerExec bool + inspectContainerExecRes *types.ContainerExecInspect + inspectContainerExecErr error + expectedError error + expectedStatus apicontainerstatus.ManagedAgentStatus + expectedStartTime time.Time + }{ + { + name: "test exec disabled", + execEnabled: false, + containers: testContainers, + }, + { + name: "test with create container error", + execEnabled: true, + containers: testContainers, + expectCreateContainerExec: true, + expectedStatus: apicontainerstatus.ManagedAgentStopped, + createContainerExecErr: mockError, + expectedError: StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [create]: %v", mockError), retryable: true}, + expectedStartTime: zeroTime, + }, + { + name: "test with start container error", + execEnabled: true, + containers: testContainers, + expectCreateContainerExec: true, + createContainerExecRes: &types.IDResponse{ + ID: testDockerExecId, + }, + expectStartContainerExec: true, + expectedStatus: apicontainerstatus.ManagedAgentStopped, + startContainerExecErr: mockError, + expectedError: StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [pre-start]: %v", mockError), retryable: true}, + expectedStartTime: zeroTime, + }, + { + name: "test with inspect container error", + execEnabled: true, + containers: testContainers, + expectCreateContainerExec: true, + createContainerExecRes: &types.IDResponse{ + ID: testDockerExecId, + }, + expectStartContainerExec: true, + startContainerExecErr: nil, // Simulate StartContainerExec succeeds + expectInspectContainerExec: true, + expectedStatus: apicontainerstatus.ManagedAgentStopped, + inspectContainerExecErr: mockError, + expectedError: StartError{error: fmt.Errorf("unable to start ExecuteCommandAgent [inspect]: %v", mockError), retryable: true}, + expectedStartTime: zeroTime, + }, + { + name: "test happy path", + execEnabled: true, + containers: testContainers, + expectCreateContainerExec: true, + createContainerExecRes: &types.IDResponse{ + ID: testDockerExecId, + }, + expectStartContainerExec: true, + startContainerExecErr: nil, // Simulate StartContainerExec succeeds + expectInspectContainerExec: true, + expectedStatus: apicontainerstatus.ManagedAgentRunning, + expectedStartTime: nowTime, + inspectContainerExecRes: &types.ContainerExecInspect{ + ExecID: testDockerExecId, + Pid: testPid1, + Running: true, + }, + }, + } + testUUID := "test-uid" + defer func() { + newUUID = uuid.New + }() + newUUID = func() string { + return testUUID + } + for _, test := range tt { + t.Run(test.name, func(t *testing.T) { + + testTask := &apitask.Task{ + Arn: "taskArn:aws:ecs:region:account-id:task/test-task-taskArn", + Containers: test.containers, + } + if test.execEnabled { + for _, c := range testTask.Containers { + c.ManagedAgentsUnsafe = []apicontainer.ManagedAgent{ + { + Name: ExecuteCommandAgentName, + ManagedAgentState: apicontainer.ManagedAgentState{ + ID: testUUID, + }, + }, + } + } + } + + times := maxRetries + retryableErr, isRetryable := test.expectedError.(errors2.RetriableError) + if test.expectedError == nil || (isRetryable && !retryableErr.Retry()) { + times = 1 + } + if test.expectCreateContainerExec { + execCfg := types.ExecConfig{ + User: specUser, + Detach: true, + Cmd: []string{specTestCmd}, + } + client.EXPECT().CreateContainerExec(gomock.Any(), testTask.Containers[0].RuntimeID, execCfg, dockerclient.ContainerExecCreateTimeout). + Return(test.createContainerExecRes, test.createContainerExecErr). + Times(times) + } + + if test.expectStartContainerExec { + client.EXPECT().StartContainerExec(gomock.Any(), testDockerExecId, gomock.Any(), dockerclient.ContainerExecStartTimeout). + Return(test.startContainerExecErr). + Times(times) + } + + if test.expectInspectContainerExec { + client.EXPECT().InspectContainerExec(gomock.Any(), testDockerExecId, dockerclient.ContainerExecInspectTimeout). + Return(test.inspectContainerExecRes, test.inspectContainerExecErr). + Times(times) + } + + mgr := newTestManager() + prevMetadata := getAgentMetadata(test.containers[0]) + err := mgr.StartAgent(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) + if test.expectedError != nil { + assert.Equal(t, test.expectedError, err, "Wrong error returned") + // When there's an error, ExecCommandAgentMetadata should not be modified + newMetadata := getAgentMetadata(test.containers[0]) + assert.Equal(t, prevMetadata, newMetadata) + } else { // No error case + assert.NoError(t, err, "No error was expected") + if !test.execEnabled { + _, ok := test.containers[0].GetManagedAgentByName(ExecuteCommandAgentName) + assert.False(t, ok) + } else { + ma, _ := test.containers[0].GetManagedAgentByName(ExecuteCommandAgentName) + execMD := getAgentMetadata(test.containers[0]) + assert.Equal(t, strconv.Itoa(testPid1), execMD.PID, "PID not equal") + assert.Equal(t, testDockerExecId, execMD.DockerExecID, "DockerExecId not equal") + assert.Equal(t, specTestCmd, execMD.CMD) + assert.Equal(t, test.expectedStatus, ma.Status, "Exec status not equal") + assert.WithinDuration(t, test.expectedStartTime, ma.LastStartedAt, 5*time.Second, "StartedAt not equal") + } + } + }) + } +} + +func TestIdempotentStartAgent(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + client := mock_dockerapi.NewMockDockerClient(ctrl) + const ( + testDockerExecId = "abc" + testPid = 111 + ) + var ( + testPidStr = strconv.Itoa(testPid) + testCmd = specTestCmd + ) + testUUID := "test-uid" + defer func() { + newUUID = uuid.New + }() + newUUID = func() string { + return testUUID + } + + testTask := &apitask.Task{ + Arn: "taskArn:aws:ecs:region:account-id:task/test-task-taskArn", + Containers: []*apicontainer.Container{{ + RuntimeID: "123", + ManagedAgentsUnsafe: []apicontainer.ManagedAgent{ + { + Name: ExecuteCommandAgentName, + ManagedAgentState: apicontainer.ManagedAgentState{ + ID: testUUID, + }, + }, + }, + }}, + } + + execCfg := types.ExecConfig{ + User: specUser, + Detach: true, + Cmd: []string{specTestCmd}, + } + client.EXPECT().CreateContainerExec(gomock.Any(), testTask.Containers[0].RuntimeID, execCfg, dockerclient.ContainerExecCreateTimeout). + Return(&types.IDResponse{ID: testDockerExecId}, nil). + Times(1) + + client.EXPECT().StartContainerExec(gomock.Any(), testDockerExecId, gomock.Any(), dockerclient.ContainerExecStartTimeout). + Return(nil). + Times(1) + + client.EXPECT().InspectContainerExec(gomock.Any(), testDockerExecId, dockerclient.ContainerExecInspectTimeout). + Return(&types.ContainerExecInspect{ + ExecID: testDockerExecId, + Pid: testPid, + Running: true, + }, nil). + Times(2) + + mgr := newTestManager() + err := mgr.StartAgent(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) + assert.NoError(t, err) + + ma, _ := testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) + execMD := getAgentMetadata(testTask.Containers[0]) + firstStart := ma.LastStartedAt + assert.NotNil(t, ma.LastStartedAt) + assert.Equal(t, testPidStr, execMD.PID) + assert.Equal(t, testDockerExecId, execMD.DockerExecID) + assert.Equal(t, testCmd, execMD.CMD) + assert.Equal(t, apicontainerstatus.ManagedAgentRunning, ma.Status) + + // Second call to start. The mock's expected call times is 1 (except for inspect); the absence of "too many calls" + // along with unchanged metadata guarantees idempotency + err = mgr.StartAgent(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) + assert.NoError(t, err) + + ma, _ = testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) + execMD = getAgentMetadata(testTask.Containers[0]) + // check StartedAt is not Zero and hasn't been updated + assert.NotNil(t, ma.LastStartedAt) + assert.False(t, ma.LastStartedAt.IsZero()) + assert.Equal(t, ma.LastStartedAt, firstStart) + assert.Equal(t, testPidStr, execMD.PID) + assert.Equal(t, testDockerExecId, execMD.DockerExecID) + assert.Equal(t, testCmd, execMD.CMD) + assert.Equal(t, apicontainerstatus.ManagedAgentRunning, ma.Status) +} + +func TestRestartAgentIfStopped(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + client := mock_dockerapi.NewMockDockerClient(ctrl) + nowTime := time.Now() + + const ( + testContainerId = "123" + testNewDockerExecID = "newDockerExecId" + testNewPID = 111 + ) + testUUID := "test-uid" + defer func() { + newUUID = uuid.New + }() + newUUID = func() string { + return testUUID + } + var ( + mockError = errors.New("mock error") + dockerTimeoutErr = &dockerapi.DockerTimeoutError{} + testExecAgentState = apicontainer.ManagedAgentState{ + ID: testUUID, + LastStartedAt: nowTime, + Metadata: map[string]interface{}{ + "PID": "456", + "DockerExecID": "789", + "CMD": "amazon-ssm-agent", + }, + } + ) + tt := []struct { + name string + execEnabled bool + expectedRestartStatus RestartStatus + execAgentState apicontainer.ManagedAgentState + containerExecInspectRes *types.ContainerExecInspect + expectedInspectErr error + expectedRestartErr error + expectedExecAgentStatus apicontainerstatus.ManagedAgentStatus + }{ + { + name: "test with exec agent disabled", + execEnabled: false, + expectedRestartStatus: NotRestarted, + }, + { + name: "test with inspect timeout error", + execEnabled: true, + execAgentState: testExecAgentState, + expectedInspectErr: dockerTimeoutErr, + expectedRestartErr: nil, + expectedRestartStatus: Unknown, + expectedExecAgentStatus: apicontainerstatus.ManagedAgentStopped, + }, + { + name: "test with other inspect error", + execEnabled: true, + execAgentState: testExecAgentState, + expectedInspectErr: mockError, + expectedRestartErr: nil, + expectedRestartStatus: Unknown, + expectedExecAgentStatus: apicontainerstatus.ManagedAgentStopped, + }, + { + name: "test with exec command still running", + execEnabled: true, + execAgentState: testExecAgentState, + containerExecInspectRes: &types.ContainerExecInspect{ + Running: true, + }, + expectedRestartStatus: NotRestarted, + expectedExecAgentStatus: apicontainerstatus.ManagedAgentRunning, + }, + { + name: "test with exec command stopped", + execEnabled: true, + execAgentState: testExecAgentState, + containerExecInspectRes: &types.ContainerExecInspect{ + Running: false, + }, + expectedRestartStatus: Restarted, + expectedExecAgentStatus: apicontainerstatus.ManagedAgentRunning, + }, + } + + for _, test := range tt { + t.Run(test.name, func(t *testing.T) { + testTask := &apitask.Task{ + Arn: "taskArn:aws:ecs:region:account-id:task/test-task-taskArn", + Containers: []*apicontainer.Container{{ + RuntimeID: testContainerId, + }}, + } + + if test.execEnabled { + testTask.Containers[0].ManagedAgentsUnsafe = []apicontainer.ManagedAgent{ + { + Name: ExecuteCommandAgentName, + ManagedAgentState: test.execAgentState, + }, + } + times := 1 + if test.expectedInspectErr == mockError { + times = maxRetries + } + execMD := getAgentMetadata(testTask.Containers[0]) + client.EXPECT().InspectContainerExec(gomock.Any(), execMD.DockerExecID, dockerclient.ContainerExecInspectTimeout). + Return(test.containerExecInspectRes, test.expectedInspectErr).Times(times) + } + + // Expect calls made by Start() + if test.containerExecInspectRes != nil && !test.containerExecInspectRes.Running { + client.EXPECT().CreateContainerExec(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(&types.IDResponse{ID: testNewDockerExecID}, nil). + Times(1) + + client.EXPECT().StartContainerExec(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil). + Times(1) + + client.EXPECT().InspectContainerExec(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&types.ContainerExecInspect{ + ExecID: testNewDockerExecID, + Pid: testNewPID, + Running: true, + }, nil). + Times(1) + } + + mgr := newTestManager() + restarted, err := mgr.RestartAgentIfStopped(context.TODO(), client, testTask, testTask.Containers[0], testTask.Containers[0].RuntimeID) + assert.Equal(t, test.expectedRestartErr, err) + assert.Equal(t, test.expectedRestartStatus, restarted, "expected: %s, actual: %s", test.expectedRestartStatus, restarted) + + if test.expectedRestartStatus != Restarted { + ma, _ := testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) + actualState := ma.ManagedAgentState + assert.Equal(t, test.execAgentState, actualState, "ExecCommandAgentMetadata was incorrectly modified") + } else { + ma, _ := testTask.Containers[0].GetManagedAgentByName(ExecuteCommandAgentName) + actualState := ma.ManagedAgentState + execMD := getAgentMetadata(testTask.Containers[0]) + assert.Equal(t, strconv.Itoa(testNewPID), execMD.PID, + "ExecCommandAgentMetadata.PID is not the newest after restart") + assert.Equal(t, testNewDockerExecID, execMD.DockerExecID, + "ExecCommandAgentMetadata.ExecID is not the newest after restart") + assert.Equal(t, specTestCmd, execMD.CMD, + "ExecCommandAgentMetadata.CMD is not the newest after restart") + assert.Equal(t, test.expectedExecAgentStatus, actualState.Status, + "ExecCommandAgentStatus is not correct after restart") + } + }) + } +} + +func newTestManager() *manager { + m := NewManager() + m.retryMaxDelay = time.Millisecond * 30 + m.retryMinDelay = time.Millisecond * 1 + m.startRetryTimeout = time.Second * 2 + m.inspectRetryTimeout = time.Second + return m +} diff --git a/agent/engine/execcmd/manager_start_windows.go b/agent/engine/execcmd/manager_start_windows.go index 641951096e9..a02ef67c34e 100644 --- a/agent/engine/execcmd/manager_start_windows.go +++ b/agent/engine/execcmd/manager_start_windows.go @@ -1,19 +1,26 @@ +// +build windows + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. package execcmd -import ( - "context" +import apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" - apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" - apitask "github.com/aws/amazon-ecs-agent/agent/api/task" - "github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi" +const ( + execAgentCmdUser = "NT AUTHORITY\\SYSTEM" + execAgentCmdBinDir = "C:\\Program Files\\Amazon\\SSM" ) -// Note: exec cmd agent is a linux/windows feature, thus implemented here as a no-op. -func (m *manager) RestartAgentIfStopped(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) (RestartStatus, error) { - return NotRestarted, nil -} - -// Note: exec cmd agent is a linux/windows feature, thus implemented here as a no-op. -func (m *manager) StartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) error { - return nil +func getExecAgentCmdBinDir(ma *apicontainer.ManagedAgent) string { + return execAgentCmdBinDir } diff --git a/agent/engine/execcmd/manager_start_windows_test.go b/agent/engine/execcmd/manager_start_windows_test.go new file mode 100644 index 00000000000..dec0d085cf2 --- /dev/null +++ b/agent/engine/execcmd/manager_start_windows_test.go @@ -0,0 +1,21 @@ +// +build windows,unit + +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execcmd + +const ( + specTestCmd = "C:\\Program Files\\Amazon\\SSM\\amazon-ssm-agent.exe" + specUser = "NT AUTHORITY\\SYSTEM" +) From e4279a35fc8d840986b0e64de3f1a64ed31b289f Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Fri, 25 Jun 2021 15:49:32 -0700 Subject: [PATCH 06/14] small bug fix --- agent/engine/execcmd/manager_init_task_windows.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/engine/execcmd/manager_init_task_windows.go b/agent/engine/execcmd/manager_init_task_windows.go index 5c7766431b0..2268acbb393 100644 --- a/agent/engine/execcmd/manager_init_task_windows.go +++ b/agent/engine/execcmd/manager_init_task_windows.go @@ -117,13 +117,13 @@ func createNewConfigDir(agentConfig, configDirPath string) error { // make individual config files agentConfigFilePath := filepath.Join(configDirPath, containerConfigFileName) - err = createNewExecAgentConfigFile(agentConfigFilePath, agentConfig) + err = createNewExecAgentConfigFile(agentConfig, agentConfigFilePath) if err != nil { return err } logConfigFilePath := filepath.Join(configDirPath, ExecAgentLogConfigFileName) - err = createNewExecAgentConfigFile(logConfigFilePath, execAgentLogConfigTemplate) + err = createNewExecAgentConfigFile(execAgentLogConfigTemplate, logConfigFilePath) if err != nil { return err } From ac220761f175fb56ae5293fed63bb4599ce81f86 Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Fri, 25 Jun 2021 12:32:20 -0700 Subject: [PATCH 07/14] Small variable name changes --- agent/engine/execcmd/manager_init_task_windows.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/agent/engine/execcmd/manager_init_task_windows.go b/agent/engine/execcmd/manager_init_task_windows.go index 2268acbb393..548dc967b0d 100644 --- a/agent/engine/execcmd/manager_init_task_windows.go +++ b/agent/engine/execcmd/manager_init_task_windows.go @@ -35,7 +35,7 @@ var ( // ecsAgentDepsBinDir is the directory where ECS Agent will read versions of SSM agent ecsAgentDepsBinDir = ecsAgentExecDepsDir + "\\bin" - containerDepsFolder = "C:\\Program Files\\Amazon\\SSM" + ContainerDepsFolder = config.AmazonProgramFiles + "\\SSM" SSMAgentBinName = "amazon-ssm-agent.exe" SSMAgentWorkerBinName = "ssm-agent-worker.exe" @@ -44,7 +44,7 @@ var ( HostLogDir = config.AmazonECSProgramData + "\\exec" ContainerLogDir = config.AmazonProgramData + "\\SSM" - SSMPluginDir = "C:\\Program Files\\Amazon\\SSM\\Plugins" + SSMPluginDir = config.AmazonProgramFiles + "\\SSM\\Plugins" // since ecs agent windows is not running in a container, the agent and host log dirs are the same ECSAgentExecLogDir = config.AmazonECSProgramData + "\\exec" @@ -167,12 +167,12 @@ func addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid string, session // Add ssm binary mount hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( latestBinVersionDir, - containerDepsFolder)) + ContainerDepsFolder)) // Add ssm configuration dir mount hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( filepath.Join(ECSAgentExecConfigDir, configDirHash), - filepath.Join(containerDepsFolder, "configuration"))) + filepath.Join(ContainerDepsFolder, "configuration"))) // Add ssm log bind mount hostConfig.Binds = append(hostConfig.Binds, getBindMountMapping( From ddb3a0d850c0e53cfa7cc975c26e880c81b5e3d7 Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Sun, 27 Jun 2021 17:17:09 -0700 Subject: [PATCH 08/14] Disable execcmd for Win2016 --- agent/app/agent_capability.go | 11 +++- agent/app/agent_capability_test.go | 10 ++++ agent/app/agent_capability_unix.go | 4 ++ agent/app/agent_capability_unspecified.go | 4 ++ agent/app/agent_capability_windows.go | 9 ++++ agent/app/agent_capability_windows_test.go | 60 ++++++++++++++++++++++ 6 files changed, 96 insertions(+), 2 deletions(-) diff --git a/agent/app/agent_capability.go b/agent/app/agent_capability.go index d0164f34ce9..cc90dc89d71 100644 --- a/agent/app/agent_capability.go +++ b/agent/app/agent_capability.go @@ -99,8 +99,9 @@ var ( // use empty struct as value type to simulate set capabilityExecInvalidSsmVersions = map[string]struct{}{} - pathExists = defaultPathExists - getSubDirectories = defaultGetSubDirectories + pathExists = defaultPathExists + getSubDirectories = defaultGetSubDirectories + isPlatformExecSupported = defaultIsPlatformExecSupported // List of capabilities that are not supported on external capacity. externalUnsupportedCapabilities = []string{ @@ -379,6 +380,12 @@ func (agent *ecsAgent) appendTaskENICapabilities(capabilities []*ecs.Attribute) } func (agent *ecsAgent) appendExecCapabilities(capabilities []*ecs.Attribute) ([]*ecs.Attribute, error) { + + // Only Windows 2019 and above are supported, all Linux supported + if platformSupported, err := isPlatformExecSupported(); err != nil || !platformSupported { + return capabilities, err + } + // for an instance to be exec-enabled, it needs resources needed by SSM (binaries, configuration files and certs) // the following bind mounts are defined in ecs-init and added to the ecs-agent container diff --git a/agent/app/agent_capability_test.go b/agent/app/agent_capability_test.go index 7f3cd283347..0753f3910c7 100644 --- a/agent/app/agent_capability_test.go +++ b/agent/app/agent_capability_test.go @@ -878,6 +878,7 @@ func TestCapabilitiesExecuteCommand(t *testing.T) { getSubDirectories func(path string) ([]string, error) invalidSsmVersions map[string]struct{} shouldHaveExecCapability bool + osPlatformNotSupported bool }{ { name: "execute-command capability should not be added if any required file is not found", @@ -917,6 +918,13 @@ func TestCapabilitiesExecuteCommand(t *testing.T) { getSubDirectories: func(path string) ([]string, error) { return []string{"3.0.236.0", "3.1.23.0"}, nil }, shouldHaveExecCapability: true, }, + { + name: "execute-command capability should not be added if os platform is not supported", + pathExists: func(path string, shouldBeDirectory bool) (bool, error) { return true, nil }, + getSubDirectories: func(path string) ([]string, error) { return []string{"3.0.236.0"}, nil }, + osPlatformNotSupported: true, + shouldHaveExecCapability: false, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -924,10 +932,12 @@ func TestCapabilitiesExecuteCommand(t *testing.T) { getSubDirectories = tc.getSubDirectories oCapabilityExecInvalidSsmVersions := capabilityExecInvalidSsmVersions capabilityExecInvalidSsmVersions = tc.invalidSsmVersions + isPlatformExecSupported = func() (bool, error) { return !tc.osPlatformNotSupported, nil } defer func() { mockPathExists(false) getSubDirectories = defaultGetSubDirectories capabilityExecInvalidSsmVersions = oCapabilityExecInvalidSsmVersions + isPlatformExecSupported = defaultIsPlatformExecSupported }() ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/agent/app/agent_capability_unix.go b/agent/app/agent_capability_unix.go index eff6534a01e..b7fed896c07 100644 --- a/agent/app/agent_capability_unix.go +++ b/agent/app/agent_capability_unix.go @@ -231,3 +231,7 @@ func (agent *ecsAgent) getTaskENIPluginVersionAttribute() (*ecs.Attribute, error Value: aws.String(version), }, nil } + +func defaultIsPlatformExecSupported() (bool, error) { + return true, nil +} diff --git a/agent/app/agent_capability_unspecified.go b/agent/app/agent_capability_unspecified.go index 0a62f9e810f..28cfb80d45a 100644 --- a/agent/app/agent_capability_unspecified.go +++ b/agent/app/agent_capability_unspecified.go @@ -140,3 +140,7 @@ func (agent *ecsAgent) appendFSxWindowsFileServerCapabilities(capabilities []*ec func (agent *ecsAgent) getTaskENIPluginVersionAttribute() (*ecs.Attribute, error) { return nil, errors.New("unsupported platform") } + +func defaultIsPlatformExecSupported() (bool, error) { + return false, nil +} diff --git a/agent/app/agent_capability_windows.go b/agent/app/agent_capability_windows.go index 25a8a7e33c4..99225922e2a 100644 --- a/agent/app/agent_capability_windows.go +++ b/agent/app/agent_capability_windows.go @@ -140,3 +140,12 @@ func (agent *ecsAgent) getTaskENIPluginVersionAttribute() (*ecs.Attribute, error Value: aws.String(version), }, nil } + +var isWindows2016 = config.IsWindows2016 + +func defaultIsPlatformExecSupported() (bool, error) { + if windows2016, err := isWindows2016(); err != nil || windows2016 { + return false, err + } + return true, nil +} diff --git a/agent/app/agent_capability_windows_test.go b/agent/app/agent_capability_windows_test.go index 281df83bc20..747a4db32f1 100644 --- a/agent/app/agent_capability_windows_test.go +++ b/agent/app/agent_capability_windows_test.go @@ -308,3 +308,63 @@ func TestAppendFSxWindowsFileServerCapabilitiesFalse(t *testing.T) { assert.Equal(t, len(expectedCapabilities), len(capabilities)) } + +func TestAppendExecCapabilities(t *testing.T) { + var inputCapabilities []*ecs.Attribute + var expectedCapabilities []*ecs.Attribute + execCapability := ecs.Attribute{ + Name: aws.String(attributePrefix + capabilityExec), + } + + expectedCapabilities = append(expectedCapabilities, + []*ecs.Attribute{}...) + testCases := []struct { + name string + pathExists func(string, bool) (bool, error) + getSubDirectories func(path string) ([]string, error) + isWindows2016Instance bool + shouldHaveExecCapability bool + }{ + { + name: "execute-command capability should not be added on Win2016 instances", + pathExists: func(path string, shouldBeDirectory bool) (bool, error) { return true, nil }, + getSubDirectories: func(path string) ([]string, error) { return []string{"3.0.236.0"}, nil }, + isWindows2016Instance: true, + shouldHaveExecCapability: false, + }, + { + name: "execute-command capability should be added if not a Win2016 instances", + pathExists: func(path string, shouldBeDirectory bool) (bool, error) { return true, nil }, + getSubDirectories: func(path string) ([]string, error) { return []string{"3.0.236.0"}, nil }, + isWindows2016Instance: false, + shouldHaveExecCapability: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + isWindows2016 = func() (bool, error) { return tc.isWindows2016Instance, nil } + pathExists = tc.pathExists + getSubDirectories = tc.getSubDirectories + + defer func() { + isWindows2016 = config.IsWindows2016 + pathExists = defaultPathExists + getSubDirectories = defaultGetSubDirectories + }() + agent := &ecsAgent{ + cfg: &config.Config{}, + } + + capabilities, err := agent.appendExecCapabilities(inputCapabilities) + + assert.NoError(t, err) + + if tc.shouldHaveExecCapability { + assert.Contains(t, capabilities, &execCapability) + } else { + assert.NotContains(t, capabilities, &execCapability) + } + }) + } +} From 6e23feb0d86cae3d9c29cc8f60eba69757b7741a Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Thu, 24 Jun 2021 10:40:09 -0700 Subject: [PATCH 09/14] Create execcmd windows test infrastructure --- misc/exec-command-agent-test/build.ps1 | 44 +++++++++++++++++++ misc/exec-command-agent-test/sleep/main.go | 2 +- .../windows.dockerfile | 13 ++++++ scripts/run-integ-tests.ps1 | 1 + 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 misc/exec-command-agent-test/build.ps1 create mode 100644 misc/exec-command-agent-test/windows.dockerfile diff --git a/misc/exec-command-agent-test/build.ps1 b/misc/exec-command-agent-test/build.ps1 new file mode 100644 index 00000000000..400010cf633 --- /dev/null +++ b/misc/exec-command-agent-test/build.ps1 @@ -0,0 +1,44 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# create the container image used to run execcmd integ tests +docker build -t "amazon/amazon-ecs-exec-command-agent-windows-test:make" -f "${PSScriptRoot}/windows.dockerfile" ${PSScriptRoot} + +$SIMULATED_ECS_AGENT_DEPS_BIN_DIR="C:\Program Files\Amazon\ECS\managed-agents\execute-command\bin\1.0.0.0" +Remove-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR -Recurse -Force -ErrorAction SilentlyContinue +New-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR -ItemType Directory -Force +go build -tags integration -installsuffix cgo -a -o $SIMULATED_ECS_AGENT_DEPS_BIN_DIR\amazon-ssm-agent.exe ${PSScriptRoot}\sleep\ +New-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR\ssm-agent-worker.exe -ItemType File -Force +New-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR\ssm-session-worker.exe -ItemType File -Force + +# Dont want to destroy local development environments plugin folder +$SIMULATED_SSM_PLUGINS_DIR="C:\Program Files\Amazon\SSM\Plugins" +if(!(Test-Path -path $SIMULATED_SSM_PLUGINS_DIR)) +{ + New-Item -Path $SIMULATED_SSM_PLUGINS_DIR -ItemType directory -Force +} + +if(!(Test-Path -path $SIMULATED_SSM_PLUGINS_DIR\SessionManagerShell)) +{ + New-Item -Path $SIMULATED_SSM_PLUGINS_DIR\SessionManagerShell -ItemType directory -Force +} + +if(!(Test-Path -path $SIMULATED_SSM_PLUGINS_DIR\awsCloudWatch)) +{ + New-Item -Path $SIMULATED_SSM_PLUGINS_DIR\awsCloudWatch -ItemType directory -Force +} + +if(!(Test-Path -path $SIMULATED_SSM_PLUGINS_DIR\awsDomainJoin)) +{ + New-Item -Path $SIMULATED_SSM_PLUGINS_DIR\awsDomainJoin -ItemType directory -Force +} \ No newline at end of file diff --git a/misc/exec-command-agent-test/sleep/main.go b/misc/exec-command-agent-test/sleep/main.go index ac18bf884b4..fb4b462a9dd 100644 --- a/misc/exec-command-agent-test/sleep/main.go +++ b/misc/exec-command-agent-test/sleep/main.go @@ -1,4 +1,4 @@ -// +build !windows,integration +// +build integration // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/misc/exec-command-agent-test/windows.dockerfile b/misc/exec-command-agent-test/windows.dockerfile new file mode 100644 index 00000000000..0eb3f4e41fe --- /dev/null +++ b/misc/exec-command-agent-test/windows.dockerfile @@ -0,0 +1,13 @@ +# escape=` + +FROM golang:1.12 as build-env +MAINTAINER Amazon Web Services, Inc. + +# There is dockerfile documentation on how to treat windows paths +WORKDIR C:\Users\Administrator\go\src\sleep +COPY ./sleep C:/Users/Administrator/go/src/sleep +RUN go build -tags integration -installsuffix cgo -a -o C:/Users/Administrator/go/src/sleep/sleep.exe . + +FROM amazon-ecs-ftest-windows-base:make +MAINTAINER Amazon Web Services, Inc. +COPY --from=build-env C:/Users/Administrator/go/src/sleep/sleep.exe C:/ diff --git a/scripts/run-integ-tests.ps1 b/scripts/run-integ-tests.ps1 index 97beaea3b02..8e01996d530 100755 --- a/scripts/run-integ-tests.ps1 +++ b/scripts/run-integ-tests.ps1 @@ -51,6 +51,7 @@ Invoke-Expression "${PSScriptRoot}\..\misc\image-cleanup-test-images\build.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\stats-windows\build.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\container-health-windows\build.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\netkitten\build.ps1" +Invoke-Expression "${PSScriptRoot}\..\misc\exec-command-agent-test\build.ps1" # Run the tests $cwd = (pwd).Path From 988cc903e46f46f53ddf32ea1314b6d144092a85 Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Fri, 25 Jun 2021 12:32:33 -0700 Subject: [PATCH 10/14] Windows execcmd engine integ tests --- agent/engine/engine_windows_integ_test.go | 387 +++++++++++++++++- .../execcmd/manager_init_task_windows.go | 2 +- 2 files changed, 383 insertions(+), 6 deletions(-) diff --git a/agent/engine/engine_windows_integ_test.go b/agent/engine/engine_windows_integ_test.go index 4416adbcf13..c2d50e06ca8 100644 --- a/agent/engine/engine_windows_integ_test.go +++ b/agent/engine/engine_windows_integ_test.go @@ -22,9 +22,17 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "strconv" "strings" "testing" + "time" + + "github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs" + + "github.com/cihub/seelog" + + "github.com/docker/docker/api/types" "github.com/aws/amazon-ecs-agent/agent/api" apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" @@ -53,11 +61,14 @@ import ( ) const ( - dockerEndpoint = "npipe:////./pipe/docker_engine" - testVolumeImage = "amazon/amazon-ecs-volumes-test:make" - testRegistryImage = "amazon/amazon-ecs-netkitten:make" - testBaseImage = "amazon-ecs-ftest-windows-base:make" - dockerVolumeDirectoryFormat = "c:\\ProgramData\\docker\\volumes\\%s\\_data" + dockerEndpoint = "npipe:////./pipe/docker_engine" + testVolumeImage = "amazon/amazon-ecs-volumes-test:make" + testRegistryImage = "amazon/amazon-ecs-netkitten:make" + testExecCommandAgentImage = "amazon/amazon-ecs-exec-command-agent-windows-test:make" + testBaseImage = "amazon-ecs-ftest-windows-base:make" + dockerVolumeDirectoryFormat = "c:\\ProgramData\\docker\\volumes\\%s\\_data" + testExecCommandAgentKillBin = "c:\\kill.exe" + testExecCommandAgentSleepBin = "c:\\sleep.exe" ) var endpoint = utils.DefaultIfBlank(os.Getenv(DockerEndpointEnvVariable), dockerEndpoint) @@ -506,3 +517,369 @@ func verifyContainerCredentialSpec(client *sdkClient.Client, id, credentialspecO return errors.New("unable to obtain credentialspec") } + +// This setup for execcmd is same as Linux, just implemented different +// TestExecCommandAgent validates ExecCommandAgent start and monitor processes. The algorithm to test is as follows: +// 1. Pre-setup: the build file in ../../misc/exec-command-agent-windows-test will create a special docker sleeper image +// based on a base windows image. This image simulates a ecs windows image and contains pre-baked /sleep and /kill binaries. +// /sleep is the main process used to launch the test container; /kill is an application that kills a process running in +// the container given a PID. +// The build file will also create a fake amazon-ssm-agent which is a go program that only sleeps for a certain time specified. +// +// 2. Setup: Create a new docker task engine with a modified path pointing to our fake amazon-ssm-agent binary +// 3. Create and start our test task using our test image +// 4. Wait for the task to start and verify that the expected ExecCommandAgent bind mounts are present in the containers +// 5. Verify that our fake amazon-ssm-agent was started inside the container using docker top, and retrieve its PID +// 6. Kill the fake amazon-ssm-agent using the PID retrieved in previous step +// 7. Verify that the engine restarted our fake amazon-ssm-agent by doing docker top one more time (a new PID should popup) +func TestExecCommandAgent(t *testing.T) { + // the execcmd feature is not supported for Win2016 + if windows2016, _ := config.IsWindows2016(); windows2016 { + return + } + const ( + testTaskId = "exec-command-agent-test-task" + testContainerName = "exec-command-agent-test-container" + sleepFor = time.Minute * 2 + ) + + client, err := sdkClient.NewClientWithOpts(sdkClient.WithHost(endpoint), sdkClient.WithVersion(sdkclientfactory.GetDefaultVersion().String())) + require.NoError(t, err, "Creating go docker client failed") + + testExecCmdHostBinDir := "C:\\Program Files\\Amazon\\ECS\\managed-agents\\execute-command\\bin" + + taskEngine, done, _ := setupEngineForExecCommandAgent(t, testExecCmdHostBinDir) + stateChangeEvents := taskEngine.StateChangeEvents() + defer done() + + testTask := createTestExecCommandAgentTask(testTaskId, testContainerName, sleepFor) + execAgentLogPath := filepath.Join("C:\\ProgramData\\Amazon\\ECS\\exec", testTaskId) + err = os.MkdirAll(execAgentLogPath, 0644) + require.NoError(t, err, "error creating execAgent log file") + _, err = os.Stat(execAgentLogPath) + require.NoError(t, err, "execAgent log dir doesn't exist") + err = os.MkdirAll(execcmd.ECSAgentExecConfigDir, 0644) + require.NoError(t, err, "error creating execAgent config dir") + + go taskEngine.AddTask(testTask) + + verifyContainerRunningStateChange(t, taskEngine) + verifyTaskRunningStateChange(t, taskEngine) + + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + containerMap, _ := taskEngine.(*DockerTaskEngine).state.ContainerMapByArn(testTask.Arn) + cid := containerMap[testTask.Containers[0].Name].DockerID + + // session limit is 2 + testconfigDirName, _ := execcmd.GetExecAgentConfigDir(2) + + // todo: change to file contents passed in + verifyExecCmdAgentExpectedMounts(t, ctx, client, testTaskId, cid, testContainerName, testExecCmdHostBinDir+"\\1.0.0.0", testconfigDirName) + pidA := verifyMockExecCommandAgentIsRunning(t, client, cid) + seelog.Infof("Verified mock ExecCommandAgent is running (pidA=%s)", pidA) + killMockExecCommandAgent(t, client, cid, pidA) + seelog.Infof("kill signal sent to ExecCommandAgent (pidA=%s)", pidA) + verifyMockExecCommandAgentIsStopped(t, client, cid, pidA) + seelog.Infof("Verified mock ExecCommandAgent was killed (pidA=%s)", pidA) + pidB := verifyMockExecCommandAgentIsRunning(t, client, cid) + seelog.Infof("Verified mock ExecCommandAgent was restarted (pidB=%s)", pidB) + require.NotEqual(t, pidA, pidB, "ExecCommandAgent PID did not change after restart") + + taskUpdate := createTestExecCommandAgentTask(testTaskId, testContainerName, sleepFor) + taskUpdate.SetDesiredStatus(apitaskstatus.TaskStopped) + go taskEngine.AddTask(taskUpdate) + + ctx, cancel = context.WithTimeout(context.Background(), time.Second*20) + go func() { + verifyTaskIsStopped(stateChangeEvents, testTask) + cancel() + }() + + <-ctx.Done() + require.NotEqual(t, context.DeadlineExceeded, ctx.Err(), "Timed out waiting for task (%s) to stop", testTaskId) + assert.NotNil(t, testTask.Containers[0].GetKnownExitCode(), "No exit code found") + // TODO: [ecs-exec] We should be able to wait for cleanup instead of calling deleteTask directly + taskEngine.(*DockerTaskEngine).deleteTask(testTask) + _, err = os.Stat(execAgentLogPath) + assert.True(t, os.IsNotExist(err), "execAgent log cleanup failed") + os.RemoveAll(execcmd.ECSAgentExecConfigDir) +} + +// TestManagedAgentEvent validates the emitted container events for a started and a stopped managed agent. +func TestManagedAgentEvent(t *testing.T) { + // the execcmd feature is not supported for Win2016 + if windows2016, _ := config.IsWindows2016(); windows2016 { + return + } + testcases := []struct { + Name string + ExpectedStatus apicontainerstatus.ManagedAgentStatus + ManagedAgentLifetime time.Duration + ShouldBeRunning bool + }{ + { + Name: "Confirmed emit RUNNING event", + ExpectedStatus: apicontainerstatus.ManagedAgentRunning, + ManagedAgentLifetime: 1, + ShouldBeRunning: true, + }, + { + Name: "Confirmed emit STOPPED event", + ExpectedStatus: apicontainerstatus.ManagedAgentStopped, + ManagedAgentLifetime: 0, + ShouldBeRunning: false, + }, + } + for _, tc := range testcases { + t.Run(tc.Name, func(t *testing.T) { + + const ( + testTaskId = "exec-command-agent-test-task" + testContainerName = "exec-command-agent-test-container" + ) + + client, err := sdkClient.NewClientWithOpts(sdkClient.WithHost(endpoint), sdkClient.WithVersion(sdkclientfactory.GetDefaultVersion().String())) + require.NoError(t, err, "Creating go docker client failed") + + testExecCmdHostBinDir := "C:\\Program Files\\Amazon\\ECS\\managed-agents\\execute-command\\bin" + + taskEngine, done, _ := setupEngineForExecCommandAgent(t, testExecCmdHostBinDir) + defer done() + + testTask := createTestExecCommandAgentTask(testTaskId, testContainerName, time.Minute*tc.ManagedAgentLifetime) + execAgentLogPath := filepath.Join("C:\\ProgramData\\Amazon\\ECS\\exec", testTaskId) + err = os.MkdirAll(execAgentLogPath, 0644) + require.NoError(t, err, "error creating execAgent log file") + _, err = os.Stat(execAgentLogPath) + require.NoError(t, err, "execAgent log dir doesn't exist") + err = os.MkdirAll(execcmd.ECSAgentExecConfigDir, 0644) + require.NoError(t, err, "error creating execAgent config dir") + + go taskEngine.AddTask(testTask) + + verifyContainerRunningStateChange(t, taskEngine) + verifyTaskRunningStateChange(t, taskEngine) + + if tc.ShouldBeRunning { + containerMap, _ := taskEngine.(*DockerTaskEngine).state.ContainerMapByArn(testTask.Arn) + cid := containerMap[testTask.Containers[0].Name].DockerID + verifyMockExecCommandAgentIsRunning(t, client, cid) + } + waitDone := make(chan struct{}) + + go verifyExecAgentStateChange(t, taskEngine, tc.ExpectedStatus, waitDone) + + timeout := false + select { + case <-waitDone: + case <-time.After(20 * time.Second): + timeout = true + } + assert.False(t, timeout) + + if tc.ShouldBeRunning { + containerMap, _ := taskEngine.(*DockerTaskEngine).state.ContainerMapByArn(testTask.Arn) + cid := containerMap[testTask.Containers[0].Name].DockerID + // Kill the existing container now + err = client.ContainerKill(context.TODO(), cid, "SIGKILL") + assert.NoError(t, err, "Could not kill container") + } + + taskEngine.(*DockerTaskEngine).deleteTask(testTask) + _, err = os.Stat(execAgentLogPath) + assert.True(t, os.IsNotExist(err), "execAgent log cleanup failed") + os.RemoveAll(execcmd.ECSAgentExecConfigDir) + }) + } +} + +func createTestExecCommandAgentTask(taskId, containerName string, sleepFor time.Duration) *apitask.Task { + testTask := createTestTask("arn:aws:ecs:us-west-2:1234567890:task/" + taskId) + testTask.PIDMode = ecs.PidModeHost + testTask.Containers[0].Name = containerName + testTask.Containers[0].Image = testExecCommandAgentImage + testTask.Containers[0].Command = []string{testExecCommandAgentSleepBin, "-time=" + sleepFor.String()} + enableExecCommandAgentForContainer(testTask.Containers[0], apicontainer.ManagedAgentState{}) + return testTask +} + +// setupEngineForExecCommandAgent creates a new TaskEngine with a custom execcmd.Manager that will attempt to read the +// host binaries from the directory passed as parameter (as opposed to the default directory). +// Additionally, it overrides the engine's monitorExecAgentsInterval to one second. +func setupEngineForExecCommandAgent(t *testing.T, hostBinDir string) (TaskEngine, func(), credentials.Manager) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + skipIntegTestIfApplicable(t) + + cfg := defaultTestConfigIntegTest() + sdkClientFactory := sdkclientfactory.NewFactory(ctx, dockerEndpoint) + dockerClient, err := dockerapi.NewDockerGoClient(sdkClientFactory, cfg, context.Background()) + if err != nil { + t.Fatalf("Error creating Docker client: %v", err) + } + credentialsManager := credentials.NewManager() + state := dockerstate.NewTaskEngineState() + imageManager := NewImageManager(cfg, dockerClient, state) + imageManager.SetDataClient(data.NewNoopClient()) + metadataManager := containermetadata.NewManager(dockerClient, cfg) + execCmdMgr := execcmd.NewManagerWithBinDir(hostBinDir) + + taskEngine := NewDockerTaskEngine(cfg, dockerClient, credentialsManager, + eventstream.NewEventStream("ENGINEINTEGTEST", context.Background()), imageManager, state, metadataManager, + nil, execCmdMgr) + taskEngine.monitorExecAgentsInterval = time.Second + taskEngine.MustInit(context.TODO()) + return taskEngine, func() { + taskEngine.Shutdown() + }, credentialsManager +} + +var ( + containerDepsDir = execcmd.ContainerDepsFolder +) + +func verifyExecCmdAgentExpectedMounts(t *testing.T, + ctx context.Context, + client *sdkClient.Client, + testTaskId, containerId, containerName, testExecCmdHostVersionedBinDir, testconfigDirName string) { + inspectState, _ := client.ContainerInspect(ctx, containerId) + + // The dockerclient only gives back lowercase paths in Windows + expectedMounts := []struct { + source string + destRegex string + readOnly bool + }{ + { + source: strings.ToLower(testExecCmdHostVersionedBinDir), + destRegex: strings.ToLower(containerDepsDir), + readOnly: true, + }, + { + source: strings.ToLower(filepath.Join(execcmd.HostExecConfigDir, testconfigDirName)), + destRegex: strings.ToLower(filepath.Join(containerDepsDir, "configuration")), + readOnly: true, + }, + { + source: strings.ToLower(filepath.Join(execcmd.HostLogDir, testTaskId, containerName)), + destRegex: strings.ToLower(execcmd.ContainerLogDir), + readOnly: false, + }, + { + source: strings.ToLower(execcmd.SSMPluginDir), + destRegex: strings.ToLower(execcmd.SSMPluginDir), + readOnly: true, + }, + } + + for _, em := range expectedMounts { + var found *types.MountPoint + for _, m := range inspectState.Mounts { + if m.Source == em.source { + found = &m + break + } + } + require.NotNil(t, found, "Expected mount point not found (%s)", em.source) + require.Equal(t, em.destRegex, found.Destination, "Destination for mount point (%s) is invalid expected: %s, actual: %s", em.source, em.destRegex, found.Destination) + if em.readOnly { + require.False(t, found.RW, found.Mode, "Destination for mount point (%s) should be read only", em.source) + } else { + require.True(t, found.RW, "Destination for mount point (%s) should be writable", em.source) + } + require.Equal(t, "bind", string(found.Type), "Destination for mount point (%s) is not of type bind", em.source) + } + + require.Equal(t, len(expectedMounts), len(inspectState.Mounts), "Wrong number of bind mounts detected in container (%s)", containerName) +} + +func verifyMockExecCommandAgentIsRunning(t *testing.T, client *sdkClient.Client, containerId string) string { + return verifyMockExecCommandAgentStatus(t, client, containerId, "", true) +} + +func verifyMockExecCommandAgentIsStopped(t *testing.T, client *sdkClient.Client, containerId, pid string) { + verifyMockExecCommandAgentStatus(t, client, containerId, pid, false) +} + +func verifyMockExecCommandAgentStatus(t *testing.T, client *sdkClient.Client, containerId, expectedPid string, checkIsRunning bool) string { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + res := make(chan string, 1) + execCmdAgentProcessRegex := execcmd.SSMAgentBinName + go func() { + for { + top, err := client.ContainerTop(ctx, containerId, nil) + if err != nil { + continue + } + cmdPos := -1 + pidPos := -1 + for i, t := range top.Titles { + if strings.ToUpper(t) == "NAME" { + cmdPos = i + } + if strings.ToUpper(t) == "PID" { + pidPos = i + } + + } + require.NotEqual(t, -1, cmdPos, "NAME title not found in the container top response") + require.NotEqual(t, -1, pidPos, "PID title not found in the container top response") + for _, proc := range top.Processes { + matched, _ := regexp.MatchString(execCmdAgentProcessRegex, proc[cmdPos]) + if matched { + res <- proc[pidPos] + return + } + } + seelog.Infof("Processes running in container: %s", top.Processes) + select { + case <-ctx.Done(): + return + case <-time.After(time.Second * 4): + } + } + }() + + var ( + isRunning bool + pid string + ) + select { + case <-ctx.Done(): + case r := <-res: + if r != "" { + pid = r + isRunning = true + if expectedPid != "" && pid != expectedPid { + isRunning = false + } + } + + } + require.Equal(t, checkIsRunning, isRunning, "SSM agent was not found in container's process list") + return pid +} + +func killMockExecCommandAgent(t *testing.T, client *sdkClient.Client, containerId, pid string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + containerAdminUser := "NT AUTHORITY\\SYSTEM" + create, err := client.ContainerExecCreate(ctx, containerId, types.ExecConfig{ + User: containerAdminUser, + Detach: true, + Cmd: []string{"cmd", "/C", "taskkill /F /IM amazon-ssm-agent.exe"}, + }) + require.NoError(t, err) + + err = client.ContainerExecStart(ctx, create.ID, types.ExecStartCheck{ + Detach: true, + }) + require.NoError(t, err) + + // windows docker exec takes longer than Linux + time.Sleep(4 * time.Second) +} diff --git a/agent/engine/execcmd/manager_init_task_windows.go b/agent/engine/execcmd/manager_init_task_windows.go index 548dc967b0d..08bcc62f262 100644 --- a/agent/engine/execcmd/manager_init_task_windows.go +++ b/agent/engine/execcmd/manager_init_task_windows.go @@ -180,7 +180,7 @@ func addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid string, session ContainerLogDir)) // add ssm plugin bind mount (needed for execcmd windows) - hostConfig.Binds = append(hostConfig.Binds, getBindMountMapping( + hostConfig.Binds = append(hostConfig.Binds, getReadOnlyBindMountMapping( SSMPluginDir, SSMPluginDir)) From 64289e10add3733ce478b325093838e0007cce6a Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Wed, 30 Jun 2021 09:30:31 -0700 Subject: [PATCH 11/14] Removed unnecessary references to kill binary --- agent/engine/engine_windows_integ_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/agent/engine/engine_windows_integ_test.go b/agent/engine/engine_windows_integ_test.go index c2d50e06ca8..48ebe2f6880 100644 --- a/agent/engine/engine_windows_integ_test.go +++ b/agent/engine/engine_windows_integ_test.go @@ -67,7 +67,6 @@ const ( testExecCommandAgentImage = "amazon/amazon-ecs-exec-command-agent-windows-test:make" testBaseImage = "amazon-ecs-ftest-windows-base:make" dockerVolumeDirectoryFormat = "c:\\ProgramData\\docker\\volumes\\%s\\_data" - testExecCommandAgentKillBin = "c:\\kill.exe" testExecCommandAgentSleepBin = "c:\\sleep.exe" ) @@ -521,16 +520,16 @@ func verifyContainerCredentialSpec(client *sdkClient.Client, id, credentialspecO // This setup for execcmd is same as Linux, just implemented different // TestExecCommandAgent validates ExecCommandAgent start and monitor processes. The algorithm to test is as follows: // 1. Pre-setup: the build file in ../../misc/exec-command-agent-windows-test will create a special docker sleeper image -// based on a base windows image. This image simulates a ecs windows image and contains pre-baked /sleep and /kill binaries. -// /sleep is the main process used to launch the test container; /kill is an application that kills a process running in -// the container given a PID. +// based on a base windows 2019 image. This image simulates a ecs windows image and contains a pre-baked C:\sleep.exe binary. +// /sleep is the main process used to launch the test container; Then use the windows "taskkill /f /im amazon-ssm-agent.exe" to +// kill the running agent process in the container // The build file will also create a fake amazon-ssm-agent which is a go program that only sleeps for a certain time specified. // // 2. Setup: Create a new docker task engine with a modified path pointing to our fake amazon-ssm-agent binary // 3. Create and start our test task using our test image // 4. Wait for the task to start and verify that the expected ExecCommandAgent bind mounts are present in the containers // 5. Verify that our fake amazon-ssm-agent was started inside the container using docker top, and retrieve its PID -// 6. Kill the fake amazon-ssm-agent using the PID retrieved in previous step +// 6. Kill the fake amazon-ssm-agent using the command in step 1 // 7. Verify that the engine restarted our fake amazon-ssm-agent by doing docker top one more time (a new PID should popup) func TestExecCommandAgent(t *testing.T) { // the execcmd feature is not supported for Win2016 @@ -579,7 +578,7 @@ func TestExecCommandAgent(t *testing.T) { verifyExecCmdAgentExpectedMounts(t, ctx, client, testTaskId, cid, testContainerName, testExecCmdHostBinDir+"\\1.0.0.0", testconfigDirName) pidA := verifyMockExecCommandAgentIsRunning(t, client, cid) seelog.Infof("Verified mock ExecCommandAgent is running (pidA=%s)", pidA) - killMockExecCommandAgent(t, client, cid, pidA) + killMockExecCommandAgent(t, client, cid) seelog.Infof("kill signal sent to ExecCommandAgent (pidA=%s)", pidA) verifyMockExecCommandAgentIsStopped(t, client, cid, pidA) seelog.Infof("Verified mock ExecCommandAgent was killed (pidA=%s)", pidA) @@ -864,12 +863,13 @@ func verifyMockExecCommandAgentStatus(t *testing.T, client *sdkClient.Client, co return pid } -func killMockExecCommandAgent(t *testing.T, client *sdkClient.Client, containerId, pid string) { +func killMockExecCommandAgent(t *testing.T, client *sdkClient.Client, containerId string) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - containerAdminUser := "NT AUTHORITY\\SYSTEM" + // this is the same user used to start the execcmd agent (ssm agent) + containerNTUser := "NT AUTHORITY\\SYSTEM" create, err := client.ContainerExecCreate(ctx, containerId, types.ExecConfig{ - User: containerAdminUser, + User: containerNTUser, Detach: true, Cmd: []string{"cmd", "/C", "taskkill /F /IM amazon-ssm-agent.exe"}, }) @@ -880,6 +880,6 @@ func killMockExecCommandAgent(t *testing.T, client *sdkClient.Client, containerI }) require.NoError(t, err) - // windows docker exec takes longer than Linux + // Windows docker exec takes longer than Linux time.Sleep(4 * time.Second) } From 8d8b23ff6263ad2f72b70232f27b8f5b17ec5765 Mon Sep 17 00:00:00 2001 From: Arun Annamalai Date: Thu, 1 Jul 2021 17:38:22 -0700 Subject: [PATCH 12/14] Update how test ensures container exits --- agent/engine/engine_windows_integ_test.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/agent/engine/engine_windows_integ_test.go b/agent/engine/engine_windows_integ_test.go index 48ebe2f6880..59bae72ac6f 100644 --- a/agent/engine/engine_windows_integ_test.go +++ b/agent/engine/engine_windows_integ_test.go @@ -679,11 +679,20 @@ func TestManagedAgentEvent(t *testing.T) { assert.False(t, timeout) if tc.ShouldBeRunning { - containerMap, _ := taskEngine.(*DockerTaskEngine).state.ContainerMapByArn(testTask.Arn) - cid := containerMap[testTask.Containers[0].Name].DockerID - // Kill the existing container now - err = client.ContainerKill(context.TODO(), cid, "SIGKILL") - assert.NoError(t, err, "Could not kill container") + stateChangeEvents := taskEngine.StateChangeEvents() + taskUpdate := createTestExecCommandAgentTask(testTaskId, testContainerName, time.Minute*tc.ManagedAgentLifetime) + taskUpdate.SetDesiredStatus(apitaskstatus.TaskStopped) + go taskEngine.AddTask(taskUpdate) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) + go func() { + verifyTaskIsStopped(stateChangeEvents, testTask) + cancel() + }() + + <-ctx.Done() + require.NotEqual(t, context.DeadlineExceeded, ctx.Err(), "Timed out waiting for task (%s) to stop", testTaskId) + assert.NotNil(t, testTask.Containers[0].GetKnownExitCode(), "No exit code found") } taskEngine.(*DockerTaskEngine).deleteTask(testTask) From 915556df81ffc761b2e2bdb30bc78f5403deaa13 Mon Sep 17 00:00:00 2001 From: Harsh Rawat Date: Fri, 8 Oct 2021 11:22:49 -0700 Subject: [PATCH 13/14] fix for integration test for ECS Exec on Windows This change fixes the following issues- - Golang base image is Windows release specific and is not scalable. The change introduced here ensures that the correct base image is being used instead of the generic golang image. Also, we will build the binary and then copy it to the container image. - We need to disable Go Module in order to build the binary. We enable it at the end of the script. --- misc/exec-command-agent-test/build.ps1 | 10 ++++++-- .../windows.dockerfile | 25 +++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/misc/exec-command-agent-test/build.ps1 b/misc/exec-command-agent-test/build.ps1 index 400010cf633..5867610a515 100644 --- a/misc/exec-command-agent-test/build.ps1 +++ b/misc/exec-command-agent-test/build.ps1 @@ -11,13 +11,16 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. +go env -w GO111MODULE=off +go build -tags integration -o ${PSScriptRoot}\sleep.exe ${PSScriptRoot}\sleep\ + # create the container image used to run execcmd integ tests docker build -t "amazon/amazon-ecs-exec-command-agent-windows-test:make" -f "${PSScriptRoot}/windows.dockerfile" ${PSScriptRoot} $SIMULATED_ECS_AGENT_DEPS_BIN_DIR="C:\Program Files\Amazon\ECS\managed-agents\execute-command\bin\1.0.0.0" Remove-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR -Recurse -Force -ErrorAction SilentlyContinue New-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR -ItemType Directory -Force -go build -tags integration -installsuffix cgo -a -o $SIMULATED_ECS_AGENT_DEPS_BIN_DIR\amazon-ssm-agent.exe ${PSScriptRoot}\sleep\ +Move-Item -Path ${PSScriptRoot}\sleep.exe -Destination $SIMULATED_ECS_AGENT_DEPS_BIN_DIR\amazon-ssm-agent.exe New-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR\ssm-agent-worker.exe -ItemType File -Force New-Item -Path $SIMULATED_ECS_AGENT_DEPS_BIN_DIR\ssm-session-worker.exe -ItemType File -Force @@ -41,4 +44,7 @@ if(!(Test-Path -path $SIMULATED_SSM_PLUGINS_DIR\awsCloudWatch)) if(!(Test-Path -path $SIMULATED_SSM_PLUGINS_DIR\awsDomainJoin)) { New-Item -Path $SIMULATED_SSM_PLUGINS_DIR\awsDomainJoin -ItemType directory -Force -} \ No newline at end of file +} + +# Set the Go Modules to auto once we have built the binaries. +go env -w GO111MODULE=auto \ No newline at end of file diff --git a/misc/exec-command-agent-test/windows.dockerfile b/misc/exec-command-agent-test/windows.dockerfile index 0eb3f4e41fe..736247d64f7 100644 --- a/misc/exec-command-agent-test/windows.dockerfile +++ b/misc/exec-command-agent-test/windows.dockerfile @@ -1,13 +1,18 @@ -# escape=` - -FROM golang:1.12 as build-env -MAINTAINER Amazon Web Services, Inc. - -# There is dockerfile documentation on how to treat windows paths -WORKDIR C:\Users\Administrator\go\src\sleep -COPY ./sleep C:/Users/Administrator/go/src/sleep -RUN go build -tags integration -installsuffix cgo -a -o C:/Users/Administrator/go/src/sleep/sleep.exe . +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License FROM amazon-ecs-ftest-windows-base:make + MAINTAINER Amazon Web Services, Inc. -COPY --from=build-env C:/Users/Administrator/go/src/sleep/sleep.exe C:/ + +ADD sleep.exe C:/sleep.exe \ No newline at end of file From 5ddc83b4f4225116770f87e47f6447c8accc5f38 Mon Sep 17 00:00:00 2001 From: Harsh Rawat Date: Mon, 18 Oct 2021 16:29:10 -0700 Subject: [PATCH 14/14] Updating the go build tags as part of Linux Golang upgrade Previously, the Linux was running on older version of Golang. With the Golang version upgrade to 1.17, the refactoring was done to change the build tags. This commit is for the same refactoring to older commits of ECS Exec on Windows feature. --- agent/engine/execcmd/manager_init_task.go | 2 +- agent/engine/execcmd/manager_init_task_test.go | 2 +- agent/engine/execcmd/manager_init_task_windows.go | 2 +- agent/engine/execcmd/manager_init_task_windows_test.go | 2 +- agent/engine/execcmd/manager_linux.go | 2 +- agent/engine/execcmd/manager_start.go | 2 +- agent/engine/execcmd/manager_start_test.go | 2 +- agent/engine/execcmd/manager_start_windows.go | 3 ++- agent/engine/execcmd/manager_start_windows_test.go | 2 +- agent/engine/execcmd/manager_windows.go | 2 +- 10 files changed, 11 insertions(+), 10 deletions(-) diff --git a/agent/engine/execcmd/manager_init_task.go b/agent/engine/execcmd/manager_init_task.go index 24808342a0d..e588baacfa4 100644 --- a/agent/engine/execcmd/manager_init_task.go +++ b/agent/engine/execcmd/manager_init_task.go @@ -1,4 +1,4 @@ -// +build linux windows +//go:build linux || windows // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_init_task_test.go b/agent/engine/execcmd/manager_init_task_test.go index 7c4548a16bd..3fe2912e3ab 100644 --- a/agent/engine/execcmd/manager_init_task_test.go +++ b/agent/engine/execcmd/manager_init_task_test.go @@ -1,4 +1,4 @@ -// +build unit +//go:build unit // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_init_task_windows.go b/agent/engine/execcmd/manager_init_task_windows.go index 08bcc62f262..39dedf02915 100644 --- a/agent/engine/execcmd/manager_init_task_windows.go +++ b/agent/engine/execcmd/manager_init_task_windows.go @@ -1,4 +1,4 @@ -// +build windows +//go:build windows // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_init_task_windows_test.go b/agent/engine/execcmd/manager_init_task_windows_test.go index 0bc6f08a9dc..f1a5e89bf4d 100644 --- a/agent/engine/execcmd/manager_init_task_windows_test.go +++ b/agent/engine/execcmd/manager_init_task_windows_test.go @@ -1,4 +1,4 @@ -// +build windows,unit +//go:build windows && unit // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_linux.go b/agent/engine/execcmd/manager_linux.go index 5984eceb97c..706d5da358e 100644 --- a/agent/engine/execcmd/manager_linux.go +++ b/agent/engine/execcmd/manager_linux.go @@ -1,4 +1,4 @@ -// +build linux +//go:build linux // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_start.go b/agent/engine/execcmd/manager_start.go index 57890c491f6..b1a46d18684 100644 --- a/agent/engine/execcmd/manager_start.go +++ b/agent/engine/execcmd/manager_start.go @@ -1,4 +1,4 @@ -// +build linux windows +//go:build linux || windows // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_start_test.go b/agent/engine/execcmd/manager_start_test.go index 3810ed25646..7f369eef24e 100644 --- a/agent/engine/execcmd/manager_start_test.go +++ b/agent/engine/execcmd/manager_start_test.go @@ -1,4 +1,4 @@ -// +build unit +//go:build unit // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_start_windows.go b/agent/engine/execcmd/manager_start_windows.go index a02ef67c34e..a111ec92bff 100644 --- a/agent/engine/execcmd/manager_start_windows.go +++ b/agent/engine/execcmd/manager_start_windows.go @@ -1,4 +1,4 @@ -// +build windows +//go:build windows // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // @@ -12,6 +12,7 @@ // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either // express or implied. See the License for the specific language governing // permissions and limitations under the License. + package execcmd import apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" diff --git a/agent/engine/execcmd/manager_start_windows_test.go b/agent/engine/execcmd/manager_start_windows_test.go index dec0d085cf2..c64c0963db0 100644 --- a/agent/engine/execcmd/manager_start_windows_test.go +++ b/agent/engine/execcmd/manager_start_windows_test.go @@ -1,4 +1,4 @@ -// +build windows,unit +//go:build windows && unit // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // diff --git a/agent/engine/execcmd/manager_windows.go b/agent/engine/execcmd/manager_windows.go index 085b52bbe26..ad364756bea 100644 --- a/agent/engine/execcmd/manager_windows.go +++ b/agent/engine/execcmd/manager_windows.go @@ -1,4 +1,4 @@ -// +build windows +//go:build windows // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. //