Merging Moby with Latest Dev into AgentMetrics

CHANGELOG.md

@@ -1,8 +1,16 @@
 # Changelog
 
-## 1.22.0-dev
+## 1.23.0-dev
+* Feature - Add support for ECS Secrets integrating with AWS Secrets Manager [#1713](https://github.com/aws/amazon-ecs-agent/pull/1713)
+* Enhancement - Add availability zone to task metadata endpoint and ECS metadata file
+* Bug - Fixed a bug where agent can register container instance back to back and gets
+  assigned two container instance ARNs [#1711](https://github.com/aws/amazon-ecs-agent/pull/1711)
+* Bug - Fixed a bug where propagated `aws:` tags are passed through RegisterContainerInstance API call [#1706](https://github.com/aws/amazon-ecs-agent/pull/1706)
+
+## 1.22.0
 * Feature - Add support for ECS Secrets integrating with AWS Systems Manager Parameter Store
 * Feature - Support for `--pid`, `--ipc` Docker run flags. [#1584](https://github.com/aws/amazon-ecs-agent/pull/1584)
+* Feature - Introduce two environment variables `ECS_CONTAINER_INSTANCE_PROPAGATE_TAGS_FROM` and `ECS_CONTAINER_INSTANCE_TAGS` to support ECS tagging [#1618](https://github.com/aws/amazon-ecs-agent/pull/1618)
 
 ## 1.21.0
 * Feature - Add v3 task metadata support for awsvpc, host and bridge network mode

Makefile

@@ -15,6 +15,7 @@ USERID=$(shell id -u)
 GO_EXECUTABLE=$(shell command -v go 2> /dev/null)
 
 .PHONY: all gobuild static docker release certs test clean netkitten test-registry namespace-tests run-functional-tests benchmark-test gogenerate run-integ-tests pause-container get-cni-sources cni-plugins test-artifacts
+BUILD_PLATFORM:=$(shell uname -m)
 
 all: docker
 
@@ -88,8 +89,13 @@ misc/certs/ca-certificates.crt:
 	docker build -t "amazon/amazon-ecs-agent-cert-source:make" misc/certs/
 	docker run "amazon/amazon-ecs-agent-cert-source:make" cat /etc/ssl/certs/ca-certificates.crt > misc/certs/ca-certificates.crt
 
-test:
+ifeq (${BUILD_PLATFORM},aarch64)
+test::
+	. ./scripts/shared_env && go test -tags unit -timeout=25s -v -cover $(shell go list ./agent/... | grep -v /vendor/)
+else
+test::
 	. ./scripts/shared_env && go test -race -tags unit -timeout=25s -v -cover $(shell go list ./agent/... | grep -v /vendor/)
+endif
 
 test-silent:
 	. ./scripts/shared_env && go test -timeout=25s -cover $(shell go list ./agent/... | grep -v /vendor/)
@@ -123,7 +129,13 @@ endef
 
 # TODO: use `go list -f` to target the test files more directly
 ALL_GO_FILES = $(shell find . -name "*.go" -print | tr "\n" " ")
+
+ifeq (${BUILD_PLATFORM},aarch64)
+GO_INTEG_TEST = go test -tags integration -c -o
+else
 GO_INTEG_TEST = go test -race -tags integration -c -o
+endif
+
 out/test-artifacts/linux-engine-tests: $(ALL_GO_FILES) .out-stamp .builder-image-stamp
 	$(call dockerbuild,$(GO_INTEG_TEST) $@ ./agent/engine)
 
@@ -234,11 +246,21 @@ cni-plugins: get-cni-sources .out-stamp
 		"amazon/amazon-ecs-build-cniplugins:make"
 	@echo "Built amazon-ecs-cni-plugins successfully."
 
+ifeq (${BUILD_PLATFORM},aarch64)
 run-integ-tests: test-registry gremlin container-health-check-image run-sudo-tests
-	. ./scripts/shared_env && go test -race -tags integration -timeout=10m -v ./agent/engine/... ./agent/stats/... ./agent/app/...
-
-run-sudo-tests:
+	. ./scripts/shared_env && go test -tags integration -timeout=20m -v ./agent/engine/... ./agent/stats/... ./agent/app/...agent
+else
+run-integ-tests: test-registry gremlin container-health-check-image run-sudo-tests
+	. ./scripts/shared_env && go test -race -tags integration -timeout=12m -v ./agent/engine/... ./agent/stats/... ./agent/app/...
+endif
+
+ifeq (${BUILD_PLATFORM},aarch64)
+run-sudo-tests::
+	. ./scripts/shared_env && sudo -E ${GO_EXECUTABLE} test -tags sudo -timeout=10m -v ./agent/engine/...
+else
+run-sudo-tests::
 	. ./scripts/shared_env && sudo -E ${GO_EXECUTABLE} test -race -tags sudo -timeout=1m -v ./agent/engine/...
+endif
 
 .PHONY: codebuild
 codebuild: test-artifacts .out-stamp

README.md

@@ -132,6 +132,8 @@ additional details on each available environment variable.
 | `ECS_ENABLE_CPU_UNBOUNDED_WINDOWS_WORKAROUND` | `true` | When `true`, ECS will allow CPU unbounded(CPU=`0`) tasks to run along with CPU bounded tasks in Windows. | Not applicable | `false` |
 | `ECS_TASK_METADATA_RPS_LIMIT` | `100,150` | Comma separated integer values for steady state and burst throttle limits for task metadata endpoint | `40,60` | `40,60` |
 | `ECS_SHARED_VOLUME_MATCH_FULL_CONFIG` | `true` | When `true`, ECS Agent will compare name, driver options, and labels to make sure volumes are identical. When `false`, Agent will short circuit shared volume comparison if the names match. This is the default Docker behavior. If a volume is shared across instances, this should be set to `false`. | `false` | `false`|
+| `ECS_CONTAINER_INSTANCE_PROPAGATE_TAGS_FROM` | `ec2_instance` | If `ec2_instance` is specified, existing tags defined on the container instance will be registered to Amazon ECS and will be discoverable using the `ListTagsForResource` API. Using this requires that the IAM role associated with the container instance have the `ec2:DescribeTags` action allowed. | `none` | `none` |
+| `ECS_CONTAINER_INSTANCE_TAGS` | `{"tag_key": "tag_val"}` | The metadata that you apply to the container instance to help you categorize and organize them. Each tag consists of a key and an optional value, both of which you define. Tag keys can have a maximum character length of 128 characters, and tag values can have a maximum length of 256 characters. If tags also exist on your container instance that are propagated using the `ECS_CONTAINER_INSTANCE_PROPAGATE_TAGS_FROM` parameter, those tags will be overwritten by the tags specified using `ECS_CONTAINER_INSTANCE_TAGS`. | `{}` | `{}` |
 
 ### Persistence
 

THIRD-PARTY

@@ -3,6 +3,7 @@
 ** github.com/containerd/continuity; version 1bed1ecb1dc42d8f4d2ac8c23e5cac64749e82c9 -- https://github.com/containerd/continuity
 ** github.com/containernetworking/cni; version v0.5.2 -- https://github.com/containernetworking/cni
 ** github.com/coreos/go-systemd; version 24036eb3df68550d24a2736c5d013f4e83366866 -- https://github.com/coreos/go-systemd
+** github.com/docker/distribution; version 749f6afb4572201e3c37325d0ffedb6f32be8950 -- https://www.docker.com
 ** github.com/docker/docker; version e4d0fe84f9ea88b0e0cfd847412c9f29442cc62d -- https://github.com/moby/moby
 ** github.com/docker/go-connections; version v0.3.0 -- https://github.com/docker/go-connections
 ** github.com/docker/go-units; version v0.3.2 -- https://github.com/docker/go-units
@@ -13,6 +14,7 @@
 ** github.com/opencontainers/image-spec; version v1.0.1 -- https://github.com/opencontainers/image-spec
 ** github.com/opencontainers/runc; version v0.1.1 -- https://github.com/opencontainers/runc
 ** github.com/opencontainers/runtime-spec; version d349388c43b01b2ea695965ae561b5bddb81318f -- https://github.com/opencontainers/runtime-spec
+** github.com/prometheus/client_golang; version 0.9.0 -- https://github.com/prometheus/client_golang
 ** github.com/vishvananda/netlink; version fe3b5664d23a11b52ba59bece4ff29c52772a56b -- https://github.com/vishvananda/netlink
 ** github.com/vishvananda/netns; version be1fbeda19366dea804f00efff2dd73a1642fdcc -- https://github.com/vishvananda/netns
 
@@ -213,6 +215,8 @@ Copyright (c) 2016-2017 the containerd authors
 Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
 * For github.com/coreos/go-systemd see also this required NOTICE:
 Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+* For github.com/docker/distribution see also this required NOTICE:
+Copyright 2013-2017 Docker, Inc.
 * For github.com/docker/docker see also this required NOTICE:
 Copyright 2013-2017 Docker, Inc.
 * For github.com/docker/go-connections see also this required NOTICE:
@@ -234,6 +238,32 @@ Copyright 2016 The Linux Foundation.
 Copyright 2014 Docker, Inc.
 * For github.com/opencontainers/runtime-spec see also this required NOTICE:
 Copyright 2015 The Linux Foundation.
+* For github.com/prometheus/client_golang see also this required NOTICE:
+Prometheus instrumentation library for Go applications
+Copyright 2012-2015 The Prometheus Authors
+
+This product includes software developed at
+SoundCloud Ltd. (http://soundcloud.com/).
+
+
+The following components are included in this product:
+
+perks - a fork of https://github.com/bmizerany/perks
+https://github.com/beorn7/perks
+Copyright 2013-2015 Blake Mizerany, Björn Rabenstein
+See https://github.com/beorn7/perks/blob/master/README.md for license
+details.
+
+Go support for Protocol Buffers - Google's data interchange format
+http://github.com/golang/protobuf/
+Copyright 2010 The Go Authors
+See source code for license details.
+
+Support for streaming Protocol Buffer messages for the Go language
+(golang).
+https://github.com/matttproud/golang_protobuf_extensions
+Copyright 2013 Matt T. Proud
+Licensed under the Apache License, Version 2.0
 * For github.com/vishvananda/netlink see also this required NOTICE:
 Copyright 2014 Vishvananda Ishaya.
 Copyright 2014 Docker, Inc.

VERSION

@@ -1 +1 @@
-1.21.0
+1.22.0

agent/Gopkg.toml

@@ -88,3 +88,7 @@ required = ["github.com/golang/mock/mockgen/model"]
 [[constraint]]
   name = "github.com/gorilla/mux"
   version = "1.6.2"
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "0.9.0"

agent/acs/model/api/api-2.json

@@ -437,7 +437,10 @@
     },
     "SecretProvider":{
       "type":"string",
-      "enum":["ssm"]
+      "enum":[
+        "ssm",
+        "asm"
+      ]
     },
     "SecretType":{
       "type":"string",

agent/api/container/container.go

@@ -57,6 +57,12 @@ const (
 
 	// SecretProviderSSM is to show secret provider being SSM
 	SecretProviderSSM = "ssm"
+
+	// SecretProviderASM is to show secret provider being ASM
+	SecretProviderASM = "asm"
+
+	// SecretTypeEnv is to show secret type being ENVIRONMENT_VARIABLE
+	SecretTypeEnv = "ENVIRONMENT_VARIABLE"
 )
 
 // DockerConfig represents additional metadata about a container to run. It's
@@ -257,9 +263,9 @@ type Secret struct {
 	Provider      string `json:"provider"`
 }
 
-// GetSSMSecretResourceCacheKey returns the key required to access the secret
+// GetSecretResourceCacheKey returns the key required to access the secret
 // from the ssmsecret resource
-func (s *Secret) GetSSMSecretResourceCacheKey() string {
+func (s *Secret) GetSecretResourceCacheKey() string {
 	return s.ValueFrom + "_" + s.Region
 }
 
@@ -752,7 +758,7 @@ func (c *Container) ShouldCreateWithSSMSecret() bool {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 
-	//Secrets field will be nil if there is no secrets for container
+	// Secrets field will be nil if there is no secrets for container
 	if c.Secrets == nil {
 		return false
 	}
@@ -765,6 +771,25 @@ func (c *Container) ShouldCreateWithSSMSecret() bool {
 	return false
 }
 
+// ShouldCreateWithASMSecret returns true if this container needs to get secret
+// value from AWS Secrets Manager
+func (c *Container) ShouldCreateWithASMSecret() bool {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	// Secrets field will be nil if there is no secrets for container
+	if c.Secrets == nil {
+		return false
+	}
+
+	for _, secret := range c.Secrets {
+		if secret.Provider == SecretProviderASM {
+			return true
+		}
+	}
+	return false
+}
+
 // MergeEnvironmentVariables appends additional envVarName:envVarValue pairs to
 // the the container's enviornment values structure
 func (c *Container) MergeEnvironmentVariables(envVars map[string]string) {
@@ -775,8 +800,23 @@ func (c *Container) MergeEnvironmentVariables(envVars map[string]string) {
 	if c.Environment == nil {
 		c.Environment = make(map[string]string)
 	}
-
 	for k, v := range envVars {
 		c.Environment[k] = v
 	}
 }
+
+func (c *Container) HasSecretAsEnv() bool {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	// Secrets field will be nil if there is no secrets for container
+	if c.Secrets == nil {
+		return false
+	}
+	for _, secret := range c.Secrets {
+		if secret.Type == SecretTypeEnv {
+			return true
+		}
+	}
+	return false
+}