Add Private Host IPv4 address to container metadata

Related issue: aws#1575 Related PR: aws#1730 PR 1730 adds the Public IP Address of the host to the container metadata file, however the EC2 host may be configured without a public address. In this case, the EC2 metadata API returns a 404 response, and the host IP is not available to containers. Example ECS Agent Log ``` [ERROR] Unable to retrieve Host Instance PublicIPv4 Address: EC2MetadataError: failed to make EC2Metadata request caused by: <?xml version="1.0" encoding="iso-8859-1"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> <head> <title>404 - Not Found</title> </head> <body> <h1>404 - Not Found</h1> </body> </html> ``` This commit adds an extra field to the container metadata json, `HostPrivateIPv4Address` which is available on EC2 hosts without a public address.
deliveroo · Apr 23, 2019 · e2918c2 · e2918c2
1 parent bd661d0
commit e2918c2
Show file tree

Hide file tree

Showing 14 changed files with 186 additions and 40 deletions.
diff --git a/agent/app/agent.go b/agent/app/agent.go
@@ -297,6 +297,7 @@ func (agent *ecsAgent) doStart(containerChangeEventStream *eventstream.EventStre
 	if agent.cfg.ContainerMetadataEnabled {
 		agent.metadataManager.SetContainerInstanceARN(agent.containerInstanceARN)
 		agent.metadataManager.SetAvailabilityZone(agent.availabilityZone)
+		agent.metadataManager.SetHostPrivateIPv4Address(agent.getHostPrivateIPv4AddressFromEC2Metadata())
 		agent.metadataManager.SetHostPublicIPv4Address(agent.getHostPublicIPv4AddressFromEC2Metadata())
 	}
 
@@ -695,10 +696,22 @@ func mergeTags(localTags []*ecs.Tag, ec2Tags []*ecs.Tag) []*ecs.Tag {
 	return utils.MapToTags(tagsMap)
 }
 
+// getHostPrivateIPv4AddressFromEC2Metadata will retrieve the PrivateIPAddress (IPv4) of this
+// instance throught the EC2 API
+func (agent *ecsAgent) getHostPrivateIPv4AddressFromEC2Metadata() string {
+	// Get instance private IP from ec2 metadata client.
+	hostPrivateIPv4Address, err := agent.ec2MetadataClient.PrivateIPv4Address()
+	if err != nil {
+		seelog.Errorf("Unable to retrieve Host Instance PrivateIPv4 Address: %v", err)
+		return ""
+	}
+	return hostPrivateIPv4Address
+}
+
 // getHostPublicIPv4AddressFromEC2Metadata will retrieve the PublicIPAddress (IPv4) of this
 // instance through the EC2 API
 func (agent *ecsAgent) getHostPublicIPv4AddressFromEC2Metadata() string {
-	// Get instance ID from ec2 metadata client.
+	// Get instance public IP from ec2 metadata client.
 	hostPublicIPv4Address, err := agent.ec2MetadataClient.PublicIPv4Address()
 	if err != nil {
 		seelog.Errorf("Unable to retrieve Host Instance PublicIPv4 Address: %v", err)

diff --git a/agent/app/agent_test.go b/agent/app/agent_test.go
@@ -51,10 +51,11 @@ import (
 )
 
 const (
-	clusterName           = "some-cluster"
-	containerInstanceARN  = "container-instance1"
-	availabilityZone      = "us-west-2b"
-	hostPublicIPv4Address = "127.0.0.1"
+	clusterName            = "some-cluster"
+	containerInstanceARN   = "container-instance1"
+	availabilityZone       = "us-west-2b"
+	hostPrivateIPv4Address = "127.0.0.1"
+	hostPublicIPv4Address  = "127.0.0.1"
 )
 
 var apiVersions = []dockerclient.DockerVersion{
@@ -304,6 +305,7 @@ func TestDoStartRegisterAvailabilityZone(t *testing.T) {
 	defer ctrl.Finish()
 
 	ec2MetadataClient := mock_ec2.NewMockEC2MetadataClient(ctrl)
+	ec2MetadataClient.EXPECT().PrivateIPv4Address().Return(hostPrivateIPv4Address, nil)
 	ec2MetadataClient.EXPECT().PublicIPv4Address().Return(hostPublicIPv4Address, nil)
 
 	var discoverEndpointsInvoked sync.WaitGroup
@@ -339,6 +341,7 @@ func TestDoStartRegisterAvailabilityZone(t *testing.T) {
 			"arn:123", availabilityZone, nil),
 		containermetadata.EXPECT().SetContainerInstanceARN("arn:123"),
 		containermetadata.EXPECT().SetAvailabilityZone(availabilityZone),
+		containermetadata.EXPECT().SetHostPrivateIPv4Address(hostPrivateIPv4Address),
 		containermetadata.EXPECT().SetHostPublicIPv4Address(hostPublicIPv4Address),
 		imageManager.EXPECT().SetSaver(gomock.Any()),
 		dockerClient.EXPECT().ContainerEvents(gomock.Any()),
@@ -1115,6 +1118,38 @@ func TestGetContainerInstanceTagsFromEC2APIFailToDescribeECSTagsForInstance(t *t
 	assert.Nil(t, resTags)
 }
 
+func TestGetHostPrivateIPv4AddressFromEC2Metadata(t *testing.T) {
+	ctrl := gomock.NetController(t)
+	defer ctrl.Finish()
+
+	ec2MetadataClient := mock_ec2.NewMockEC2MetadataClient(ctrl)
+	ec2Client := mock_ec2.NewMockClient(ctrl)
+
+	agent := &ecsAgent{
+		ec2MetadataClient: ec2MetadataClient,
+		ec2Client:         ec2Client,
+	}
+	ec2MetadataClient.EXPECT().PrivateIPv4Address().Return(hostPrivateIPv4Address, nil)
+
+	assert.Equal(t, hostPrivateIPv4Address, agent.getHostPrivateIPv4AddressFromEC2Metadata())
+}
+
+func TestGetHostPrivateIPv4AddressFromEC2MetadataFailWithError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	ec2MetadataClient := mock_ec2.NewMockEC2MetadataClient(ctrl)
+	ec2Client := mock_ec2.NewMockClient(ctrl)
+
+	agent := &ecsAgent{
+		ec2MetadataClient: ec2MetadataClient,
+		ec2Client:         ec2Client,
+	}
+	ec2MetadataClient.EXPECT().PrivateIPv4Address().Return("", errors.New("Unable to get IP Address"))
+
+	assert.Empty(t, agent.getHostPrivateIPv4AddressFromEC2Metadata())
+}
+
 func TestGetHostPublicIPv4AddressFromEC2Metadata(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()

diff --git a/agent/containermetadata/manager.go b/agent/containermetadata/manager.go
@@ -40,6 +40,7 @@ const (
 type Manager interface {
 	SetContainerInstanceARN(string)
 	SetAvailabilityZone(string)
+	SetHostPrivateIPv4Address(string)
 	SetHostPublicIPv4Address(string)
 	Create(*dockercontainer.Config, *dockercontainer.HostConfig, *apitask.Task, string) error
 	Update(context.Context, string, *apitask.Task, string) error
@@ -67,7 +68,9 @@ type metadataManager struct {
 	ioutilWrap ioutilwrapper.IOUtil
 	// availabilityZone is the availabiltyZone where task is in
 	availabilityZone string
-	// hostPublicIPv4Address is the public IPv4 address associated with the EC2 instance ID
+	// hostPrivateIPv4Address is the private IPv4 address associated with the EC2 instance
+	hostPrivateIPv4Address string
+	// hostPublicIPv4Address is the public IPv4 address associated with the EC2 instance
 	hostPublicIPv4Address string
 }
 
@@ -95,6 +98,12 @@ func (manager *metadataManager) SetAvailabilityZone(availabilityZone string) {
 	manager.availabilityZone = availabilityZone
 }
 
+// SetHostPrivateIPv4Address sets the metadataManager's hostPrivateIPv4Address which is not available
+// at its creation as this information is not present immediately at the agent's startup
+func (manager *metadataManager) SetHostPrivateIPv4Address(ipv4address string) {
+	manager.hostPrivateIPv4Address = ipv4address
+}
+
 // SetHostPublicIPv4Address sets the metadataManager's hostPublicIPv4Address which is not available
 // at its creation as this information is not present immediately at the agent's startup
 func (manager *metadataManager) SetHostPublicIPv4Address(ipv4address string) {

diff --git a/agent/containermetadata/manager_test.go b/agent/containermetadata/manager_test.go
@@ -41,6 +41,7 @@ const (
 	containerName          = "container"
 	dataDir                = "ecs_mockdata"
 	availabilityZone       = "us-west-2b"
+	hostPrivateIPv4Address = "127.0.0.1"
 	hostPublicIPv4Address  = "127.0.0.1"
 )
 
@@ -75,6 +76,15 @@ func TestSetAvailabilityZone(t *testing.T) {
 	assert.Equal(t, mockAvailabilityZone, newManager.availabilityZone)
 }
 
+// TestSetHostPrivateIPv4Address checks whether the container hostPublicIPv4Address is set correctly.
+func TestSetHostPrivateIPv4Address(t *testing.T) {
+	_, _, _, _, done := managerSetup(t)
+	defer done()
+	newManager := &metadataManager{}
+	newManager.SetHostPrivateIPv4Address(hostPublicIPv4Address)
+	assert.Equal(t, hostPrivateIPv4Address, newManager.hostPrivateIPv4Address)
+}
+
 // TestSetHostPublicIPv4Address checks whether the container hostPublicIPv4Address is set correctly.
 func TestSetHostPublicIPv4Address(t *testing.T) {
 	_, _, _, _, done := managerSetup(t)

diff --git a/agent/containermetadata/mocks/containermetadata_mocks.go b/agent/containermetadata/mocks/containermetadata_mocks.go
diff --git a/agent/containermetadata/parse_metadata.go b/agent/containermetadata/parse_metadata.go
@@ -38,10 +38,11 @@ func (manager *metadataManager) parseMetadataAtContainerCreate(task *apitask.Tas
 			taskDefinitionFamily:   task.Family,
 			taskDefinitionRevision: task.Version,
 		},
-		containerInstanceARN:  manager.containerInstanceARN,
-		metadataStatus:        MetadataInitial,
-		availabilityZone:      manager.availabilityZone,
-		hostPublicIPv4Address: manager.hostPublicIPv4Address,
+		containerInstanceARN:   manager.containerInstanceARN,
+		metadataStatus:         MetadataInitial,
+		availabilityZone:       manager.availabilityZone,
+		hostPrivateIPv4Address: manager.hostPrivateIPv4Address,
+		hostPublicIPv4Address:  manager.hostPublicIPv4Address,
 	}
 }
 
@@ -63,6 +64,7 @@ func (manager *metadataManager) parseMetadata(dockerContainer *types.ContainerJS
 		containerInstanceARN:    manager.containerInstanceARN,
 		metadataStatus:          MetadataReady,
 		availabilityZone:        manager.availabilityZone,
+		hostPrivateIPv4Address:  manager.hostPrivateIPv4Address,
 		hostPublicIPv4Address:   manager.hostPublicIPv4Address,
 	}
 }