Added EFSVolumeConfiguration (#2234)

* Added EFSVolumeConfiguration models

* Translate EFS volumes from ACS to Docker volume type

* fix gocyclo failure

* code review comments

* remove readonly config

* remove readonly options from code

* code review comments

* code review

* naming is hard

agent/acs/model/api/api-2.json

@@ -300,6 +300,13 @@
       }
     },
 
+    "EFSVolumeConfiguration": {
+      "type":"structure",
+      "members":{
+        "fileSystemId":{"shape":"String"},
+        "rootDirectory":{"shape":"String"}
+      }
+    },
     "ElasticNetworkInterface":{
       "type":"structure",
       "members":{
@@ -669,7 +676,8 @@
         "name":{"shape":"String"},
         "type":{"shape":"VolumeType"},
         "host":{"shape":"HostVolumeProperties"},
-        "dockerVolumeConfiguration":{"shape":"DockerVolumeConfiguration"}
+        "dockerVolumeConfiguration":{"shape":"DockerVolumeConfiguration"},
+        "EFSVolumeConfiguration":{"shape":"EFSVolumeConfiguration"}
       }
     },
     "VolumeFrom":{
@@ -691,7 +699,8 @@
       "type":"string",
       "enum":[
         "host",
-        "docker"
+        "docker",
+        "efs"
       ]
     },
     "TaskIdentifier": {

agent/api/task/task.go

@@ -284,6 +284,22 @@ func TaskFromACS(acsTask *ecsacs.Task, envelope *ecsacs.PayloadMessage) (*Task,
 	return task, nil
 }
 
+func (task *Task) initializeVolumes(cfg *config.Config, dockerClient dockerapi.DockerClient, ctx context.Context) error {
+	err := task.initializeDockerLocalVolumes(dockerClient, ctx)
+	if err != nil {
+		return apierrors.NewResourceInitError(task.Arn, err)
+	}
+	err = task.initializeDockerVolumes(cfg.SharedVolumeMatchFullConfig, dockerClient, ctx)
+	if err != nil {
+		return apierrors.NewResourceInitError(task.Arn, err)
+	}
+	err = task.initializeEFSVolumes(cfg, dockerClient, ctx)
+	if err != nil {
+		return apierrors.NewResourceInitError(task.Arn, err)
+	}
+	return nil
+}
+
 // PostUnmarshalTask is run after a task has been unmarshalled, but before it has been
 // run. It is possible it will be subsequently called after that and should be
 // able to handle such an occurrence appropriately (e.g. behave idempotently).
@@ -322,16 +338,17 @@ func (task *Task) PostUnmarshalTask(cfg *config.Config,
 		task.initializeASMSecretResource(credentialsManager, resourceFields)
 	}
 
-	if err := task.initializeDockerLocalVolumes(dockerClient, ctx); err != nil {
-		return apierrors.NewResourceInitError(task.Arn, err)
-	}
-	if err := task.initializeDockerVolumes(cfg.SharedVolumeMatchFullConfig, dockerClient, ctx); err != nil {
-		return apierrors.NewResourceInitError(task.Arn, err)
+	if err := task.initializeVolumes(cfg, dockerClient, ctx); err != nil {
+		return err
 	}
 
-	if err := task.addGPUResource(cfg); err != nil {
-		seelog.Errorf("Task [%s]: could not initialize GPU associations: %v", task.Arn, err)
-		return apierrors.NewResourceInitError(task.Arn, err)
+	if cfg.GPUSupportEnabled {
+		err = task.addGPUResource()
+		if err != nil {
+			seelog.Errorf("Task [%s]: could not initialize GPU associations: %v", task.Arn, err)
+			return apierrors.NewResourceInitError(task.Arn, err)
+		}
+		task.NvidiaRuntime = cfg.NvidiaRuntime
 	}
 
 	task.initializeCredentialsEndpoint(credentialsManager)
@@ -496,6 +513,68 @@ func (task *Task) initializeDockerVolumes(sharedVolumeMatchFullConfig bool, dock
 	return nil
 }
 
+// initializeEFSVolumes inspects the volume definitions in the task definition.
+// If it finds EFS volumes in the task definition, then it converts it to a docker
+// volume definition.
+func (task *Task) initializeEFSVolumes(cfg *config.Config, dockerClient dockerapi.DockerClient, ctx context.Context) error {
+	for i, vol := range task.Volumes {
+		// No need to do this for non-efs volume, eg: host bind/empty volume
+		if vol.Type != EFSVolumeType {
+			continue
+		}
+
+		efsvol, ok := vol.Volume.(*taskresourcevolume.EFSVolumeConfig)
+		if !ok {
+			return errors.New("task volume: volume configuration does not match the type 'efs'")
+		}
+
+		err := task.addEFSVolumes(ctx, cfg, dockerClient, &task.Volumes[i], efsvol)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// addEFSVolumes converts the EFS task definition into an internal docker 'local' volume
+// mounted with NFS struct and updates container dependency
+func (task *Task) addEFSVolumes(
+	ctx context.Context,
+	cfg *config.Config,
+	dockerClient dockerapi.DockerClient,
+	vol *TaskVolume,
+	efsvol *taskresourcevolume.EFSVolumeConfig,
+) error {
+	// TODO CN and gov partition logic
+	// These are the NFS options recommended by EFS, see:
+	// https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-mount-cmd-general.html
+	ostr := fmt.Sprintf("addr=%s.efs.%s.amazonaws.com,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport", efsvol.Filesystem, cfg.AWSRegion)
+	devstr := fmt.Sprintf(":%s", efsvol.RootDirectory)
+	volumeResource, err := taskresourcevolume.NewVolumeResource(
+		ctx,
+		vol.Name,
+		task.volumeName(vol.Name),
+		"task",
+		false,
+		"local",
+		map[string]string{
+			"type":   "nfs",
+			"device": devstr,
+			"o":      ostr,
+		},
+		map[string]string{},
+		dockerClient,
+	)
+	if err != nil {
+		return err
+	}
+
+	vol.Volume = &volumeResource.VolumeConfig
+	task.AddResource(resourcetype.DockerVolumeKey, volumeResource)
+	task.updateContainerVolumeDependency(vol.Name)
+	return nil
+}
+
 // addTaskScopedVolumes adds the task scoped volume into task resources and updates container dependency
 func (task *Task) addTaskScopedVolumes(ctx context.Context, dockerClient dockerapi.DockerClient,
 	vol *TaskVolume) error {

agent/api/task/taskvolume.go

@@ -24,6 +24,7 @@ import (
 const (
 	HostVolumeType   = "host"
 	DockerVolumeType = "docker"
+	EFSVolumeType    = "efs"
 )
 
 // TaskVolume is a definition of all the volumes available for containers to
@@ -64,8 +65,10 @@ func (tv *TaskVolume) UnmarshalJSON(b []byte) error {
 		return tv.unmarshalHostVolume(intermediate["host"])
 	case DockerVolumeType:
 		return tv.unmarshalDockerVolume(intermediate["dockerVolumeConfiguration"])
+	case EFSVolumeType:
+		return tv.unmarshalEFSVolume(intermediate["EFSVolumeConfiguration"])
 	default:
-		return errors.Errorf("invalid Volume: type must be docker or host, got %q", tv.Type)
+		return errors.Errorf("unrecognized volume type: %q", tv.Type)
 	}
 }
 
@@ -85,6 +88,8 @@ func (tv *TaskVolume) MarshalJSON() ([]byte, error) {
 		result["dockerVolumeConfiguration"] = tv.Volume
 	case HostVolumeType:
 		result["host"] = tv.Volume
+	case EFSVolumeType:
+		result["EFSVolumeConfiguration"] = tv.Volume
 	default:
 		return nil, errors.Errorf("unrecognized volume type: %q", tv.Type)
 	}
@@ -106,6 +111,20 @@ func (tv *TaskVolume) unmarshalDockerVolume(data json.RawMessage) error {
 	return nil
 }
 
+func (tv *TaskVolume) unmarshalEFSVolume(data json.RawMessage) error {
+	if data == nil {
+		return errors.New("invalid volume: empty volume configuration")
+	}
+	var efsVolumeConfig taskresourcevolume.EFSVolumeConfig
+	err := json.Unmarshal(data, &efsVolumeConfig)
+	if err != nil {
+		return err
+	}
+
+	tv.Volume = &efsVolumeConfig
+	return nil
+}
+
 func (tv *TaskVolume) unmarshalHostVolume(data json.RawMessage) error {
 	if data == nil {
 		return errors.New("invalid volume: empty volume configuration")