From c2ce8863e097d8768265af766cd671fc56eb0e13 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Fri, 26 Apr 2019 15:51:29 -0400 Subject: [PATCH 1/3] resource/aws_emr_cluster: Add master_instance_group and core_instance_group configuration blocks, deprecate other instance group configuration methods Reference: https://github.com/terraform-providers/terraform-provider-aws/issues/8245 Output from acceptance testing: ``` --- PASS: TestAccAWSEMRCluster_CoreInstanceGroup_AutoscalingPolicy (487.78s) --- PASS: TestAccAWSEMRCluster_CoreInstanceGroup_BidPrice (1006.42s) --- PASS: TestAccAWSEMRCluster_CoreInstanceGroup_InstanceCount (1004.90s) --- PASS: TestAccAWSEMRCluster_CoreInstanceGroup_InstanceType (869.47s) --- PASS: TestAccAWSEMRCluster_CoreInstanceGroup_Migration_CoreInstanceType (415.45s) --- PASS: TestAccAWSEMRCluster_CoreInstanceGroup_Migration_InstanceGroup (485.50s) --- PASS: TestAccAWSEMRCluster_CoreInstanceGroup_Name (769.30s) --- PASS: TestAccAWSEMRCluster_MasterInstanceGroup_BidPrice (849.98s) --- PASS: TestAccAWSEMRCluster_MasterInstanceGroup_InstanceType (756.42s) --- PASS: TestAccAWSEMRCluster_MasterInstanceGroup_Migration_InstanceGroup (414.36s) --- PASS: TestAccAWSEMRCluster_MasterInstanceGroup_Migration_MasterInstanceType (423.02s) --- PASS: TestAccAWSEMRCluster_MasterInstanceGroup_Name (735.58s) ``` --- aws/resource_aws_emr_cluster.go | 564 +++++-- aws/resource_aws_emr_cluster_test.go | 1352 +++++++++++++++-- website/aws.erb | 4 + website/docs/guides/version-3-upgrade.html.md | 186 +++ website/docs/r/emr_cluster.html.markdown | 54 +- 5 files changed, 1908 insertions(+), 252 deletions(-) create mode 100644 website/docs/guides/version-3-upgrade.html.md diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 7a1a56113f34..1920b103ec9d 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -95,10 +95,12 @@ func resourceAwsEMRCluster() *schema.Resource { Required: true, }, "master_instance_type": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Computed: true, + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Computed: true, + ConflictsWith: []string{"master_instance_group"}, + Deprecated: "use `master_instance_group` configuration block `instance_type` argument instead", }, "additional_info": { Type: schema.TypeString, @@ -112,16 +114,20 @@ func resourceAwsEMRCluster() *schema.Resource { }, }, "core_instance_type": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Computed: true, + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Computed: true, + ConflictsWith: []string{"core_instance_group"}, + Deprecated: "use `core_instance_group` configuration block `instance_type` argument instead", }, "core_instance_count": { - Type: schema.TypeInt, - Optional: true, - ValidateFunc: validation.IntAtLeast(1), - Computed: true, + Type: schema.TypeInt, + Optional: true, + ValidateFunc: validation.IntAtLeast(1), + Computed: true, + ConflictsWith: []string{"core_instance_group"}, + Deprecated: "use `core_instance_group` configuration block `instance_count` argument instead", }, "cluster_state": { Type: schema.TypeString, @@ -249,11 +255,153 @@ func resourceAwsEMRCluster() *schema.Resource { }, }, }, + "core_instance_group": { + Type: schema.TypeList, + Optional: true, + Computed: true, + ForceNew: true, + MaxItems: 1, + ConflictsWith: []string{"core_instance_count", "core_instance_type", "instance_group"}, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "autoscaling_policy": { + Type: schema.TypeString, + Optional: true, + DiffSuppressFunc: suppressEquivalentJsonDiffs, + ValidateFunc: validation.ValidateJsonString, + }, + "bid_price": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "ebs_config": { + Type: schema.TypeSet, + Optional: true, + Computed: true, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "iops": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + }, + "size": { + Type: schema.TypeInt, + Required: true, + ForceNew: true, + }, + "type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validateAwsEmrEbsVolumeType(), + }, + "volumes_per_instance": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Default: 1, + }, + }, + }, + Set: resourceAwsEMRClusterEBSConfigHash, + }, + "id": { + Type: schema.TypeString, + Computed: true, + }, + "instance_count": { + Type: schema.TypeInt, + Optional: true, + Default: 1, + ValidateFunc: validation.IntAtLeast(1), + }, + "instance_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + }, + }, + }, + "master_instance_group": { + Type: schema.TypeList, + Optional: true, + Computed: true, + ForceNew: true, + MaxItems: 1, + ConflictsWith: []string{"master_instance_type", "instance_group"}, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "bid_price": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "ebs_config": { + Type: schema.TypeSet, + Optional: true, + Computed: true, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "iops": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + }, + "size": { + Type: schema.TypeInt, + Required: true, + ForceNew: true, + }, + "type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validateAwsEmrEbsVolumeType(), + }, + "volumes_per_instance": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Default: 1, + }, + }, + }, + Set: resourceAwsEMRClusterEBSConfigHash, + }, + "id": { + Type: schema.TypeString, + Computed: true, + }, + "instance_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + }, + }, + }, "instance_group": { - Type: schema.TypeSet, - Optional: true, - ForceNew: true, - Computed: true, + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Computed: true, + ConflictsWith: []string{"core_instance_group", "master_instance_group"}, + Deprecated: "use `master_instance_group` configuration block, `core_instance_group` configuration block, and `aws_emr_instance_group` resource(s) instead", Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "bid_price": { @@ -494,6 +642,59 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error TerminationProtected: aws.Bool(terminationProtection), } + if l := d.Get("master_instance_group").([]interface{}); len(l) > 0 && l[0] != nil { + m := l[0].(map[string]interface{}) + + instanceGroup := &emr.InstanceGroupConfig{ + InstanceCount: aws.Int64(1), + InstanceRole: aws.String(emr.InstanceRoleTypeMaster), + InstanceType: aws.String(m["instance_type"].(string)), + Market: aws.String(emr.MarketTypeOnDemand), + Name: aws.String(m["name"].(string)), + } + + if v, ok := m["bid_price"]; ok && v.(string) != "" { + instanceGroup.BidPrice = aws.String(v.(string)) + instanceGroup.Market = aws.String(emr.MarketTypeSpot) + } + + expandEbsConfig(m, instanceGroup) + + instanceConfig.InstanceGroups = append(instanceConfig.InstanceGroups, instanceGroup) + } + + if l := d.Get("core_instance_group").([]interface{}); len(l) > 0 && l[0] != nil { + m := l[0].(map[string]interface{}) + + instanceGroup := &emr.InstanceGroupConfig{ + InstanceCount: aws.Int64(int64(m["instance_count"].(int))), + InstanceRole: aws.String(emr.InstanceRoleTypeCore), + InstanceType: aws.String(m["instance_type"].(string)), + Market: aws.String(emr.MarketTypeOnDemand), + Name: aws.String(m["name"].(string)), + } + + if v, ok := m["autoscaling_policy"]; ok && v.(string) != "" { + var autoScalingPolicy *emr.AutoScalingPolicy + + if err := json.Unmarshal([]byte(v.(string)), &autoScalingPolicy); err != nil { + return fmt.Errorf("error parsing core_instance_group Auto Scaling Policy JSON: %s", err) + } + + instanceGroup.AutoScalingPolicy = autoScalingPolicy + } + + if v, ok := m["bid_price"]; ok && v.(string) != "" { + instanceGroup.BidPrice = aws.String(v.(string)) + instanceGroup.Market = aws.String(emr.MarketTypeSpot) + } + + expandEbsConfig(m, instanceGroup) + + instanceConfig.InstanceGroups = append(instanceConfig.InstanceGroups, instanceGroup) + } + + // DEPRECATED: Remove in a future major version if v, ok := d.GetOk("master_instance_type"); ok { masterInstanceGroupConfig := &emr.InstanceGroupConfig{ InstanceRole: aws.String("MASTER"), @@ -503,6 +704,7 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.InstanceGroups = append(instanceConfig.InstanceGroups, masterInstanceGroupConfig) } + // DEPRECATED: Remove in a future major version var coreInstanceType string if v, ok := d.GetOk("core_instance_type"); ok { coreInstanceType = v.(string) @@ -565,6 +767,8 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.ServiceAccessSecurityGroup = aws.String(v.(string)) } } + + // DEPRECATED: Remove in a future major version if v, ok := d.GetOk("instance_group"); ok { instanceGroupConfigs := v.(*schema.Set).List() instanceGroups, err := expandInstanceGroupConfigs(instanceGroupConfigs) @@ -741,24 +945,47 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error { } instanceGroups, err := fetchAllEMRInstanceGroups(emrconn, d.Id()) - if err == nil { - coreGroup := emrCoreInstanceGroup(instanceGroups) - if coreGroup != nil { - d.Set("core_instance_type", coreGroup.InstanceType) - d.Set("core_instance_count", coreGroup.RequestedInstanceCount) - } - masterGroup := findMasterGroup(instanceGroups) - if masterGroup != nil { - d.Set("master_instance_type", masterGroup.InstanceType) - } - flattenedInstanceGroups, err := flattenInstanceGroups(instanceGroups) - if err != nil { - return fmt.Errorf("error flattening instance groups: %+v", err) - } - if err := d.Set("instance_group", flattenedInstanceGroups); err != nil { - return fmt.Errorf("[ERR] Error setting EMR instance groups: %s", err) - } + if err != nil { + return err + } + + coreGroup := emrCoreInstanceGroup(instanceGroups) + masterGroup := findMasterGroup(instanceGroups) + + d.Set("core_instance_count", 0) + d.Set("core_instance_type", "") + d.Set("master_instance_type", "") + + if coreGroup != nil { + d.Set("core_instance_type", coreGroup.InstanceType) + d.Set("core_instance_count", coreGroup.RequestedInstanceCount) + } + + if masterGroup != nil { + d.Set("master_instance_type", masterGroup.InstanceType) + } + + flattenedInstanceGroups, err := flattenInstanceGroups(instanceGroups) + if err != nil { + return fmt.Errorf("error flattening instance groups: %s", err) + } + if err := d.Set("instance_group", flattenedInstanceGroups); err != nil { + return fmt.Errorf("error setting instance_group: %s", err) + } + + flattenedCoreInstanceGroup, err := flattenEmrCoreInstanceGroup(coreGroup) + + if err != nil { + return fmt.Errorf("error flattening core_instance_group: %s", err) + } + + if err := d.Set("core_instance_group", flattenedCoreInstanceGroup); err != nil { + return fmt.Errorf("error setting core_instance_group: %s", err) + } + + if err := d.Set("master_instance_group", flattenEmrMasterInstanceGroup(masterGroup)); err != nil { + return fmt.Errorf("error setting master_instance_group: %s", err) } d.Set("name", cluster.Name) @@ -925,6 +1152,102 @@ func resourceAwsEMRClusterUpdate(d *schema.ResourceData, meta interface{}) error } } + if d.HasChange("core_instance_group.0.autoscaling_policy") { + autoscalingPolicyStr := d.Get("core_instance_group.0.autoscaling_policy").(string) + instanceGroupID := d.Get("core_instance_group.0.id").(string) + + if autoscalingPolicyStr != "" { + var autoScalingPolicy *emr.AutoScalingPolicy + + if err := json.Unmarshal([]byte(autoscalingPolicyStr), &autoScalingPolicy); err != nil { + return fmt.Errorf("error parsing core_instance_group Auto Scaling Policy JSON: %s", err) + } + + input := &emr.PutAutoScalingPolicyInput{ + ClusterId: aws.String(d.Id()), + AutoScalingPolicy: autoScalingPolicy, + InstanceGroupId: aws.String(instanceGroupID), + } + + if _, err := conn.PutAutoScalingPolicy(input); err != nil { + return fmt.Errorf("error updating EMR Cluster (%s) Instance Group (%s) Auto Scaling Policy: %s", d.Id(), instanceGroupID, err) + } + } else { + input := &emr.RemoveAutoScalingPolicyInput{ + ClusterId: aws.String(d.Id()), + InstanceGroupId: aws.String(instanceGroupID), + } + + if _, err := conn.RemoveAutoScalingPolicy(input); err != nil { + return fmt.Errorf("error removing EMR Cluster (%s) Instance Group (%s) Auto Scaling Policy: %s", d.Id(), instanceGroupID, err) + } + + // RemoveAutoScalingPolicy seems to have eventual consistency. + // Retry reading Instance Group configuration until the policy is removed. + err := resource.Retry(1*time.Minute, func() *resource.RetryError { + autoscalingPolicy, err := getEmrCoreInstanceGroupAutoscalingPolicy(conn, d.Id()) + + if err != nil { + return resource.NonRetryableError(err) + } + + if autoscalingPolicy != nil { + return resource.RetryableError(fmt.Errorf("EMR Cluster (%s) Instance Group (%s) Auto Scaling Policy still exists", d.Id(), instanceGroupID)) + } + + return nil + }) + + if isResourceTimeoutError(err) { + var autoscalingPolicy *emr.AutoScalingPolicyDescription + + autoscalingPolicy, err = getEmrCoreInstanceGroupAutoscalingPolicy(conn, d.Id()) + + if autoscalingPolicy != nil { + err = fmt.Errorf("EMR Cluster (%s) Instance Group (%s) Auto Scaling Policy still exists", d.Id(), instanceGroupID) + } + } + + if err != nil { + return fmt.Errorf("error waiting for EMR Cluster (%s) Instance Group (%s) Auto Scaling Policy removal: %s", d.Id(), instanceGroupID, err) + } + } + } + + if d.HasChange("core_instance_group.0.instance_count") { + instanceGroupID := d.Get("core_instance_group.0.id").(string) + + input := &emr.ModifyInstanceGroupsInput{ + InstanceGroups: []*emr.InstanceGroupModifyConfig{ + { + InstanceGroupId: aws.String(instanceGroupID), + InstanceCount: aws.Int64(int64(d.Get("core_instance_group.0.instance_count").(int))), + }, + }, + } + + if _, err := conn.ModifyInstanceGroups(input); err != nil { + return fmt.Errorf("error modifying EMR Cluster (%s) Instance Group (%s): %s", d.Id(), instanceGroupID, err) + } + + stateConf := &resource.StateChangeConf{ + Pending: []string{ + emr.InstanceGroupStateBootstrapping, + emr.InstanceGroupStateProvisioning, + emr.InstanceGroupStateReconfiguring, + emr.InstanceGroupStateResizing, + }, + Target: []string{emr.InstanceGroupStateRunning}, + Refresh: instanceGroupStateRefresh(conn, d.Id(), instanceGroupID), + Timeout: 20 * time.Minute, + Delay: 10 * time.Second, + } + + if _, err := stateConf.WaitForState(); err != nil { + return fmt.Errorf("error waiting for EMR Cluster (%s) Instance Group (%s) modification: %s", d.Id(), instanceGroupID, err) + } + } + if d.HasChange("instance_group") { o, n := d.GetChange("instance_group") oSet := o.(*schema.Set).List() @@ -938,6 +1261,11 @@ func resourceAwsEMRClusterUpdate(d *schema.ResourceData, meta interface{}) error continue } + // Prevent duplicate PutAutoScalingPolicy from earlier update logic + if nInstanceGroup["id"] == d.Get("core_instance_group.0.id").(string) && d.HasChange("core_instance_group.0.autoscaling_policy") { + continue + } + if v, ok := nInstanceGroup["autoscaling_policy"]; ok && v.(string) != "" { var autoScalingPolicy *emr.AutoScalingPolicy @@ -1091,6 +1419,103 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ return result } +func flattenEmrAutoScalingPolicyDescription(policy *emr.AutoScalingPolicyDescription) (string, error) { + if policy == nil { + return "", nil + } + + // AutoScalingPolicy has an additional Status field and null values that are causing a new hashcode to be generated + // for `instance_group`. + // We are purposefully omitting that field and the null values here when we flatten the autoscaling policy string + // for the statefile. + for i, rule := range policy.Rules { + for j, dimension := range rule.Trigger.CloudWatchAlarmDefinition.Dimensions { + if *dimension.Key == "JobFlowId" { + tmpDimensions := append(policy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions[:j], policy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions[j+1:]...) + policy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions = tmpDimensions + } + } + if len(policy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions) == 0 { + policy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions = nil + } + } + + tmpAutoScalingPolicy := emr.AutoScalingPolicy{ + Constraints: policy.Constraints, + Rules: policy.Rules, + } + autoscalingPolicyConstraintsBytes, err := json.Marshal(tmpAutoScalingPolicy.Constraints) + if err != nil { + return "", fmt.Errorf("error parsing EMR Cluster Instance Groups AutoScalingPolicy Constraints: %s", err) + } + autoscalingPolicyConstraintsString := string(autoscalingPolicyConstraintsBytes) + + autoscalingPolicyRulesBytes, err := json.Marshal(tmpAutoScalingPolicy.Rules) + if err != nil { + return "", fmt.Errorf("error parsing EMR Cluster Instance Groups AutoScalingPolicy Rules: %s", err) + } + + var rules []map[string]interface{} + if err := json.Unmarshal(autoscalingPolicyRulesBytes, &rules); err != nil { + return "", err + } + + var cleanRules []map[string]interface{} + for _, rule := range rules { + cleanRules = append(cleanRules, removeNil(rule)) + } + + withoutNulls, err := json.Marshal(cleanRules) + if err != nil { + return "", err + } + autoscalingPolicyRulesString := string(withoutNulls) + + autoscalingPolicyString := fmt.Sprintf("{\"Constraints\":%s,\"Rules\":%s}", autoscalingPolicyConstraintsString, autoscalingPolicyRulesString) + + return autoscalingPolicyString, nil +} + +func flattenEmrCoreInstanceGroup(instanceGroup *emr.InstanceGroup) ([]interface{}, error) { + if instanceGroup == nil { + return []interface{}{}, nil + } + + autoscalingPolicy, err := flattenEmrAutoScalingPolicyDescription(instanceGroup.AutoScalingPolicy) + + if err != nil { + return nil, err + } + + m := map[string]interface{}{ + "autoscaling_policy": autoscalingPolicy, + "bid_price": aws.StringValue(instanceGroup.BidPrice), + "ebs_config": flattenEBSConfig(instanceGroup.EbsBlockDevices), + "id": aws.StringValue(instanceGroup.Id), + "instance_count": aws.Int64Value(instanceGroup.RequestedInstanceCount), + "instance_type": aws.StringValue(instanceGroup.InstanceType), + "name": aws.StringValue(instanceGroup.Name), + } + + return []interface{}{m}, nil +} + +func flattenEmrMasterInstanceGroup(instanceGroup *emr.InstanceGroup) []interface{} { + if instanceGroup == nil { + return []interface{}{} + } + + m := map[string]interface{}{ + "bid_price": aws.StringValue(instanceGroup.BidPrice), + "ebs_config": flattenEBSConfig(instanceGroup.EbsBlockDevices), + "id": aws.StringValue(instanceGroup.Id), + "instance_type": aws.StringValue(instanceGroup.InstanceType), + "name": aws.StringValue(instanceGroup.Name), + } + + return []interface{}{m} +} + func flattenEmrKerberosAttributes(d *schema.ResourceData, kerberosAttributes *emr.KerberosAttributes) []map[string]interface{} { l := make([]map[string]interface{}, 0) @@ -1183,61 +1608,14 @@ func flattenInstanceGroup(ig *emr.InstanceGroup) (map[string]interface{}, error) attrs["name"] = *ig.Name } - if ig.AutoScalingPolicy != nil { - // AutoScalingPolicy has an additional Status field and null values that are causing a new hashcode to be generated - // for `instance_group`. - // We are purposefully omitting that field and the null values here when we flatten the autoscaling policy string - // for the statefile. - for i, rule := range ig.AutoScalingPolicy.Rules { - for j, dimension := range rule.Trigger.CloudWatchAlarmDefinition.Dimensions { - if *dimension.Key == "JobFlowId" { - tmpDimensions := append(ig.AutoScalingPolicy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions[:j], ig.AutoScalingPolicy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions[j+1:]...) - ig.AutoScalingPolicy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions = tmpDimensions - } - } - if len(ig.AutoScalingPolicy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions) == 0 { - ig.AutoScalingPolicy.Rules[i].Trigger.CloudWatchAlarmDefinition.Dimensions = nil - } - } - - tmpAutoScalingPolicy := emr.AutoScalingPolicy{ - Constraints: ig.AutoScalingPolicy.Constraints, - Rules: ig.AutoScalingPolicy.Rules, - } - autoscalingPolicyConstraintsBytes, err := json.Marshal(tmpAutoScalingPolicy.Constraints) - if err != nil { - return nil, fmt.Errorf("error parsing EMR Cluster Instance Groups AutoScalingPolicy Constraints: %s", err) - } - autoscalingPolicyConstraintsString := string(autoscalingPolicyConstraintsBytes) - - autoscalingPolicyRulesBytes, err := json.Marshal(tmpAutoScalingPolicy.Rules) - if err != nil { - return nil, fmt.Errorf("error parsing EMR Cluster Instance Groups AutoScalingPolicy Rules: %s", err) - } - - var rules []map[string]interface{} - if err := json.Unmarshal(autoscalingPolicyRulesBytes, &rules); err != nil { - return nil, err - } - - var cleanRules []map[string]interface{} - for _, rule := range rules { - cleanRules = append(cleanRules, removeNil(rule)) - } - - withoutNulls, err := json.Marshal(cleanRules) - if err != nil { - return nil, err - } - autoscalingPolicyRulesString := string(withoutNulls) + autoscalingPolicy, err := flattenEmrAutoScalingPolicyDescription(ig.AutoScalingPolicy) - autoscalingPolicyString := fmt.Sprintf("{\"Constraints\":%s,\"Rules\":%s}", autoscalingPolicyConstraintsString, autoscalingPolicyRulesString) - - attrs["autoscaling_policy"] = autoscalingPolicyString - } else { - attrs["autoscaling_policy"] = "" + if err != nil { + return nil, err } + attrs["autoscaling_policy"] = autoscalingPolicy + if attrs["name"] != nil { attrs["name"] = *ig.Name } @@ -1720,3 +2098,19 @@ func resourceAwsEMRClusterEBSConfigHash(v interface{}) int { } return hashcode.String(buf.String()) } + +func getEmrCoreInstanceGroupAutoscalingPolicy(conn *emr.EMR, clusterID string) (*emr.AutoScalingPolicyDescription, error) { + instanceGroups, err := fetchAllEMRInstanceGroups(conn, clusterID) + + if err != nil { + return nil, err + } + + coreGroup := emrCoreInstanceGroup(instanceGroups) + + if coreGroup == nil { + return nil, fmt.Errorf("EMR Cluster (%s) Core Instance Group not found", clusterID) + } + + return coreGroup.AutoScalingPolicy, nil +} diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index 6014497b0764..d71414b33635 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -154,6 +154,368 @@ func TestAccAWSEMRCluster_configurationsJson(t *testing.T) { }) } +func TestAccAWSEMRCluster_CoreInstanceGroup_AutoscalingPolicy(t *testing.T) { + var cluster1, cluster2, cluster3 emr.Cluster + autoscalingPolicy1 := ` +{ + "Constraints": { + "MinCapacity": 1, + "MaxCapacity": 2 + }, + "Rules": [ + { + "Name": "ScaleOutMemoryPercentage", + "Description": "Scale out if YARNMemoryAvailablePercentage is less than 15", + "Action": { + "SimpleScalingPolicyConfiguration": { + "AdjustmentType": "CHANGE_IN_CAPACITY", + "ScalingAdjustment": 1, + "CoolDown": 300 + } + }, + "Trigger": { + "CloudWatchAlarmDefinition": { + "ComparisonOperator": "LESS_THAN", + "EvaluationPeriods": 1, + "MetricName": "YARNMemoryAvailablePercentage", + "Namespace": "AWS/ElasticMapReduce", + "Period": 300, + "Statistic": "AVERAGE", + "Threshold": 15.0, + "Unit": "PERCENT" + } + } + } + ] +} +` + autoscalingPolicy2 := ` +{ + "Constraints": { + "MinCapacity": 1, + "MaxCapacity": 3 + }, + "Rules": [ + { + "Name": "ScaleOutMemoryPercentage", + "Description": "Scale out if YARNMemoryAvailablePercentage is less than 15", + "Action": { + "SimpleScalingPolicyConfiguration": { + "AdjustmentType": "CHANGE_IN_CAPACITY", + "ScalingAdjustment": 1, + "CoolDown": 300 + } + }, + "Trigger": { + "CloudWatchAlarmDefinition": { + "ComparisonOperator": "LESS_THAN", + "EvaluationPeriods": 1, + "MetricName": "YARNMemoryAvailablePercentage", + "Namespace": "AWS/ElasticMapReduce", + "Period": 300, + "Statistic": "AVERAGE", + "Threshold": 15.0, + "Unit": "PERCENT" + } + } + } + ] +} +` + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupAutoscalingPolicy(rName, autoscalingPolicy1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestMatchResourceAttr(resourceName, "core_instance_group.0.autoscaling_policy", regexp.MustCompile(`"MaxCapacity": ?2`)), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupAutoscalingPolicy(rName, autoscalingPolicy2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestMatchResourceAttr(resourceName, "core_instance_group.0.autoscaling_policy", regexp.MustCompile(`"MaxCapacity": ?3`)), + ), + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupAutoscalingPolicyRemoved(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster3), + testAccCheckAWSEmrClusterNotRecreated(&cluster2, &cluster3), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.autoscaling_policy", ""), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_CoreInstanceGroup_BidPrice(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupBidPrice(rName, "0.50"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.bid_price", "0.50"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupBidPrice(rName, "0.51"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.bid_price", "0.51"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_CoreInstanceGroup_InstanceCount(t *testing.T) { + var cluster1, cluster2, cluster3 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceCount(rName, 2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_count", "2"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceCount(rName, 1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_count", "1"), + ), + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceCount(rName, 2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster3), + testAccCheckAWSEmrClusterNotRecreated(&cluster2, &cluster3), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_count", "2"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_CoreInstanceGroup_InstanceType(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_type", "m4.large"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceType(rName, "m4.xlarge"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_type", "m4.xlarge"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_CoreInstanceGroup_Migration_CoreInstanceType(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigCoreInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "core_instance_type", "m4.large"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_type", "m4.large"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_CoreInstanceGroup_Migration_InstanceGroup(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigInstanceGroupCoreInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "instance_group.#", "2"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.instance_type", "m4.large"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_CoreInstanceGroup_Name(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupName(rName, "name1"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.name", "name1"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigCoreInstanceGroupName(rName, "name2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "core_instance_group.0.name", "name2"), + ), + }, + }, + }) +} + func TestAccAWSEMRCluster_instance_group(t *testing.T) { var cluster emr.Cluster r := acctest.RandInt() @@ -325,6 +687,204 @@ func TestAccAWSEMRCluster_Kerberos_ClusterDedicatedKdc(t *testing.T) { }) } +func TestAccAWSEMRCluster_MasterInstanceGroup_BidPrice(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupBidPrice(rName, "0.50"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.bid_price", "0.50"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupBidPrice(rName, "0.51"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.bid_price", "0.51"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_MasterInstanceGroup_InstanceType(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.instance_type", "m4.large"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, "m4.xlarge"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.instance_type", "m4.xlarge"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_MasterInstanceGroup_Migration_InstanceGroup(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigInstanceGroupMasterInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "instance_group.#", "1"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.instance_type", "m4.large"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_MasterInstanceGroup_Migration_MasterInstanceType(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigMasterInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "master_instance_type", "m4.large"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupInstanceType(rName, "m4.large"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterNotRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.instance_type", "m4.large"), + ), + }, + }, + }) +} + +func TestAccAWSEMRCluster_MasterInstanceGroup_Name(t *testing.T) { + var cluster1, cluster2 emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupName(rName, "name1"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster1), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.name", "name1"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{ + "configurations", + "keep_job_flow_alive_when_no_steps", + }, + }, + { + Config: testAccAWSEmrClusterConfigMasterInstanceGroupName(rName, "name2"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster2), + testAccCheckAWSEmrClusterRecreated(&cluster1, &cluster2), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.#", "1"), + resource.TestCheckResourceAttr(resourceName, "master_instance_group.0.name", "name2"), + ), + }, + }, + }) +} + func TestAccAWSEMRCluster_security_config(t *testing.T) { var cluster emr.Cluster r := acctest.RandInt() @@ -837,38 +1397,132 @@ func testAccCheckAWSEmrDestroy(s *terraform.State) error { return nil } -func testAccCheckAWSEmrClusterExists(n string, v *emr.Cluster) resource.TestCheckFunc { - return func(s *terraform.State) error { - rs, ok := s.RootModule().Resources[n] - if !ok { - return fmt.Errorf("Not found: %s", n) - } - if rs.Primary.ID == "" { - return fmt.Errorf("No cluster id set") - } - conn := testAccProvider.Meta().(*AWSClient).emrconn - describe, err := conn.DescribeCluster(&emr.DescribeClusterInput{ - ClusterId: aws.String(rs.Primary.ID), - }) - if err != nil { - return fmt.Errorf("EMR error: %v", err) - } +func testAccCheckAWSEmrClusterExists(n string, v *emr.Cluster) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + if rs.Primary.ID == "" { + return fmt.Errorf("No cluster id set") + } + conn := testAccProvider.Meta().(*AWSClient).emrconn + describe, err := conn.DescribeCluster(&emr.DescribeClusterInput{ + ClusterId: aws.String(rs.Primary.ID), + }) + if err != nil { + return fmt.Errorf("EMR error: %v", err) + } + + if describe.Cluster == nil || *describe.Cluster.Id != rs.Primary.ID { + return fmt.Errorf("EMR cluster %q not found", rs.Primary.ID) + } + + *v = *describe.Cluster + + if describe.Cluster.Status != nil { + state := aws.StringValue(describe.Cluster.Status.State) + if state != emr.ClusterStateRunning && state != emr.ClusterStateWaiting { + return fmt.Errorf("EMR cluster %q is not RUNNING or WAITING, currently: %s", rs.Primary.ID, state) + } + } + + return nil + } +} + +func testAccCheckAWSEmrClusterNotRecreated(i, j *emr.Cluster) resource.TestCheckFunc { + return func(s *terraform.State) error { + if aws.StringValue(i.Id) != aws.StringValue(j.Id) { + return fmt.Errorf("EMR Cluster recreated: %s -> %s", aws.StringValue(i.Id), aws.StringValue(j.Id)) + } + + return nil + } +} + +func testAccCheckAWSEmrClusterRecreated(i, j *emr.Cluster) resource.TestCheckFunc { + return func(s *terraform.State) error { + if aws.StringValue(i.Id) == aws.StringValue(j.Id) { + return fmt.Errorf("EMR Cluster not recreated: %s", aws.StringValue(i.Id)) + } + + return nil + } +} + +func testAccAWSEmrClusterConfigBaseVpc() string { + return fmt.Sprintf(` +data "aws_availability_zones" "current" {} + +resource "aws_vpc" "test" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + + tags = { + Name = "tf-acc-test-emr-cluster" + } +} + +resource "aws_internet_gateway" "test" { + vpc_id = "${aws_vpc.test.id}" + + tags = { + Name = "tf-acc-test-emr-cluster" + } +} + +resource "aws_security_group" "test" { + vpc_id = "${aws_vpc.test.id}" + + ingress { + from_port = 0 + protocol = "-1" + self = true + to_port = 0 + } + + egress { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + to_port = 0 + } + + tags = { + Name = "tf-acc-test-emr-cluster" + } + + # EMR will modify ingress rules + lifecycle { + ignore_changes = ["ingress"] + } +} + +resource "aws_subnet" "test" { + availability_zone = "${data.aws_availability_zones.current.names[0]}" + cidr_block = "10.0.0.0/24" + vpc_id = "${aws_vpc.test.id}" - if describe.Cluster == nil || *describe.Cluster.Id != rs.Primary.ID { - return fmt.Errorf("EMR cluster %q not found", rs.Primary.ID) - } + tags = { + Name = "tf-acc-test-emr-cluster" + } +} - *v = *describe.Cluster +resource "aws_route_table" "test" { + vpc_id = "${aws_vpc.test.id}" - if describe.Cluster.Status != nil { - state := aws.StringValue(describe.Cluster.Status.State) - if state != emr.ClusterStateRunning && state != emr.ClusterStateWaiting { - return fmt.Errorf("EMR cluster %q is not RUNNING or WAITING, currently: %s", rs.Primary.ID, state) - } - } + route { + cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.test.id}" + } +} - return nil - } +resource "aws_route_table_association" "test" { + route_table_id = "${aws_route_table.test.id}" + subnet_id = "${aws_subnet.test.id}" +} +`) } func testAccAWSEmrClusterConfig_bootstrap(r string) string { @@ -2054,10 +2708,6 @@ EOT func testAccAWSEmrClusterConfig_Kerberos_ClusterDedicatedKdc(r int, password string) string { return fmt.Sprintf(` -provider "aws" { - region = "us-west-2" -} - data "aws_availability_zones" "available" {} resource "aws_emr_security_configuration" "foo" { @@ -2178,10 +2828,6 @@ resource "aws_main_route_table_association" "a" { func testAccAWSEmrClusterConfig_SecurityConfiguration(r int) string { return fmt.Sprintf(` -provider "aws" { - region = "us-west-2" -} - resource "aws_emr_cluster" "tf-test-cluster" { name = "emr-test-%d" release_label = "emr-5.5.0" @@ -2523,161 +3169,478 @@ resource "aws_kms_key" "foo" { } POLICY } -`, r, r, r, r, r, r, r, r, r) +`, r, r, r, r, r, r, r, r, r) +} + +const testAccAWSEmrClusterConfig_Step_DebugLoggingStep = ` + # Example from: https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-plan-debugging.html + step { + action_on_failure = "TERMINATE_CLUSTER" + name = "Setup Hadoop Debugging" + + hadoop_jar_step { + jar = "command-runner.jar" + args = ["state-pusher-script"] + } + } +` + +const testAccAWSEmrClusterConfig_Step_SparkStep = ` + # Example from: https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-submit-step.html + step { + action_on_failure = "CONTINUE" + name = "Spark Step" + + hadoop_jar_step { + jar = "command-runner.jar" + args = ["spark-example", "SparkPi", "10"] + } + } +` + +func testAccAWSEmrClusterConfig_Step_Multiple(rInt int) string { + stepConfig := testAccAWSEmrClusterConfig_Step_DebugLoggingStep + testAccAWSEmrClusterConfig_Step_SparkStep + return testAccAWSEmrClusterConfig_Step(rInt, stepConfig) +} + +func testAccAWSEmrClusterConfig_Step_NoBlocks(rInt int) string { + return testAccAWSEmrClusterConfig_Step(rInt, "") +} + +func testAccAWSEmrClusterConfig_Step_Single(rInt int) string { + return testAccAWSEmrClusterConfig_Step(rInt, testAccAWSEmrClusterConfig_Step_DebugLoggingStep) +} + +func testAccAWSEmrClusterConfig_Step_Zeroed(rInt int) string { + return testAccAWSEmrClusterConfig_Step(rInt, "step = []") +} + +func testAccAWSEmrClusterConfig_Step(rInt int, stepConfig string) string { + return fmt.Sprintf(` +data "aws_availability_zones" "available" {} + +resource "aws_emr_cluster" "tf-test-cluster" { + applications = ["Spark"] + core_instance_count = 1 + core_instance_type = "c4.large" + keep_job_flow_alive_when_no_steps = true + log_uri = "s3://${aws_s3_bucket.test.bucket}/" + master_instance_type = "c4.large" + name = "emr-test-%[1]d" + release_label = "emr-5.12.0" + service_role = "EMR_DefaultRole" + termination_protection = false + + ec2_attributes { + emr_managed_master_security_group = "${aws_security_group.allow_all.id}" + emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" + instance_profile = "EMR_EC2_DefaultRole" + subnet_id = "${aws_subnet.main.0.id}" + } + +%[2]s + + depends_on = ["aws_main_route_table_association.a"] +} + +resource "aws_s3_bucket" "test" { + bucket = "tf-acc-test-%[1]d" + force_destroy = true +} + +resource "aws_security_group" "allow_all" { + name = "allow_all_%[1]d" + description = "Allow all inbound traffic" + vpc_id = "${aws_vpc.main.id}" + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + depends_on = ["aws_subnet.main"] + + lifecycle { + ignore_changes = ["ingress", "egress"] + } + + tags = { + Name = "emr_test" + } +} + +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + + tags = { + Name = "terraform-testacc-emr-cluster-step" + } +} + +resource "aws_subnet" "main" { + availability_zone = "${element(data.aws_availability_zones.available.names, count.index)}" + count = 2 + cidr_block = "10.0.${count.index}.0/24" + vpc_id = "${aws_vpc.main.id}" + + tags = { + Name = "terraform-testacc-emr-cluster-step" + } +} + +resource "aws_internet_gateway" "gw" { + vpc_id = "${aws_vpc.main.id}" + + tags = { + Name = "terraform-testacc-emr-cluster-step" + } +} + +resource "aws_route_table" "r" { + vpc_id = "${aws_vpc.main.id}" + + route { + cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.gw.id}" + } +} + +resource "aws_main_route_table_association" "a" { + route_table_id = "${aws_route_table.r.id}" + vpc_id = "${aws_vpc.main.id}" +} +`, rInt, stepConfig) +} + +func testAccAWSEmrClusterConfigCoreInstanceGroupAutoscalingPolicy(rName, autoscalingPolicy string) string { + return testAccAWSEmrClusterConfigBaseVpc() + fmt.Sprintf(` +data "aws_iam_policy_document" "test" { + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + identifiers = [ + "application-autoscaling.amazonaws.com", + "elasticmapreduce.amazonaws.com", + ] + type = "Service" + } + } +} + +data "aws_partition" "current" {} + +resource "aws_iam_role" "test" { + name = %[1]q + assume_role_policy = "${data.aws_iam_policy_document.test.json}" +} + +resource "aws_iam_role_policy_attachment" "test" { + role = "${aws_iam_role.test.name}" + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" +} + +resource "aws_emr_cluster" "test" { + applications = ["Spark"] + autoscaling_role = "${aws_iam_role.test.arn}" + keep_job_flow_alive_when_no_steps = true + name = %[1]q + release_label = "emr-5.12.0" + service_role = "EMR_DefaultRole" + + ec2_attributes { + emr_managed_master_security_group = "${aws_security_group.test.id}" + emr_managed_slave_security_group = "${aws_security_group.test.id}" + instance_profile = "EMR_EC2_DefaultRole" + subnet_id = "${aws_subnet.test.id}" + } + + master_instance_group { + instance_type = "m4.large" + } + + core_instance_group { + autoscaling_policy = <AWS Provider Version 2 Upgrade +
  • + AWS Provider Version 3 Upgrade +
    • +
  • Custom Service Endpoints
    • diff --git a/website/docs/guides/version-3-upgrade.html.md b/website/docs/guides/version-3-upgrade.html.md new file mode 100644 index 000000000000..5ffe10028648 --- /dev/null +++ b/website/docs/guides/version-3-upgrade.html.md @@ -0,0 +1,186 @@ +--- +layout: "aws" +page_title: "Terraform AWS Provider Version 3 Upgrade Guide" +sidebar_current: "docs-aws-guide-version-2-upgrade" +description: |- + Terraform AWS Provider Version 3 Upgrade Guide +--- + +# Terraform AWS Provider Version 3 Upgrade Guide + +~> **NOTE:** This upgrade guide is a work in progress and will not be completed until the release of version 3.0.0 of the provider in the coming months. Many of the topics discussed, except for the actual provider upgrade, can be performed using the most recent 2.X version of the provider. + +Version 3.0.0 of the AWS provider for Terraform is a major release and includes some changes that you will need to consider when upgrading. This guide is intended to help with that process and focuses only on changes from version 1.X to version 3.0.0. + +Most of the changes outlined in this guide have been previously marked as deprecated in the Terraform plan/apply output throughout previous provider releases. These changes, such as deprecation notices, can always be found in the [Terraform AWS Provider CHANGELOG](https://github.com/terraform-providers/terraform-provider-aws/blob/master/CHANGELOG.md). + +Upgrade topics: + + + +- [Provider Version Configuration](#provider-version-configuration) +- [Resource: aws_emr_cluster](#resource-aws_emr_cluster) + + + +## Provider Version Configuration + +!> **WARNING:** This topic is placeholder documentation until version 3.0.0 is released in the coming months. + +-> Before upgrading to version 3.0.0, it is recommended to upgrade to the most recent 2.X version of the provider and ensure that your environment successfully runs [`terraform plan`](https://www.terraform.io/docs/commands/plan.html) without unexpected changes or deprecation notices. + +It is recommended to use [version constraints when configuring Terraform providers](https://www.terraform.io/docs/configuration/providers.html#provider-versions). If you are following that recommendation, update the version constraints in your Terraform configuration and run [`terraform init`](https://www.terraform.io/docs/commands/init.html) to download the new version. + +For example, given this previous configuration: + +```hcl +provider "aws" { + # ... other configuration ... + + version = "~> 2.8" +} +``` + +Update to latest 3.X version: + +```hcl +provider "aws" { + # ... other configuration ... + + version = "~> 3.0" +} +``` + +## Resource: aws_emr_cluster + +### core_instance_count Argument Removal + +Switch your Terraform configuration to the `core_instance_group` configuration block instead. + +For example, given this previous configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + core_instance_count = 2 +} +``` + +An updated configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + core_instance_group { + # ... other configuration ... + + instance_count = 2 + } +} +``` + +### core_instance_type Argument Removal + +Switch your Terraform configuration to the `core_instance_group` configuration block instead. + +For example, given this previous configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + core_instance_type = "m4.large" +} +``` + +An updated configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + core_instance_group { + instance_type = "m4.large" + } +} +``` + +### instance_group Configuration Block Removal + +Switch your Terraform configuration to the `master_instance_group` and `core_instance_group` configuration blocks instead. For any task instance groups, use the `aws_emr_instance_group` resource. + +For example, given this previous configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + instance_group { + instance_role = "MASTER" + instance_type = "m4.large" + } + + instance_group { + instance_count = 1 + instance_role = "CORE" + instance_type = "c4.large" + } + + instance_group { + instance_count = 2 + instance_role = "TASK" + instance_type = "c4.xlarge" + } +} +``` + +An updated configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + master_instance_group { + instance_type = "m4.large" + } + + core_instance_group { + instance_count = 1 + instance_type = "c4.large" + } +} + +resource "aws_emr_instance_group" "example" { + cluster_id = "${aws_emr_cluster.example.id}" + instance_count = 2 + instance_type = "c4.xlarge" +} +``` + +### master_instance_type Argument Removal + +Switch your Terraform configuration to the `master_instance_group` configuration block instead. + +For example, given this previous configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + master_instance_type = "m4.large" +} +``` + +An updated configuration: + +```hcl +resource "aws_emr_cluster" "example" { + # ... other configuration ... + + master_instance_group { + instance_type = "m4.large" + } +} +``` diff --git a/website/docs/r/emr_cluster.html.markdown b/website/docs/r/emr_cluster.html.markdown index b6d3ca97938b..298131ace082 100644 --- a/website/docs/r/emr_cluster.html.markdown +++ b/website/docs/r/emr_cluster.html.markdown @@ -12,6 +12,10 @@ Provides an Elastic MapReduce Cluster, a web service that makes it easy to process large amounts of data efficiently. See [Amazon Elastic MapReduce Documentation](https://aws.amazon.com/documentation/elastic-mapreduce/) for more information. +To configure [Instance Groups](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-group-configuration.html#emr-plan-instance-groups) for [task nodes](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-master-core-task-nodes.html#emr-plan-task), see the [`aws_emr_instance_group` resource](/docs/providers/aws/r/emr_instance_group.html). + +-> Support for [Instance Fleets](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-group-configuration.html#emr-plan-instance-fleets) will be made available in an upcoming release. + ## Example Usage ```hcl @@ -38,10 +42,13 @@ EOF instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" } - instance_group { - instance_role = "CORE" + master_instance_group { + instance_type = "m4.large" + } + + core_instance_group { instance_type = "c4.large" - instance_count = "1" + instance_count = 1 ebs_config { size = "40" type = "gp2" @@ -84,10 +91,6 @@ EOF } ebs_root_volume_size = 100 - master_instance_type = "m5.xlarge" - core_instance_type = "m5.xlarge" - core_instance_count = 1 - tags = { role = "rolename" env = "env" @@ -173,14 +176,16 @@ The following arguments are supported: * `name` - (Required) The name of the job flow * `release_label` - (Required) The release label for the Amazon EMR release -* `master_instance_type` - (Optional) The EC2 instance type of the master node. Exactly one of `master_instance_type` and `instance_group` must be specified. +* `master_instance_group` - (Optional) Configuration block to use an [Instance Group](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-group-configuration.html#emr-plan-instance-groups) for the [master node type](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-master-core-task-nodes.html#emr-plan-master). Cannot be specified if `master_instance_type` argument or `instance_group` configuration blocks are set. Detailed below. +* `master_instance_type` - (Optional, **DEPRECATED**) Use the `master_instance_group` configuration block `instance_type` argument instead. The EC2 instance type of the master node. Cannot be specified if `master_instance_group` or `instance_group` configuration blocks are set. * `scale_down_behavior` - (Optional) The way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an `instance group` is resized. * `additional_info` - (Optional) A JSON string for selecting additional features such as adding proxy information. Note: Currently there is no API to retrieve the value of this argument after EMR cluster creation from provider, therefore Terraform cannot detect drift from the actual EMR cluster if its value is changed outside Terraform. * `service_role` - (Required) IAM role that will be assumed by the Amazon EMR service to access AWS resources * `security_configuration` - (Optional) The security configuration name to attach to the EMR cluster. Only valid for EMR clusters with `release_label` 4.8.0 or greater -* `core_instance_type` - (Optional) The EC2 instance type of the slave nodes. Cannot be specified if `instance_groups` is set -* `core_instance_count` - (Optional) Number of Amazon EC2 instances used to execute the job flow. EMR will use one node as the cluster's master node and use the remainder of the nodes (`core_instance_count`-1) as core nodes. Cannot be specified if `instance_groups` is set. Default `1` -* `instance_group` - (Optional) A list of `instance_group` objects for each instance group in the cluster. Exactly one of `master_instance_type` and `instance_group` must be specified. If `instance_group` is set, then it must contain a configuration block for at least the `MASTER` instance group type (as well as any additional instance groups). Defined below +* `core_instance_group` - (Optional) Configuration block to use an [Instance Group](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-instance-group-configuration.html#emr-plan-instance-groups) for the [core node type](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-master-core-task-nodes.html#emr-plan-core). Cannot be specified if `core_instance_count` argument, `core_instance_type` argument, or `instance_group` configuration blocks are set. Detailed below. +* `core_instance_type` - (Optional, **DEPRECATED**) Use the `core_instance_group` configuration block `instance_type` argument instead. The EC2 instance type of the slave nodes. Cannot be specified if `core_instance_group` or `instance_group` configuration blocks are set. +* `core_instance_count` - (Optional, **DEPRECATED**) Use the `core_instance_group` configuration block `instance_count` argument instead. Number of Amazon EC2 instances used to execute the job flow. EMR will use one node as the cluster's master node and use the remainder of the nodes (`core_instance_count`-1) as core nodes. Cannot be specified if `core_instance_group` or `instance_group` configuration blocks are set. Default `1` +* `instance_group` - (Optional, **DEPRECATED**) Use the `master_instance_group` configuration block, `core_instance_group` configuration block and [`aws_emr_instance_group` resource(s)](/docs/providers/aws/r/emr_instance_group.html) instead. A list of `instance_group` objects for each instance group in the cluster. Exactly one of `master_instance_type` and `instance_group` must be specified. If `instance_group` is set, then it must contain a configuration block for at least the `MASTER` instance group type (as well as any additional instance groups). Cannot be specified if `master_instance_group` or `core_instance_group` configuration blocks are set. Defined below * `log_uri` - (Optional) S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created * `applications` - (Optional) A list of applications for the cluster. Valid values are: `Flink`, `Hadoop`, `Hive`, `Mahout`, `Pig`, `Spark`, and `JupyterHub` (as of EMR 5.14.0). Case insensitive * `termination_protection` - (Optional) Switch on/off termination protection (default is off) @@ -220,6 +225,17 @@ EOF * `step` - (Optional) List of steps to run when creating the cluster. Defined below. It is highly recommended to utilize the [lifecycle configuration block](/docs/configuration/resources.html) with `ignore_changes` if other steps are being managed outside of Terraform. This argument is processed in [attribute-as-blocks mode](/docs/configuration/attr-as-blocks.html). * `tags` - (Optional) list of tags to apply to the EMR Cluster +## core_instance_group Configuration Block + +Supported nested arguments for the `core_instance_group` configuration block: + +* `instance_type` - (Required) EC2 instance type for all instances in the instance group. +* `autoscaling_policy` - (Optional) String containing the [EMR Auto Scaling Policy](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-automatic-scaling.html) JSON. +* `bid_price` - (Optional) Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances. +* `ebs_config` - (Optional) Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below. +* `instance_count` - (Optional) Target number of instances for the instance group. Must be at least 1. Defaults to 1. +* `name` - (Optional) Friendly name given to the instance group. + ## ec2_attributes Attributes for the Amazon EC2 instances running the job flow @@ -270,6 +286,15 @@ Attributes for each task instance group in the cluster * `ebs_config` - (Optional) A list of attributes for the EBS volumes attached to each instance in the instance group. Each `ebs_config` defined will result in additional EBS volumes being attached to _each_ instance in the instance group. Defined below * `autoscaling_policy` - (Optional) The autoscaling policy document. This is a JSON formatted string. See [EMR Auto Scaling](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-automatic-scaling.html) +## master_instance_group Configuration Block + +Supported nested arguments for the `master_instance_group` configuration block: + +* `instance_type` - (Required) EC2 instance type for all instances in the instance group. +* `bid_price` - (Optional) Bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances. +* `ebs_config` - (Optional) Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below. +* `name` - (Optional) Friendly name given to the instance group. + ## ebs_config Attributes for the EBS volumes attached to each EC2 instance in the `instance_group` @@ -309,10 +334,9 @@ In addition to all arguments above, the following attributes are exported: * `id` - The ID of the EMR Cluster * `name` - The name of the cluster. * `release_label` - The release label for the Amazon EMR release. -* `master_instance_type` - The EC2 instance type of the master node. +* `master_instance_group.0.id` - Master node type Instance Group ID, if using Instance Group for this node type. * `master_public_dns` - The public DNS name of the master EC2 instance. -* `core_instance_type` - The EC2 instance type of the slave nodes. -* `core_instance_count` The number of slave nodes, i.e. EC2 instance nodes. +* `core_instance_group.0.id` - Core node type Instance Group ID, if using Instance Group for this node type. * `log_uri` - The path to the Amazon S3 location where logs for this cluster are stored. * `applications` - The applications installed on this cluster. * `ec2_attributes` - Provides information about the EC2 instances in a cluster grouped by category: key name, subnet ID, IAM instance profile, and so on. @@ -322,8 +346,6 @@ In addition to all arguments above, the following attributes are exported: * `visible_to_all_users` - Indicates whether the job flow is visible to all IAM users of the AWS account associated with the job flow. * `tags` - The list of tags associated with a cluster. -For any instance_group the id is exported IN `aws_emr_cluster.instance_group.HASHCODE.id` format, e.g. `aws_emr_cluster.example.instance_group.12345678.id` - ## Example bootable config **NOTE:** This configuration demonstrates a minimal configuration needed to From 4e00927a63e919ec9f14fbfe2194532babef3057 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Mon, 13 May 2019 08:37:03 -0400 Subject: [PATCH 2/3] Apply suggestions from code review Co-Authored-By: Wilken Rivera --- website/docs/guides/version-3-upgrade.html.md | 2 +- website/docs/r/emr_cluster.html.markdown | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/guides/version-3-upgrade.html.md b/website/docs/guides/version-3-upgrade.html.md index 5ffe10028648..a5e8370bd6f9 100644 --- a/website/docs/guides/version-3-upgrade.html.md +++ b/website/docs/guides/version-3-upgrade.html.md @@ -1,7 +1,7 @@ --- layout: "aws" page_title: "Terraform AWS Provider Version 3 Upgrade Guide" -sidebar_current: "docs-aws-guide-version-2-upgrade" +sidebar_current: "docs-aws-guide-version-3-upgrade" description: |- Terraform AWS Provider Version 3 Upgrade Guide --- diff --git a/website/docs/r/emr_cluster.html.markdown b/website/docs/r/emr_cluster.html.markdown index 298131ace082..7e20815518bb 100644 --- a/website/docs/r/emr_cluster.html.markdown +++ b/website/docs/r/emr_cluster.html.markdown @@ -227,7 +227,7 @@ EOF ## core_instance_group Configuration Block -Supported nested arguments for the `core_instance_group` configuration block: +Supported arguments for the `core_instance_group` configuration block: * `instance_type` - (Required) EC2 instance type for all instances in the instance group. * `autoscaling_policy` - (Optional) String containing the [EMR Auto Scaling Policy](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-automatic-scaling.html) JSON. From 66faff887bd895bd5c0c5da0289bc45c69507031 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Mon, 13 May 2019 08:56:13 -0400 Subject: [PATCH 3/3] tests/resource/aws_emr_cluster: Switch testAccAWSEmrClusterConfig_SecurityConfiguration to index notation Reference: https://github.com/terraform-providers/terraform-provider-aws/pull/8459/files#r282175062 Output from acceptance testing: ``` --- PASS: TestAccAWSEMRCluster_security_config (404.88s) ``` --- aws/resource_aws_emr_cluster_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index d71414b33635..d1c532802fa4 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -2826,10 +2826,10 @@ resource "aws_main_route_table_association" "a" { `, r, password) } -func testAccAWSEmrClusterConfig_SecurityConfiguration(r int) string { +func testAccAWSEmrClusterConfig_SecurityConfiguration(rInt int) string { return fmt.Sprintf(` resource "aws_emr_cluster" "tf-test-cluster" { - name = "emr-test-%d" + name = "emr-test-%[1]d" release_label = "emr-5.5.0" applications = ["Spark"] @@ -2871,7 +2871,7 @@ resource "aws_emr_cluster" "tf-test-cluster" { } resource "aws_security_group" "allow_all" { - name = "allow_all_%d" + name = "allow_all_%[1]d" description = "Allow all inbound traffic" vpc_id = "${aws_vpc.main.id}" @@ -2944,7 +2944,7 @@ resource "aws_main_route_table_association" "a" { # IAM role for EMR Service resource "aws_iam_role" "iam_emr_default_role" { - name = "iam_emr_default_role_%d" + name = "iam_emr_default_role_%[1]d" assume_role_policy = <