Add glue_version property to the aws_glue_job resource

aws/resource_aws_glue_job.go

@@ -3,6 +3,7 @@ package aws
 import (
 	"fmt"
 	"log"
+	"regexp"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/glue"
@@ -82,6 +83,12 @@ func resourceAwsGlueJob() *schema.Resource {
 					},
 				},
 			},
+			"glue_version": {
+				Type:         schema.TypeString,
+				Optional:     true,
+				Computed:     true,
+				ValidateFunc: validation.StringMatch(regexp.MustCompile(`\d\.\d`), "must be 'major.minor', e.g. '0.9' or '1.0'"),
+			},
 			"max_capacity": {
 				Type:          schema.TypeFloat,
 				Optional:      true,
@@ -159,6 +166,10 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error {
 		input.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{}))
 	}
 
+	if v, ok := d.GetOk("glue_version"); ok {
+		input.GlueVersion = aws.String(v.(string))
+	}
+
 	if v, ok := d.GetOk("max_retries"); ok {
 		input.MaxRetries = aws.Int64(int64(v.(int)))
 	}
@@ -216,6 +227,7 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
 	if err := d.Set("execution_property", flattenGlueExecutionProperty(job.ExecutionProperty)); err != nil {
 		return fmt.Errorf("error setting execution_property: %s", err)
 	}
+	d.Set("glue_version", job.GlueVersion)
 	d.Set("max_capacity", aws.Float64Value(job.MaxCapacity))
 	d.Set("max_retries", int(aws.Int64Value(job.MaxRetries)))
 	d.Set("name", job.Name)
@@ -271,6 +283,10 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error {
 		jobUpdate.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{}))
 	}
 
+	if v, ok := d.GetOk("glue_version"); ok {
+		jobUpdate.GlueVersion = aws.String(v.(string))
+	}
+
 	if v, ok := d.GetOk("max_retries"); ok {
 		jobUpdate.MaxRetries = aws.Int64(int64(v.(int)))
 	}

aws/resource_aws_glue_job_test.go

@@ -273,6 +273,44 @@ func TestAccAWSGlueJob_ExecutionProperty(t *testing.T) {
 	})
 }
 
+func TestAccAWSGlueJob_GlueVersion(t *testing.T) {
+	var job glue.Job
+
+	rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5))
+	resourceName := "aws_glue_job.test"
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckAWSGlueJobDestroy,
+		Steps: []resource.TestStep{
+			{
+				Config:      testAccAWSGlueJobConfig_GlueVersion(rName, "1"),
+				ExpectError: regexp.MustCompile(`must be 'major.minor', e.g. '0.9' or '1.0'`),
+			},
+			{
+				Config: testAccAWSGlueJobConfig_GlueVersion(rName, "0.9"),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckAWSGlueJobExists(resourceName, &job),
+					resource.TestCheckResourceAttr(resourceName, "glue_version", "0.9"),
+				),
+			},
+			{
+				Config: testAccAWSGlueJobConfig_GlueVersion(rName, "1.0"),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckAWSGlueJobExists(resourceName, &job),
+					resource.TestCheckResourceAttr(resourceName, "glue_version", "1.0"),
+				),
+			},
+			{
+				ResourceName:      resourceName,
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+		},
+	})
+}
+
 func TestAccAWSGlueJob_MaxRetries(t *testing.T) {
 	var job glue.Job
 
@@ -667,6 +705,25 @@ resource "aws_glue_job" "test" {
 `, testAccAWSGlueJobConfig_Base(rName), rName, maxConcurrentRuns)
 }
 
+func testAccAWSGlueJobConfig_GlueVersion(rName string, glueVersion string) string {
+	return fmt.Sprintf(`
+%s
+
+resource "aws_glue_job" "test" {
+  name               = "%s"
+  role_arn           = "${aws_iam_role.test.arn}"
+  allocated_capacity = 10
+  glue_version       = "%s"
+
+  command {
+    script_location = "testscriptlocation"
+  }
+
+  depends_on = ["aws_iam_role_policy_attachment.test"]
+}
+`, testAccAWSGlueJobConfig_Base(rName), rName, glueVersion)
+}
+
 func testAccAWSGlueJobConfig_MaxRetries(rName string, maxRetries int) string {
 	return fmt.Sprintf(`
 %s

website/docs/r/glue_job.html.markdown

@@ -54,6 +54,7 @@ be removed in future releases, please use `max_capacity` instead.
 * `default_arguments` – (Optional) The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide.
 * `description` – (Optional) Description of the job.
 * `execution_property` – (Optional) Execution property of the job. Defined below.
+* `glue_version` - (Optional) The glue version determines the spark and python version, see [glue version table](https://docs.aws.amazon.com/en_pv/glue/latest/dg/add-job.html#glue-version-table). Current valid values are '0.9' and '1.0'.
 * `max_capacity` – (Optional) The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
 * `max_retries` – (Optional) The maximum number of times to retry this job if it fails.
 * `name` – (Required) The name you assign to this job. It must be unique in your account.