From 2e78485dd717ba6c6bd1507f099a557a14fba423 Mon Sep 17 00:00:00 2001 From: bdoepf Date: Sun, 20 Oct 2019 12:13:00 +0200 Subject: [PATCH] Add glue_version property to the aws_glue_job resource --- aws/resource_aws_glue_job.go | 16 ++++++++ aws/resource_aws_glue_job_test.go | 57 +++++++++++++++++++++++++++ website/docs/r/glue_job.html.markdown | 1 + 3 files changed, 74 insertions(+) diff --git a/aws/resource_aws_glue_job.go b/aws/resource_aws_glue_job.go index 07aab379c36a..045cebb8a9ae 100644 --- a/aws/resource_aws_glue_job.go +++ b/aws/resource_aws_glue_job.go @@ -3,6 +3,7 @@ package aws import ( "fmt" "log" + "regexp" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/glue" @@ -82,6 +83,12 @@ func resourceAwsGlueJob() *schema.Resource { }, }, }, + "glue_version": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ValidateFunc: validation.StringMatch(regexp.MustCompile(`\d\.\d`), "must be 'major.minor', e.g. '0.9' or '1.0'"), + }, "max_capacity": { Type: schema.TypeFloat, Optional: true, @@ -159,6 +166,10 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error { input.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{})) } + if v, ok := d.GetOk("glue_version"); ok { + input.GlueVersion = aws.String(v.(string)) + } + if v, ok := d.GetOk("max_retries"); ok { input.MaxRetries = aws.Int64(int64(v.(int))) } @@ -216,6 +227,7 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error { if err := d.Set("execution_property", flattenGlueExecutionProperty(job.ExecutionProperty)); err != nil { return fmt.Errorf("error setting execution_property: %s", err) } + d.Set("glue_version", job.GlueVersion) d.Set("max_capacity", aws.Float64Value(job.MaxCapacity)) d.Set("max_retries", int(aws.Int64Value(job.MaxRetries))) d.Set("name", job.Name) @@ -271,6 +283,10 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error { jobUpdate.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{})) } + if v, ok := d.GetOk("glue_version"); ok { + jobUpdate.GlueVersion = aws.String(v.(string)) + } + if v, ok := d.GetOk("max_retries"); ok { jobUpdate.MaxRetries = aws.Int64(int64(v.(int))) } diff --git a/aws/resource_aws_glue_job_test.go b/aws/resource_aws_glue_job_test.go index 2e549d22d21d..1ff7a19c136e 100644 --- a/aws/resource_aws_glue_job_test.go +++ b/aws/resource_aws_glue_job_test.go @@ -273,6 +273,44 @@ func TestAccAWSGlueJob_ExecutionProperty(t *testing.T) { }) } +func TestAccAWSGlueJob_GlueVersion(t *testing.T) { + var job glue.Job + + rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5)) + resourceName := "aws_glue_job.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueJobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSGlueJobConfig_GlueVersion(rName, "1"), + ExpectError: regexp.MustCompile(`must be 'major.minor', e.g. '0.9' or '1.0'`), + }, + { + Config: testAccAWSGlueJobConfig_GlueVersion(rName, "0.9"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "glue_version", "0.9"), + ), + }, + { + Config: testAccAWSGlueJobConfig_GlueVersion(rName, "1.0"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "glue_version", "1.0"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func TestAccAWSGlueJob_MaxRetries(t *testing.T) { var job glue.Job @@ -667,6 +705,25 @@ resource "aws_glue_job" "test" { `, testAccAWSGlueJobConfig_Base(rName), rName, maxConcurrentRuns) } +func testAccAWSGlueJobConfig_GlueVersion(rName string, glueVersion string) string { + return fmt.Sprintf(` +%s + +resource "aws_glue_job" "test" { + name = "%s" + role_arn = "${aws_iam_role.test.arn}" + allocated_capacity = 10 + glue_version = "%s" + + command { + script_location = "testscriptlocation" + } + + depends_on = ["aws_iam_role_policy_attachment.test"] +} +`, testAccAWSGlueJobConfig_Base(rName), rName, glueVersion) +} + func testAccAWSGlueJobConfig_MaxRetries(rName string, maxRetries int) string { return fmt.Sprintf(` %s diff --git a/website/docs/r/glue_job.html.markdown b/website/docs/r/glue_job.html.markdown index 381882a16a20..17c23a5409c1 100644 --- a/website/docs/r/glue_job.html.markdown +++ b/website/docs/r/glue_job.html.markdown @@ -54,6 +54,7 @@ be removed in future releases, please use `max_capacity` instead. * `default_arguments` – (Optional) The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide. * `description` – (Optional) Description of the job. * `execution_property` – (Optional) Execution property of the job. Defined below. +* `glue_version` - (Optional) The glue version determines the spark and python version, see [glue version table](https://docs.aws.amazon.com/en_pv/glue/latest/dg/add-job.html#glue-version-table). Current valid values are '0.9' and '1.0'. * `max_capacity` – (Optional) The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. * `max_retries` – (Optional) The maximum number of times to retry this job if it fails. * `name` – (Required) The name you assign to this job. It must be unique in your account.