feat(glue): Added value to PythonVersion enum (#21670)

PR to fix #21568. Extended the PythonVersion enum to include 3.9, as it already seems to be supported everywhere (CloudFormation, SDK). #21568 (comment) ---- ### All Submissions: * [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) ### Adding new Unconventional Dependencies: * [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies) ### New Features * [x] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)? * [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Aug 29, 2022 · 9774d4c · 9774d4c
1 parent f9ca639
commit 9774d4c
Show file tree

Hide file tree

Showing 9 changed files with 445 additions and 15 deletions.
diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md
@@ -72,7 +72,11 @@ new glue.Job(this, 'PythonSparkStreamingJob', {
 ### Python Shell Jobs
 
 A Python shell job runs Python scripts as a shell and supports a Python version that depends on the AWS Glue version you are using.
-This can be used to schedule and run tasks that don't require an Apache Spark environment.
+This can be used to schedule and run tasks that don't require an Apache Spark environment. Currently, three flavors are supported:
+
+* PythonVersion.TWO (2.7; EOL)
+* PythonVersion.THREE (3.6)
+* PythonVersion.THREE_NINE (3.9)
 
 ```ts
 declare const bucket: s3.Bucket;

diff --git a/packages/@aws-cdk/aws-glue/lib/job-executable.ts b/packages/@aws-cdk/aws-glue/lib/job-executable.ts
@@ -75,6 +75,11 @@ export enum PythonVersion {
    * Python 3 (the exact version depends on GlueVersion and JobCommand used)
    */
   THREE = '3',
+
+  /**
+   * Python 3.9 (the exact version depends on GlueVersion and JobCommand used)
+   */
+  THREE_NINE = '3.9',
 }
 
 /**
@@ -300,6 +305,9 @@ export class JobExecutable {
     if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) {
       throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON');
     }
+    if (config.pythonVersion === PythonVersion.THREE_NINE && config.type !== JobType.PYTHON_SHELL) {
+      throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell');
+    }
     this.config = config;
   }
 

diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.ts b/packages/@aws-cdk/aws-glue/test/integ.job.ts
@@ -86,4 +86,20 @@ new glue.Job(stack, 'ShellJob', {
   },
 });
 
+new glue.Job(stack, 'ShellJob39', {
+  jobName: 'ShellJob39',
+  executable: glue.JobExecutable.pythonShell({
+    glueVersion: glue.GlueVersion.V1_0,
+    pythonVersion: glue.PythonVersion.THREE_NINE,
+    script,
+  }),
+  defaultArguments: {
+    arg1: 'value1',
+    arg2: 'value2',
+  },
+  tags: {
+    key: 'value',
+  },
+});
+
 app.synth();
diff --git a/packages/@aws-cdk/aws-glue/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue/test/job-executable.test.ts
@@ -14,6 +14,14 @@ describe('GlueVersion', () => {
   test('of(customVersion) should set the name correctly', () => expect(glue.GlueVersion.of('CustomVersion').name).toEqual('CustomVersion'));
 });
 
+describe('PythonVersion', () => {
+  test('.TWO should set the name correctly', () => expect(glue.PythonVersion.TWO).toEqual('2'));
+
+  test('.THREE should set the name correctly', () => expect(glue.PythonVersion.THREE).toEqual('3'));
+
+  test('.THREE_NINE should set the name correctly', () => expect(glue.PythonVersion.THREE_NINE).toEqual('3.9'));
+});
+
 describe('JobType', () => {
   test('.ETL should set the name correctly', () => expect(glue.JobType.ETL.name).toEqual('glueetl'));
 
@@ -102,5 +110,25 @@ describe('JobExecutable', () => {
         })).toThrow(`Specified GlueVersion ${glueVersion.name} does not support PythonVersion 2`);
       });
     });
+
+    test('with PythonVersion set to PythonVersion.THREE_NINE and JobType not pythonshell should throw', () => {
+      expect(() => glue.JobExecutable.of({
+        type: glue.JobType.ETL,
+        language: glue.JobLanguage.PYTHON,
+        pythonVersion: glue.PythonVersion.THREE_NINE,
+        script,
+        glueVersion: glue.GlueVersion.V1_0,
+      })).toThrow('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell');
+    });
+
+    test('with PythonVersion PythonVersion.THREE_NINE and JobType pythonshell should succeed', () => {
+      expect(glue.JobExecutable.of({
+        type: glue.JobType.PYTHON_SHELL,
+        glueVersion: glue.GlueVersion.V1_0,
+        language: glue.JobLanguage.PYTHON,
+        pythonVersion: glue.PythonVersion.THREE_NINE,
+        script,
+      })).toBeDefined();
+    });
   });
 });
diff --git a/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/aws-glue-job.assets.json
@@ -1,5 +1,5 @@
 {
-  "version": "17.0.0",
+  "version": "20.0.0",
   "files": {
     "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": {
       "source": {
@@ -14,15 +14,15 @@
         }
       }
     },
-    "9621659bccc489f1130729575d70f1c13a90dc7584eac90773642d7ad0cc1914": {
+    "a44de72f7ea39efc17f8c7abc7e0c595ae4651b3e239851c49461fecf09bb151": {
       "source": {
         "path": "aws-glue-job.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "9621659bccc489f1130729575d70f1c13a90dc7584eac90773642d7ad0cc1914.json",
+          "objectKey": "a44de72f7ea39efc17f8c7abc7e0c595ae4651b3e239851c49461fecf09bb151.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

diff --git a/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/aws-glue-job.template.json
@@ -556,6 +556,157 @@
      "key": "value"
     }
    }
+  },
+  "ShellJob39ServiceRole2F6F3768": {
+   "Type": "AWS::IAM::Role",
+   "Properties": {
+    "AssumeRolePolicyDocument": {
+     "Statement": [
+      {
+       "Action": "sts:AssumeRole",
+       "Effect": "Allow",
+       "Principal": {
+        "Service": "glue.amazonaws.com"
+       }
+      }
+     ],
+     "Version": "2012-10-17"
+    },
+    "ManagedPolicyArns": [
+     {
+      "Fn::Join": [
+       "",
+       [
+        "arn:",
+        {
+         "Ref": "AWS::Partition"
+        },
+        ":iam::aws:policy/service-role/AWSGlueServiceRole"
+       ]
+      ]
+     }
+    ]
+   }
+  },
+  "ShellJob39ServiceRoleDefaultPolicy38A33919": {
+   "Type": "AWS::IAM::Policy",
+   "Properties": {
+    "PolicyDocument": {
+     "Statement": [
+      {
+       "Action": [
+        "s3:GetBucket*",
+        "s3:GetObject*",
+        "s3:List*"
+       ],
+       "Effect": "Allow",
+       "Resource": [
+        {
+         "Fn::Join": [
+          "",
+          [
+           "arn:",
+           {
+            "Ref": "AWS::Partition"
+           },
+           ":s3:::",
+           {
+            "Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3Bucket4E517469"
+           },
+           "/*"
+          ]
+         ]
+        },
+        {
+         "Fn::Join": [
+          "",
+          [
+           "arn:",
+           {
+            "Ref": "AWS::Partition"
+           },
+           ":s3:::",
+           {
+            "Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3Bucket4E517469"
+           }
+          ]
+         ]
+        }
+       ]
+      }
+     ],
+     "Version": "2012-10-17"
+    },
+    "PolicyName": "ShellJob39ServiceRoleDefaultPolicy38A33919",
+    "Roles": [
+     {
+      "Ref": "ShellJob39ServiceRole2F6F3768"
+     }
+    ]
+   }
+  },
+  "ShellJob390C141361": {
+   "Type": "AWS::Glue::Job",
+   "Properties": {
+    "Command": {
+     "Name": "pythonshell",
+     "PythonVersion": "3.9",
+     "ScriptLocation": {
+      "Fn::Join": [
+       "",
+       [
+        "s3://",
+        {
+         "Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3Bucket4E517469"
+        },
+        "/",
+        {
+         "Fn::Select": [
+          0,
+          {
+           "Fn::Split": [
+            "||",
+            {
+             "Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3VersionKeyF7753763"
+            }
+           ]
+          }
+         ]
+        },
+        {
+         "Fn::Select": [
+          1,
+          {
+           "Fn::Split": [
+            "||",
+            {
+             "Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3VersionKeyF7753763"
+            }
+           ]
+          }
+         ]
+        }
+       ]
+      ]
+     }
+    },
+    "Role": {
+     "Fn::GetAtt": [
+      "ShellJob39ServiceRole2F6F3768",
+      "Arn"
+     ]
+    },
+    "DefaultArguments": {
+     "--job-language": "python",
+     "arg1": "value1",
+     "arg2": "value2"
+    },
+    "GlueVersion": "1.0",
+    "Name": "ShellJob39",
+    "Tags": {
+     "key": "value"
+    }
+   }
   }
  },
  "Parameters": {

diff --git a/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/cdk.out b/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/cdk.out
@@ -1 +1 @@
-{"version":"17.0.0"}
+{"version":"20.0.0"}
diff --git a/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/job.integ.snapshot/manifest.json
@@ -1,5 +1,5 @@
 {
-  "version": "17.0.0",
+  "version": "20.0.0",
   "artifacts": {
     "Tree": {
       "type": "cdk:tree",
@@ -112,6 +112,24 @@
             "type": "aws:cdk:logicalId",
             "data": "ShellJob42E81F95"
           }
+        ],
+        "/aws-glue-job/ShellJob39/ServiceRole/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "ShellJob39ServiceRole2F6F3768"
+          }
+        ],
+        "/aws-glue-job/ShellJob39/ServiceRole/DefaultPolicy/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "ShellJob39ServiceRoleDefaultPolicy38A33919"
+          }
+        ],
+        "/aws-glue-job/ShellJob39/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "ShellJob390C141361"
+          }
         ]
       },
       "displayName": "aws-glue-job"