From cde94afa4b34e732f55b365429a910486c336b36 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Tue, 9 Jan 2024 18:14:34 +0900 Subject: [PATCH 1/2] fix(glue): added RayExecutableProps which supports s3PythonModules --- .../aws-glue-alpha/lib/job-executable.ts | 32 +- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 3 + ...2cab491bb6985900b80e05d634fceef328c5c4.zip | Bin 0 -> 886 bytes ...c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py | 4 + .../aws-glue-job.assets.json | 32 +- .../aws-glue-job.template.json | 134 +++-- .../test/integ.job.js.snapshot/cdk.out | 2 +- .../test/integ.job.js.snapshot/integ.json | 2 +- .../test/integ.job.js.snapshot/manifest.json | 5 +- .../test/integ.job.js.snapshot/tree.json | 554 ++++++++++-------- .../@aws-cdk/aws-glue-alpha/test/integ.job.ts | 5 +- .../test/job-executable.test.ts | 24 + .../test/job-script/resolve_options.py | 4 + .../@aws-cdk/aws-glue-alpha/test/job.test.ts | 2 +- .../aws-glue-alpha/test/module/utils.zip | Bin 0 -> 886 bytes 15 files changed, 486 insertions(+), 317 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index 15cb5757e88b3..7ab79b0cc1a45 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -177,6 +177,22 @@ interface PythonExecutableProps { readonly extraPythonFiles?: Code[]; } +interface RayExecutableProps { + /** + * The Python version to use. + */ + readonly pythonVersion: PythonVersion; + + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * + * @default - no extra python files and argument is not set + * + * @see `--s3-py-modules` in https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; +} + interface SharedJobExecutableProps { /** * Runtime. It is required for Ray jobs. @@ -253,7 +269,7 @@ export interface PythonShellExecutableProps extends SharedJobExecutableProps, Py /** * Props for creating a Python Ray job executable */ -export interface PythonRayExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {} +export interface PythonRayExecutableProps extends SharedJobExecutableProps, RayExecutableProps {} /** * The executable properties related to the Glue job's GlueVersion, JobType and code @@ -377,6 +393,9 @@ export class JobExecutable { if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) { throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON'); } + if (config.extraPythonFiles && type === JobType.RAY.name) { + throw new Error('extraPythonFiles is not supported for Ray jobs'); + } if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) { throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray'); } @@ -384,7 +403,7 @@ export class JobExecutable { throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray'); } if (config.runtime === undefined && type === JobType.RAY.name) { - throw new Error('Runtime is required for Ray jobs.'); + throw new Error('Runtime is required for Ray jobs'); } this.config = config; } @@ -466,6 +485,15 @@ export interface JobExecutableConfig { */ readonly extraPythonFiles?: Code[]; + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * + * @default - no extra python files specified. + * + * @see `--s3-py-modules` in https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; + /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. * diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 229d02173dd58..5cbb53bcdd2ff 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -801,6 +801,9 @@ export class Job extends JobBase { if (config.extraPythonFiles && config.extraPythonFiles.length > 0) { args['--extra-py-files'] = config.extraPythonFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } + if (config.s3PythonModules && config.s3PythonModules.length > 0) { + args['--s3-py-modules'] = config.s3PythonModules.map(code => this.codeS3ObjectUrl(code)).join(','); + } if (config.extraFiles && config.extraFiles.length > 0) { args['--extra-files'] = config.extraFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip new file mode 100644 index 0000000000000000000000000000000000000000..3056cd4db8f271cbcf6400230172cb0ba47ebe86 GIT binary patch literal 886 zcmWIWW@Zs#-~htAU78ULP;il%0VtxtP+F3iQ><4|85+XNz@Ef3H`R)LZmIjp{sc=uGXABJZSFP8uC_-g!MAL;&4W!| z!CxEBJ8s=-qj7GV&f+x*kK%i-Z`gRO_khpReSE9FX1@~K@^OxB=&?uZ7O9sm>$w-G znKWOTRrFhO-oGoqgWY~_{JVKgO!ZX@y=h$BRrOLV-&7PieoS@#=PZ!>)Y`z)pzzIA z;bm>5-51LaCT$FoXgJ7x+29F(KxwbhKTDS#Gg95>9j)kms^qPA`hzp$IrH0Z*>+17 zPkwaSP3P-r9{Cf^E|SW@5nYov&R640l-Ra~**5J%hu9PG0B?4V#}{U>6$9e{6kP#W zBa$5`D-4Xt_;_DOXaC>`eZ6?Bar%=JBTkbOM3d7Kk`g}n`hea z$Ho@e65|+H7nv286j!9hEbYv4{Nk#nt#R?oj`-|8obxQwea`L+jLOW1UsgKCeBN^? z6zCsz4xg2U>GiGa{)X`yG^2VPHukh=tXYSQ0L}smL~h!VLzNG@b;y Xfs9ZL@MdKL8NvjF+kx~-CJ+w*MNT#7 literal 0 HcmV?d00001 diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index b371535051dc3..4be1daf9e3601 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,33 @@ } } }, - "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a": { + "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4": { + "source": { + "path": "asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7": { + "source": { + "path": "asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "objectKey": "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index e524ee21d34da..6ed2d571b6a3e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -149,12 +149,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -187,6 +181,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -338,12 +338,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -352,6 +346,12 @@ "GlueVersion": "2.0", "Name": "StreamingJob2.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -507,12 +507,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -545,6 +539,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -696,12 +696,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -710,6 +704,12 @@ "GlueVersion": "3.0", "Name": "StreamingJob3.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -865,12 +865,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -903,6 +897,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1054,12 +1054,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1068,6 +1062,12 @@ "GlueVersion": "4.0", "Name": "StreamingJob4.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1181,12 +1181,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1194,6 +1188,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob", + "Role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1306,12 +1306,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1319,6 +1313,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob39", + "Role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1427,25 +1427,37 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] } }, - "Role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "GlueVersion": "4.0", "Name": "RayJob", "NumberOfWorkers": 2, + "Role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1559,12 +1571,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python" }, @@ -1572,6 +1578,12 @@ "GlueVersion": "3.0", "Name": "EtlJobWithFLEX", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "WorkerType": "G.1X" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out index f0b901e7c06e5..1f0068d32659a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"32.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json index 3e404e817257e..9618d2538fb71 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "testCases": { "integ.job": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index f729d71dca775..e0a0106226e31 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "aws-glue-job.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "aws-glue-job.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 1aad2f99aa438..6d88a4cb8fd4a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -20,8 +20,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -59,8 +59,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -165,20 +165,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -193,40 +193,40 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, - "Code161d2d609eea130c2e9c740956bb950c": { - "id": "Code161d2d609eea130c2e9c740956bb950c", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c", + "Code2fe0fc936e45d982e718ad516d9c48b5": { + "id": "Code2fe0fc936e45d982e718ad516d9c48b5", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/Stage", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/Stage", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" } }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/AssetBucket", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -251,12 +251,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -289,6 +283,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -297,8 +297,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -346,20 +346,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob2.0": { @@ -374,8 +374,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -413,8 +413,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -482,20 +482,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -520,12 +520,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -534,6 +528,12 @@ "glueVersion": "2.0", "name": "StreamingJob2.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -541,14 +541,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob3.0": { @@ -563,8 +563,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -602,8 +602,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -708,20 +708,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -736,14 +736,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -768,12 +768,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -806,6 +800,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -814,8 +814,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -863,20 +863,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob3.0": { @@ -891,8 +891,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -930,8 +930,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -999,20 +999,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1037,12 +1037,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1051,6 +1045,12 @@ "glueVersion": "3.0", "name": "StreamingJob3.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1058,14 +1058,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob4.0": { @@ -1080,8 +1080,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1119,8 +1119,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1225,20 +1225,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -1253,14 +1253,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -1285,12 +1285,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -1323,6 +1317,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1331,8 +1331,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -1380,20 +1380,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob4.0": { @@ -1408,8 +1408,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1447,8 +1447,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1516,20 +1516,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1554,12 +1554,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1568,6 +1562,12 @@ "glueVersion": "4.0", "name": "StreamingJob4.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1575,14 +1575,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob": { @@ -1597,8 +1597,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1636,8 +1636,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1705,20 +1705,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1743,12 +1743,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1756,20 +1750,26 @@ }, "glueVersion": "1.0", "name": "ShellJob", + "role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob39": { @@ -1784,8 +1784,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob39/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1823,8 +1823,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1892,20 +1892,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1930,12 +1930,6 @@ }, "pythonVersion": "3.9" }, - "role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1943,20 +1937,26 @@ }, "glueVersion": "1.0", "name": "ShellJob39", + "role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "RayJob": { @@ -1971,8 +1971,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/RayJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2010,8 +2010,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2079,20 +2079,72 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Code0115baecff23f24504399384f987872e": { + "id": "Code0115baecff23f24504399384f987872e", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Code7e4c8ab0af31dc38a92c1c69bf696244": { + "id": "Code7e4c8ab0af31dc38a92c1c69bf696244", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -2111,27 +2163,39 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] }, "pythonVersion": "3.9", "runtime": "Ray2.4" }, - "role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "glueVersion": "4.0", "name": "RayJob", "numberOfWorkers": 2, + "role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -2139,14 +2203,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJobWithFLEX": { @@ -2161,8 +2225,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2200,8 +2264,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2269,20 +2333,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -2307,12 +2371,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python" }, @@ -2320,40 +2378,46 @@ "glueVersion": "3.0", "name": "EtlJobWithFLEX", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "workerType": "G.1X" } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-glue-job/BootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" } }, "CheckBootstrapVersion": { "id": "CheckBootstrapVersion", "path": "aws-glue-job/CheckBootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" } }, "Tree": { @@ -2361,13 +2425,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index aa420c52eccf4..e4a24d1c9dc7f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -22,6 +22,8 @@ const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-glue-job'); const script = glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')); +const scriptResolveOptions = glue.Code.fromAsset(path.join(__dirname, 'job-script/resolve_options.py')); +const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module/utils.zip')); [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { const etlJob = new glue.Job(stack, 'EtlJob' + glueVersion.name, { @@ -113,7 +115,8 @@ new glue.Job(stack, 'RayJob', { glueVersion: glue.GlueVersion.V4_0, pythonVersion: glue.PythonVersion.THREE_NINE, runtime: glue.Runtime.RAY_TWO_FOUR, - script, + s3PythonModules: [moduleUtils], + script: scriptResolveOptions, }), workerType: glue.WorkerType.Z_2X, workerCount: 2, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts index 28c6225c542b6..d00faa55091ba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts @@ -254,5 +254,29 @@ describe('JobExecutable', () => { script, })).toBeDefined(); }); + + test('with JobTypeof("glueray") and extraPythonFiles set should throw', () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, + extraPythonFiles: [script], + script, + })).toThrow(/extraPythonFiles is not supported for Ray jobs/); + }); + + test('with JobType ray and s3PythonModules should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + s3PythonModules: [script], + runtime: glue.Runtime.RAY_TWO_FOUR, + script, + })).toBeDefined(); + }); }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index 748d89b5668a2..48e395a3fedc9 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -871,7 +871,7 @@ describe('Job', () => { }), workerType: glue.WorkerType.Z_2X, workerCount: 2, - })).toThrow('Runtime is required for Ray jobs.'); + })).toThrow('Runtime is required for Ray jobs'); }); }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip new file mode 100644 index 0000000000000000000000000000000000000000..3056cd4db8f271cbcf6400230172cb0ba47ebe86 GIT binary patch literal 886 zcmWIWW@Zs#-~htAU78ULP;il%0VtxtP+F3iQ><4|85+XNz@Ef3H`R)LZmIjp{sc=uGXABJZSFP8uC_-g!MAL;&4W!| z!CxEBJ8s=-qj7GV&f+x*kK%i-Z`gRO_khpReSE9FX1@~K@^OxB=&?uZ7O9sm>$w-G znKWOTRrFhO-oGoqgWY~_{JVKgO!ZX@y=h$BRrOLV-&7PieoS@#=PZ!>)Y`z)pzzIA z;bm>5-51LaCT$FoXgJ7x+29F(KxwbhKTDS#Gg95>9j)kms^qPA`hzp$IrH0Z*>+17 zPkwaSP3P-r9{Cf^E|SW@5nYov&R640l-Ra~**5J%hu9PG0B?4V#}{U>6$9e{6kP#W zBa$5`D-4Xt_;_DOXaC>`eZ6?Bar%=JBTkbOM3d7Kk`g}n`hea z$Ho@e65|+H7nv286j!9hEbYv4{Nk#nt#R?oj`-|8obxQwea`L+jLOW1UsgKCeBN^? z6zCsz4xg2U>GiGa{)X`yG^2VPHukh=tXYSQ0L}smL~h!VLzNG@b;y Xfs9ZL@MdKL8NvjF+kx~-CJ+w*MNT#7 literal 0 HcmV?d00001 From 91ab22e26cdd370abf8b14b09918ee17720a03cc Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Wed, 14 Feb 2024 16:11:14 +0900 Subject: [PATCH 2/2] @see expects a url after the tag, and nothing else --- .../aws-glue-alpha/lib/job-executable.ts | 39 ++++++++++++------- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 7 ++-- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index 7ab79b0cc1a45..4bee0a054bcd8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -169,10 +169,11 @@ interface PythonExecutableProps { /** * Additional Python files that AWS Glue adds to the Python path before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files and argument is not set * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; } @@ -185,10 +186,11 @@ interface RayExecutableProps { /** * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. * * @default - no extra python files and argument is not set * - * @see `--s3-py-modules` in https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html */ readonly s3PythonModules?: Code[]; } @@ -215,10 +217,11 @@ interface SharedJobExecutableProps { /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-files`. * * @default [] - no extra files are copied to the working directory * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; } @@ -227,19 +230,21 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-jars`. * * @default [] - no extra jars are added to the classpath * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default false - priority is not given to user-provided jars * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } @@ -250,8 +255,9 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { export interface ScalaJobExecutableProps extends SharedSparkJobExecutableProps { /** * The fully qualified Scala class name that serves as the entry point for the job. + * Equivalent to a job parameter `--class`. * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className: string; } @@ -429,8 +435,9 @@ export interface JobExecutableConfig { /** * The language of the job (Scala or Python). + * Equivalent to a job parameter `--job-language`. * - * @see `--job-language` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly language: JobLanguage; @@ -460,55 +467,61 @@ export interface JobExecutableConfig { /** * The Scala class that serves as the entry point for the job. This applies only if your the job langauage is Scala. + * Equivalent to a job parameter `--class`. * * @default - no scala className specified * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className?: string; /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. + * Equivalent to a job parameter `--extra-jars`. * * @default - no extra jars specified. * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Additional Python files that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files specified. * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; /** * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. * * @default - no extra python files specified. * - * @see `--s3-py-modules` in https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html */ readonly s3PythonModules?: Code[]; /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. + * Equivalent to a job parameter `--extra-files`. * * @default - no extra files specified. * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default - extra jars are not prioritized. * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 5cbb53bcdd2ff..f4d2aa74c2676 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -598,10 +598,11 @@ export interface JobProps { /** * Enables the collection of metrics for job profiling. + * Equivalent to a job parameter `--enable-metrics`. * * @default - no profiling metrics emitted. * - * @see `--enable-metrics` at https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly enableProfilingMetrics? :boolean; @@ -889,8 +890,8 @@ export class Job extends JobBase { } /** - * Create a CloudWatch Metric that's based on Glue Job events - * {@see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types} + * Create a CloudWatch Metric that's based on Glue Job events. + * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types * The metric has namespace = 'AWS/Events', metricName = 'TriggeredRules' and RuleName = rule.ruleName dimension. * * @param rule for use in setting RuleName dimension value