diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index 15cb5757e88b3..4bee0a054bcd8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -169,14 +169,32 @@ interface PythonExecutableProps { /** * Additional Python files that AWS Glue adds to the Python path before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files and argument is not set * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; } +interface RayExecutableProps { + /** + * The Python version to use. + */ + readonly pythonVersion: PythonVersion; + + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files and argument is not set + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; +} + interface SharedJobExecutableProps { /** * Runtime. It is required for Ray jobs. @@ -199,10 +217,11 @@ interface SharedJobExecutableProps { /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-files`. * * @default [] - no extra files are copied to the working directory * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; } @@ -211,19 +230,21 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-jars`. * * @default [] - no extra jars are added to the classpath * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default false - priority is not given to user-provided jars * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } @@ -234,8 +255,9 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { export interface ScalaJobExecutableProps extends SharedSparkJobExecutableProps { /** * The fully qualified Scala class name that serves as the entry point for the job. + * Equivalent to a job parameter `--class`. * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className: string; } @@ -253,7 +275,7 @@ export interface PythonShellExecutableProps extends SharedJobExecutableProps, Py /** * Props for creating a Python Ray job executable */ -export interface PythonRayExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {} +export interface PythonRayExecutableProps extends SharedJobExecutableProps, RayExecutableProps {} /** * The executable properties related to the Glue job's GlueVersion, JobType and code @@ -377,6 +399,9 @@ export class JobExecutable { if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) { throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON'); } + if (config.extraPythonFiles && type === JobType.RAY.name) { + throw new Error('extraPythonFiles is not supported for Ray jobs'); + } if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) { throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray'); } @@ -384,7 +409,7 @@ export class JobExecutable { throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray'); } if (config.runtime === undefined && type === JobType.RAY.name) { - throw new Error('Runtime is required for Ray jobs.'); + throw new Error('Runtime is required for Ray jobs'); } this.config = config; } @@ -410,8 +435,9 @@ export interface JobExecutableConfig { /** * The language of the job (Scala or Python). + * Equivalent to a job parameter `--job-language`. * - * @see `--job-language` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly language: JobLanguage; @@ -441,46 +467,61 @@ export interface JobExecutableConfig { /** * The Scala class that serves as the entry point for the job. This applies only if your the job langauage is Scala. + * Equivalent to a job parameter `--class`. * * @default - no scala className specified * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className?: string; /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. + * Equivalent to a job parameter `--extra-jars`. * * @default - no extra jars specified. * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Additional Python files that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files specified. * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files specified. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; + /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. + * Equivalent to a job parameter `--extra-files`. * * @default - no extra files specified. * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default - extra jars are not prioritized. * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 526cb774017c3..8420d380bb675 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -598,10 +598,11 @@ export interface JobProps { /** * Enables the collection of metrics for job profiling. + * Equivalent to a job parameter `--enable-metrics`. * * @default - no profiling metrics emitted. * - * @see `--enable-metrics` at https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly enableProfilingMetrics? :boolean; @@ -801,6 +802,9 @@ export class Job extends JobBase { if (config.extraPythonFiles && config.extraPythonFiles.length > 0) { args['--extra-py-files'] = config.extraPythonFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } + if (config.s3PythonModules && config.s3PythonModules.length > 0) { + args['--s3-py-modules'] = config.s3PythonModules.map(code => this.codeS3ObjectUrl(code)).join(','); + } if (config.extraFiles && config.extraFiles.length > 0) { args['--extra-files'] = config.extraFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } @@ -886,8 +890,8 @@ export class Job extends JobBase { } /** - * Create a CloudWatch Metric that's based on Glue Job events - * {@see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types} + * Create a CloudWatch Metric that's based on Glue Job events. + * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types * The metric has namespace = 'AWS/Events', metricName = 'TriggeredRules' and RuleName = rule.ruleName dimension. * * @param rule for use in setting RuleName dimension value diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip new file mode 100644 index 0000000000000..3056cd4db8f27 Binary files /dev/null and b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip differ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index b371535051dc3..4be1daf9e3601 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,33 @@ } } }, - "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a": { + "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4": { + "source": { + "path": "asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7": { + "source": { + "path": "asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "objectKey": "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index e524ee21d34da..6ed2d571b6a3e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -149,12 +149,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -187,6 +181,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -338,12 +338,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -352,6 +346,12 @@ "GlueVersion": "2.0", "Name": "StreamingJob2.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -507,12 +507,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -545,6 +539,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -696,12 +696,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -710,6 +704,12 @@ "GlueVersion": "3.0", "Name": "StreamingJob3.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -865,12 +865,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -903,6 +897,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1054,12 +1054,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1068,6 +1062,12 @@ "GlueVersion": "4.0", "Name": "StreamingJob4.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1181,12 +1181,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1194,6 +1188,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob", + "Role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1306,12 +1306,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1319,6 +1313,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob39", + "Role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1427,25 +1427,37 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] } }, - "Role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "GlueVersion": "4.0", "Name": "RayJob", "NumberOfWorkers": 2, + "Role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1559,12 +1571,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python" }, @@ -1572,6 +1578,12 @@ "GlueVersion": "3.0", "Name": "EtlJobWithFLEX", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "WorkerType": "G.1X" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out index f0b901e7c06e5..1f0068d32659a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"32.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json index 3e404e817257e..9618d2538fb71 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "testCases": { "integ.job": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index f729d71dca775..e0a0106226e31 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "aws-glue-job.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "aws-glue-job.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 1aad2f99aa438..6d88a4cb8fd4a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -20,8 +20,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -59,8 +59,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -165,20 +165,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -193,40 +193,40 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, - "Code161d2d609eea130c2e9c740956bb950c": { - "id": "Code161d2d609eea130c2e9c740956bb950c", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c", + "Code2fe0fc936e45d982e718ad516d9c48b5": { + "id": "Code2fe0fc936e45d982e718ad516d9c48b5", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/Stage", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/Stage", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" } }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/AssetBucket", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -251,12 +251,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -289,6 +283,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -297,8 +297,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -346,20 +346,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob2.0": { @@ -374,8 +374,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -413,8 +413,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -482,20 +482,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -520,12 +520,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -534,6 +528,12 @@ "glueVersion": "2.0", "name": "StreamingJob2.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -541,14 +541,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob3.0": { @@ -563,8 +563,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -602,8 +602,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -708,20 +708,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -736,14 +736,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -768,12 +768,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -806,6 +800,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -814,8 +814,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -863,20 +863,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob3.0": { @@ -891,8 +891,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -930,8 +930,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -999,20 +999,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1037,12 +1037,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1051,6 +1045,12 @@ "glueVersion": "3.0", "name": "StreamingJob3.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1058,14 +1058,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob4.0": { @@ -1080,8 +1080,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1119,8 +1119,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1225,20 +1225,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -1253,14 +1253,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -1285,12 +1285,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -1323,6 +1317,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1331,8 +1331,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -1380,20 +1380,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob4.0": { @@ -1408,8 +1408,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1447,8 +1447,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1516,20 +1516,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1554,12 +1554,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1568,6 +1562,12 @@ "glueVersion": "4.0", "name": "StreamingJob4.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1575,14 +1575,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob": { @@ -1597,8 +1597,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1636,8 +1636,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1705,20 +1705,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1743,12 +1743,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1756,20 +1750,26 @@ }, "glueVersion": "1.0", "name": "ShellJob", + "role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob39": { @@ -1784,8 +1784,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob39/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1823,8 +1823,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1892,20 +1892,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1930,12 +1930,6 @@ }, "pythonVersion": "3.9" }, - "role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1943,20 +1937,26 @@ }, "glueVersion": "1.0", "name": "ShellJob39", + "role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "RayJob": { @@ -1971,8 +1971,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/RayJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2010,8 +2010,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2079,20 +2079,72 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Code0115baecff23f24504399384f987872e": { + "id": "Code0115baecff23f24504399384f987872e", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Code7e4c8ab0af31dc38a92c1c69bf696244": { + "id": "Code7e4c8ab0af31dc38a92c1c69bf696244", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -2111,27 +2163,39 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] }, "pythonVersion": "3.9", "runtime": "Ray2.4" }, - "role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "glueVersion": "4.0", "name": "RayJob", "numberOfWorkers": 2, + "role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -2139,14 +2203,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJobWithFLEX": { @@ -2161,8 +2225,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2200,8 +2264,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2269,20 +2333,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -2307,12 +2371,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python" }, @@ -2320,40 +2378,46 @@ "glueVersion": "3.0", "name": "EtlJobWithFLEX", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "workerType": "G.1X" } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-glue-job/BootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" } }, "CheckBootstrapVersion": { "id": "CheckBootstrapVersion", "path": "aws-glue-job/CheckBootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" } }, "Tree": { @@ -2361,13 +2425,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index c16b5f9691ae1..7b405d53ca490 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -22,6 +22,8 @@ const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-glue-job'); const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')); +const scriptResolveOptions = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'resolve_options.py')); +const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module', 'utils.zip')); [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { const etlJob = new glue.Job(stack, 'EtlJob' + glueVersion.name, { @@ -113,7 +115,8 @@ new glue.Job(stack, 'RayJob', { glueVersion: glue.GlueVersion.V4_0, pythonVersion: glue.PythonVersion.THREE_NINE, runtime: glue.Runtime.RAY_TWO_FOUR, - script, + s3PythonModules: [moduleUtils], + script: scriptResolveOptions, }), workerType: glue.WorkerType.Z_2X, workerCount: 2, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts index 28c6225c542b6..d00faa55091ba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts @@ -254,5 +254,29 @@ describe('JobExecutable', () => { script, })).toBeDefined(); }); + + test('with JobTypeof("glueray") and extraPythonFiles set should throw', () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, + extraPythonFiles: [script], + script, + })).toThrow(/extraPythonFiles is not supported for Ray jobs/); + }); + + test('with JobType ray and s3PythonModules should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + s3PythonModules: [script], + runtime: glue.Runtime.RAY_TWO_FOUR, + script, + })).toBeDefined(); + }); }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index 748d89b5668a2..48e395a3fedc9 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -871,7 +871,7 @@ describe('Job', () => { }), workerType: glue.WorkerType.Z_2X, workerCount: 2, - })).toThrow('Runtime is required for Ray jobs.'); + })).toThrow('Runtime is required for Ray jobs'); }); }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip new file mode 100644 index 0000000000000..3056cd4db8f27 Binary files /dev/null and b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip differ