From a595d9f656bc356d263b979a8df627b8e25fddca Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Tue, 5 Mar 2024 07:56:37 +0900 Subject: [PATCH 1/8] fix(glue): S3 path specified in needs to end with slash --- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 2 +- .../aws-glue-job.assets.json | 6 +- .../aws-glue-job.template.json | 129 ++--- .../test/integ.job.js.snapshot/cdk.out | 2 +- .../test/integ.job.js.snapshot/integ.json | 2 +- .../test/integ.job.js.snapshot/manifest.json | 5 +- .../test/integ.job.js.snapshot/tree.json | 497 +++++++++--------- .../@aws-cdk/aws-glue-alpha/test/job.test.ts | 2 +- 8 files changed, 326 insertions(+), 319 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 526cb774017c3..edc30fca6217b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -822,7 +822,7 @@ export class Job extends JobBase { bucket.grantReadWrite(role, this.cleanPrefixForGrant(props.prefix)); const args = { '--enable-spark-ui': 'true', - '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix), + '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix).replace(/\/?$/, '/'), }; return { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index b371535051dc3..56cf58bb92671 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,7 @@ } } }, - "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a": { + "8141964ecc338ee42a348c99cebc2a452e2267523a0975464e042883687d65b6": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "objectKey": "8141964ecc338ee42a348c99cebc2a452e2267523a0975464e042883687d65b6.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index e524ee21d34da..e447db5ce889f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -149,12 +149,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -168,7 +162,8 @@ "s3://", { "Ref": "EtlJob20SparkUIBucketFD07FBD8" - } + }, + "/" ] ] }, @@ -187,6 +182,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -338,12 +339,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -352,6 +347,12 @@ "GlueVersion": "2.0", "Name": "StreamingJob2.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -507,12 +508,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -526,7 +521,8 @@ "s3://", { "Ref": "EtlJob30SparkUIBucket9D789346" - } + }, + "/" ] ] }, @@ -545,6 +541,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -696,12 +698,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -710,6 +706,12 @@ "GlueVersion": "3.0", "Name": "StreamingJob3.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -865,12 +867,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -884,7 +880,8 @@ "s3://", { "Ref": "EtlJob40SparkUIBucket02F50B0D" - } + }, + "/" ] ] }, @@ -903,6 +900,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1054,12 +1057,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1068,6 +1065,12 @@ "GlueVersion": "4.0", "Name": "StreamingJob4.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1181,12 +1184,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1194,6 +1191,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob", + "Role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1306,12 +1309,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1319,6 +1316,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob39", + "Role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1432,12 +1435,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1446,6 +1443,12 @@ "GlueVersion": "4.0", "Name": "RayJob", "NumberOfWorkers": 2, + "Role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1559,12 +1562,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python" }, @@ -1572,6 +1569,12 @@ "GlueVersion": "3.0", "Name": "EtlJobWithFLEX", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "WorkerType": "G.1X" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out index f0b901e7c06e5..1f0068d32659a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"32.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json index 3e404e817257e..9618d2538fb71 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "testCases": { "integ.job": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index f729d71dca775..d5d3085d8c35b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "aws-glue-job.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "aws-glue-job.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/8141964ecc338ee42a348c99cebc2a452e2267523a0975464e042883687d65b6.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 1aad2f99aa438..186e4dbed610a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -20,8 +20,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -59,8 +59,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -165,20 +165,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -193,40 +193,40 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, - "Code161d2d609eea130c2e9c740956bb950c": { - "id": "Code161d2d609eea130c2e9c740956bb950c", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c", + "Code2fe0fc936e45d982e718ad516d9c48b5": { + "id": "Code2fe0fc936e45d982e718ad516d9c48b5", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/Stage", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/Stage", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" } }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/AssetBucket", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -251,12 +251,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -270,7 +264,8 @@ "s3://", { "Ref": "EtlJob20SparkUIBucketFD07FBD8" - } + }, + "/" ] ] }, @@ -289,6 +284,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -297,8 +298,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -346,20 +347,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob2.0": { @@ -374,8 +375,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -413,8 +414,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -482,20 +483,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -520,12 +521,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -534,6 +529,12 @@ "glueVersion": "2.0", "name": "StreamingJob2.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -541,14 +542,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob3.0": { @@ -563,8 +564,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -602,8 +603,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -708,20 +709,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -736,14 +737,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -768,12 +769,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -787,7 +782,8 @@ "s3://", { "Ref": "EtlJob30SparkUIBucket9D789346" - } + }, + "/" ] ] }, @@ -806,6 +802,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -814,8 +816,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -863,20 +865,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob3.0": { @@ -891,8 +893,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -930,8 +932,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -999,20 +1001,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1037,12 +1039,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1051,6 +1047,12 @@ "glueVersion": "3.0", "name": "StreamingJob3.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1058,14 +1060,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob4.0": { @@ -1080,8 +1082,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1119,8 +1121,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1225,20 +1227,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -1253,14 +1255,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -1285,12 +1287,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -1304,7 +1300,8 @@ "s3://", { "Ref": "EtlJob40SparkUIBucket02F50B0D" - } + }, + "/" ] ] }, @@ -1323,6 +1320,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1331,8 +1334,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -1380,20 +1383,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob4.0": { @@ -1408,8 +1411,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1447,8 +1450,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1516,20 +1519,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1554,12 +1557,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1568,6 +1565,12 @@ "glueVersion": "4.0", "name": "StreamingJob4.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1575,14 +1578,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob": { @@ -1597,8 +1600,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1636,8 +1639,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1705,20 +1708,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1743,12 +1746,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1756,20 +1753,26 @@ }, "glueVersion": "1.0", "name": "ShellJob", + "role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob39": { @@ -1784,8 +1787,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob39/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1823,8 +1826,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1892,20 +1895,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1930,12 +1933,6 @@ }, "pythonVersion": "3.9" }, - "role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1943,20 +1940,26 @@ }, "glueVersion": "1.0", "name": "ShellJob39", + "role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "RayJob": { @@ -1971,8 +1974,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/RayJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2010,8 +2013,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2079,20 +2082,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -2118,12 +2121,6 @@ "pythonVersion": "3.9", "runtime": "Ray2.4" }, - "role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -2132,6 +2129,12 @@ "glueVersion": "4.0", "name": "RayJob", "numberOfWorkers": 2, + "role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -2139,14 +2142,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJobWithFLEX": { @@ -2161,8 +2164,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2200,8 +2203,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2269,20 +2272,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -2307,12 +2310,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python" }, @@ -2320,40 +2317,46 @@ "glueVersion": "3.0", "name": "EtlJobWithFLEX", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "workerType": "G.1X" } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-glue-job/BootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" } }, "CheckBootstrapVersion": { "id": "CheckBootstrapVersion", "path": "aws-glue-job/CheckBootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" } }, "Tree": { @@ -2361,13 +2364,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index 748d89b5668a2..d7b3f05fdb90e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -625,7 +625,7 @@ describe('Job', () => { Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { DefaultArguments: { '--enable-spark-ui': 'true', - '--spark-event-logs-path': `s3://${sparkUIBucketName}`, + '--spark-event-logs-path': `s3://${sparkUIBucketName}/`, }, }); }); From fe3555ff7191c05725c64bb39350e13e165f72ef Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Thu, 7 Mar 2024 15:43:33 +0900 Subject: [PATCH 2/8] Added trailingSlash option to s3UrlForObject --- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 2 +- .../aws-glue-job.assets.json | 4 +- .../aws-glue-job.template.json | 165 +++++++++++++ .../test/integ.job.js.snapshot/manifest.json | 20 +- .../test/integ.job.js.snapshot/tree.json | 216 ++++++++++++++++++ .../@aws-cdk/aws-glue-alpha/test/integ.job.ts | 3 + packages/aws-cdk-lib/aws-s3/lib/bucket.ts | 16 +- .../aws-cdk-lib/aws-s3/test/bucket.test.ts | 48 ++++ 8 files changed, 466 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index edc30fca6217b..b3d76dc44ee04 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -822,7 +822,7 @@ export class Job extends JobBase { bucket.grantReadWrite(role, this.cleanPrefixForGrant(props.prefix)); const args = { '--enable-spark-ui': 'true', - '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix).replace(/\/?$/, '/'), + '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix, true), }; return { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index 56cf58bb92671..991a6c7e93738 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -14,7 +14,7 @@ } } }, - "8141964ecc338ee42a348c99cebc2a452e2267523a0975464e042883687d65b6": { + "5ff83e42e9cbebab4d9d764b8daa4a715172668bb1a6bec030b46694573c74ab": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "8141964ecc338ee42a348c99cebc2a452e2267523a0975464e042883687d65b6.json", + "objectKey": "5ff83e42e9cbebab4d9d764b8daa4a715172668bb1a6bec030b46694573c74ab.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index e447db5ce889f..378abedae5b7f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -268,6 +268,43 @@ "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -320,6 +357,11 @@ ] } }, + "StreamingJob20SparkUIBucket92EF3706": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, "StreamingJob20355B58C7": { "Type": "AWS::Glue::Job", "Properties": { @@ -341,6 +383,19 @@ }, "DefaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob20SparkUIBucket92EF3706" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -627,6 +682,43 @@ "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -679,6 +771,11 @@ ] } }, + "StreamingJob30SparkUIBucketEFBF52D3": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, "StreamingJob30E005FBEB": { "Type": "AWS::Glue::Job", "Properties": { @@ -700,6 +797,19 @@ }, "DefaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob30SparkUIBucketEFBF52D3" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -986,6 +1096,43 @@ "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -1038,6 +1185,11 @@ ] } }, + "StreamingJob40SparkUIBucketA97E24C6": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, "StreamingJob40E284A782": { "Type": "AWS::Glue::Job", "Properties": { @@ -1059,6 +1211,19 @@ }, "DefaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob40SparkUIBucketA97E24C6" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index d5d3085d8c35b..8fcf83d9f0012 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/8141964ecc338ee42a348c99cebc2a452e2267523a0975464e042883687d65b6.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5ff83e42e9cbebab4d9d764b8daa4a715172668bb1a6bec030b46694573c74ab.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -76,6 +76,12 @@ "data": "StreamingJob20ServiceRoleDefaultPolicy3CF74B2D" } ], + "/aws-glue-job/StreamingJob2.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob20SparkUIBucket92EF3706" + } + ], "/aws-glue-job/StreamingJob2.0/Resource": [ { "type": "aws:cdk:logicalId", @@ -124,6 +130,12 @@ "data": "StreamingJob30ServiceRoleDefaultPolicy0C15D010" } ], + "/aws-glue-job/StreamingJob3.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob30SparkUIBucketEFBF52D3" + } + ], "/aws-glue-job/StreamingJob3.0/Resource": [ { "type": "aws:cdk:logicalId", @@ -172,6 +184,12 @@ "data": "StreamingJob40ServiceRoleDefaultPolicy0667C434" } ], + "/aws-glue-job/StreamingJob4.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob40SparkUIBucketA97E24C6" + } + ], "/aws-glue-job/StreamingJob4.0/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 186e4dbed610a..c60f466337bb2 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -430,6 +430,43 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -499,6 +536,28 @@ "version": "0.0.0" } }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/StreamingJob2.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob2.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob2.0/Resource", @@ -523,6 +582,19 @@ }, "defaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob20SparkUIBucket92EF3706" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -948,6 +1020,43 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -1017,6 +1126,28 @@ "version": "0.0.0" } }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/StreamingJob3.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob3.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob3.0/Resource", @@ -1041,6 +1172,19 @@ }, "defaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob30SparkUIBucketEFBF52D3" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -1466,6 +1610,43 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -1535,6 +1716,28 @@ "version": "0.0.0" } }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/StreamingJob4.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob4.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob4.0/Resource", @@ -1559,6 +1762,19 @@ }, "defaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob40SparkUIBucketA97E24C6" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index c16b5f9691ae1..6db000eb656e0 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -69,6 +69,9 @@ const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_wor arg1: 'value1', arg2: 'value2', }, + sparkUI: { + enabled: true, + }, tags: { key: 'value', }, diff --git a/packages/aws-cdk-lib/aws-s3/lib/bucket.ts b/packages/aws-cdk-lib/aws-s3/lib/bucket.ts index a799062afc692..3ba39ddbf5eea 100644 --- a/packages/aws-cdk-lib/aws-s3/lib/bucket.ts +++ b/packages/aws-cdk-lib/aws-s3/lib/bucket.ts @@ -164,9 +164,10 @@ export interface IBucket extends IResource { * - `s3://bucket/key` * @param key The S3 key of the object. If not specified, the S3 URL of the * bucket is returned. + * @param trailingSlash Whether to add a trailing slash when the url does not have it. * @returns an ObjectS3Url token */ - s3UrlForObject(key?: string): string; + s3UrlForObject(key?: string, trailingSlash?: boolean): string; /** * Returns an ARN that represents all objects within the bucket that match @@ -732,14 +733,21 @@ export abstract class BucketBase extends Resource implements IBucket { * * @param key The S3 key of the object. If not specified, the S3 URL of the * bucket is returned. + * @param trailingSlash Whether to add a trailing slash when the url does not have it. * @returns an ObjectS3Url token */ - public s3UrlForObject(key?: string): string { + public s3UrlForObject(key?: string, trailingSlash?: boolean): string { const prefix = 's3://'; + let url = undefined; if (typeof key !== 'string') { - return this.urlJoin(prefix, this.bucketName); + url = this.urlJoin(prefix, this.bucketName); + } else { + url = this.urlJoin(prefix, this.bucketName, key); } - return this.urlJoin(prefix, this.bucketName, key); + if (trailingSlash) { + url = url.replace(/\/?$/, '/'); + } + return url; } /** diff --git a/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts b/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts index 956f66e7d12e6..6a5ff9c2727e2 100644 --- a/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts +++ b/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts @@ -2395,6 +2395,54 @@ describe('bucket', () => { }); }); + test('s3UrlForObject returns the S3 URL with a trailing slash when trailingSlash is set to true', () => { + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'MyBucket'); + + new cdk.CfnOutput(stack, 'BucketS3URL', { value: bucket.s3UrlForObject(null, true) }); + new cdk.CfnOutput(stack, 'MyFolderS3URL', { value: bucket.s3UrlForObject('my/folder', true) }); + + Template.fromStack(stack).templateMatches({ + 'Resources': { + 'MyBucketF68F3FF0': { + 'Type': 'AWS::S3::Bucket', + 'DeletionPolicy': 'Retain', + 'UpdateReplacePolicy': 'Retain', + }, + }, + 'Outputs': { + 'BucketS3URL': { + 'Value': { + 'Fn::Join': [ + '', + [ + 's3://', + { + 'Ref': 'MyBucketF68F3FF0', + }, + '/', + ], + ], + }, + }, + 'MyFolderS3URL': { + 'Value': { + 'Fn::Join': [ + '', + [ + 's3://', + { + 'Ref': 'MyBucketF68F3FF0', + }, + '/my/folder/', + ], + ], + }, + }, + }, + }); + }); + describe('grantPublicAccess', () => { test('by default, grants s3:GetObject to all objects', () => { // GIVEN From af84fd6490013573f3dbc6fcf46e1c8ee8c958ee Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Thu, 7 Mar 2024 16:13:09 +0900 Subject: [PATCH 3/8] Fixed unit test for job --- packages/@aws-cdk/aws-glue-alpha/test/job.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index d7b3f05fdb90e..fd0cc8922b4b5 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -542,6 +542,7 @@ describe('Job', () => { { Ref: 'JobSparkUIBucket8E6A0139', }, + '/', ], ], }, From 3ee4a9c07f36eadb169e4a84fdab9b7e797c8a72 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Mon, 11 Mar 2024 17:14:35 +0900 Subject: [PATCH 4/8] Resolved conflicts with PR #28625 --- .../aws-glue-alpha/lib/job-executable.ts | 67 ++++++++++++++---- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 10 ++- ...2cab491bb6985900b80e05d634fceef328c5c4.zip | Bin 0 -> 886 bytes ...c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py | 4 ++ .../aws-glue-job.assets.json | 30 +++++++- .../aws-glue-job.template.json | 14 +++- .../test/integ.job.js.snapshot/manifest.json | 2 +- .../test/integ.job.js.snapshot/tree.json | 66 ++++++++++++++++- .../@aws-cdk/aws-glue-alpha/test/integ.job.ts | 5 +- .../test/job-executable.test.ts | 24 +++++++ .../test/job-script/resolve_options.py | 4 ++ .../@aws-cdk/aws-glue-alpha/test/job.test.ts | 2 +- .../aws-glue-alpha/test/module/utils.zip | Bin 0 -> 886 bytes 13 files changed, 205 insertions(+), 23 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index 15cb5757e88b3..4bee0a054bcd8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -169,14 +169,32 @@ interface PythonExecutableProps { /** * Additional Python files that AWS Glue adds to the Python path before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files and argument is not set * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; } +interface RayExecutableProps { + /** + * The Python version to use. + */ + readonly pythonVersion: PythonVersion; + + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files and argument is not set + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; +} + interface SharedJobExecutableProps { /** * Runtime. It is required for Ray jobs. @@ -199,10 +217,11 @@ interface SharedJobExecutableProps { /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-files`. * * @default [] - no extra files are copied to the working directory * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; } @@ -211,19 +230,21 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-jars`. * * @default [] - no extra jars are added to the classpath * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default false - priority is not given to user-provided jars * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } @@ -234,8 +255,9 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { export interface ScalaJobExecutableProps extends SharedSparkJobExecutableProps { /** * The fully qualified Scala class name that serves as the entry point for the job. + * Equivalent to a job parameter `--class`. * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className: string; } @@ -253,7 +275,7 @@ export interface PythonShellExecutableProps extends SharedJobExecutableProps, Py /** * Props for creating a Python Ray job executable */ -export interface PythonRayExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {} +export interface PythonRayExecutableProps extends SharedJobExecutableProps, RayExecutableProps {} /** * The executable properties related to the Glue job's GlueVersion, JobType and code @@ -377,6 +399,9 @@ export class JobExecutable { if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) { throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON'); } + if (config.extraPythonFiles && type === JobType.RAY.name) { + throw new Error('extraPythonFiles is not supported for Ray jobs'); + } if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) { throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray'); } @@ -384,7 +409,7 @@ export class JobExecutable { throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray'); } if (config.runtime === undefined && type === JobType.RAY.name) { - throw new Error('Runtime is required for Ray jobs.'); + throw new Error('Runtime is required for Ray jobs'); } this.config = config; } @@ -410,8 +435,9 @@ export interface JobExecutableConfig { /** * The language of the job (Scala or Python). + * Equivalent to a job parameter `--job-language`. * - * @see `--job-language` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly language: JobLanguage; @@ -441,46 +467,61 @@ export interface JobExecutableConfig { /** * The Scala class that serves as the entry point for the job. This applies only if your the job langauage is Scala. + * Equivalent to a job parameter `--class`. * * @default - no scala className specified * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className?: string; /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. + * Equivalent to a job parameter `--extra-jars`. * * @default - no extra jars specified. * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Additional Python files that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files specified. * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files specified. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; + /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. + * Equivalent to a job parameter `--extra-files`. * * @default - no extra files specified. * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default - extra jars are not prioritized. * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index b3d76dc44ee04..4fd745612f3e1 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -598,10 +598,11 @@ export interface JobProps { /** * Enables the collection of metrics for job profiling. + * Equivalent to a job parameter `--enable-metrics`. * * @default - no profiling metrics emitted. * - * @see `--enable-metrics` at https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly enableProfilingMetrics? :boolean; @@ -801,6 +802,9 @@ export class Job extends JobBase { if (config.extraPythonFiles && config.extraPythonFiles.length > 0) { args['--extra-py-files'] = config.extraPythonFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } + if (config.s3PythonModules && config.s3PythonModules.length > 0) { + args['--s3-py-modules'] = config.s3PythonModules.map(code => this.codeS3ObjectUrl(code)).join(','); + } if (config.extraFiles && config.extraFiles.length > 0) { args['--extra-files'] = config.extraFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } @@ -886,8 +890,8 @@ export class Job extends JobBase { } /** - * Create a CloudWatch Metric that's based on Glue Job events - * {@see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types} + * Create a CloudWatch Metric that's based on Glue Job events. + * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types * The metric has namespace = 'AWS/Events', metricName = 'TriggeredRules' and RuleName = rule.ruleName dimension. * * @param rule for use in setting RuleName dimension value diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip new file mode 100644 index 0000000000000000000000000000000000000000..3056cd4db8f271cbcf6400230172cb0ba47ebe86 GIT binary patch literal 886 zcmWIWW@Zs#-~htAU78ULP;il%0VtxtP+F3iQ><4|85+XNz@Ef3H`R)LZmIjp{sc=uGXABJZSFP8uC_-g!MAL;&4W!| z!CxEBJ8s=-qj7GV&f+x*kK%i-Z`gRO_khpReSE9FX1@~K@^OxB=&?uZ7O9sm>$w-G znKWOTRrFhO-oGoqgWY~_{JVKgO!ZX@y=h$BRrOLV-&7PieoS@#=PZ!>)Y`z)pzzIA z;bm>5-51LaCT$FoXgJ7x+29F(KxwbhKTDS#Gg95>9j)kms^qPA`hzp$IrH0Z*>+17 zPkwaSP3P-r9{Cf^E|SW@5nYov&R640l-Ra~**5J%hu9PG0B?4V#}{U>6$9e{6kP#W zBa$5`D-4Xt_;_DOXaC>`eZ6?Bar%=JBTkbOM3d7Kk`g}n`hea z$Ho@e65|+H7nv286j!9hEbYv4{Nk#nt#R?oj`-|8obxQwea`L+jLOW1UsgKCeBN^? z6zCsz4xg2U>GiGa{)X`yG^2VPHukh=tXYSQ0L}smL~h!VLzNG@b;y Xfs9ZL@MdKL8NvjF+kx~-CJ+w*MNT#7 literal 0 HcmV?d00001 diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index 991a6c7e93738..79fcfa4848c78 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -14,7 +14,33 @@ } } }, - "5ff83e42e9cbebab4d9d764b8daa4a715172668bb1a6bec030b46694573c74ab": { + "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4": { + "source": { + "path": "asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7": { + "source": { + "path": "asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "5ff83e42e9cbebab4d9d764b8daa4a715172668bb1a6bec030b46694573c74ab.json", + "objectKey": "dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index 378abedae5b7f..a198186232829 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -1595,13 +1595,25 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] } }, "DefaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index 8fcf83d9f0012..1d980ef4d57bb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/5ff83e42e9cbebab4d9d764b8daa4a715172668bb1a6bec030b46694573c74ab.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index c60f466337bb2..91a8fd605214a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -2314,6 +2314,58 @@ "version": "0.0.0" } }, + "Code0115baecff23f24504399384f987872e": { + "id": "Code0115baecff23f24504399384f987872e", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Code7e4c8ab0af31dc38a92c1c69bf696244": { + "id": "Code7e4c8ab0af31dc38a92c1c69bf696244", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/RayJob/Resource", @@ -2330,7 +2382,7 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] }, @@ -2339,6 +2391,18 @@ }, "defaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index 6db000eb656e0..91bf9bab212fc 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -22,6 +22,8 @@ const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-glue-job'); const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')); +const scriptResolveOptions = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'resolve_options.py')); +const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module', 'utils.zip')); [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { const etlJob = new glue.Job(stack, 'EtlJob' + glueVersion.name, { @@ -116,7 +118,8 @@ new glue.Job(stack, 'RayJob', { glueVersion: glue.GlueVersion.V4_0, pythonVersion: glue.PythonVersion.THREE_NINE, runtime: glue.Runtime.RAY_TWO_FOUR, - script, + s3PythonModules: [moduleUtils], + script: scriptResolveOptions, }), workerType: glue.WorkerType.Z_2X, workerCount: 2, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts index 28c6225c542b6..d00faa55091ba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts @@ -254,5 +254,29 @@ describe('JobExecutable', () => { script, })).toBeDefined(); }); + + test('with JobTypeof("glueray") and extraPythonFiles set should throw', () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, + extraPythonFiles: [script], + script, + })).toThrow(/extraPythonFiles is not supported for Ray jobs/); + }); + + test('with JobType ray and s3PythonModules should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + s3PythonModules: [script], + runtime: glue.Runtime.RAY_TWO_FOUR, + script, + })).toBeDefined(); + }); }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index fd0cc8922b4b5..0e6db582c1d71 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -872,7 +872,7 @@ describe('Job', () => { }), workerType: glue.WorkerType.Z_2X, workerCount: 2, - })).toThrow('Runtime is required for Ray jobs.'); + })).toThrow('Runtime is required for Ray jobs'); }); }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip new file mode 100644 index 0000000000000000000000000000000000000000..3056cd4db8f271cbcf6400230172cb0ba47ebe86 GIT binary patch literal 886 zcmWIWW@Zs#-~htAU78ULP;il%0VtxtP+F3iQ><4|85+XNz@Ef3H`R)LZmIjp{sc=uGXABJZSFP8uC_-g!MAL;&4W!| z!CxEBJ8s=-qj7GV&f+x*kK%i-Z`gRO_khpReSE9FX1@~K@^OxB=&?uZ7O9sm>$w-G znKWOTRrFhO-oGoqgWY~_{JVKgO!ZX@y=h$BRrOLV-&7PieoS@#=PZ!>)Y`z)pzzIA z;bm>5-51LaCT$FoXgJ7x+29F(KxwbhKTDS#Gg95>9j)kms^qPA`hzp$IrH0Z*>+17 zPkwaSP3P-r9{Cf^E|SW@5nYov&R640l-Ra~**5J%hu9PG0B?4V#}{U>6$9e{6kP#W zBa$5`D-4Xt_;_DOXaC>`eZ6?Bar%=JBTkbOM3d7Kk`g}n`hea z$Ho@e65|+H7nv286j!9hEbYv4{Nk#nt#R?oj`-|8obxQwea`L+jLOW1UsgKCeBN^? z6zCsz4xg2U>GiGa{)X`yG^2VPHukh=tXYSQ0L}smL~h!VLzNG@b;y Xfs9ZL@MdKL8NvjF+kx~-CJ+w*MNT#7 literal 0 HcmV?d00001 From bebd67ffd02a5cfbbf4e60f4f5a3b2085c3f2044 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Wed, 13 Mar 2024 10:50:26 +0900 Subject: [PATCH 5/8] Revert S3 side change for s3UrlForObject --- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 2 +- .../aws-glue-job.assets.json | 4 +- .../aws-glue-job.template.json | 165 ------------- .../test/integ.job.js.snapshot/manifest.json | 20 +- .../test/integ.job.js.snapshot/tree.json | 216 ------------------ .../@aws-cdk/aws-glue-alpha/test/integ.job.ts | 3 - packages/aws-cdk-lib/aws-s3/lib/bucket.ts | 16 +- .../aws-cdk-lib/aws-s3/test/bucket.test.ts | 48 ---- 8 files changed, 8 insertions(+), 466 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 4fd745612f3e1..ff71ebf882d11 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -826,7 +826,7 @@ export class Job extends JobBase { bucket.grantReadWrite(role, this.cleanPrefixForGrant(props.prefix)); const args = { '--enable-spark-ui': 'true', - '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix, true), + '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix).replace(/\/?$/, '/'), }; return { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index 79fcfa4848c78..58e35ee257212 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -40,7 +40,7 @@ } } }, - "dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7": { + "30c658bb3cfa20d3e01112705df055414f1af71c39615b6f43131af59cb4c208": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -48,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7.json", + "objectKey": "30c658bb3cfa20d3e01112705df055414f1af71c39615b6f43131af59cb4c208.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index a198186232829..76e36f0b7a378 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -268,43 +268,6 @@ "Properties": { "PolicyDocument": { "Statement": [ - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "StreamingJob20SparkUIBucket92EF3706", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "StreamingJob20SparkUIBucket92EF3706", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - }, { "Action": [ "s3:GetBucket*", @@ -357,11 +320,6 @@ ] } }, - "StreamingJob20SparkUIBucket92EF3706": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" - }, "StreamingJob20355B58C7": { "Type": "AWS::Glue::Job", "Properties": { @@ -383,19 +341,6 @@ }, "DefaultArguments": { "--job-language": "python", - "--enable-spark-ui": "true", - "--spark-event-logs-path": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "StreamingJob20SparkUIBucket92EF3706" - }, - "/" - ] - ] - }, "arg1": "value1", "arg2": "value2" }, @@ -682,43 +627,6 @@ "Properties": { "PolicyDocument": { "Statement": [ - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "StreamingJob30SparkUIBucketEFBF52D3", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "StreamingJob30SparkUIBucketEFBF52D3", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - }, { "Action": [ "s3:GetBucket*", @@ -771,11 +679,6 @@ ] } }, - "StreamingJob30SparkUIBucketEFBF52D3": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" - }, "StreamingJob30E005FBEB": { "Type": "AWS::Glue::Job", "Properties": { @@ -797,19 +700,6 @@ }, "DefaultArguments": { "--job-language": "python", - "--enable-spark-ui": "true", - "--spark-event-logs-path": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "StreamingJob30SparkUIBucketEFBF52D3" - }, - "/" - ] - ] - }, "arg1": "value1", "arg2": "value2" }, @@ -1096,43 +986,6 @@ "Properties": { "PolicyDocument": { "Statement": [ - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "StreamingJob40SparkUIBucketA97E24C6", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "StreamingJob40SparkUIBucketA97E24C6", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - }, { "Action": [ "s3:GetBucket*", @@ -1185,11 +1038,6 @@ ] } }, - "StreamingJob40SparkUIBucketA97E24C6": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" - }, "StreamingJob40E284A782": { "Type": "AWS::Glue::Job", "Properties": { @@ -1211,19 +1059,6 @@ }, "DefaultArguments": { "--job-language": "python", - "--enable-spark-ui": "true", - "--spark-event-logs-path": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "StreamingJob40SparkUIBucketA97E24C6" - }, - "/" - ] - ] - }, "arg1": "value1", "arg2": "value2" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index 1d980ef4d57bb..7b38c258f7eaa 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/30c658bb3cfa20d3e01112705df055414f1af71c39615b6f43131af59cb4c208.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -76,12 +76,6 @@ "data": "StreamingJob20ServiceRoleDefaultPolicy3CF74B2D" } ], - "/aws-glue-job/StreamingJob2.0/SparkUIBucket/Resource": [ - { - "type": "aws:cdk:logicalId", - "data": "StreamingJob20SparkUIBucket92EF3706" - } - ], "/aws-glue-job/StreamingJob2.0/Resource": [ { "type": "aws:cdk:logicalId", @@ -130,12 +124,6 @@ "data": "StreamingJob30ServiceRoleDefaultPolicy0C15D010" } ], - "/aws-glue-job/StreamingJob3.0/SparkUIBucket/Resource": [ - { - "type": "aws:cdk:logicalId", - "data": "StreamingJob30SparkUIBucketEFBF52D3" - } - ], "/aws-glue-job/StreamingJob3.0/Resource": [ { "type": "aws:cdk:logicalId", @@ -184,12 +172,6 @@ "data": "StreamingJob40ServiceRoleDefaultPolicy0667C434" } ], - "/aws-glue-job/StreamingJob4.0/SparkUIBucket/Resource": [ - { - "type": "aws:cdk:logicalId", - "data": "StreamingJob40SparkUIBucketA97E24C6" - } - ], "/aws-glue-job/StreamingJob4.0/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index ce6127ed97d50..46a642e17080b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -430,43 +430,6 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "StreamingJob20SparkUIBucket92EF3706", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "StreamingJob20SparkUIBucket92EF3706", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - }, { "Action": [ "s3:GetBucket*", @@ -536,28 +499,6 @@ "version": "0.0.0" } }, - "SparkUIBucket": { - "id": "SparkUIBucket", - "path": "aws-glue-job/StreamingJob2.0/SparkUIBucket", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job/StreamingJob2.0/SparkUIBucket/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::S3::Bucket", - "aws:cdk:cloudformation:props": {} - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.CfnBucket", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.Bucket", - "version": "0.0.0" - } - }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob2.0/Resource", @@ -582,19 +523,6 @@ }, "defaultArguments": { "--job-language": "python", - "--enable-spark-ui": "true", - "--spark-event-logs-path": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "StreamingJob20SparkUIBucket92EF3706" - }, - "/" - ] - ] - }, "arg1": "value1", "arg2": "value2" }, @@ -1020,43 +948,6 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "StreamingJob30SparkUIBucketEFBF52D3", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "StreamingJob30SparkUIBucketEFBF52D3", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - }, { "Action": [ "s3:GetBucket*", @@ -1126,28 +1017,6 @@ "version": "0.0.0" } }, - "SparkUIBucket": { - "id": "SparkUIBucket", - "path": "aws-glue-job/StreamingJob3.0/SparkUIBucket", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job/StreamingJob3.0/SparkUIBucket/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::S3::Bucket", - "aws:cdk:cloudformation:props": {} - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.CfnBucket", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.Bucket", - "version": "0.0.0" - } - }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob3.0/Resource", @@ -1172,19 +1041,6 @@ }, "defaultArguments": { "--job-language": "python", - "--enable-spark-ui": "true", - "--spark-event-logs-path": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "StreamingJob30SparkUIBucketEFBF52D3" - }, - "/" - ] - ] - }, "arg1": "value1", "arg2": "value2" }, @@ -1610,43 +1466,6 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "StreamingJob40SparkUIBucketA97E24C6", - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "StreamingJob40SparkUIBucketA97E24C6", - "Arn" - ] - }, - "/*" - ] - ] - } - ] - }, { "Action": [ "s3:GetBucket*", @@ -1716,28 +1535,6 @@ "version": "0.0.0" } }, - "SparkUIBucket": { - "id": "SparkUIBucket", - "path": "aws-glue-job/StreamingJob4.0/SparkUIBucket", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job/StreamingJob4.0/SparkUIBucket/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::S3::Bucket", - "aws:cdk:cloudformation:props": {} - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.CfnBucket", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.Bucket", - "version": "0.0.0" - } - }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob4.0/Resource", @@ -1762,19 +1559,6 @@ }, "defaultArguments": { "--job-language": "python", - "--enable-spark-ui": "true", - "--spark-event-logs-path": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "StreamingJob40SparkUIBucketA97E24C6" - }, - "/" - ] - ] - }, "arg1": "value1", "arg2": "value2" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index 91bf9bab212fc..7b405d53ca490 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -71,9 +71,6 @@ const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module', 'utils.zi arg1: 'value1', arg2: 'value2', }, - sparkUI: { - enabled: true, - }, tags: { key: 'value', }, diff --git a/packages/aws-cdk-lib/aws-s3/lib/bucket.ts b/packages/aws-cdk-lib/aws-s3/lib/bucket.ts index 3ba39ddbf5eea..a799062afc692 100644 --- a/packages/aws-cdk-lib/aws-s3/lib/bucket.ts +++ b/packages/aws-cdk-lib/aws-s3/lib/bucket.ts @@ -164,10 +164,9 @@ export interface IBucket extends IResource { * - `s3://bucket/key` * @param key The S3 key of the object. If not specified, the S3 URL of the * bucket is returned. - * @param trailingSlash Whether to add a trailing slash when the url does not have it. * @returns an ObjectS3Url token */ - s3UrlForObject(key?: string, trailingSlash?: boolean): string; + s3UrlForObject(key?: string): string; /** * Returns an ARN that represents all objects within the bucket that match @@ -733,21 +732,14 @@ export abstract class BucketBase extends Resource implements IBucket { * * @param key The S3 key of the object. If not specified, the S3 URL of the * bucket is returned. - * @param trailingSlash Whether to add a trailing slash when the url does not have it. * @returns an ObjectS3Url token */ - public s3UrlForObject(key?: string, trailingSlash?: boolean): string { + public s3UrlForObject(key?: string): string { const prefix = 's3://'; - let url = undefined; if (typeof key !== 'string') { - url = this.urlJoin(prefix, this.bucketName); - } else { - url = this.urlJoin(prefix, this.bucketName, key); - } - if (trailingSlash) { - url = url.replace(/\/?$/, '/'); + return this.urlJoin(prefix, this.bucketName); } - return url; + return this.urlJoin(prefix, this.bucketName, key); } /** diff --git a/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts b/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts index 6a5ff9c2727e2..956f66e7d12e6 100644 --- a/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts +++ b/packages/aws-cdk-lib/aws-s3/test/bucket.test.ts @@ -2395,54 +2395,6 @@ describe('bucket', () => { }); }); - test('s3UrlForObject returns the S3 URL with a trailing slash when trailingSlash is set to true', () => { - const stack = new cdk.Stack(); - const bucket = new s3.Bucket(stack, 'MyBucket'); - - new cdk.CfnOutput(stack, 'BucketS3URL', { value: bucket.s3UrlForObject(null, true) }); - new cdk.CfnOutput(stack, 'MyFolderS3URL', { value: bucket.s3UrlForObject('my/folder', true) }); - - Template.fromStack(stack).templateMatches({ - 'Resources': { - 'MyBucketF68F3FF0': { - 'Type': 'AWS::S3::Bucket', - 'DeletionPolicy': 'Retain', - 'UpdateReplacePolicy': 'Retain', - }, - }, - 'Outputs': { - 'BucketS3URL': { - 'Value': { - 'Fn::Join': [ - '', - [ - 's3://', - { - 'Ref': 'MyBucketF68F3FF0', - }, - '/', - ], - ], - }, - }, - 'MyFolderS3URL': { - 'Value': { - 'Fn::Join': [ - '', - [ - 's3://', - { - 'Ref': 'MyBucketF68F3FF0', - }, - '/my/folder/', - ], - ], - }, - }, - }, - }); - }); - describe('grantPublicAccess', () => { test('by default, grants s3:GetObject to all objects', () => { // GIVEN From ffebc56754b4e655db29845e47c8f324e4f0172c Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Wed, 13 Mar 2024 14:33:10 +0900 Subject: [PATCH 6/8] Made change in integ.job --- packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index 7b405d53ca490..91bf9bab212fc 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -71,6 +71,9 @@ const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module', 'utils.zi arg1: 'value1', arg2: 'value2', }, + sparkUI: { + enabled: true, + }, tags: { key: 'value', }, From 0a1ce0cbb3e9379bd8bb78f5bfb0ce744959a15c Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Wed, 13 Mar 2024 17:11:02 +0900 Subject: [PATCH 7/8] Updated snapshot --- .../aws-glue-job.assets.json | 4 +- .../aws-glue-job.template.json | 165 +++++++++++ .../test/integ.job.js.snapshot/manifest.json | 20 +- .../test/integ.job.js.snapshot/tree.json | 268 ++++++++++++++---- 4 files changed, 402 insertions(+), 55 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index 58e35ee257212..79fcfa4848c78 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -40,7 +40,7 @@ } } }, - "30c658bb3cfa20d3e01112705df055414f1af71c39615b6f43131af59cb4c208": { + "dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -48,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "30c658bb3cfa20d3e01112705df055414f1af71c39615b6f43131af59cb4c208.json", + "objectKey": "dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index 76e36f0b7a378..a198186232829 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -268,6 +268,43 @@ "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -320,6 +357,11 @@ ] } }, + "StreamingJob20SparkUIBucket92EF3706": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, "StreamingJob20355B58C7": { "Type": "AWS::Glue::Job", "Properties": { @@ -341,6 +383,19 @@ }, "DefaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob20SparkUIBucket92EF3706" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -627,6 +682,43 @@ "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -679,6 +771,11 @@ ] } }, + "StreamingJob30SparkUIBucketEFBF52D3": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, "StreamingJob30E005FBEB": { "Type": "AWS::Glue::Job", "Properties": { @@ -700,6 +797,19 @@ }, "DefaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob30SparkUIBucketEFBF52D3" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -986,6 +1096,43 @@ "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -1038,6 +1185,11 @@ ] } }, + "StreamingJob40SparkUIBucketA97E24C6": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, "StreamingJob40E284A782": { "Type": "AWS::Glue::Job", "Properties": { @@ -1059,6 +1211,19 @@ }, "DefaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob40SparkUIBucketA97E24C6" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index 7b38c258f7eaa..1d980ef4d57bb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/30c658bb3cfa20d3e01112705df055414f1af71c39615b6f43131af59cb4c208.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dd8261e4900491c4aaaec8bcd31c57b1f30085f1d579f19089bd55a4bdb023b7.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -76,6 +76,12 @@ "data": "StreamingJob20ServiceRoleDefaultPolicy3CF74B2D" } ], + "/aws-glue-job/StreamingJob2.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob20SparkUIBucket92EF3706" + } + ], "/aws-glue-job/StreamingJob2.0/Resource": [ { "type": "aws:cdk:logicalId", @@ -124,6 +130,12 @@ "data": "StreamingJob30ServiceRoleDefaultPolicy0C15D010" } ], + "/aws-glue-job/StreamingJob3.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob30SparkUIBucketEFBF52D3" + } + ], "/aws-glue-job/StreamingJob3.0/Resource": [ { "type": "aws:cdk:logicalId", @@ -172,6 +184,12 @@ "data": "StreamingJob40ServiceRoleDefaultPolicy0667C434" } ], + "/aws-glue-job/StreamingJob4.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob40SparkUIBucketA97E24C6" + } + ], "/aws-glue-job/StreamingJob4.0/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 46a642e17080b..91a8fd605214a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -430,6 +430,43 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob20SparkUIBucket92EF3706", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -499,6 +536,28 @@ "version": "0.0.0" } }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/StreamingJob2.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob2.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob2.0/Resource", @@ -523,6 +582,19 @@ }, "defaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob20SparkUIBucket92EF3706" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -948,6 +1020,43 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob30SparkUIBucketEFBF52D3", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -1017,6 +1126,28 @@ "version": "0.0.0" } }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/StreamingJob3.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob3.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob3.0/Resource", @@ -1041,6 +1172,19 @@ }, "defaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob30SparkUIBucketEFBF52D3" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -1466,6 +1610,43 @@ "aws:cdk:cloudformation:props": { "policyDocument": { "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StreamingJob40SparkUIBucketA97E24C6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, { "Action": [ "s3:GetBucket*", @@ -1535,6 +1716,28 @@ "version": "0.0.0" } }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/StreamingJob4.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob4.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/StreamingJob4.0/Resource", @@ -1559,6 +1762,19 @@ }, "defaultArguments": { "--job-language": "python", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "StreamingJob40SparkUIBucketA97E24C6" + }, + "/" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, @@ -2124,58 +2340,6 @@ "version": "0.0.0" } }, - "Code7e4c8ab0af31dc38a92c1c69bf696244": { - "id": "Code7e4c8ab0af31dc38a92c1c69bf696244", - "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "Code0115baecff23f24504399384f987872e": { - "id": "Code0115baecff23f24504399384f987872e", - "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, "Code7e4c8ab0af31dc38a92c1c69bf696244": { "id": "Code7e4c8ab0af31dc38a92c1c69bf696244", "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244", From 2a1674121687ca8e0762bd5db0f7aae8fcc287a2 Mon Sep 17 00:00:00 2001 From: Parker Scanlon <69879391+scanlonp@users.noreply.github.com> Date: Mon, 18 Mar 2024 22:23:15 -0700 Subject: [PATCH 8/8] Update packages/@aws-cdk/aws-glue-alpha/lib/job.ts --- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index ff71ebf882d11..813894f0b6898 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -826,7 +826,7 @@ export class Job extends JobBase { bucket.grantReadWrite(role, this.cleanPrefixForGrant(props.prefix)); const args = { '--enable-spark-ui': 'true', - '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix).replace(/\/?$/, '/'), + '--spark-event-logs-path': bucket.s3UrlForObject(props.prefix).replace(/\/?$/, '/'), // path will always end with a slash }; return {