From 6d74002aebb817c6abecd77b3318b40a46cc8072 Mon Sep 17 00:00:00 2001 From: awstools Date: Thu, 12 Dec 2024 19:32:12 +0000 Subject: [PATCH] feat(client-glue): To support customer-managed encryption in Data Quality to allow customers encrypt data with their own KMS key, we will add a DataQualityEncryption field to the SecurityConfiguration API where customers can provide their KMS keys. --- .../CreateSecurityConfigurationCommand.ts | 4 + .../src/commands/CreateTriggerCommand.ts | 1 + .../src/commands/GetJobRunCommand.ts | 2 +- .../src/commands/GetJobRunsCommand.ts | 2 + .../GetSecurityConfigurationCommand.ts | 4 + .../GetSecurityConfigurationsCommand.ts | 4 + .../src/commands/UpdateTriggerCommand.ts | 1 + clients/client-glue/src/models/models_0.ts | 12 +-- clients/client-glue/src/models/models_1.ts | 78 +++++++++-------- clients/client-glue/src/models/models_2.ts | 63 +++++++------- clients/client-glue/src/models/models_3.ts | 83 +++++++++++++------ .../client-glue/src/protocols/Aws_json1_1.ts | 7 +- codegen/sdk-codegen/aws-models/glue.json | 71 ++++++++++++---- 13 files changed, 218 insertions(+), 114 deletions(-) diff --git a/clients/client-glue/src/commands/CreateSecurityConfigurationCommand.ts b/clients/client-glue/src/commands/CreateSecurityConfigurationCommand.ts index 7230560e60d6..0f087c5476f3 100644 --- a/clients/client-glue/src/commands/CreateSecurityConfigurationCommand.ts +++ b/clients/client-glue/src/commands/CreateSecurityConfigurationCommand.ts @@ -54,6 +54,10 @@ export interface CreateSecurityConfigurationCommandOutput * JobBookmarksEncryptionMode: "DISABLED" || "CSE-KMS", * KmsKeyArn: "STRING_VALUE", * }, + * DataQualityEncryption: { // DataQualityEncryption + * DataQualityEncryptionMode: "DISABLED" || "SSE-KMS", + * KmsKeyArn: "STRING_VALUE", + * }, * }, * }; * const command = new CreateSecurityConfigurationCommand(input); diff --git a/clients/client-glue/src/commands/CreateTriggerCommand.ts b/clients/client-glue/src/commands/CreateTriggerCommand.ts index 345e3b86483a..780f2200246b 100644 --- a/clients/client-glue/src/commands/CreateTriggerCommand.ts +++ b/clients/client-glue/src/commands/CreateTriggerCommand.ts @@ -29,6 +29,7 @@ export interface CreateTriggerCommandOutput extends CreateTriggerResponse, __Met /** *

Creates a new trigger.

+ *

Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to keep them within the Job.

* @example * Use a bare-bones client and the command you need to make an API call. * ```javascript diff --git a/clients/client-glue/src/commands/GetJobRunCommand.ts b/clients/client-glue/src/commands/GetJobRunCommand.ts index e7a65198c665..ba86e1f8b2cf 100644 --- a/clients/client-glue/src/commands/GetJobRunCommand.ts +++ b/clients/client-glue/src/commands/GetJobRunCommand.ts @@ -28,7 +28,7 @@ export interface GetJobRunCommandInput extends GetJobRunRequest {} export interface GetJobRunCommandOutput extends GetJobRunResponse, __MetadataBearer {} /** - *

Retrieves the metadata for a given job run. Job run history is accessible for 90 days for your workflow and job run.

+ *

Retrieves the metadata for a given job run. Job run history is accessible for 365 days for your workflow and job run.

* @example * Use a bare-bones client and the command you need to make an API call. * ```javascript diff --git a/clients/client-glue/src/commands/GetJobRunsCommand.ts b/clients/client-glue/src/commands/GetJobRunsCommand.ts index 5c8202bf41ec..1a22a4439e48 100644 --- a/clients/client-glue/src/commands/GetJobRunsCommand.ts +++ b/clients/client-glue/src/commands/GetJobRunsCommand.ts @@ -29,6 +29,8 @@ export interface GetJobRunsCommandOutput extends GetJobRunsResponse, __MetadataB /** *

Retrieves metadata for all runs of a given job definition.

+ *

+ * GetJobRuns returns the job runs in chronological order, with the newest jobs returned first.

* @example * Use a bare-bones client and the command you need to make an API call. * ```javascript diff --git a/clients/client-glue/src/commands/GetSecurityConfigurationCommand.ts b/clients/client-glue/src/commands/GetSecurityConfigurationCommand.ts index 87c9d4444468..8573f350420c 100644 --- a/clients/client-glue/src/commands/GetSecurityConfigurationCommand.ts +++ b/clients/client-glue/src/commands/GetSecurityConfigurationCommand.ts @@ -59,6 +59,10 @@ export interface GetSecurityConfigurationCommandOutput extends GetSecurityConfig * // JobBookmarksEncryptionMode: "DISABLED" || "CSE-KMS", * // KmsKeyArn: "STRING_VALUE", * // }, + * // DataQualityEncryption: { // DataQualityEncryption + * // DataQualityEncryptionMode: "DISABLED" || "SSE-KMS", + * // KmsKeyArn: "STRING_VALUE", + * // }, * // }, * // }, * // }; diff --git a/clients/client-glue/src/commands/GetSecurityConfigurationsCommand.ts b/clients/client-glue/src/commands/GetSecurityConfigurationsCommand.ts index f53150329775..b62b83eeff76 100644 --- a/clients/client-glue/src/commands/GetSecurityConfigurationsCommand.ts +++ b/clients/client-glue/src/commands/GetSecurityConfigurationsCommand.ts @@ -61,6 +61,10 @@ export interface GetSecurityConfigurationsCommandOutput extends GetSecurityConfi * // JobBookmarksEncryptionMode: "DISABLED" || "CSE-KMS", * // KmsKeyArn: "STRING_VALUE", * // }, + * // DataQualityEncryption: { // DataQualityEncryption + * // DataQualityEncryptionMode: "DISABLED" || "SSE-KMS", + * // KmsKeyArn: "STRING_VALUE", + * // }, * // }, * // }, * // ], diff --git a/clients/client-glue/src/commands/UpdateTriggerCommand.ts b/clients/client-glue/src/commands/UpdateTriggerCommand.ts index d4fb35fe3964..b554f7f4fdad 100644 --- a/clients/client-glue/src/commands/UpdateTriggerCommand.ts +++ b/clients/client-glue/src/commands/UpdateTriggerCommand.ts @@ -29,6 +29,7 @@ export interface UpdateTriggerCommandOutput extends UpdateTriggerResponse, __Met /** *

Updates a trigger definition.

+ *

Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to keep them within the Job.

* @example * Use a bare-bones client and the command you need to make an API call. * ```javascript diff --git a/clients/client-glue/src/models/models_0.ts b/clients/client-glue/src/models/models_0.ts index 073301bcb87f..728aa362fc4c 100644 --- a/clients/client-glue/src/models/models_0.ts +++ b/clients/client-glue/src/models/models_0.ts @@ -8823,22 +8823,22 @@ export interface JobRun { * G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

* * @public diff --git a/clients/client-glue/src/models/models_1.ts b/clients/client-glue/src/models/models_1.ts index fd5cbdfb90fe..eb2aa3f24a6f 100644 --- a/clients/client-glue/src/models/models_1.ts +++ b/clients/client-glue/src/models/models_1.ts @@ -3496,6 +3496,39 @@ export interface CloudWatchEncryption { KmsKeyArn?: string | undefined; } +/** + * @public + * @enum + */ +export const DataQualityEncryptionMode = { + DISABLED: "DISABLED", + SSEKMS: "SSE-KMS", +} as const; + +/** + * @public + */ +export type DataQualityEncryptionMode = (typeof DataQualityEncryptionMode)[keyof typeof DataQualityEncryptionMode]; + +/** + *

Specifies how Data Quality assets in your account should be encrypted.

+ * @public + */ +export interface DataQualityEncryption { + /** + *

The encryption mode to use for encrypting Data Quality assets. These assets include data quality rulesets, results, statistics, anomaly detection models and observations.

+ *

Valid values are SSEKMS for encryption using a customer-managed KMS key, or DISABLED.

+ * @public + */ + DataQualityEncryptionMode?: DataQualityEncryptionMode | undefined; + + /** + *

The Amazon Resource Name (ARN) of the KMS key to be used to encrypt the data.

+ * @public + */ + KmsKeyArn?: string | undefined; +} + /** * @public * @enum @@ -3583,6 +3616,12 @@ export interface EncryptionConfiguration { * @public */ JobBookmarksEncryption?: JobBookmarksEncryption | undefined; + + /** + *

The encryption configuration for Glue Data Quality assets.

+ * @public + */ + DataQualityEncryption?: DataQualityEncryption | undefined; } /** @@ -3716,19 +3755,19 @@ export interface CreateSessionRequest { * G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts the value Z.2X for Ray notebooks.

* * @public @@ -4606,6 +4645,7 @@ export interface CreateWorkflowRequest { /** *

A collection of properties to be used as part of each execution of the workflow.

+ *

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

* @public */ DefaultRunProperties?: Record | undefined; @@ -7880,34 +7920,6 @@ export interface ExecutionAttempt { ErrorMessage?: string | undefined; } -/** - * @public - * @enum - */ -export const ScheduleType = { - AUTO: "AUTO", - CRON: "CRON", -} as const; - -/** - * @public - */ -export type ScheduleType = (typeof ScheduleType)[keyof typeof ScheduleType]; - -/** - * @public - * @enum - */ -export const SettingSource = { - CATALOG: "CATALOG", - TABLE: "TABLE", -} as const; - -/** - * @public - */ -export type SettingSource = (typeof SettingSource)[keyof typeof SettingSource]; - /** * @internal */ diff --git a/clients/client-glue/src/models/models_2.ts b/clients/client-glue/src/models/models_2.ts index d8b59f7f1db5..6b74e6d0fb78 100644 --- a/clients/client-glue/src/models/models_2.ts +++ b/clients/client-glue/src/models/models_2.ts @@ -64,11 +64,9 @@ import { RegistryId, RegistryStatus, ResourceUri, - ScheduleType, SchemaStatus, SchemaVersionStatus, Session, - SettingSource, SourceProcessingProperties, SourceTableConfig, Tag, @@ -81,6 +79,34 @@ import { ViewDialect, } from "./models_1"; +/** + * @public + * @enum + */ +export const ScheduleType = { + AUTO: "AUTO", + CRON: "CRON", +} as const; + +/** + * @public + */ +export type ScheduleType = (typeof ScheduleType)[keyof typeof ScheduleType]; + +/** + * @public + * @enum + */ +export const SettingSource = { + CATALOG: "CATALOG", + TABLE: "TABLE", +} as const; + +/** + * @public + */ +export type SettingSource = (typeof SettingSource)[keyof typeof SettingSource]; + /** *

The settings for a column statistics task.

* @public @@ -8118,6 +8144,7 @@ export interface PutWorkflowRunPropertiesRequest { /** *

The properties to put for the specified run.

+ *

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

* @public */ RunProperties: Record | undefined; @@ -8559,38 +8586,6 @@ export interface PropertyPredicate { Comparator?: Comparator | undefined; } -/** - * @public - * @enum - */ -export const Sort = { - ASCENDING: "ASC", - DESCENDING: "DESC", -} as const; - -/** - * @public - */ -export type Sort = (typeof Sort)[keyof typeof Sort]; - -/** - *

Specifies a field to sort by and a sort order.

- * @public - */ -export interface SortCriterion { - /** - *

The name of the field on which to sort.

- * @public - */ - FieldName?: string | undefined; - - /** - *

An ascending or descending sort.

- * @public - */ - Sort?: Sort | undefined; -} - /** * @internal */ diff --git a/clients/client-glue/src/models/models_3.ts b/clients/client-glue/src/models/models_3.ts index 6242bd1ffcbe..4edb7c636460 100644 --- a/clients/client-glue/src/models/models_3.ts +++ b/clients/client-glue/src/models/models_3.ts @@ -138,11 +138,42 @@ import { ResourceShareType, ResourceState, SchemaVersionNumber, - SortCriterion, ViewDefinition, ViewValidation, } from "./models_2"; +/** + * @public + * @enum + */ +export const Sort = { + ASCENDING: "ASC", + DESCENDING: "DESC", +} as const; + +/** + * @public + */ +export type Sort = (typeof Sort)[keyof typeof Sort]; + +/** + *

Specifies a field to sort by and a sort order.

+ * @public + */ +export interface SortCriterion { + /** + *

The name of the field on which to sort.

+ * @public + */ + FieldName?: string | undefined; + + /** + *

An ascending or descending sort.

+ * @public + */ + Sort?: Sort | undefined; +} + /** * @public */ @@ -724,22 +755,22 @@ export interface StartJobRunRequest { * G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

*
    *
  • - *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    + *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    *
    • *
  • - *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    + *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    *
    • *
  • - *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    + *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    *
    • *
  • - *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    + *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    *
    • *
* @public @@ -883,6 +914,7 @@ export interface StartWorkflowRunRequest { /** *

The workflow run properties for the new workflow run.

+ *

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

* @public */ RunProperties?: Record | undefined; @@ -2808,6 +2840,7 @@ export interface UpdateWorkflowRequest { /** *

A collection of properties to be used as part of each execution of the workflow.

+ *

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

* @public */ DefaultRunProperties?: Record | undefined; @@ -3949,22 +3982,22 @@ export interface CreateJobRequest { * G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

*
    *
  • - *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    + *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    *
    • *
  • - *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    + *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    *
    • *
  • - *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    + *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    *
    • *
  • - *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    + *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    *
    • *
* @public @@ -4192,22 +4225,22 @@ export interface Job { * G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

*
    *
  • - *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    + *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    *
    • *
  • - *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    + *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    *
    • *
  • - *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    + *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    *
    • *
  • - *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    + *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    *
    • *
* @public @@ -4439,22 +4472,22 @@ export interface JobUpdate { * G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

*
    *
  • - *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    + *

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    *
    • *
  • - *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    + *

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    *
    • *
  • - *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    + *

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    *
    • *
  • - *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    + *

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    *
    • *
  • - *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    + *

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    *
    • *
* @public diff --git a/clients/client-glue/src/protocols/Aws_json1_1.ts b/clients/client-glue/src/protocols/Aws_json1_1.ts index 82690c50b194..06dc18b783a0 100644 --- a/clients/client-glue/src/protocols/Aws_json1_1.ts +++ b/clients/client-glue/src/protocols/Aws_json1_1.ts @@ -893,6 +893,7 @@ import { DatabaseInput, DataLakeAccessProperties, DataLakePrincipal, + DataQualityEncryption, DataQualityTargetTable, DateColumnStatisticsData, DecimalColumnStatisticsData, @@ -1211,7 +1212,6 @@ import { SchemaVersionNumber, SecurityConfiguration, Segment, - SortCriterion, Statement, StatisticModelResult, StatisticSummary, @@ -1256,6 +1256,7 @@ import { SchedulerRunningException, SearchTablesRequest, SearchTablesResponse, + SortCriterion, StartBlueprintRunRequest, StartColumnStatisticsTaskRunRequest, StartColumnStatisticsTaskRunScheduleRequest, @@ -11178,6 +11179,8 @@ const se_CreateTableRequest = (input: CreateTableRequest, context: __SerdeContex // se_DatapointInclusionAnnotation omitted. +// se_DataQualityEncryption omitted. + // se_DataQualityEvaluationRunAdditionalRunOptions omitted. /** @@ -14024,6 +14027,8 @@ const de_DataQualityAnalyzerResults = (output: any, context: __SerdeContext): Da return retVal; }; +// de_DataQualityEncryption omitted. + // de_DataQualityEvaluationRunAdditionalRunOptions omitted. /** diff --git a/codegen/sdk-codegen/aws-models/glue.json b/codegen/sdk-codegen/aws-models/glue.json index 011d626b10c5..c9962af623f7 100644 --- a/codegen/sdk-codegen/aws-models/glue.json +++ b/codegen/sdk-codegen/aws-models/glue.json @@ -10749,7 +10749,7 @@ "WorkerType": { "target": "com.amazonaws.glue#WorkerType", "traits": { - "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" + "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" } }, "CodeGenConfigurationNodes": { @@ -11627,7 +11627,7 @@ "WorkerType": { "target": "com.amazonaws.glue#WorkerType", "traits": { - "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts the value Z.2X for Ray notebooks.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" + "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts the value Z.2X for Ray notebooks.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" } }, "SecurityConfiguration": { @@ -11894,7 +11894,7 @@ } ], "traits": { - "smithy.api#documentation": "

Creates a new trigger.

" + "smithy.api#documentation": "

Creates a new trigger.

\n

Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to keep them within the Job.

" } }, "com.amazonaws.glue#CreateTriggerRequest": { @@ -12184,7 +12184,7 @@ "DefaultRunProperties": { "target": "com.amazonaws.glue#WorkflowRunProperties", "traits": { - "smithy.api#documentation": "

A collection of properties to be used as part of each execution of the workflow.

" + "smithy.api#documentation": "

A collection of properties to be used as part of each execution of the workflow.

\n

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

" } }, "Tags": { @@ -12915,6 +12915,43 @@ } } }, + "com.amazonaws.glue#DataQualityEncryption": { + "type": "structure", + "members": { + "DataQualityEncryptionMode": { + "target": "com.amazonaws.glue#DataQualityEncryptionMode", + "traits": { + "smithy.api#documentation": "

The encryption mode to use for encrypting Data Quality assets. These assets include data quality rulesets, results, statistics, anomaly detection models and observations.

\n

Valid values are SSEKMS for encryption using a customer-managed KMS key, or DISABLED.

" + } + }, + "KmsKeyArn": { + "target": "com.amazonaws.glue#KmsKeyArn", + "traits": { + "smithy.api#documentation": "

The Amazon Resource Name (ARN) of the KMS key to be used to encrypt the data.

" + } + } + }, + "traits": { + "smithy.api#documentation": "

Specifies how Data Quality assets in your account should be encrypted.

" + } + }, + "com.amazonaws.glue#DataQualityEncryptionMode": { + "type": "enum", + "members": { + "DISABLED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "DISABLED" + } + }, + "SSEKMS": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "SSE-KMS" + } + } + } + }, "com.amazonaws.glue#DataQualityEvaluationRunAdditionalRunOptions": { "type": "structure", "members": { @@ -17152,6 +17189,12 @@ "traits": { "smithy.api#documentation": "

The encryption configuration for job bookmarks.

" } + }, + "DataQualityEncryption": { + "target": "com.amazonaws.glue#DataQualityEncryption", + "traits": { + "smithy.api#documentation": "

The encryption configuration for Glue Data Quality assets.

" + } } }, "traits": { @@ -21378,7 +21421,7 @@ } ], "traits": { - "smithy.api#documentation": "

Retrieves the metadata for a given job run. Job run history is accessible for 90 days for your workflow and job run.

" + "smithy.api#documentation": "

Retrieves the metadata for a given job run. Job run history is accessible for 365 days for your workflow and job run.

" } }, "com.amazonaws.glue#GetJobRunRequest": { @@ -21447,7 +21490,7 @@ } ], "traits": { - "smithy.api#documentation": "

Retrieves metadata for all runs of a given job definition.

", + "smithy.api#documentation": "

Retrieves metadata for all runs of a given job definition.

\n

\n GetJobRuns returns the job runs in chronological order, with the newest jobs returned first.

", "smithy.api#paginated": { "inputToken": "NextToken", "outputToken": "NextToken", @@ -27027,7 +27070,7 @@ "WorkerType": { "target": "com.amazonaws.glue#WorkerType", "traits": { - "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" + "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" } }, "NumberOfWorkers": { @@ -27383,7 +27426,7 @@ "WorkerType": { "target": "com.amazonaws.glue#WorkerType", "traits": { - "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" + "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" } }, "NumberOfWorkers": { @@ -27617,7 +27660,7 @@ "WorkerType": { "target": "com.amazonaws.glue#WorkerType", "traits": { - "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" + "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" } }, "NumberOfWorkers": { @@ -33645,7 +33688,7 @@ "RunProperties": { "target": "com.amazonaws.glue#WorkflowRunProperties", "traits": { - "smithy.api#documentation": "

The properties to put for the specified run.

", + "smithy.api#documentation": "

The properties to put for the specified run.

\n

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

", "smithy.api#required": {} } } @@ -38494,7 +38537,7 @@ "WorkerType": { "target": "com.amazonaws.glue#WorkerType", "traits": { - "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 128GB disk (approximately 77GB free), and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free), and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk (approximately 120GB free), and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" + "smithy.api#documentation": "

The type of predefined worker that is allocated when a job runs. Accepts a value of\n G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.

\n
    \n
  • \n

    For the G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.

    \n
    • \n
  • \n

    For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).

    \n
    • \n
  • \n

    For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.

    \n
    • \n
  • \n

    For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.

    \n
    • \n
  • \n

    For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.

    \n
    • \n
" } }, "NumberOfWorkers": { @@ -38760,7 +38803,7 @@ "RunProperties": { "target": "com.amazonaws.glue#WorkflowRunProperties", "traits": { - "smithy.api#documentation": "

The workflow run properties for the new workflow run.

" + "smithy.api#documentation": "

The workflow run properties for the new workflow run.

\n

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

" } } }, @@ -44096,7 +44139,7 @@ } ], "traits": { - "smithy.api#documentation": "

Updates a trigger definition.

" + "smithy.api#documentation": "

Updates a trigger definition.

\n

Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to keep them within the Job.

" } }, "com.amazonaws.glue#UpdateTriggerRequest": { @@ -44328,7 +44371,7 @@ "DefaultRunProperties": { "target": "com.amazonaws.glue#WorkflowRunProperties", "traits": { - "smithy.api#documentation": "

A collection of properties to be used as part of each execution of the workflow.

" + "smithy.api#documentation": "

A collection of properties to be used as part of each execution of the workflow.

\n

Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.

" } }, "MaxConcurrentRuns": {