Skip to content

Commit

Permalink
feat(client-glue): This release enables customers to create new Apach…
Browse files Browse the repository at this point in the history
…e Iceberg tables and associated metadata in Amazon S3 by using native AWS Glue CreateTable operation.
  • Loading branch information
awstools committed Jul 7, 2023
1 parent 29a8b11 commit f4ee5d6
Show file tree
Hide file tree
Showing 7 changed files with 178 additions and 68 deletions.
6 changes: 6 additions & 0 deletions clients/client-glue/src/commands/CreateTableCommand.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,12 @@ export interface CreateTableCommandOutput extends CreateTableResponse, __Metadat
* },
* ],
* TransactionId: "STRING_VALUE",
* OpenTableFormatInput: { // OpenTableFormatInput
* IcebergInput: { // IcebergInput
* MetadataOperation: "CREATE", // required
* Version: "STRING_VALUE",
* },
* },
* };
* const command = new CreateTableCommand(input);
* const response = await client.send(command);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import {
} from "@smithy/types";

import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { GetUnfilteredTableMetadataRequest, GetUnfilteredTableMetadataResponse } from "../models/models_1";
import { GetUnfilteredTableMetadataRequest } from "../models/models_1";
import { GetUnfilteredTableMetadataResponse } from "../models/models_2";
import { de_GetUnfilteredTableMetadataCommand, se_GetUnfilteredTableMetadataCommand } from "../protocols/Aws_json1_1";

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ import {
} from "@smithy/types";

import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { GetUserDefinedFunctionRequest } from "../models/models_1";
import { GetUserDefinedFunctionResponse } from "../models/models_2";
import { GetUserDefinedFunctionRequest, GetUserDefinedFunctionResponse } from "../models/models_2";
import { de_GetUserDefinedFunctionCommand, se_GetUserDefinedFunctionCommand } from "../protocols/Aws_json1_1";

/**
Expand Down
108 changes: 45 additions & 63 deletions clients/client-glue/src/models/models_1.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1017,6 +1017,46 @@ export interface CreateSessionResponse {
Session?: Session;
}

/**
* @public
* @enum
*/
export const MetadataOperation = {
CREATE: "CREATE",
} as const;

/**
* @public
*/
export type MetadataOperation = (typeof MetadataOperation)[keyof typeof MetadataOperation];

/**
* @public
* <p>A structure that defines an Apache Iceberg metadata table to create in the catalog.</p>
*/
export interface IcebergInput {
/**
* <p>A required metadata operation. Can only be set to <code>CREATE</code>.</p>
*/
MetadataOperation: MetadataOperation | string | undefined;

/**
* <p>The table version for the Iceberg table. Defaults to 2.</p>
*/
Version?: string;
}

/**
* @public
* <p>A structure representing an open format table.</p>
*/
export interface OpenTableFormatInput {
/**
* <p>Specifies an <code>IcebergInput</code> structure that defines an Apache Iceberg metadata table.</p>
*/
IcebergInput?: IcebergInput;
}

/**
* @public
* <p>A structure that describes a target table for resource linking.</p>
Expand Down Expand Up @@ -1170,6 +1210,11 @@ export interface CreateTableRequest {
* <p>The ID of the transaction.</p>
*/
TransactionId?: string;

/**
* <p>Specifies an <code>OpenTableFormatInput</code> structure when creating an open format table.</p>
*/
OpenTableFormatInput?: OpenTableFormatInput;
}

/**
Expand Down Expand Up @@ -7169,66 +7214,3 @@ export interface GetUnfilteredTableMetadataRequest {
*/
SupportedPermissionTypes: (PermissionType | string)[] | undefined;
}

/**
* @public
* <p>A filter that uses both column-level and row-level filtering.</p>
*/
export interface ColumnRowFilter {
/**
* <p>A string containing the name of the column.</p>
*/
ColumnName?: string;

/**
* <p>A string containing the row-level filter expression.</p>
*/
RowFilterExpression?: string;
}

/**
* @public
*/
export interface GetUnfilteredTableMetadataResponse {
/**
* <p>A Table object containing the table metadata.</p>
*/
Table?: Table;

/**
* <p>A list of column names that the user has been granted access to.</p>
*/
AuthorizedColumns?: string[];

/**
* <p>A Boolean value that indicates whether the partition location is registered
* with Lake Formation.</p>
*/
IsRegisteredWithLakeFormation?: boolean;

/**
* <p>A list of column row filters.</p>
*/
CellFilters?: ColumnRowFilter[];
}

/**
* @public
*/
export interface GetUserDefinedFunctionRequest {
/**
* <p>The ID of the Data Catalog where the function to be retrieved is located. If none is
* provided, the Amazon Web Services account ID is used by default.</p>
*/
CatalogId?: string;

/**
* <p>The name of the catalog database where the function is located.</p>
*/
DatabaseName: string | undefined;

/**
* <p>The name of the function.</p>
*/
FunctionName: string | undefined;
}
63 changes: 63 additions & 0 deletions clients/client-glue/src/models/models_2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,69 @@ import {
UserDefinedFunctionInput,
} from "./models_1";

/**
* @public
* <p>A filter that uses both column-level and row-level filtering.</p>
*/
export interface ColumnRowFilter {
/**
* <p>A string containing the name of the column.</p>
*/
ColumnName?: string;

/**
* <p>A string containing the row-level filter expression.</p>
*/
RowFilterExpression?: string;
}

/**
* @public
*/
export interface GetUnfilteredTableMetadataResponse {
/**
* <p>A Table object containing the table metadata.</p>
*/
Table?: Table;

/**
* <p>A list of column names that the user has been granted access to.</p>
*/
AuthorizedColumns?: string[];

/**
* <p>A Boolean value that indicates whether the partition location is registered
* with Lake Formation.</p>
*/
IsRegisteredWithLakeFormation?: boolean;

/**
* <p>A list of column row filters.</p>
*/
CellFilters?: ColumnRowFilter[];
}

/**
* @public
*/
export interface GetUserDefinedFunctionRequest {
/**
* <p>The ID of the Data Catalog where the function to be retrieved is located. If none is
* provided, the Amazon Web Services account ID is used by default.</p>
*/
CatalogId?: string;

/**
* <p>The name of the catalog database where the function is located.</p>
*/
DatabaseName: string | undefined;

/**
* <p>The name of the function.</p>
*/
FunctionName: string | undefined;
}

/**
* @public
* <p>Represents the equivalent of a Hive user-defined function
Expand Down
11 changes: 9 additions & 2 deletions clients/client-glue/src/protocols/Aws_json1_1.ts
Original file line number Diff line number Diff line change
Expand Up @@ -831,16 +831,16 @@ import {
GetUnfilteredPartitionsMetadataRequest,
GetUnfilteredPartitionsMetadataResponse,
GetUnfilteredTableMetadataRequest,
GetUnfilteredTableMetadataResponse,
GetUserDefinedFunctionRequest,
GluePolicy,
GrokClassifier,
IcebergInput,
JobBookmarksEncryption,
JsonClassifier,
Location,
LongColumnStatisticsData,
MappingEntry,
MLTransform,
OpenTableFormatInput,
PartitionIndex,
PermissionType,
PermissionTypeMismatchException,
Expand Down Expand Up @@ -892,6 +892,8 @@ import {
DevEndpointCustomLibraries,
GetJobResponse,
GetJobsResponse,
GetUnfilteredTableMetadataResponse,
GetUserDefinedFunctionRequest,
GetUserDefinedFunctionResponse,
GetUserDefinedFunctionsRequest,
GetUserDefinedFunctionsResponse,
Expand Down Expand Up @@ -16117,6 +16119,7 @@ const se_CreateTableRequest = (input: CreateTableRequest, context: __SerdeContex
return take(input, {
CatalogId: [],
DatabaseName: [],
OpenTableFormatInput: _json,
PartitionIndexes: _json,
TableInput: (_) => se_TableInput(_, context),
TransactionId: [],
Expand Down Expand Up @@ -16656,6 +16659,8 @@ const se_GetTablesRequest = (input: GetTablesRequest, context: __SerdeContext):

// se_GovernedCatalogTarget omitted.

// se_IcebergInput omitted.

// se_IcebergTarget omitted.

// se_IcebergTargetList omitted.
Expand Down Expand Up @@ -16947,6 +16952,8 @@ const se_Mappings = (input: Mapping[], context: __SerdeContext): any => {

// se_OneInput omitted.

// se_OpenTableFormatInput omitted.

// se_Option omitted.

// se_OptionList omitted.
Expand Down
52 changes: 52 additions & 0 deletions codegen/sdk-codegen/aws-models/glue.json
Original file line number Diff line number Diff line change
Expand Up @@ -9120,6 +9120,12 @@
"traits": {
"smithy.api#documentation": "<p>The ID of the transaction.</p>"
}
},
"OpenTableFormatInput": {
"target": "com.amazonaws.glue#OpenTableFormatInput",
"traits": {
"smithy.api#documentation": "<p>Specifies an <code>OpenTableFormatInput</code> structure when creating an open format table.</p>"
}
}
},
"traits": {
Expand Down Expand Up @@ -19764,6 +19770,27 @@
}
}
},
"com.amazonaws.glue#IcebergInput": {
"type": "structure",
"members": {
"MetadataOperation": {
"target": "com.amazonaws.glue#MetadataOperation",
"traits": {
"smithy.api#documentation": "<p>A required metadata operation. Can only be set to <code>CREATE</code>.</p>",
"smithy.api#required": {}
}
},
"Version": {
"target": "com.amazonaws.glue#VersionString",
"traits": {
"smithy.api#documentation": "<p>The table version for the Iceberg table. Defaults to 2.</p>"
}
}
},
"traits": {
"smithy.api#documentation": "<p>A structure that defines an Apache Iceberg metadata table to create in the catalog.</p>"
}
},
"com.amazonaws.glue#IcebergTarget": {
"type": "structure",
"members": {
Expand Down Expand Up @@ -23862,6 +23889,17 @@
"target": "com.amazonaws.glue#MetadataKeyValuePair"
}
},
"com.amazonaws.glue#MetadataOperation": {
"type": "enum",
"members": {
"CREATE": {
"target": "smithy.api#Unit",
"traits": {
"smithy.api#enumValue": "CREATE"
}
}
}
},
"com.amazonaws.glue#MetadataValueString": {
"type": "string",
"traits": {
Expand Down Expand Up @@ -24301,6 +24339,20 @@
}
}
},
"com.amazonaws.glue#OpenTableFormatInput": {
"type": "structure",
"members": {
"IcebergInput": {
"target": "com.amazonaws.glue#IcebergInput",
"traits": {
"smithy.api#documentation": "<p>Specifies an <code>IcebergInput</code> structure that defines an Apache Iceberg metadata table.</p>"
}
}
},
"traits": {
"smithy.api#documentation": "<p>A structure representing an open format table.</p>"
}
},
"com.amazonaws.glue#OperationTimeoutException": {
"type": "structure",
"members": {
Expand Down

0 comments on commit f4ee5d6

Please sign in to comment.