diff --git a/packages/@aws-cdk/aws-redshift/README.md b/packages/@aws-cdk/aws-redshift/README.md index 8ff734a6be255..7fd769670d808 100644 --- a/packages/@aws-cdk/aws-redshift/README.md +++ b/packages/@aws-cdk/aws-redshift/README.md @@ -167,6 +167,34 @@ new Table(this, 'Table', { }); ``` +The table can be configured to have distStyle attribute and a distKey column: + +```ts fixture=cluster +new Table(this, 'Table', { + tableColumns: [ + { name: 'col1', dataType: 'varchar(4)', distKey: true }, + { name: 'col2', dataType: 'float' }, + ], + cluster: cluster, + databaseName: 'databaseName', + distStyle: TableDistStyle.KEY, +}); +``` + +The table can also be configured to have sortStyle attribute and sortKey columns: + +```ts fixture=cluster +new Table(this, 'Table', { + tableColumns: [ + { name: 'col1', dataType: 'varchar(4)', sortKey: true }, + { name: 'col2', dataType: 'float', sortKey: true }, + ], + cluster: cluster, + databaseName: 'databaseName', + sortStyle: TableSortStyle.COMPOUND, +}); +``` + ### Granting Privileges You can give a user privileges to perform certain actions on a table by using the diff --git a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/privileges.ts b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/privileges.ts index 9f2064d0e5e5a..9bbfb56754e3e 100644 --- a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/privileges.ts +++ b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/privileges.ts @@ -1,7 +1,8 @@ /* eslint-disable-next-line import/no-unresolved */ import * as AWSLambda from 'aws-lambda'; import { TablePrivilege, UserTablePrivilegesHandlerProps } from '../handler-props'; -import { ClusterProps, executeStatement, makePhysicalId } from './util'; +import { ClusterProps } from './types'; +import { executeStatement, makePhysicalId } from './util'; export async function handler(props: UserTablePrivilegesHandlerProps & ClusterProps, event: AWSLambda.CloudFormationCustomResourceEvent) { const username = props.username; diff --git a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/table.ts b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/table.ts index bc0c1d44971ff..197617757ba63 100644 --- a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/table.ts +++ b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/table.ts @@ -1,20 +1,20 @@ /* eslint-disable-next-line import/no-unresolved */ import * as AWSLambda from 'aws-lambda'; import { Column } from '../../table'; -import { TableHandlerProps } from '../handler-props'; -import { ClusterProps, executeStatement } from './util'; +import { ClusterProps, TableAndClusterProps, TableSortStyle } from './types'; +import { areColumnsEqual, executeStatement, getDistKeyColumn, getSortKeyColumns } from './util'; -export async function handler(props: TableHandlerProps & ClusterProps, event: AWSLambda.CloudFormationCustomResourceEvent) { +export async function handler(props: TableAndClusterProps, event: AWSLambda.CloudFormationCustomResourceEvent) { const tableNamePrefix = props.tableName.prefix; const tableNameSuffix = props.tableName.generateSuffix === 'true' ? `${event.RequestId.substring(0, 8)}` : ''; const tableColumns = props.tableColumns; - const clusterProps = props; + const tableAndClusterProps = props; if (event.RequestType === 'Create') { - const tableName = await createTable(tableNamePrefix, tableNameSuffix, tableColumns, clusterProps); + const tableName = await createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps); return { PhysicalResourceId: tableName }; } else if (event.RequestType === 'Delete') { - await dropTable(event.PhysicalResourceId, clusterProps); + await dropTable(event.PhysicalResourceId, tableAndClusterProps); return; } else if (event.RequestType === 'Update') { const tableName = await updateTable( @@ -22,8 +22,8 @@ export async function handler(props: TableHandlerProps & ClusterProps, event: AW tableNamePrefix, tableNameSuffix, tableColumns, - clusterProps, - event.OldResourceProperties as TableHandlerProps & ClusterProps, + tableAndClusterProps, + event.OldResourceProperties as TableAndClusterProps, ); return { PhysicalResourceId: tableName }; } else { @@ -32,10 +32,33 @@ export async function handler(props: TableHandlerProps & ClusterProps, event: AW } } -async function createTable(tableNamePrefix: string, tableNameSuffix: string, tableColumns: Column[], clusterProps: ClusterProps): Promise { +async function createTable( + tableNamePrefix: string, + tableNameSuffix: string, + tableColumns: Column[], + tableAndClusterProps: TableAndClusterProps, +): Promise { const tableName = tableNamePrefix + tableNameSuffix; const tableColumnsString = tableColumns.map(column => `${column.name} ${column.dataType}`).join(); - await executeStatement(`CREATE TABLE ${tableName} (${tableColumnsString})`, clusterProps); + + let statement = `CREATE TABLE ${tableName} (${tableColumnsString})`; + + if (tableAndClusterProps.distStyle) { + statement += ` DISTSTYLE ${tableAndClusterProps.distStyle}`; + } + + const distKeyColumn = getDistKeyColumn(tableColumns); + if (distKeyColumn) { + statement += ` DISTKEY(${distKeyColumn.name})`; + } + + const sortKeyColumns = getSortKeyColumns(tableColumns); + if (sortKeyColumns.length > 0) { + const sortKeyColumnsString = getSortKeyColumnsString(sortKeyColumns); + statement += ` ${tableAndClusterProps.sortStyle} SORTKEY(${sortKeyColumnsString})`; + } + + await executeStatement(statement, tableAndClusterProps); return tableName; } @@ -48,28 +71,79 @@ async function updateTable( tableNamePrefix: string, tableNameSuffix: string, tableColumns: Column[], - clusterProps: ClusterProps, - oldResourceProperties: TableHandlerProps & ClusterProps, + tableAndClusterProps: TableAndClusterProps, + oldResourceProperties: TableAndClusterProps, ): Promise { + const alterationStatements: string[] = []; + const oldClusterProps = oldResourceProperties; - if (clusterProps.clusterName !== oldClusterProps.clusterName || clusterProps.databaseName !== oldClusterProps.databaseName) { - return createTable(tableNamePrefix, tableNameSuffix, tableColumns, clusterProps); + if (tableAndClusterProps.clusterName !== oldClusterProps.clusterName || tableAndClusterProps.databaseName !== oldClusterProps.databaseName) { + return createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps); } const oldTableNamePrefix = oldResourceProperties.tableName.prefix; if (tableNamePrefix !== oldTableNamePrefix) { - return createTable(tableNamePrefix, tableNameSuffix, tableColumns, clusterProps); + return createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps); } const oldTableColumns = oldResourceProperties.tableColumns; if (!oldTableColumns.every(oldColumn => tableColumns.some(column => column.name === oldColumn.name && column.dataType === oldColumn.dataType))) { - return createTable(tableNamePrefix, tableNameSuffix, tableColumns, clusterProps); + return createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps); } - const additions = tableColumns.filter(column => { + const columnAdditions = tableColumns.filter(column => { return !oldTableColumns.some(oldColumn => column.name === oldColumn.name && column.dataType === oldColumn.dataType); }).map(column => `ADD ${column.name} ${column.dataType}`); - await Promise.all(additions.map(addition => executeStatement(`ALTER TABLE ${tableName} ${addition}`, clusterProps))); + if (columnAdditions.length > 0) { + alterationStatements.push(...columnAdditions.map(addition => `ALTER TABLE ${tableName} ${addition}`)); + } + + const oldDistStyle = oldResourceProperties.distStyle; + if ((!oldDistStyle && tableAndClusterProps.distStyle) || + (oldDistStyle && !tableAndClusterProps.distStyle)) { + return createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps); + } else if (oldDistStyle !== tableAndClusterProps.distStyle) { + alterationStatements.push(`ALTER TABLE ${tableName} ALTER DISTSTYLE ${tableAndClusterProps.distStyle}`); + } + + const oldDistKey = getDistKeyColumn(oldTableColumns)?.name; + const newDistKey = getDistKeyColumn(tableColumns)?.name; + if ((!oldDistKey && newDistKey ) || (oldDistKey && !newDistKey)) { + return createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps); + } else if (oldDistKey !== newDistKey) { + alterationStatements.push(`ALTER TABLE ${tableName} ALTER DISTKEY ${newDistKey}`); + } + + const oldSortKeyColumns = getSortKeyColumns(oldTableColumns); + const newSortKeyColumns = getSortKeyColumns(tableColumns); + const oldSortStyle = oldResourceProperties.sortStyle; + const newSortStyle = tableAndClusterProps.sortStyle; + if ((oldSortStyle === newSortStyle && !areColumnsEqual(oldSortKeyColumns, newSortKeyColumns)) + || (oldSortStyle !== newSortStyle)) { + switch (newSortStyle) { + case TableSortStyle.INTERLEAVED: + // INTERLEAVED sort key addition requires replacement. + // https://docs.aws.amazon.com/redshift/latest/dg/r_ALTER_TABLE.html + return createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps); + + case TableSortStyle.COMPOUND: { + const sortKeyColumnsString = getSortKeyColumnsString(newSortKeyColumns); + alterationStatements.push(`ALTER TABLE ${tableName} ALTER ${newSortStyle} SORTKEY(${sortKeyColumnsString})`); + break; + } + + case TableSortStyle.AUTO: { + alterationStatements.push(`ALTER TABLE ${tableName} ALTER SORTKEY ${newSortStyle}`); + break; + } + } + } + + await Promise.all(alterationStatements.map(statement => executeStatement(statement, tableAndClusterProps))); return tableName; } + +function getSortKeyColumnsString(sortKeyColumns: Column[]) { + return sortKeyColumns.map(column => column.name).join(); +} diff --git a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/types.ts b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/types.ts new file mode 100644 index 0000000000000..6d80398b7f41b --- /dev/null +++ b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/types.ts @@ -0,0 +1,26 @@ +import { DatabaseQueryHandlerProps, TableHandlerProps } from '../handler-props'; + +export type ClusterProps = Omit; +export type TableAndClusterProps = TableHandlerProps & ClusterProps; + +/** + * The sort style of a table. + * This has been duplicated here to exporting private types. + */ +export enum TableSortStyle { + /** + * Amazon Redshift assigns an optimal sort key based on the table data. + */ + AUTO = 'AUTO', + + /** + * Specifies that the data is sorted using a compound key made up of all of the listed columns, + * in the order they are listed. + */ + COMPOUND = 'COMPOUND', + + /** + * Specifies that the data is sorted using an interleaved sort key. + */ + INTERLEAVED = 'INTERLEAVED', +} diff --git a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/user.ts b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/user.ts index 707af78714e43..c1763048a9057 100644 --- a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/user.ts +++ b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/user.ts @@ -3,7 +3,8 @@ import * as AWSLambda from 'aws-lambda'; /* eslint-disable-next-line import/no-extraneous-dependencies */ import * as SecretsManager from 'aws-sdk/clients/secretsmanager'; import { UserHandlerProps } from '../handler-props'; -import { ClusterProps, executeStatement, makePhysicalId } from './util'; +import { ClusterProps } from './types'; +import { executeStatement, makePhysicalId } from './util'; const secretsManager = new SecretsManager(); diff --git a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/util.ts b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/util.ts index d834cd474f986..1cc1d2033dcc2 100644 --- a/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/util.ts +++ b/packages/@aws-cdk/aws-redshift/lib/private/database-query-provider/util.ts @@ -1,11 +1,10 @@ /* eslint-disable-next-line import/no-extraneous-dependencies */ import * as RedshiftData from 'aws-sdk/clients/redshiftdata'; -import { DatabaseQueryHandlerProps } from '../handler-props'; +import { Column } from '../../table'; +import { ClusterProps } from './types'; const redshiftData = new RedshiftData(); -export type ClusterProps = Omit; - export async function executeStatement(statement: string, clusterProps: ClusterProps): Promise { const executeStatementProps = { ClusterIdentifier: clusterProps.clusterName, @@ -38,3 +37,30 @@ async function waitForStatementComplete(statementId: string): Promise { export function makePhysicalId(resourceName: string, clusterProps: ClusterProps, requestId: string): string { return `${clusterProps.clusterName}:${clusterProps.databaseName}:${resourceName}:${requestId}`; } + +export function getDistKeyColumn(columns: Column[]): Column | undefined { + // string comparison is required for custom resource since everything is passed as string + const distKeyColumns = columns.filter(column => column.distKey === true || (column.distKey as unknown as string) === 'true'); + + if (distKeyColumns.length === 0) { + return undefined; + } else if (distKeyColumns.length > 1) { + throw new Error('Multiple dist key columns found'); + } + + return distKeyColumns[0]; +} + +export function getSortKeyColumns(columns: Column[]): Column[] { + // string comparison is required for custom resource since everything is passed as string + return columns.filter(column => column.sortKey === true || (column.sortKey as unknown as string) === 'true'); +} + +export function areColumnsEqual(columnsA: Column[], columnsB: Column[]): boolean { + if (columnsA.length !== columnsB.length) { + return false; + } + return columnsA.every(columnA => { + return columnsB.find(column => column.name === columnA.name && column.dataType === columnA.dataType); + }); +} diff --git a/packages/@aws-cdk/aws-redshift/lib/private/handler-props.ts b/packages/@aws-cdk/aws-redshift/lib/private/handler-props.ts index 0cc1c49066bf7..97089078f00a2 100644 --- a/packages/@aws-cdk/aws-redshift/lib/private/handler-props.ts +++ b/packages/@aws-cdk/aws-redshift/lib/private/handler-props.ts @@ -1,4 +1,4 @@ -import { Column } from '../table'; +import { Column, TableDistStyle, TableSortStyle } from '../table'; export interface DatabaseQueryHandlerProps { readonly handler: string; @@ -18,6 +18,8 @@ export interface TableHandlerProps { readonly generateSuffix: string; }; readonly tableColumns: Column[]; + readonly distStyle?: TableDistStyle; + readonly sortStyle: TableSortStyle; } export interface TablePrivilege { diff --git a/packages/@aws-cdk/aws-redshift/lib/table.ts b/packages/@aws-cdk/aws-redshift/lib/table.ts index a2d4904f0361a..c9bf7c4c46ac2 100644 --- a/packages/@aws-cdk/aws-redshift/lib/table.ts +++ b/packages/@aws-cdk/aws-redshift/lib/table.ts @@ -4,6 +4,7 @@ import { ICluster } from './cluster'; import { DatabaseOptions } from './database-options'; import { DatabaseQuery } from './private/database-query'; import { HandlerName } from './private/database-query-provider/handler-name'; +import { getDistKeyColumn, getSortKeyColumns } from './private/database-query-provider/util'; import { TableHandlerProps } from './private/handler-props'; import { IUser } from './user'; @@ -66,6 +67,20 @@ export interface Column { * The data type of the column. */ readonly dataType: string; + + /** + * Boolean value that indicates whether the column is to be configured as DISTKEY. + * + * @default - column is not DISTKEY + */ + readonly distKey?: boolean; + + /** + * Boolean value that indicates whether the column is to be configured as SORTKEY. + * + * @default - column is not a SORTKEY + */ + readonly sortKey?: boolean; } /** @@ -84,6 +99,20 @@ export interface TableProps extends DatabaseOptions { */ readonly tableColumns: Column[]; + /** + * The distribution style of the table. + * + * @default TableDistStyle.AUTO + */ + readonly distStyle?: TableDistStyle; + + /** + * The sort style of the table. + * + * @default TableSortStyle.AUTO if no sort key is specified, TableSortStyle.COMPOUND if a sort key is specified + */ + readonly sortStyle?: TableSortStyle; + /** * The policy to apply when this resource is removed from the application. * @@ -183,6 +212,14 @@ export class Table extends TableBase { constructor(scope: Construct, id: string, props: TableProps) { super(scope, id); + this.validateDistKeyColumns(props.tableColumns); + if (props.distStyle) { + this.validateDistStyle(props.distStyle, props.tableColumns); + } + if (props.sortStyle) { + this.validateSortStyle(props.sortStyle, props.tableColumns); + } + this.tableColumns = props.tableColumns; this.cluster = props.cluster; this.databaseName = props.databaseName; @@ -197,6 +234,8 @@ export class Table extends TableBase { generateSuffix: !props.tableName ? 'true' : 'false', }, tableColumns: this.tableColumns, + distStyle: props.distStyle, + sortStyle: props.sortStyle ?? this.getDefaultSortStyle(props.tableColumns), }, }); @@ -219,4 +258,83 @@ export class Table extends TableBase { public applyRemovalPolicy(policy: cdk.RemovalPolicy): void { this.resource.applyRemovalPolicy(policy); } + + private validateDistKeyColumns(columns: Column[]): void { + try { + getDistKeyColumn(columns); + } catch (err) { + throw new Error('Only one column can be configured as distKey.'); + } + } + + private validateDistStyle(distStyle: TableDistStyle, columns: Column[]): void { + const distKeyColumn = getDistKeyColumn(columns); + if (distKeyColumn && distStyle !== TableDistStyle.KEY) { + throw new Error(`Only 'TableDistStyle.KEY' can be configured when distKey is also configured. Found ${distStyle}`); + } + if (!distKeyColumn && distStyle === TableDistStyle.KEY) { + throw new Error('distStyle of "TableDistStyle.KEY" can only be configured when distKey is also configured.'); + } + } + + private validateSortStyle(sortStyle: TableSortStyle, columns: Column[]): void { + const sortKeyColumns = getSortKeyColumns(columns); + if (sortKeyColumns.length === 0 && sortStyle !== TableSortStyle.AUTO) { + throw new Error(`sortStyle of '${sortStyle}' can only be configured when sortKey is also configured.`); + } + if (sortKeyColumns.length > 0 && sortStyle === TableSortStyle.AUTO) { + throw new Error(`sortStyle of '${TableSortStyle.AUTO}' cannot be configured when sortKey is also configured.`); + } + } + + private getDefaultSortStyle(columns: Column[]): TableSortStyle { + const sortKeyColumns = getSortKeyColumns(columns); + return (sortKeyColumns.length === 0) ? TableSortStyle.AUTO : TableSortStyle.COMPOUND; + } +} + +/** + * The data distribution style of a table. + */ +export enum TableDistStyle { + /** + * Amazon Redshift assigns an optimal distribution style based on the table data + */ + AUTO = 'AUTO', + + /** + * The data in the table is spread evenly across the nodes in a cluster in a round-robin distribution. + */ + EVEN = 'EVEN', + + /** + * The data is distributed by the values in the DISTKEY column. + */ + KEY = 'KEY', + + /** + * A copy of the entire table is distributed to every node. + */ + ALL = 'ALL', +} + +/** + * The sort style of a table. + */ +export enum TableSortStyle { + /** + * Amazon Redshift assigns an optimal sort key based on the table data. + */ + AUTO = 'AUTO', + + /** + * Specifies that the data is sorted using a compound key made up of all of the listed columns, + * in the order they are listed. + */ + COMPOUND = 'COMPOUND', + + /** + * Specifies that the data is sorted using an interleaved sort key. + */ + INTERLEAVED = 'INTERLEAVED', } diff --git a/packages/@aws-cdk/aws-redshift/rosetta/cluster.ts-fixture b/packages/@aws-cdk/aws-redshift/rosetta/cluster.ts-fixture index 82d98ca3e381e..4c7ab6ccdb771 100644 --- a/packages/@aws-cdk/aws-redshift/rosetta/cluster.ts-fixture +++ b/packages/@aws-cdk/aws-redshift/rosetta/cluster.ts-fixture @@ -1,7 +1,7 @@ // Fixture with cluster already created import { Construct, SecretValue, Stack } from '@aws-cdk/core'; import { Vpc } from '@aws-cdk/aws-ec2'; -import { Cluster, Table, TableAction, User } from '@aws-cdk/aws-redshift'; +import { Cluster, Table, TableAction, TableDistStyle, TableSortStyle, User } from '@aws-cdk/aws-redshift'; class Fixture extends Stack { constructor(scope: Construct, id: string) { diff --git a/packages/@aws-cdk/aws-redshift/test/database-query-provider/table.test.ts b/packages/@aws-cdk/aws-redshift/test/database-query-provider/table.test.ts index 40456bcf6d7ca..7c5534d59a785 100644 --- a/packages/@aws-cdk/aws-redshift/test/database-query-provider/table.test.ts +++ b/packages/@aws-cdk/aws-redshift/test/database-query-provider/table.test.ts @@ -1,18 +1,30 @@ /* eslint-disable-next-line import/no-unresolved */ import type * as AWSLambda from 'aws-lambda'; +const mockExecuteStatement = jest.fn(() => ({ promise: jest.fn(() => ({ Id: 'statementId' })) })); +jest.mock('aws-sdk/clients/redshiftdata', () => class { + executeStatement = mockExecuteStatement; + describeStatement = () => ({ promise: jest.fn(() => ({ Status: 'FINISHED' })) }); +}); +import { Column, TableDistStyle, TableSortStyle } from '../../lib'; +import { handler as manageTable } from '../../lib/private/database-query-provider/table'; +import { TableAndClusterProps } from '../../lib/private/database-query-provider/types'; + +type ResourcePropertiesType = TableAndClusterProps & { ServiceToken: string }; + const tableNamePrefix = 'tableNamePrefix'; const tableColumns = [{ name: 'col1', dataType: 'varchar(1)' }]; const clusterName = 'clusterName'; const adminUserArn = 'adminUserArn'; const databaseName = 'databaseName'; const physicalResourceId = 'PhysicalResourceId'; -const resourceProperties = { +const resourceProperties: ResourcePropertiesType = { tableName: { prefix: tableNamePrefix, generateSuffix: 'true', }, tableColumns, + sortStyle: TableSortStyle.AUTO, clusterName, adminUserArn, databaseName, @@ -30,13 +42,6 @@ const genericEvent: AWSLambda.CloudFormationCustomResourceEventCommon = { ResourceType: '', }; -const mockExecuteStatement = jest.fn(() => ({ promise: jest.fn(() => ({ Id: 'statementId' })) })); -jest.mock('aws-sdk/clients/redshiftdata', () => class { - executeStatement = mockExecuteStatement; - describeStatement = () => ({ promise: jest.fn(() => ({ Status: 'FINISHED' })) }); -}); -import { handler as manageTable } from '../../lib/private/database-query-provider/table'; - beforeEach(() => { jest.clearAllMocks(); }); @@ -75,6 +80,60 @@ describe('create', () => { Sql: `CREATE TABLE ${tableNamePrefix} (col1 varchar(1))`, })); }); + + test('serializes distKey and distStyle in statement', async () => { + const event = baseEvent; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: [{ name: 'col1', dataType: 'varchar(1)', distKey: true }], + distStyle: TableDistStyle.KEY, + }; + + await manageTable(newResourceProperties, event); + + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1)) DISTSTYLE KEY DISTKEY(col1)`, + })); + }); + + test('serializes sortKeys and sortStyle in statement', async () => { + const event = baseEvent; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: [ + { name: 'col1', dataType: 'varchar(1)', sortKey: true }, + { name: 'col2', dataType: 'varchar(1)' }, + { name: 'col3', dataType: 'varchar(1)', sortKey: true }, + ], + sortStyle: TableSortStyle.COMPOUND, + }; + + await manageTable(newResourceProperties, event); + + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1),col2 varchar(1),col3 varchar(1)) COMPOUND SORTKEY(col1,col3)`, + })); + }); + + test('serializes distKey and sortKeys as string booleans', async () => { + const event = baseEvent; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: [ + { name: 'col1', dataType: 'varchar(4)', distKey: 'true' as unknown as boolean }, + { name: 'col2', dataType: 'float', sortKey: 'true' as unknown as boolean }, + { name: 'col3', dataType: 'float', sortKey: 'true' as unknown as boolean }, + ], + distStyle: TableDistStyle.KEY, + sortStyle: TableSortStyle.COMPOUND, + }; + + await manageTable(newResourceProperties, event); + + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(4),col2 float,col3 float) DISTSTYLE KEY DISTKEY(col1) COMPOUND SORTKEY(col2,col3)`, + })); + }); }); describe('delete', () => { @@ -199,4 +258,251 @@ describe('update', () => { Sql: `ALTER TABLE ${physicalResourceId} ADD ${newTableColumnName} ${newTableColumnDataType}`, })); }); + + describe('distStyle and distKey', () => { + test('replaces if distStyle is added', async () => { + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + distStyle: TableDistStyle.EVEN, + }; + + await expect(manageTable(newResourceProperties, event)).resolves.not.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1)) DISTSTYLE EVEN`, + })); + }); + + test('replaces if distStyle is removed', async () => { + const newEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + distStyle: TableDistStyle.EVEN, + }, + }; + const newResourceProperties = { + ...resourceProperties, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.not.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1))`, + })); + }); + + test('does not replace if distStyle is changed', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + distStyle: TableDistStyle.EVEN, + }, + }; + const newDistStyle = TableDistStyle.ALL; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + distStyle: newDistStyle, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `ALTER TABLE ${physicalResourceId} ALTER DISTSTYLE ${newDistStyle}`, + })); + }); + + test('replaces if distKey is added', async () => { + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: [{ name: 'col1', dataType: 'varchar(1)', distKey: true }], + }; + + await expect(manageTable(newResourceProperties, event)).resolves.not.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1)) DISTKEY(col1)`, + })); + }); + + test('replaces if distKey is removed', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + tableColumns: [{ name: 'col1', dataType: 'varchar(1)', distKey: true }], + }, + }; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.not.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1))`, + })); + }); + + test('does not replace if distKey is changed', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + tableColumns: [ + { name: 'col1', dataType: 'varchar(1)', distKey: true }, + { name: 'col2', dataType: 'varchar(1)' }, + ], + }, + }; + const newDistKey = 'col2'; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: [ + { name: 'col1', dataType: 'varchar(1)' }, + { name: 'col2', dataType: 'varchar(1)', distKey: true }, + ], + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `ALTER TABLE ${physicalResourceId} ALTER DISTKEY ${newDistKey}`, + })); + }); + }); + + describe('sortStyle and sortKeys', () => { + const oldTableColumnsWithSortKeys: Column[] = [ + { name: 'col1', dataType: 'varchar(1)', sortKey: true }, + { name: 'col2', dataType: 'varchar(1)' }, + ]; + const newTableColumnsWithSortKeys: Column[] = [ + { name: 'col1', dataType: 'varchar(1)' }, + { name: 'col2', dataType: 'varchar(1)', sortKey: true }, + ]; + + test('replaces when same sortStyle, different sortKey columns: INTERLEAVED', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.INTERLEAVED, + }, + }; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: newTableColumnsWithSortKeys, + sortStyle: TableSortStyle.INTERLEAVED, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.not.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1),col2 varchar(1)) INTERLEAVED SORTKEY(col2)`, + })); + }); + + test('replaces when differnt sortStyle: INTERLEAVED', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.AUTO, + }, + }; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.INTERLEAVED, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.not.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1),col2 varchar(1)) INTERLEAVED SORTKEY(col1)`, + })); + }); + + test('does not replace when same sortStyle, different sortKey columns: COMPOUND', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.COMPOUND, + }, + }; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: newTableColumnsWithSortKeys, + sortStyle: TableSortStyle.COMPOUND, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `ALTER TABLE ${physicalResourceId} ALTER COMPOUND SORTKEY(col2)`, + })); + }); + + test('does not replace when differnt sortStyle: COMPOUND', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.AUTO, + }, + }; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.COMPOUND, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `ALTER TABLE ${physicalResourceId} ALTER COMPOUND SORTKEY(col1)`, + })); + }); + + test('does not replace when differnt sortStyle: AUTO', async () => { + const newEvent: AWSLambda.CloudFormationCustomResourceEvent = { + ...event, + OldResourceProperties: { + ...event.OldResourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.COMPOUND, + }, + }; + const newResourceProperties: ResourcePropertiesType = { + ...resourceProperties, + tableColumns: oldTableColumnsWithSortKeys, + sortStyle: TableSortStyle.AUTO, + }; + + await expect(manageTable(newResourceProperties, newEvent)).resolves.toMatchObject({ + PhysicalResourceId: physicalResourceId, + }); + expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({ + Sql: `ALTER TABLE ${physicalResourceId} ALTER SORTKEY AUTO`, + })); + }); + }); + }); diff --git a/packages/@aws-cdk/aws-redshift/test/integ.database.expected.json b/packages/@aws-cdk/aws-redshift/test/integ.database.expected.json index f38a1d5b3818e..6e909192a7f3d 100644 --- a/packages/@aws-cdk/aws-redshift/test/integ.database.expected.json +++ b/packages/@aws-cdk/aws-redshift/test/integ.database.expected.json @@ -1167,7 +1167,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4fS3Bucket3B967306" + "Ref": "AssetParameters85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066S3Bucket0B347C2E" }, "S3Key": { "Fn::Join": [ @@ -1180,7 +1180,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4fS3VersionKeyC171429B" + "Ref": "AssetParameters85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066S3VersionKey932D0479" } ] } @@ -1193,7 +1193,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4fS3VersionKeyC171429B" + "Ref": "AssetParameters85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066S3VersionKey932D0479" } ] } @@ -1374,30 +1374,39 @@ "tableColumns": [ { "name": "col1", - "dataType": "varchar(4)" + "dataType": "varchar(4)", + "distKey": true }, { "name": "col2", - "dataType": "float" + "dataType": "float", + "sortKey": true + }, + { + "name": "col3", + "dataType": "float", + "sortKey": true } - ] + ], + "distStyle": "KEY", + "sortStyle": "INTERLEAVED" }, "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" } }, "Parameters": { - "AssetParameters7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4fS3Bucket3B967306": { + "AssetParameters85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066S3Bucket0B347C2E": { "Type": "String", - "Description": "S3 bucket for asset \"7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4f\"" + "Description": "S3 bucket for asset \"85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066\"" }, - "AssetParameters7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4fS3VersionKeyC171429B": { + "AssetParameters85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066S3VersionKey932D0479": { "Type": "String", - "Description": "S3 key for asset version \"7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4f\"" + "Description": "S3 key for asset version \"85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066\"" }, - "AssetParameters7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4fArtifactHash0EE8CD3D": { + "AssetParameters85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066ArtifactHash78689978": { "Type": "String", - "Description": "Artifact hash for asset \"7eb6a250bd5ce32c07f08f536377d71e59ad43e16e25b9aa6e50f6fc20fdfc4f\"" + "Description": "Artifact hash for asset \"85597bcd6a07abd4673fe02c7e92e21df5859eee0d831e9db67f4d2e74d4d066\"" }, "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1": { "Type": "String", diff --git a/packages/@aws-cdk/aws-redshift/test/integ.database.ts b/packages/@aws-cdk/aws-redshift/test/integ.database.ts index 6e4893c0c0089..d5079b83f0c1b 100644 --- a/packages/@aws-cdk/aws-redshift/test/integ.database.ts +++ b/packages/@aws-cdk/aws-redshift/test/integ.database.ts @@ -39,7 +39,13 @@ const databaseOptions = { const user = new redshift.User(stack, 'User', databaseOptions); const table = new redshift.Table(stack, 'Table', { ...databaseOptions, - tableColumns: [{ name: 'col1', dataType: 'varchar(4)' }, { name: 'col2', dataType: 'float' }], + tableColumns: [ + { name: 'col1', dataType: 'varchar(4)', distKey: true }, + { name: 'col2', dataType: 'float', sortKey: true }, + { name: 'col3', dataType: 'float', sortKey: true }, + ], + distStyle: redshift.TableDistStyle.KEY, + sortStyle: redshift.TableSortStyle.INTERLEAVED, }); table.grant(user, redshift.TableAction.INSERT, redshift.TableAction.DELETE); diff --git a/packages/@aws-cdk/aws-redshift/test/table.test.ts b/packages/@aws-cdk/aws-redshift/test/table.test.ts index dda05eb9cc063..571a87fff5227 100644 --- a/packages/@aws-cdk/aws-redshift/test/table.test.ts +++ b/packages/@aws-cdk/aws-redshift/test/table.test.ts @@ -5,7 +5,10 @@ import * as redshift from '../lib'; describe('cluster table', () => { const tableName = 'tableName'; - const tableColumns = [{ name: 'col1', dataType: 'varchar(4)' }, { name: 'col2', dataType: 'float' }]; + const tableColumns: redshift.Column[] = [ + { name: 'col1', dataType: 'varchar(4)' }, + { name: 'col2', dataType: 'float' }, + ]; let stack: cdk.Stack; let vpc: ec2.Vpc; @@ -135,4 +138,110 @@ describe('cluster table', () => { DeletionPolicy: 'Delete', }); }); + + describe('distKey and distStyle', () => { + it('throws if more than one distKeys are configured', () => { + const updatedTableColumns: redshift.Column[] = [ + ...tableColumns, + { name: 'col3', dataType: 'varchar(4)', distKey: true }, + { name: 'col4', dataType: 'float', distKey: true }, + ]; + + expect( + () => new redshift.Table(stack, 'Table', { + ...databaseOptions, + tableColumns: updatedTableColumns, + }), + ).toThrow(/Only one column can be configured as distKey./); + }); + + it('throws if distStyle other than KEY is configured with configured distKey column', () => { + const updatedTableColumns: redshift.Column[] = [ + ...tableColumns, + { name: 'col3', dataType: 'varchar(4)', distKey: true }, + ]; + + expect( + () => new redshift.Table(stack, 'Table', { + ...databaseOptions, + tableColumns: updatedTableColumns, + distStyle: redshift.TableDistStyle.EVEN, + }), + ).toThrow(`Only 'TableDistStyle.KEY' can be configured when distKey is also configured. Found ${redshift.TableDistStyle.EVEN}`); + }); + + it('throws if KEY distStyle is configired with no distKey column', () => { + expect( + () => new redshift.Table(stack, 'Table', { + ...databaseOptions, + tableColumns, + distStyle: redshift.TableDistStyle.KEY, + }), + ).toThrow('distStyle of "TableDistStyle.KEY" can only be configured when distKey is also configured.'); + }); + }); + + describe('sortKeys and sortStyle', () => { + it('configures default sortStyle based on sortKeys if no sortStyle is passed: AUTO', () => { + // GIVEN + const tableColumnsWithoutSortKey = tableColumns; + + // WHEN + new redshift.Table(stack, 'Table', { + ...databaseOptions, + tableColumns: tableColumnsWithoutSortKey, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('Custom::RedshiftDatabaseQuery', { + sortStyle: redshift.TableSortStyle.AUTO, + }); + }); + + it('configures default sortStyle based on sortKeys if no sortStyle is passed: COMPOUND', () => { + // GIVEN + const tableColumnsWithSortKey: redshift.Column[] = [ + ...tableColumns, + { name: 'col3', dataType: 'varchar(4)', sortKey: true }, + ]; + + // WHEN + new redshift.Table(stack, 'Table', { + ...databaseOptions, + tableColumns: tableColumnsWithSortKey, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('Custom::RedshiftDatabaseQuery', { + sortStyle: redshift.TableSortStyle.COMPOUND, + }); + }); + + it('throws if sortStlye other than AUTO is passed with no configured sortKeys', () => { + expect( + () => new redshift.Table(stack, 'Table', { + ...databaseOptions, + tableColumns, + sortStyle: redshift.TableSortStyle.COMPOUND, + }), + ).toThrow(`sortStyle of '${redshift.TableSortStyle.COMPOUND}' can only be configured when sortKey is also configured.`); + }); + + it('throws if sortStlye of AUTO is passed with some configured sortKeys', () => { + // GIVEN + const tableColumnsWithSortKey: redshift.Column[] = [ + ...tableColumns, + { name: 'col3', dataType: 'varchar(4)', sortKey: true }, + ]; + + // THEN + expect( + () => new redshift.Table(stack, 'Table', { + ...databaseOptions, + tableColumns: tableColumnsWithSortKey, + sortStyle: redshift.TableSortStyle.AUTO, + }), + ).toThrow(`sortStyle of '${redshift.TableSortStyle.AUTO}' cannot be configured when sortKey is also configured.`); + }); + }); });