From 97b47bf87a03ef30462678e06caefe64498f2bfd Mon Sep 17 00:00:00 2001 From: Adam Ruka Date: Mon, 14 Oct 2019 18:05:06 -0700 Subject: [PATCH] feat(vpc): allow Vpc.fromLookup() to discover asymmetric subnets Previously, Vpc.fromLookup() required every subnet group to be in the same Availability Zones, and for all subnets in all groups to cover all of those Availability Zones. This loosens this requirement, so that now Vpc.fromLookup() works for VPCs of all shapes. This also add a way to customize which tag of the subnet is considered when grouping subnets into groups when calling fromLookup(). Fixes #3407 --- packages/@aws-cdk/aws-ec2/lib/vpc-lookup.ts | 10 + packages/@aws-cdk/aws-ec2/lib/vpc.ts | 116 ++++- .../aws-ec2/test/test.vpc.from-lookup.ts | 148 ++++++ packages/@aws-cdk/aws-ec2/test/test.vpc.ts | 29 -- .../@aws-cdk/core/lib/context-provider.ts | 5 + packages/@aws-cdk/core/test/test.context.ts | 27 ++ packages/@aws-cdk/cx-api/lib/context/vpc.ts | 85 ++++ packages/@aws-cdk/cx-api/lib/versioning.ts | 9 +- .../__snapshots__/cloud-assembly.test.js.snap | 2 +- .../aws-cdk/lib/context-providers/vpcs.ts | 54 ++- .../context-providers/test.asymmetric-vpcs.ts | 455 ++++++++++++++++++ .../test/context-providers/test.vpcs.ts | 9 +- tools/cdk-integ-tools/lib/integ-helpers.ts | 31 +- 13 files changed, 924 insertions(+), 56 deletions(-) create mode 100644 packages/@aws-cdk/aws-ec2/test/test.vpc.from-lookup.ts create mode 100644 packages/aws-cdk/test/context-providers/test.asymmetric-vpcs.ts diff --git a/packages/@aws-cdk/aws-ec2/lib/vpc-lookup.ts b/packages/@aws-cdk/aws-ec2/lib/vpc-lookup.ts index 573ff75538e2e..3f9f49dba2540 100644 --- a/packages/@aws-cdk/aws-ec2/lib/vpc-lookup.ts +++ b/packages/@aws-cdk/aws-ec2/lib/vpc-lookup.ts @@ -38,4 +38,14 @@ export interface VpcLookupOptions { * @default Don't care whether we return the default VPC */ readonly isDefault?: boolean; + + /** + * Optional tag for subnet group name. + * If not provided, we'll look at the aws-cdk:subnet-name tag. + * If the subnet does not have the specified tag, + * we'll use its type as the name. + * + * @default aws-cdk:subnet-name + */ + readonly subnetGroupNameTag?: string; } diff --git a/packages/@aws-cdk/aws-ec2/lib/vpc.ts b/packages/@aws-cdk/aws-ec2/lib/vpc.ts index 1a809380d38ab..678eed082f915 100644 --- a/packages/@aws-cdk/aws-ec2/lib/vpc.ts +++ b/packages/@aws-cdk/aws-ec2/lib/vpc.ts @@ -847,13 +847,17 @@ export class Vpc extends VpcBase { filter.isDefault = options.isDefault ? 'true' : 'false'; } - const attributes = ContextProvider.getValue(scope, { + const attributes: cxapi.VpcContextResponse = ContextProvider.getValue(scope, { provider: cxapi.VPC_PROVIDER, - props: { filter } as cxapi.VpcContextQuery, - dummyValue: undefined + props: { + filter, + returnAsymmetricSubnets: true, + subnetGroupNameTag: options.subnetGroupNameTag, + } as cxapi.VpcContextQuery, + dummyValue: undefined, }).value; - return new ImportedVpc(scope, id, attributes || DUMMY_VPC_PROPS, attributes === undefined); + return new LookedUpVpc(scope, id, attributes || DUMMY_VPC_PROPS, attributes === undefined); /** * Prefixes all keys in the argument with `tag:`.` @@ -1486,6 +1490,61 @@ class ImportedVpc extends VpcBase { } } +class LookedUpVpc extends VpcBase { + public readonly vpcId: string; + public readonly vpnGatewayId?: string; + public readonly internetConnectivityEstablished: IDependable = new ConcreteDependable(); + public readonly availabilityZones: string[]; + public readonly publicSubnets: ISubnet[]; + public readonly privateSubnets: ISubnet[]; + public readonly isolatedSubnets: ISubnet[]; + + constructor(scope: Construct, id: string, props: cxapi.VpcContextResponse, isIncomplete: boolean) { + super(scope, id); + + this.vpcId = props.vpcId; + this.vpnGatewayId = props.vpnGatewayId; + this.incompleteSubnetDefinition = isIncomplete; + + const subnetGroups = props.subnetGroups || []; + const availabilityZones = Array.from(new Set(flatMap(subnetGroups, subnetGroup => { + return subnetGroup.subnets.map(subnet => subnet.availabilityZone); + }))); + availabilityZones.sort((az1, az2) => az1.localeCompare(az2)); + this.availabilityZones = availabilityZones; + + this.publicSubnets = this.extractSubnetsOfType(subnetGroups, cxapi.VpcSubnetGroupType.PUBLIC); + this.privateSubnets = this.extractSubnetsOfType(subnetGroups, cxapi.VpcSubnetGroupType.PRIVATE); + this.isolatedSubnets = this.extractSubnetsOfType(subnetGroups, cxapi.VpcSubnetGroupType.ISOLATED); + } + + private extractSubnetsOfType(subnetGroups: cxapi.VpcSubnetGroup[], subnetGroupType: cxapi.VpcSubnetGroupType): ISubnet[] { + return flatMap(subnetGroups.filter(subnetGroup => subnetGroup.type === subnetGroupType), + subnetGroup => this.subnetGroupToSubnets(subnetGroup)); + } + + private subnetGroupToSubnets(subnetGroup: cxapi.VpcSubnetGroup): ISubnet[] { + const ret = new Array(); + for (let i = 0; i < subnetGroup.subnets.length; i++) { + const vpcSubnet = subnetGroup.subnets[i]; + ret.push(Subnet.fromSubnetAttributes(this, `${subnetGroup.name}Subnet${i + 1}`, { + availabilityZone: vpcSubnet.availabilityZone, + subnetId: vpcSubnet.subnetId, + routeTableId: vpcSubnet.routeTableId, + })); + } + return ret; + } +} + +function flatMap(xs: T[], fn: (x: T) => U[]): U[] { + const ret = new Array(); + for (const x of xs) { + ret.push(...fn(x)); + } + return ret; +} + class CompositeDependable implements IDependable { private readonly dependables = new Array(); @@ -1589,10 +1648,49 @@ function determineNatGatewayCount(requestedCount: number | undefined, subnetConf * It's only used for testing and on the first run-through. */ const DUMMY_VPC_PROPS: cxapi.VpcContextResponse = { - availabilityZones: ['dummy-1a', 'dummy-1b'], + availabilityZones: [], + isolatedSubnetIds: undefined, + isolatedSubnetNames: undefined, + isolatedSubnetRouteTableIds: undefined, + privateSubnetIds: undefined, + privateSubnetNames: undefined, + privateSubnetRouteTableIds: undefined, + publicSubnetIds: undefined, + publicSubnetNames: undefined, + publicSubnetRouteTableIds: undefined, + subnetGroups: [ + { + name: 'Public', + type: cxapi.VpcSubnetGroupType.PUBLIC, + subnets: [ + { + availabilityZone: 'dummy-1a', + subnetId: 's-12345', + routeTableId: 'rtb-12345s', + }, + { + availabilityZone: 'dummy-1b', + subnetId: 's-67890', + routeTableId: 'rtb-67890s', + }, + ], + }, + { + name: 'Private', + type: cxapi.VpcSubnetGroupType.PRIVATE, + subnets: [ + { + availabilityZone: 'dummy-1a', + subnetId: 'p-12345', + routeTableId: 'rtb-12345p', + }, + { + availabilityZone: 'dummy-1b', + subnetId: 'p-67890', + routeTableId: 'rtb-57890p', + }, + ], + }, + ], vpcId: 'vpc-12345', - publicSubnetIds: ['s-12345', 's-67890'], - publicSubnetRouteTableIds: ['rtb-12345s', 'rtb-67890s'], - privateSubnetIds: ['p-12345', 'p-67890'], - privateSubnetRouteTableIds: ['rtb-12345p', 'rtb-57890p'], }; diff --git a/packages/@aws-cdk/aws-ec2/test/test.vpc.from-lookup.ts b/packages/@aws-cdk/aws-ec2/test/test.vpc.from-lookup.ts new file mode 100644 index 0000000000000..8ab533010098b --- /dev/null +++ b/packages/@aws-cdk/aws-ec2/test/test.vpc.from-lookup.ts @@ -0,0 +1,148 @@ +import { Construct, ContextProvider, GetContextValueOptions, GetContextValueResult, Lazy, Stack } from "@aws-cdk/core"; +import cxapi = require('@aws-cdk/cx-api'); +import { Test } from 'nodeunit'; +import { Vpc } from "../lib"; + +export = { + 'Vpc.fromLookup()': { + 'requires concrete values'(test: Test) { + // GIVEN + const stack = new Stack(); + + test.throws(() => { + Vpc.fromLookup(stack, 'Vpc', { + vpcId: Lazy.stringValue({ produce: () => 'some-id' }) + }); + + }, 'All arguments to Vpc.fromLookup() must be concrete'); + + test.done(); + }, + + 'selecting subnets by name from a looked-up VPC does not throw'(test: Test) { + // GIVEN + const stack = new Stack(undefined, undefined, { env: { region: 'us-east-1', account: '123456789012' }}); + const vpc = Vpc.fromLookup(stack, 'VPC', { + vpcId: 'vpc-1234' + }); + + // WHEN + vpc.selectSubnets({ subnetName: 'Bleep' }); + + // THEN: no exception + + test.done(); + }, + + 'accepts asymmetric subnets'(test: Test) { + const previous = mockVpcContextProviderWith(test, { + vpcId: 'vpc-1234', + subnetGroups: [ + { + name: 'Public', + type: cxapi.VpcSubnetGroupType.PUBLIC, + subnets: [ + { + subnetId: 'pub-sub-in-us-east-1a', + availabilityZone: 'us-east-1a', + routeTableId: 'rt-123', + }, + { + subnetId: 'pub-sub-in-us-east-1b', + availabilityZone: 'us-east-1b', + routeTableId: 'rt-123', + }, + ], + }, + { + name: 'Private', + type: cxapi.VpcSubnetGroupType.PRIVATE, + subnets: [ + { + subnetId: 'pri-sub-1-in-us-east-1c', + availabilityZone: 'us-east-1c', + routeTableId: 'rt-123', + }, + { + subnetId: 'pri-sub-2-in-us-east-1c', + availabilityZone: 'us-east-1c', + routeTableId: 'rt-123', + }, + { + subnetId: 'pri-sub-1-in-us-east-1d', + availabilityZone: 'us-east-1d', + routeTableId: 'rt-123', + }, + { + subnetId: 'pri-sub-2-in-us-east-1d', + availabilityZone: 'us-east-1d', + routeTableId: 'rt-123', + }, + ], + }, + ], + }, options => { + test.deepEqual(options.filter, { + isDefault: 'true', + }); + + test.equal(options.subnetGroupNameTag, undefined); + }); + + const stack = new Stack(); + const vpc = Vpc.fromLookup(stack, 'Vpc', { + isDefault: true, + }); + + test.deepEqual(vpc.availabilityZones, ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d']); + test.equal(vpc.publicSubnets.length, 2); + test.equal(vpc.privateSubnets.length, 4); + test.equal(vpc.isolatedSubnets.length, 0); + + restoreContextProvider(previous); + test.done(); + }, + }, +}; + +interface MockVcpContextResponse { + readonly vpcId: string; + readonly subnetGroups: cxapi.VpcSubnetGroup[]; +} + +function mockVpcContextProviderWith(test: Test, response: MockVcpContextResponse, + paramValidator?: (options: cxapi.VpcContextQuery) => void) { + const previous = ContextProvider.getValue; + ContextProvider.getValue = (_scope: Construct, options: GetContextValueOptions) => { + // do some basic sanity checks + test.equal(options.provider, cxapi.VPC_PROVIDER, + `Expected provider to be: '${cxapi.VPC_PROVIDER}', got: '${options.provider}'`); + test.equal((options.props || {}).returnAsymmetricSubnets, true, + `Expected options.props.returnAsymmetricSubnets to be true, got: '${(options.props || {}).returnAsymmetricSubnets}'`); + + if (paramValidator) { + paramValidator(options.props as any); + } + + return { + value: { + availabilityZones: [], + isolatedSubnetIds: undefined, + isolatedSubnetNames: undefined, + isolatedSubnetRouteTableIds: undefined, + privateSubnetIds: undefined, + privateSubnetNames: undefined, + privateSubnetRouteTableIds: undefined, + publicSubnetIds: undefined, + publicSubnetNames: undefined, + publicSubnetRouteTableIds: undefined, + ...response, + } as cxapi.VpcContextResponse, + }; + }; + return previous; +} + +function restoreContextProvider(previous: (scope: Construct, options: GetContextValueOptions) => GetContextValueResult): void { + ContextProvider.getValue = previous; +} diff --git a/packages/@aws-cdk/aws-ec2/test/test.vpc.ts b/packages/@aws-cdk/aws-ec2/test/test.vpc.ts index 4d6021ed9902d..28d2f3017ae9c 100644 --- a/packages/@aws-cdk/aws-ec2/test/test.vpc.ts +++ b/packages/@aws-cdk/aws-ec2/test/test.vpc.ts @@ -907,35 +907,6 @@ export = { test.done(); } }, - - 'fromLookup() requires concrete values'(test: Test) { - // GIVEN - const stack = new Stack(); - - test.throws(() => { - Vpc.fromLookup(stack, 'Vpc', { - vpcId: Lazy.stringValue({ produce: () => 'some-id' }) - }); - - }, 'All arguments to Vpc.fromLookup() must be concrete'); - - test.done(); - }, - - 'selecting subnets by name from a looked-up VPC does not throw'(test: Test) { - // GIVEN - const stack = new Stack(undefined, undefined, { env: { region: 'us-east-1', account: '123456789012' }}); - const vpc = Vpc.fromLookup(stack, 'VPC', { - vpcId: 'vpc-1234' - }); - - // WHEN - vpc.selectSubnets({ subnetName: 'Bleep' }); - - // THEN: no exception - - test.done(); - }, }; function getTestStack(): Stack { diff --git a/packages/@aws-cdk/core/lib/context-provider.ts b/packages/@aws-cdk/core/lib/context-provider.ts index 5045dcfb6f795..14ae005f53962 100644 --- a/packages/@aws-cdk/core/lib/context-provider.ts +++ b/packages/@aws-cdk/core/lib/context-provider.ts @@ -137,6 +137,11 @@ function propsToArray(props: {[key: string]: any}, keyPrefix = ''): string[] { const ret: string[] = []; for (const key of Object.keys(props)) { + // skip undefined values + if (props[key] === undefined) { + continue; + } + switch (typeof props[key]) { case 'object': { ret.push(...propsToArray(props[key], `${keyPrefix}${key}.`)); diff --git a/packages/@aws-cdk/core/test/test.context.ts b/packages/@aws-cdk/core/test/test.context.ts index 76d90489762ae..fa2fbeb372b31 100644 --- a/packages/@aws-cdk/core/test/test.context.ts +++ b/packages/@aws-cdk/core/test/test.context.ts @@ -112,6 +112,33 @@ export = { test.done(); }, + 'Keys with undefined values are not serialized'(test: Test) { + // GIVEN + const stack = new Stack(undefined, 'TestStack', { env: { account: '12345', region: 'us-east-1' } }); + + // WHEN + const result = ContextProvider.getKey(stack, { + provider: 'provider', + props: { + p1: 42, + p2: undefined, + }, + }); + + // THEN + test.deepEqual(result, { + key: 'provider:account=12345:p1=42:region=us-east-1', + props: { + account: '12345', + region: 'us-east-1', + p1: 42, + p2: undefined, + }, + }); + + test.done(); + }, + 'context provider errors are attached to tree'(test: Test) { const contextProps = { provider: 'bloop' }; const contextKey = 'bloop:account=12345:region=us-east-1'; // Depends on the mangling algo diff --git a/packages/@aws-cdk/cx-api/lib/context/vpc.ts b/packages/@aws-cdk/cx-api/lib/context/vpc.ts index e82f6e90f5bbd..ef0f779e0f402 100644 --- a/packages/@aws-cdk/cx-api/lib/context/vpc.ts +++ b/packages/@aws-cdk/cx-api/lib/context/vpc.ts @@ -22,6 +22,80 @@ export interface VpcContextQuery { * @see https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html */ readonly filter: {[key: string]: string}; + + /** + * Whether to populate the subnetGroups field of the {@link VpcContextResponse}, + * which contains potentially asymmetric subnet groups. + * + * @default false + */ + readonly returnAsymmetricSubnets?: boolean; + + /** + * Optional tag for subnet group name. + * If not provided, we'll look at the aws-cdk:subnet-name tag. + * If the subnet does not have the specified tag, + * we'll use its type as the name. + * + * @default 'aws-cdk:subnet-name' + */ + readonly subnetGroupNameTag?: string; +} + +/** + * The type of subnet group. + * Same as SubnetType in the @aws-cdk/aws-ec2 package, + * but we can't use that because of cyclical dependencies. + */ +export enum VpcSubnetGroupType { + /** Public subnet group type. */ + PUBLIC = 'Public', + + /** Private subnet group type. */ + PRIVATE = 'Private', + + /** Isolated subnet group type. */ + ISOLATED = 'Isolated', +} + +/** + * A subnet representation that the VPC provider uses. + */ +export interface VpcSubnet { + /** The identifier of the subnet. */ + readonly subnetId: string; + + /** + * The code of the availability zone this subnet is in + * (for example, 'us-west-2a'). + */ + readonly availabilityZone: string; + + /** The identifier of the route table for this subnet. */ + readonly routeTableId: string; +} + +/** + * A group of subnets returned by the VPC provider. + * The included subnets do NOT have to be symmetric! + */ +export interface VpcSubnetGroup { + /** + * The name of the subnet group, + * determined by looking at the tags of of the subnets + * that belong to it. + */ + readonly name: string; + + /** The type of the subnet group. */ + readonly type: VpcSubnetGroupType; + + /** + * The subnets that are part of this group. + * There is no condition that the subnets have to be symmetric + * in the group. + */ + readonly subnets: VpcSubnet[]; } /** @@ -106,4 +180,15 @@ export interface VpcContextResponse { * The VPN gateway ID */ readonly vpnGatewayId?: string; + + /** + * The subnet groups discovered for the given VPC. + * Unlike the above properties, this will include asymmetric subnets, + * if the VPC has any. + * This property will only be populated if {@link VpcContextQuery.returnAsymmetricSubnets} + * is true. + * + * @default - no subnet groups will be returned unless {@link VpcContextQuery.returnAsymmetricSubnets} is true + */ + readonly subnetGroups?: VpcSubnetGroup[]; } diff --git a/packages/@aws-cdk/cx-api/lib/versioning.ts b/packages/@aws-cdk/cx-api/lib/versioning.ts index 3ea2dbda6b429..17dc186fae9a4 100644 --- a/packages/@aws-cdk/cx-api/lib/versioning.ts +++ b/packages/@aws-cdk/cx-api/lib/versioning.ts @@ -31,7 +31,7 @@ import { AssemblyManifest } from './cloud-assembly'; * Note that the versions are not compared in a semver way, they are used as * opaque ordered tokens. */ -export const CLOUD_ASSEMBLY_VERSION = '1.10.0'; +export const CLOUD_ASSEMBLY_VERSION = '1.16.0'; /** * Look at the type of response we get and upgrade it to the latest expected version @@ -64,6 +64,11 @@ export function upgradeAssemblyManifest(manifest: AssemblyManifest): AssemblyMan manifest = justUpgradeVersion(manifest, '1.10.0'); } + if (manifest.version === '1.10.0') { + // backwards-compatible changes to the VPC provider + manifest = justUpgradeVersion(manifest, '1.16.0'); + } + return manifest; } @@ -84,4 +89,4 @@ function parseSemver(version: string) { */ function justUpgradeVersion(manifest: AssemblyManifest, version: string): AssemblyManifest { return Object.assign({}, manifest, { version }); -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/cx-api/test/__snapshots__/cloud-assembly.test.js.snap b/packages/@aws-cdk/cx-api/test/__snapshots__/cloud-assembly.test.js.snap index 745e536859f55..3549e8352e60e 100644 --- a/packages/@aws-cdk/cx-api/test/__snapshots__/cloud-assembly.test.js.snap +++ b/packages/@aws-cdk/cx-api/test/__snapshots__/cloud-assembly.test.js.snap @@ -48,7 +48,7 @@ Array [ exports[`empty assembly 1`] = ` Object { - "version": "1.10.0", + "version": "1.16.0", } `; diff --git a/packages/aws-cdk/lib/context-providers/vpcs.ts b/packages/aws-cdk/lib/context-providers/vpcs.ts index e065d89c3b636..c92f8a37d3ce1 100644 --- a/packages/aws-cdk/lib/context-providers/vpcs.ts +++ b/packages/aws-cdk/lib/context-providers/vpcs.ts @@ -17,7 +17,7 @@ export class VpcNetworkContextProviderPlugin implements ContextProviderPlugin { const vpcId = await this.findVpc(ec2, args); - return await this.readVpcProps(ec2, vpcId); + return await this.readVpcProps(ec2, vpcId, args); } private async findVpc(ec2: AWS.EC2, args: cxapi.VpcContextQuery): Promise { @@ -38,7 +38,7 @@ export class VpcNetworkContextProviderPlugin implements ContextProviderPlugin { return vpcs[0].VpcId!; } - private async readVpcProps(ec2: AWS.EC2, vpcId: string): Promise { + private async readVpcProps(ec2: AWS.EC2, vpcId: string, args: cxapi.VpcContextQuery): Promise { debug(`Describing VPC ${vpcId}`); const filters = { Filters: [{ Name: 'vpc-id', Values: [vpcId] }] }; @@ -71,7 +71,7 @@ export class VpcNetworkContextProviderPlugin implements ContextProviderPlugin { throw new Error(`Subnet ${subnet.SubnetArn} has invalid subnet type ${type} (must be ${SubnetType.Public}, ${SubnetType.Private} or ${SubnetType.Isolated})`); } - const name = getTag('aws-cdk:subnet-name', subnet.Tags) || type; + const name = getTag(args.subnetGroupNameTag || 'aws-cdk:subnet-name', subnet.Tags) || type; const routeTableId = routeTables.routeTableIdForSubnetId(subnet.SubnetId); if (!routeTableId) { @@ -87,7 +87,15 @@ export class VpcNetworkContextProviderPlugin implements ContextProviderPlugin { }; }); - const grouped = groupSubnets(subnets); + let grouped: SubnetGroups; + let assymetricSubnetGroups: cxapi.VpcSubnetGroup[] | undefined; + if (args.returnAsymmetricSubnets) { + grouped = { azs: [], groups: [] }; + assymetricSubnetGroups = groupAsymmetricSubnets(subnets); + } else { + grouped = groupSubnets(subnets); + assymetricSubnetGroups = undefined; + } // Find attached+available VPN gateway for this VPC const vpnGatewayResponse = await ec2.describeVpnGateways({ @@ -123,6 +131,7 @@ export class VpcNetworkContextProviderPlugin implements ContextProviderPlugin { publicSubnetNames: collapse(flatMap(findGroups(SubnetType.Public, grouped), group => group.name ? [group.name] : [])), publicSubnetRouteTableIds: collapse(flatMap(findGroups(SubnetType.Public, grouped), group => group.subnets.map(s => s.routeTableId))), vpnGatewayId, + subnetGroups: assymetricSubnetGroups, }; } } @@ -197,6 +206,39 @@ function groupSubnets(subnets: Subnet[]): SubnetGroups { return { azs, groups }; } +function groupAsymmetricSubnets(subnets: Subnet[]): cxapi.VpcSubnetGroup[] { + const grouping: { [key: string]: Subnet[] } = {}; + for (const subnet of subnets) { + const key = [subnet.type, subnet.name].toString(); + if (!(key in grouping)) { + grouping[key] = []; + } + grouping[key].push(subnet); + } + + return Object.values(grouping).map(subnetArray => { + subnetArray.sort((subnet1: Subnet, subnet2: Subnet) => subnet1.az.localeCompare(subnet2.az)); + + return { + name: subnetArray[0].name, + type: subnetTypeToVpcSubnetType(subnetArray[0].type), + subnets: subnetArray.map(subnet => ({ + subnetId: subnet.subnetId, + availabilityZone: subnet.az, + routeTableId: subnet.routeTableId, + })), + }; + }); +} + +function subnetTypeToVpcSubnetType(type: SubnetType): cxapi.VpcSubnetGroupType { + switch (type) { + case SubnetType.Isolated: return cxapi.VpcSubnetGroupType.ISOLATED; + case SubnetType.Private: return cxapi.VpcSubnetGroupType.PRIVATE; + case SubnetType.Public: return cxapi.VpcSubnetGroupType.PUBLIC; + } +} + enum SubnetType { Public = 'Public', Private = 'Private', @@ -212,14 +254,14 @@ function isValidSubnetType(val: string): val is SubnetType { interface Subnet { az: string; type: SubnetType; - name?: string; + name: string; routeTableId: string; subnetId: string; } interface SubnetGroup { type: SubnetType; - name?: string; + name: string; subnets: Subnet[]; } diff --git a/packages/aws-cdk/test/context-providers/test.asymmetric-vpcs.ts b/packages/aws-cdk/test/context-providers/test.asymmetric-vpcs.ts new file mode 100644 index 0000000000000..640eb110ce2c3 --- /dev/null +++ b/packages/aws-cdk/test/context-providers/test.asymmetric-vpcs.ts @@ -0,0 +1,455 @@ +import aws = require('aws-sdk'); +import AWS = require('aws-sdk-mock'); +import nodeunit = require('nodeunit'); +import { ISDK } from '../../lib/api'; +import { VpcNetworkContextProviderPlugin } from '../../lib/context-providers/vpcs'; + +AWS.setSDKInstance(aws); + +const mockSDK: ISDK = { + defaultAccount: () => Promise.resolve('123456789012'), + defaultRegion: () => Promise.resolve('bermuda-triangle-1337'), + cloudFormation: () => { throw new Error('Not Mocked'); }, + ec2: () => Promise.resolve(new aws.EC2()), + ecr: () => { throw new Error('Not Mocked'); }, + route53: () => { throw new Error('Not Mocked'); }, + s3: () => { throw new Error('Not Mocked'); }, + ssm: () => { throw new Error('Not Mocked'); }, +}; + +type AwsCallback = (err: Error | null, val: T) => void; + +export = nodeunit.testCase({ + async 'looks up the requested (symmetric) VPC'(test: nodeunit.Test) { + mockVpcLookup(test, { + subnets: [ + { SubnetId: 'sub-123456', AvailabilityZone: 'bermuda-triangle-1337', MapPublicIpOnLaunch: true }, + { SubnetId: 'sub-789012', AvailabilityZone: 'bermuda-triangle-1337', MapPublicIpOnLaunch: false } + ], + routeTables: [ + { Associations: [{ SubnetId: 'sub-123456' }], RouteTableId: 'rtb-123456', }, + { Associations: [{ SubnetId: 'sub-789012' }], RouteTableId: 'rtb-789012', } + ], + vpnGateways: [{ VpnGatewayId: 'gw-abcdef' }] + + }); + + const result = await new VpcNetworkContextProviderPlugin(mockSDK).getValue({ + filter: { foo: 'bar' }, + returnAsymmetricSubnets: true, + }); + + test.deepEqual(result, { + availabilityZones: [], + isolatedSubnetIds: undefined, + isolatedSubnetNames: undefined, + isolatedSubnetRouteTableIds: undefined, + privateSubnetIds: undefined, + privateSubnetNames: undefined, + privateSubnetRouteTableIds: undefined, + publicSubnetIds: undefined, + publicSubnetNames: undefined, + publicSubnetRouteTableIds: undefined, + subnetGroups: [ + { + name: 'Public', + type: 'Public', + subnets: [ + { + subnetId: 'sub-123456', + availabilityZone: 'bermuda-triangle-1337', + routeTableId: 'rtb-123456', + }, + ], + }, + { + name: 'Private', + type: 'Private', + subnets: [ + { + subnetId: 'sub-789012', + availabilityZone: 'bermuda-triangle-1337', + routeTableId: 'rtb-789012', + }, + ], + }, + ], + vpcId: 'vpc-1234567', + vpnGatewayId: 'gw-abcdef' + }); + + AWS.restore(); + test.done(); + }, + + async 'throws when no such VPC is found'(test: nodeunit.Test) { + AWS.mock('EC2', 'describeVpcs', (params: aws.EC2.DescribeVpcsRequest, cb: AwsCallback) => { + test.deepEqual(params.Filters, [{ Name: 'foo', Values: ['bar'] }]); + return cb(null, {}); + }); + + try { + await new VpcNetworkContextProviderPlugin(mockSDK).getValue({ + filter: { foo: 'bar' }, + returnAsymmetricSubnets: true, + }); + + throw Error('The expected exception was not raised!'); + } catch (e) { + test.throws(() => { throw e; }, /Could not find any VPCs matching/); + } + + AWS.restore(); + test.done(); + }, + + async 'throws when multiple VPCs are found'(test: nodeunit.Test) { + // GIVEN + AWS.mock('EC2', 'describeVpcs', (params: aws.EC2.DescribeVpcsRequest, cb: AwsCallback) => { + test.deepEqual(params.Filters, [{ Name: 'foo', Values: ['bar'] }]); + return cb(null, { Vpcs: [{ VpcId: 'vpc-1' }, { VpcId: 'vpc-2' }]}); + }); + + // WHEN + try { + await new VpcNetworkContextProviderPlugin(mockSDK).getValue({ + filter: { foo: 'bar' }, + returnAsymmetricSubnets: true, + }); + + throw Error('The expected exception was not raised!'); + } catch (e) { + test.throws(() => { throw e; }, /Found 2 VPCs matching/); + } + + AWS.restore(); + test.done(); + }, + + async 'uses the VPC main route table when a subnet has no specific association'(test: nodeunit.Test) { + mockVpcLookup(test, { + subnets: [ + { SubnetId: 'sub-123456', AvailabilityZone: 'bermuda-triangle-1337', MapPublicIpOnLaunch: true }, + { SubnetId: 'sub-789012', AvailabilityZone: 'bermuda-triangle-1337', MapPublicIpOnLaunch: false } + ], + routeTables: [ + { Associations: [{ SubnetId: 'sub-123456' }], RouteTableId: 'rtb-123456', }, + { Associations: [{ Main: true }], RouteTableId: 'rtb-789012', } + ], + vpnGateways: [{ VpnGatewayId: 'gw-abcdef' }] + }); + + const result = await new VpcNetworkContextProviderPlugin(mockSDK).getValue({ + filter: { foo: 'bar' }, + returnAsymmetricSubnets: true, + }); + + test.deepEqual(result, { + availabilityZones: [], + isolatedSubnetIds: undefined, + isolatedSubnetNames: undefined, + isolatedSubnetRouteTableIds: undefined, + privateSubnetIds: undefined, + privateSubnetNames: undefined, + privateSubnetRouteTableIds: undefined, + publicSubnetIds: undefined, + publicSubnetNames: undefined, + publicSubnetRouteTableIds: undefined, + subnetGroups: [ + { + name: 'Public', + type: 'Public', + subnets: [ + { + subnetId: 'sub-123456', + availabilityZone: 'bermuda-triangle-1337', + routeTableId: 'rtb-123456', + }, + ], + }, + { + name: 'Private', + type: 'Private', + subnets: [ + { + subnetId: 'sub-789012', + availabilityZone: 'bermuda-triangle-1337', + routeTableId: 'rtb-789012', + }, + ], + }, + ], + vpcId: 'vpc-1234567', + vpnGatewayId: 'gw-abcdef' + }); + + test.done(); + AWS.restore(); + }, + + async 'Recognize public subnet by route table'(test: nodeunit.Test) { + // GIVEN + mockVpcLookup(test, { + subnets: [ + { SubnetId: 'sub-123456', AvailabilityZone: 'bermuda-triangle-1337', MapPublicIpOnLaunch: false }, + ], + routeTables: [ + { + Associations: [{ SubnetId: 'sub-123456' }], + RouteTableId: 'rtb-123456', + Routes: [ + { + DestinationCidrBlock: "10.0.2.0/26", + Origin: "CreateRoute", + State: "active", + VpcPeeringConnectionId: "pcx-xxxxxx" + }, + { + DestinationCidrBlock: "10.0.1.0/24", + GatewayId: "local", + Origin: "CreateRouteTable", + State: "active" + }, + { + DestinationCidrBlock: "0.0.0.0/0", + GatewayId: "igw-xxxxxx", + Origin: "CreateRoute", + State: "active" + } + ], + }, + ], + }); + + // WHEN + const result = await new VpcNetworkContextProviderPlugin(mockSDK).getValue({ + filter: { foo: 'bar' }, + returnAsymmetricSubnets: true, + }); + + // THEN + test.deepEqual(result, { + availabilityZones: [], + isolatedSubnetIds: undefined, + isolatedSubnetNames: undefined, + isolatedSubnetRouteTableIds: undefined, + privateSubnetIds: undefined, + privateSubnetNames: undefined, + privateSubnetRouteTableIds: undefined, + publicSubnetIds: undefined, + publicSubnetNames: undefined, + publicSubnetRouteTableIds: undefined, + subnetGroups: [ + { + name: 'Public', + type: 'Public', + subnets: [ + { + subnetId: 'sub-123456', + availabilityZone: 'bermuda-triangle-1337', + routeTableId: 'rtb-123456', + }, + ], + }, + ], + vpcId: 'vpc-1234567', + vpnGatewayId: undefined, + }); + + AWS.restore(); + test.done(); + }, + + async 'works for asymmetric subnets (not spanning the same Availability Zones)'(test: nodeunit.Test) { + // GIVEN + mockVpcLookup(test, { + subnets: [ + { SubnetId: 'pri-sub-in-1b', AvailabilityZone: 'us-west-1b', MapPublicIpOnLaunch: false }, + { SubnetId: 'pub-sub-in-1c', AvailabilityZone: 'us-west-1c', MapPublicIpOnLaunch: true }, + { SubnetId: 'pub-sub-in-1b', AvailabilityZone: 'us-west-1b', MapPublicIpOnLaunch: true }, + { SubnetId: 'pub-sub-in-1a', AvailabilityZone: 'us-west-1a', MapPublicIpOnLaunch: true }, + ], + routeTables: [ + { Associations: [{ Main: true }], RouteTableId: 'rtb-123' }, + ], + }); + + // WHEN + const result = await new VpcNetworkContextProviderPlugin(mockSDK).getValue({ + filter: { foo: 'bar' }, + returnAsymmetricSubnets: true, + }); + + // THEN + test.deepEqual(result, { + availabilityZones: [], + isolatedSubnetIds: undefined, + isolatedSubnetNames: undefined, + isolatedSubnetRouteTableIds: undefined, + privateSubnetIds: undefined, + privateSubnetNames: undefined, + privateSubnetRouteTableIds: undefined, + publicSubnetIds: undefined, + publicSubnetNames: undefined, + publicSubnetRouteTableIds: undefined, + subnetGroups: [ + { + name: 'Private', + type: 'Private', + subnets: [ + { + subnetId: 'pri-sub-in-1b', + availabilityZone: 'us-west-1b', + routeTableId: 'rtb-123', + }, + ], + }, + { + name: 'Public', + type: 'Public', + subnets: [ + { + subnetId: 'pub-sub-in-1a', + availabilityZone: 'us-west-1a', + routeTableId: 'rtb-123', + }, + { + subnetId: 'pub-sub-in-1b', + availabilityZone: 'us-west-1b', + routeTableId: 'rtb-123', + }, + { + subnetId: 'pub-sub-in-1c', + availabilityZone: 'us-west-1c', + routeTableId: 'rtb-123', + }, + ], + }, + ], + vpcId: 'vpc-1234567', + vpnGatewayId: undefined, + }); + + AWS.restore(); + test.done(); + }, + + async 'allows specifying the subnet group name tag'(test: nodeunit.Test) { + // GIVEN + mockVpcLookup(test, { + subnets: [ + { + SubnetId: 'pri-sub-in-1b', AvailabilityZone: 'us-west-1b', MapPublicIpOnLaunch: false, Tags: [ + { Key: 'Tier', Value: 'restricted' }, + ] }, + { + SubnetId: 'pub-sub-in-1c', AvailabilityZone: 'us-west-1c', MapPublicIpOnLaunch: true, Tags: [ + { Key: 'Tier', Value: 'connectivity' }, + ] }, + { + SubnetId: 'pub-sub-in-1b', AvailabilityZone: 'us-west-1b', MapPublicIpOnLaunch: true, Tags: [ + { Key: 'Tier', Value: 'connectivity' }, + ] }, + { + SubnetId: 'pub-sub-in-1a', AvailabilityZone: 'us-west-1a', MapPublicIpOnLaunch: true, Tags: [ + { Key: 'Tier', Value: 'connectivity' }, + ] }, + ], + routeTables: [ + { Associations: [{ Main: true }], RouteTableId: 'rtb-123' }, + ], + }); + + const result = await new VpcNetworkContextProviderPlugin(mockSDK).getValue({ + filter: { foo: 'bar' }, + returnAsymmetricSubnets: true, + subnetGroupNameTag: 'Tier', + }); + + test.deepEqual(result, { + availabilityZones: [], + isolatedSubnetIds: undefined, + isolatedSubnetNames: undefined, + isolatedSubnetRouteTableIds: undefined, + privateSubnetIds: undefined, + privateSubnetNames: undefined, + privateSubnetRouteTableIds: undefined, + publicSubnetIds: undefined, + publicSubnetNames: undefined, + publicSubnetRouteTableIds: undefined, + subnetGroups: [ + { + name: 'restricted', + type: 'Private', + subnets: [ + { + subnetId: 'pri-sub-in-1b', + availabilityZone: 'us-west-1b', + routeTableId: 'rtb-123', + }, + ], + }, + { + name: 'connectivity', + type: 'Public', + subnets: [ + { + subnetId: 'pub-sub-in-1a', + availabilityZone: 'us-west-1a', + routeTableId: 'rtb-123', + }, + { + subnetId: 'pub-sub-in-1b', + availabilityZone: 'us-west-1b', + routeTableId: 'rtb-123', + }, + { + subnetId: 'pub-sub-in-1c', + availabilityZone: 'us-west-1c', + routeTableId: 'rtb-123', + }, + ], + }, + ], + vpcId: 'vpc-1234567', + vpnGatewayId: undefined, + }); + + AWS.restore(); + test.done(); + }, +}); + +interface VpcLookupOptions { + subnets: aws.EC2.Subnet[]; + routeTables: aws.EC2.RouteTable[]; + vpnGateways?: aws.EC2.VpnGateway[]; +} + +function mockVpcLookup(test: nodeunit.Test, options: VpcLookupOptions) { + const VpcId = 'vpc-1234567'; + + AWS.mock('EC2', 'describeVpcs', (params: aws.EC2.DescribeVpcsRequest, cb: AwsCallback) => { + test.deepEqual(params.Filters, [{ Name: 'foo', Values: ['bar'] }]); + return cb(null, { Vpcs: [{ VpcId }] }); + }); + + AWS.mock('EC2', 'describeSubnets', (params: aws.EC2.DescribeSubnetsRequest, cb: AwsCallback) => { + test.deepEqual(params.Filters, [{ Name: 'vpc-id', Values: [VpcId] }]); + return cb(null, { Subnets: options.subnets }); + }); + + AWS.mock('EC2', 'describeRouteTables', (params: aws.EC2.DescribeRouteTablesRequest, cb: AwsCallback) => { + test.deepEqual(params.Filters, [{ Name: 'vpc-id', Values: [VpcId] }]); + return cb(null, { RouteTables: options.routeTables }); + }); + + AWS.mock('EC2', 'describeVpnGateways', (params: aws.EC2.DescribeVpnGatewaysRequest, cb: AwsCallback) => { + test.deepEqual(params.Filters, [ + { Name: 'attachment.vpc-id', Values: [ VpcId ] }, + { Name: 'attachment.state', Values: [ 'attached' ] }, + { Name: 'state', Values: [ 'available' ] } + ]); + return cb(null, { VpnGateways: options.vpnGateways }); + }); +} diff --git a/packages/aws-cdk/test/context-providers/test.vpcs.ts b/packages/aws-cdk/test/context-providers/test.vpcs.ts index 9883d71a7b10c..e80735da85d19 100644 --- a/packages/aws-cdk/test/context-providers/test.vpcs.ts +++ b/packages/aws-cdk/test/context-providers/test.vpcs.ts @@ -54,7 +54,8 @@ export = nodeunit.testCase({ publicSubnetIds: ['sub-123456'], publicSubnetNames: ['Public'], publicSubnetRouteTableIds: ['rtb-123456'], - vpnGatewayId: 'gw-abcdef' + vpnGatewayId: 'gw-abcdef', + subnetGroups: undefined, }); AWS.restore(); @@ -138,7 +139,8 @@ export = nodeunit.testCase({ publicSubnetIds: ['sub-123456'], publicSubnetNames: ['Public'], publicSubnetRouteTableIds: ['rtb-123456'], - vpnGatewayId: 'gw-abcdef' + vpnGatewayId: 'gw-abcdef', + subnetGroups: undefined, }); test.done(); @@ -199,6 +201,7 @@ export = nodeunit.testCase({ publicSubnetNames: ['Public'], publicSubnetRouteTableIds: ['rtb-123456'], vpnGatewayId: undefined, + subnetGroups: undefined, }); AWS.restore(); @@ -238,4 +241,4 @@ function mockVpcLookup(test: nodeunit.Test, options: VpcLookupOptions) { ]); return cb(null, { VpnGateways: options.vpnGateways }); }); -} \ No newline at end of file +} diff --git a/tools/cdk-integ-tools/lib/integ-helpers.ts b/tools/cdk-integ-tools/lib/integ-helpers.ts index bc9266bb93a00..2d53c07814a86 100644 --- a/tools/cdk-integ-tools/lib/integ-helpers.ts +++ b/tools/cdk-integ-tools/lib/integ-helpers.ts @@ -183,13 +183,32 @@ export const DEFAULT_SYNTH_OPTIONS = { "ssm:account=12345678:parameterName=/aws/service/ami-amazon-linux-latest/amzn-ami-hvm-x86_64-gp2:region=test-region": "ami-1234", "ssm:account=12345678:parameterName=/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2:region=test-region": "ami-1234", "ssm:account=12345678:parameterName=/aws/service/ecs/optimized-ami/amazon-linux/recommended:region=test-region": "{\"image_id\": \"ami-1234\"}", - "vpc-provider:account=12345678:filter.isDefault=true:region=test-region": { + "vpc-provider:account=12345678:filter.isDefault=true:region=test-region:returnAsymmetricSubnets=true": { vpcId: "vpc-60900905", - availabilityZones: [ "us-east-1a", "us-east-1b", "us-east-1c" ], - publicSubnetIds: ["subnet-e19455ca", "subnet-e0c24797", "subnet-ccd77395"], - publicSubnetRouteTableIds: ["rtb-e19455ca", "rtb-e0c24797", "rtb-ccd77395"], - publicSubnetNames: [ "Public" ] - } + subnetGroups: [ + { + type: "Public", + name: "Public", + subnets: [ + { + subnetId: "subnet-e19455ca", + availabilityZone: "us-east-1a", + routeTableId: "rtb-e19455ca", + }, + { + subnetId: "subnet-e0c24797", + availabilityZone: "us-east-1b", + routeTableId: "rtb-e0c24797", + }, + { + subnetId: "subnet-ccd77395", + availabilityZone: "us-east-1c", + routeTableId: "rtb-ccd77395", + }, + ], + }, + ], + }, }, env: { CDK_INTEG_ACCOUNT: "12345678",