Skip to content

Commit

Permalink
Add --node-private-networking flag
Browse files Browse the repository at this point in the history 
- add docs
- include NAT Gateway
- ensure control plane also connects to private subnets
- ensure `kubernetes.io/role/internal-elb=1` tag is set properly
- ensure SSH port is only accessible inside the VPC
- use boolean ng.PrivateNetworking field and helper method to get string
  instead of string ng.SubnetTopology field
  • Loading branch information
@errordeveloper
errordeveloper committed Oct 30, 2018
1 parent b90c2d6 commit 9544a48
Show file tree
Hide file tree
Showing 9 changed files with 118 additions and 57 deletions.
6 changes: 6 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,12 @@ change it. You cannot use just any sort of CIDR, there only certain ranges that

[vpcsizing]: https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing


#### use private subnets for initial nodegroup

If you prefer to isolate initial nodegroup from the public internet, you can use `--node-private-networking` flag.
When used in conjunction with `--ssh-access` flag, SSH port can only be accessed inside the VPC.

#### use existing VPC: shared with kops

You can use a VPC of an existing Kubernetes cluster managed by kops. This is feature is provided to facilitate migration and/or
Expand Down
7 changes: 7 additions & 0 deletions pkg/cfn/builder/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@ const (
templateDescriptionSuffix = " [created and managed by eksctl]"
)

type awsCloudFormationResource struct {
Type string
Properties map[string]interface{}
UpdatePolicy map[string]map[string]string `json:",omitempty"`
DependsOn []string `json:",omitempty"`
}

// ResourceSet is an interface which cluster and nodegroup builders
// must implement
type ResourceSet interface {
Expand Down
8 changes: 4 additions & 4 deletions pkg/cfn/builder/api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,10 @@ var _ = Describe("CloudFormation template builder API", func() {
},
NodeGroups: []*api.NodeGroup{
{
AMI: "",
AMIFamily: "AmazonLinux2",
InstanceType: "t2.medium",
SubnetTopology: "Public",
AMI: "",
AMIFamily: "AmazonLinux2",
InstanceType: "t2.medium",
PrivateNetworking: false,
},
},
}
Expand Down
18 changes: 11 additions & 7 deletions pkg/cfn/builder/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,18 @@ func (c *ClusterResourceSet) newResource(name string, resource interface{}) *gfn
}

func (c *ClusterResourceSet) addResourcesForControlPlane(version string) {
clusterVPC := &gfn.AWSEKSCluster_ResourcesVpcConfig{
SecurityGroupIds: c.securityGroups,
}
for topology := range c.spec.VPC.Subnets {
clusterVPC.SubnetIds = append(clusterVPC.SubnetIds, c.subnets[topology]...)
}

c.newResource("ControlPlane", &gfn.AWSEKSCluster{
Name: gfn.NewString(c.spec.ClusterName),
RoleArn: gfn.MakeFnGetAttString("ServiceRole.Arn"),
Version: gfn.NewString(version),
ResourcesVpcConfig: &gfn.AWSEKSCluster_ResourcesVpcConfig{
SubnetIds: c.subnets[api.SubnetTopologyPublic],
SecurityGroupIds: c.securityGroups,
},
Name: gfn.NewString(c.spec.ClusterName),
RoleArn: gfn.MakeFnGetAttString("ServiceRole.Arn"),
Version: gfn.NewString(version),
ResourcesVpcConfig: clusterVPC,
})

c.rs.newOutputFromAtt(cfnOutputClusterCertificateAuthorityData, "ControlPlane.CertificateAuthorityData", false)
Expand Down
27 changes: 12 additions & 15 deletions pkg/cfn/builder/nodegroup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,6 @@ type NodeGroupResourceSet struct {
userData *gfn.Value
}

type awsCloudFormationResource struct {
Type string
Properties map[string]interface{}
UpdatePolicy map[string]map[string]string
}

// NewNodeGroupResourceSet returns a resource set for the new node group
func NewNodeGroupResourceSet(spec *api.ClusterConfig, clusterStackName string, id int) *NodeGroupResourceSet {
return &NodeGroupResourceSet{
Expand Down Expand Up @@ -85,17 +79,20 @@ func (n *NodeGroupResourceSet) newResource(name string, resource interface{}) *g

func (n *NodeGroupResourceSet) addResourcesForNodeGroup() {
lc := &gfn.AWSAutoScalingLaunchConfiguration{
AssociatePublicIpAddress: gfn.True(),
IamInstanceProfile: n.instanceProfile,
SecurityGroups: n.securityGroups,

ImageId: gfn.NewString(n.spec.AMI),
InstanceType: gfn.NewString(n.spec.InstanceType),
UserData: n.userData,
IamInstanceProfile: n.instanceProfile,
SecurityGroups: n.securityGroups,
ImageId: gfn.NewString(n.spec.AMI),
InstanceType: gfn.NewString(n.spec.InstanceType),
UserData: n.userData,
}
if n.spec.AllowSSH {
lc.KeyName = gfn.NewString(n.spec.SSHPublicKeyName)
}
if n.spec.PrivateNetworking {
lc.AssociatePublicIpAddress = gfn.False()
} else {
lc.AssociatePublicIpAddress = gfn.True()
}
if n.spec.VolumeSize > 0 {
lc.BlockDeviceMappings = []gfn.AWSAutoScalingLaunchConfiguration_BlockDeviceMapping{
{
Expand All @@ -111,12 +108,12 @@ func (n *NodeGroupResourceSet) addResourcesForNodeGroup() {
// and tags don't have `PropagateAtLaunch` field, so we have a custom method here until this gets resolved
var vpcZoneIdentifier interface{}
if len(n.spec.AvailabilityZones) > 0 {
vpcZoneIdentifier = n.clusterSpec.SubnetIDs(api.SubnetTopologyPublic)
vpcZoneIdentifier = n.clusterSpec.SubnetIDs(n.spec.SubnetTopology())
} else {
vpcZoneIdentifier = map[string][]interface{}{
gfn.FnSplit: []interface{}{
",",
makeImportValue(n.clusterStackName, cfnOutputClusterSubnets+string(api.SubnetTopologyPublic)),
makeImportValue(n.clusterStackName, cfnOutputClusterSubnets+string(n.spec.SubnetTopology())),
},
}
}
Expand Down
90 changes: 63 additions & 27 deletions pkg/cfn/builder/vpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,20 @@ import (

func (c *ClusterResourceSet) addSubnets(refRT *gfn.Value, topology api.SubnetTopology) {
for az, subnet := range c.spec.VPC.Subnets[topology] {
alias := strings.ToUpper(strings.Join(strings.Split(az, "-"), ""))
refSubnet := c.newResource("Subnet"+string(topology)+alias, &gfn.AWSEC2Subnet{
alias := string(topology) + strings.ToUpper(strings.Join(strings.Split(az, "-"), ""))
subnet := &gfn.AWSEC2Subnet{
AvailabilityZone: gfn.NewString(az),
CidrBlock: gfn.NewString(subnet.CIDR.String()),
VpcId: c.vpc,
})
c.newResource("RouteTableAssociation"+string(topology)+alias, &gfn.AWSEC2SubnetRouteTableAssociation{
}
if topology == api.SubnetTopologyPrivate {
subnet.Tags = []gfn.Tag{{
Key: gfn.NewString("kubernetes.io/role/internal-elb"),
Value: gfn.NewString("1"),
}}
}
refSubnet := c.newResource("Subnet"+alias, subnet)
c.newResource("RouteTableAssociation"+alias, &gfn.AWSEC2SubnetRouteTableAssociation{
SubnetId: refSubnet,
RouteTableId: refRT,
})
Expand All @@ -25,6 +32,8 @@ func (c *ClusterResourceSet) addSubnets(refRT *gfn.Value, topology api.SubnetTop

//nolint:interfacer
func (c *ClusterResourceSet) addResourcesForVPC() {
internetCIDR := gfn.NewString("0.0.0.0/0")

c.vpc = c.newResource("VPC", &gfn.AWSEC2VPC{
CidrBlock: gfn.NewString(c.spec.VPC.CIDR.String()),
EnableDnsSupport: gfn.True(),
Expand All @@ -39,23 +48,39 @@ func (c *ClusterResourceSet) addResourcesForVPC() {
VpcId: c.vpc,
})

refPrivateRT := c.newResource("PrivateRouteTable", &gfn.AWSEC2RouteTable{
VpcId: c.vpc,
})

c.addSubnets(refPrivateRT, api.SubnetTopologyPrivate)

refPublicRT := c.newResource("PublicRouteTable", &gfn.AWSEC2RouteTable{
VpcId: c.vpc,
})

c.newResource("PublicSubnetRoute", &gfn.AWSEC2Route{
RouteTableId: refPublicRT,
DestinationCidrBlock: gfn.NewString("0.0.0.0/0"),
DestinationCidrBlock: internetCIDR,
GatewayId: refIG,
})

c.addSubnets(refPublicRT, api.SubnetTopologyPublic)

c.newResource("NATIP", &gfn.AWSEC2EIP{
Domain: gfn.NewString("vpc"),
})
refNG := c.newResource("NATGateway", &gfn.AWSEC2NatGateway{
AllocationId: gfn.MakeFnGetAttString("NATIP.AllocationId"),
// A multi-AZ NAT Gateway is possible, but it's not very
// clear from the docs how to achieve it
SubnetId: c.subnets[api.SubnetTopologyPublic][0],
})

refPrivateRT := c.newResource("PrivateRouteTable", &gfn.AWSEC2RouteTable{
VpcId: c.vpc,
})

c.newResource("PrivateSubnetRoute", &gfn.AWSEC2Route{
RouteTableId: refPrivateRT,
DestinationCidrBlock: internetCIDR,
NatGatewayId: refNG,
})

c.addSubnets(refPrivateRT, api.SubnetTopologyPrivate)
}

func (c *ClusterResourceSet) importResourcesForVPC() {
Expand Down Expand Up @@ -158,21 +183,32 @@ func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() {
ToPort: apiPort,
})
if n.spec.AllowSSH {
n.newResource("SSHIPv4", &gfn.AWSEC2SecurityGroupIngress{
GroupId: refSG,
CidrIp: anywhereIPv4,
Description: gfn.NewString("Allow SSH access to " + desc),
IpProtocol: tcp,
FromPort: sshPort,
ToPort: sshPort,
})
n.newResource("SSHIPv6", &gfn.AWSEC2SecurityGroupIngress{
GroupId: refSG,
CidrIpv6: anywhereIPv6,
Description: gfn.NewString("Allow SSH access to " + desc),
IpProtocol: tcp,
FromPort: sshPort,
ToPort: sshPort,
})
if n.spec.PrivateNetworking {
n.newResource("SSHIPv4", &gfn.AWSEC2SecurityGroupIngress{
GroupId: refSG,
CidrIp: gfn.NewString(n.clusterSpec.VPC.CIDR.String()),
Description: gfn.NewString("Allow SSH access to " + desc + " (private, only inside VPC)"),
IpProtocol: tcp,
FromPort: sshPort,
ToPort: sshPort,
})
} else {
n.newResource("SSHIPv4", &gfn.AWSEC2SecurityGroupIngress{
GroupId: refSG,
CidrIp: anywhereIPv4,
Description: gfn.NewString("Allow SSH access to " + desc),
IpProtocol: tcp,
FromPort: sshPort,
ToPort: sshPort,
})
n.newResource("SSHIPv6", &gfn.AWSEC2SecurityGroupIngress{
GroupId: refSG,
CidrIpv6: anywhereIPv6,
Description: gfn.NewString("Allow SSH access to " + desc),
IpProtocol: tcp,
FromPort: sshPort,
ToPort: sshPort,
})
}
}
}
2 changes: 2 additions & 0 deletions pkg/ctl/create/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ func createClusterCmd() *cobra.Command {

fs.IPNetVar(cfg.VPC.CIDR, "vpc-cidr", api.DefaultCIDR(), "global CIDR to use for VPC")

fs.BoolVarP(&ng.PrivateNetworking, "node-private-networking", "P", false, "whether to make initial nodegroup networking private")

return cmd
}

Expand Down
15 changes: 12 additions & 3 deletions pkg/eks/api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@ func NewClusterConfig() *ClusterConfig {
// it returns pointer to the nodegroup for convenience
func (c *ClusterConfig) NewNodeGroup() *NodeGroup {
ng := &NodeGroup{
ID: len(c.NodeGroups),
SubnetTopology: SubnetTopologyPublic,
ID: len(c.NodeGroups),
PrivateNetworking: false,
}

c.NodeGroups = append(c.NodeGroups, ng)
Expand All @@ -108,7 +108,7 @@ type NodeGroup struct {
InstanceType string
AvailabilityZones []string
Tags map[string]string
SubnetTopology SubnetTopology
PrivateNetworking bool

DesiredCapacity int
MinSize int
Expand All @@ -127,6 +127,15 @@ type NodeGroup struct {
SSHPublicKeyName string
}

// SubnetTopology check which topology is used for the subnet of
// the given nodegroup
func (n *NodeGroup) SubnetTopology() SubnetTopology {
if n.PrivateNetworking {
return SubnetTopologyPrivate
}
return SubnetTopologyPublic
}

type (
// ClusterAddons provides addons for the created EKS cluster
ClusterAddons struct {
Expand Down
2 changes: 1 addition & 1 deletion pkg/utils/kubeconfig/kubeconfig_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ var _ = Describe("Kubeconfig", func() {
AMI: "",
InstanceType: "m5.large",
AvailabilityZones: []string{"us-west-2b", "us-west-2a", "us-west-2c"},
SubnetTopology: "Public",
PrivateNetworking: false,
AllowSSH: false,
SSHPublicKeyPath: "~/.ssh/id_rsa.pub",
SSHPublicKey: []uint8(nil),
Expand Down

0 comments on commit 9544a48

Please sign in to comment.