Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated EKS AMI with bootstrap script #16

Merged
merged 1 commit into from
Aug 2, 2018
Merged

Updated EKS AMI with bootstrap script #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 8 additions & 151 deletions amazon-eks-nodegroup.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,126 +104,6 @@ Parameters:
Description: The subnets where workers can be created.
Type: List<AWS::EC2::Subnet::Id>

Mappings:
# Mapping is calculated from AWS ENI documentation, with the following modifications:
# * First IP on each ENI is not used for pods
# * 2 additional host-networking pods (AWS ENI and kube-proxy) are accounted for
#
# # of ENI * (# of IPv4 per ENI - 1) + 2
#
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI
MaxPodsPerNode:
c4.large:
MaxPods: 29
c4.xlarge:
MaxPods: 58
c4.2xlarge:
MaxPods: 58
c4.4xlarge:
MaxPods: 234
c4.8xlarge:
MaxPods: 234
c5.large:
MaxPods: 29
c5.xlarge:
MaxPods: 58
c5.2xlarge:
MaxPods: 58
c5.4xlarge:
MaxPods: 234
c5.9xlarge:
MaxPods: 234
c5.18xlarge:
MaxPods: 737
i3.large:
MaxPods: 29
i3.xlarge:
MaxPods: 58
i3.2xlarge:
MaxPods: 58
i3.4xlarge:
MaxPods: 234
i3.8xlarge:
MaxPods: 234
i3.16xlarge:
MaxPods: 737
m3.medium:
MaxPods: 12
m3.large:
MaxPods: 29
m3.xlarge:
MaxPods: 58
m3.2xlarge:
MaxPods: 118
m4.large:
MaxPods: 20
m4.xlarge:
MaxPods: 58
m4.2xlarge:
MaxPods: 58
m4.4xlarge:
MaxPods: 234
m4.10xlarge:
MaxPods: 234
m5.large:
MaxPods: 29
m5.xlarge:
MaxPods: 58
m5.2xlarge:
MaxPods: 58
m5.4xlarge:
MaxPods: 234
m5.12xlarge:
MaxPods: 234
m5.24xlarge:
MaxPods: 737
p2.xlarge:
MaxPods: 58
p2.8xlarge:
MaxPods: 234
p2.16xlarge:
MaxPods: 234
p3.2xlarge:
MaxPods: 58
p3.8xlarge:
MaxPods: 234
p3.16xlarge:
MaxPods: 234
r3.xlarge:
MaxPods: 58
r3.2xlarge:
MaxPods: 58
r3.4xlarge:
MaxPods: 234
r3.8xlarge:
MaxPods: 234
r4.large:
MaxPods: 29
r4.xlarge:
MaxPods: 58
r4.2xlarge:
MaxPods: 58
r4.4xlarge:
MaxPods: 234
r4.8xlarge:
MaxPods: 234
r4.16xlarge:
MaxPods: 737
t2.small:
MaxPods: 8
t2.medium:
MaxPods: 17
t2.large:
MaxPods: 35
t2.xlarge:
MaxPods: 44
t2.2xlarge:
MaxPods: 44
x1.16xlarge:
MaxPods: 234
x1.32xlarge:
MaxPods: 234

Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
Expand Down Expand Up @@ -364,39 +244,16 @@ Resources:
- !Ref NodeSecurityGroup
UserData:
Fn::Base64:
Fn::Join: [
"",
[
"#!/bin/bash -xe\n",
"CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki", "\n",
"CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt", "\n",
"mkdir -p $CA_CERTIFICATE_DIRECTORY", "\n",
"aws eks describe-cluster --region=", { Ref: "AWS::Region" }," --name=", { Ref: ClusterName }," --query 'cluster.{certificateAuthorityData: certificateAuthority.data, endpoint: endpoint}' > /tmp/describe_cluster_result.json", "\n",
"cat /tmp/describe_cluster_result.json | grep certificateAuthorityData | awk '{print $2}' | sed 's/[,\"]//g' | base64 -d > $CA_CERTIFICATE_FILE_PATH", "\n",
"MASTER_ENDPOINT=$(cat /tmp/describe_cluster_result.json | grep endpoint | awk '{print $2}' | sed 's/[,\"]//g')", "\n",
"INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)", "\n",
"sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /var/lib/kubelet/kubeconfig", "\n",
"sed -i s,CLUSTER_NAME,", { Ref: ClusterName }, ",g /var/lib/kubelet/kubeconfig", "\n",
"sed -i s,REGION,", { Ref: "AWS::Region" }, ",g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,MAX_PODS,", { "Fn::FindInMap": [ MaxPodsPerNode, { Ref: NodeInstanceType }, MaxPods ] }, ",g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,INTERNAL_IP,$INTERNAL_IP,g /etc/systemd/system/kubelet.service", "\n",
"DNS_CLUSTER_IP=10.100.0.10", "\n",
"if [[ $INTERNAL_IP == 10.* ]] ; then DNS_CLUSTER_IP=172.20.0.10; fi", "\n",
"sed -i s,DNS_CLUSTER_IP,$DNS_CLUSTER_IP,g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,CERTIFICATE_AUTHORITY_FILE,$CA_CERTIFICATE_FILE_PATH,g /var/lib/kubelet/kubeconfig" , "\n",
"sed -i s,CLIENT_CA_FILE,$CA_CERTIFICATE_FILE_PATH,g /etc/systemd/system/kubelet.service" , "\n",
"systemctl daemon-reload", "\n",
"systemctl restart kubelet", "\n",
"/opt/aws/bin/cfn-signal -e $? ",
" --stack ", { Ref: "AWS::StackName" },
" --resource NodeGroup ",
" --region ", { Ref: "AWS::Region" }, "\n"
]
]
!Sub |
#!/bin/bash
set -o xtrace
/etc/eks/bootstrap.sh ${ClusterName}
/opt/aws/bin/cfn-signal --exit-code $? \
--stack ${AWS::StackName} \
--resource NodeGroup \
--region ${AWS::Region}

Outputs:
NodeInstanceRole:
Description: The node instance role
Value: !GetAtt NodeInstanceRole.Arn

132 changes: 132 additions & 0 deletions files/bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
#!/usr/bin/env bash

set -o pipefail
set -o nounset
set -o errexit

IFS=$'\n\t'

function print_help {
echo "usage: $0 [options] <cluster-name>"
echo "Bootstraps an instance into an EKS cluster"
echo ""
echo "-h,--help print this help"
echo "--use-max-pods Sets --max-pods for the kubelet when true. (default: true)"
echo "--b64-cluster-ca The base64 encoded cluster CA content. Only valid when used with --apiserver-endpoint. Bypasses calling \"aws eks describe-cluster\""
echo "--apiserver-endpoint The EKS cluster API Server endpoint. Only valid when used with --b64-cluster-ca. Bypasses calling \"aws eks describe-cluster\""
echo "--kubelet-extra-args Extra arguments to add to the kubelet. Useful for adding labels or taints."
}

POSITIONAL=()

while [[ $# -gt 0 ]]; do
key="$1"
case $key in
-h|--help)
print_help
exit 1
;;
--use-max-pods)
USE_MAX_PODS="$2"
shift
shift
;;
--b64-cluster-ca)
B64_CLUSTER_CA=$2
shift
shift
;;
--apiserver-endpoint)
APISERVER_ENDPOINT=$2
shift
shift
;;
--kubelet-extra-args)
KUBELET_EXTRA_ARGS=$2
shift
shift
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done

set +u
set -- "${POSITIONAL[@]}" # restore positional parameters
CLUSTER_NAME="$1"
set -u

USE_MAX_PODS="${USE_MAX_PODS:-true}"
B64_CLUSTER_CA="${B64_CLUSTER_CA:-}"
APISERVER_ENDPOINT="${APISERVER_ENDPOINT:-}"
KUBELET_EXTRA_ARGS="${KUBELET_EXTRA_ARGS:-}"

if [ -z "$CLUSTER_NAME" ]; then
echo "CLUSTER_NAME is not defined"
exit 1
fi

ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
AWS_DEFAULT_REGION=$(echo $ZONE | awk '{print substr($0, 1, length($0)-1)}')

### kubelet kubeconfig

CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki
CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt
mkdir -p $CA_CERTIFICATE_DIRECTORY
if [[ -z "${B64_CLUSTER_CA}" ]] && [[ -z "${APISERVER_ENDPOINT}" ]]; then
DESCRIBE_CLUSTER_RESULT="/tmp/describe_cluster_result.txt"
aws eks describe-cluster \
--region=${AWS_DEFAULT_REGION} \
--name=${CLUSTER_NAME} \
--output=text \
--query 'cluster.{certificateAuthorityData: certificateAuthority.data, endpoint: endpoint}' > $DESCRIBE_CLUSTER_RESULT
B64_CLUSTER_CA=$(cat $DESCRIBE_CLUSTER_RESULT | awk '{print $1}')
APISERVER_ENDPOINT=$(cat $DESCRIBE_CLUSTER_RESULT | awk '{print $2}')
fi

echo $B64_CLUSTER_CA | base64 -d > $CA_CERTIFICATE_FILE_PATH

kubectl config \
--kubeconfig /var/lib/kubelet/kubeconfig \
set-cluster \
kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.crt \
--server=$APISERVER_ENDPOINT
sed -i s,CLUSTER_NAME,$CLUSTER_NAME,g /var/lib/kubelet/kubeconfig

### kubelet.service configuration

INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type)
DNS_CLUSTER_IP=10.100.0.10
if [[ $INTERNAL_IP == 10.* ]] ; then
DNS_CLUSTER_IP=172.20.0.10;
fi

if [[ "$USE_MAX_PODS" = "true" ]]; then
MAX_PODS_FILE="/etc/eks/eni-max-pods.txt"
MAX_PODS=$(grep $INSTANCE_TYPE $MAX_PODS_FILE | awk '{print $2}')
cat <<EOF > /etc/systemd/system/kubelet.service.d/20-max-pods.conf
[Service]
Environment='KUBELET_MAX_PODS=--max-pods=$MAX_PODS'
EOF
fi

cat <<EOF > /etc/systemd/system/kubelet.service.d/10-kubelet-args.conf
[Service]
Environment='KUBELET_ARGS=--node-ip=$INTERNAL_IP --cluster-dns=$DNS_CLUSTER_IP --pod-infra-container-image=602401143452.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/eks/pause-amd64:3.1'
EOF

if [[ -n "$KUBELET_EXTRA_ARGS" ]]; then
cat <<EOF > /etc/systemd/system/kubelet.service.d/30-kubelet-extra-args.conf
[Service]
Environment='KUBELET_EXTRA_ARGS=$KUBELET_EXTRA_ARGS'
EOF
fi

systemctl daemon-reload
systemctl enable kubelet
systemctl start kubelet
Loading