feat: use IMDS for autoscaling information

Signed-off-by: Jonah Back <jonah@jonahback.com>
keikoproj · Apr 12, 2022 · 26d23f3 · 26d23f3
1 parent 8084fc4
commit 26d23f3
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 15 deletions.
diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go
@@ -57,7 +57,6 @@ var (
 	AllowedOsFamilies         = []string{OsFamilyWindows, OsFamilyBottleRocket, OsFamilyAmazonLinux2}
 	DefaultManagedPolicies    = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
 	CNIManagedPolicy          = "AmazonEKS_CNI_Policy"
-	AutoscalingReadOnlyPolicy = "AutoScalingReadOnlyAccess"
 	SupportedArchitectures    = []string{"x86_64", "arm64"}
 )
 

diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go
@@ -137,7 +137,7 @@ func (ctx *EksInstanceGroupContext) GetBasicUserData(clusterName, args string, k
   [string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
   [string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
   [string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
-  [string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
+  [string]$Lifecycle=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
   if ($Lifecycle -like "*Warmed*") {
     Echo "Not starting Kubelet due to warmed state."
     & C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule
@@ -182,7 +182,7 @@ if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
 	TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
 	INSTANCE_ID=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
 	REGION=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region)
-	LIFECYCLE=$(aws autoscaling describe-auto-scaling-instances --region $REGION --instance-id $INSTANCE_ID | jq ".AutoScalingInstances[].LifecycleState" || true)
+	LIFECYCLE=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
 	if [[ $LIFECYCLE == *"Warmed"* ]]; then
 		rm /var/lib/cloud/instances/$INSTANCE_ID/sem/config_scripts_user
 		exit 0
@@ -1018,7 +1018,6 @@ func (ctx *EksInstanceGroupContext) UpdateLifecycleHooks(asgName string) error {
 func (ctx *EksInstanceGroupContext) GetManagedPoliciesList(additionalPolicies []string) []string {
 	var (
 		instanceGroup = ctx.GetInstanceGroup()
-		spec          = instanceGroup.GetEKSSpec()
 		annotations   = instanceGroup.GetAnnotations()
 	)
 
@@ -1049,10 +1048,6 @@ func (ctx *EksInstanceGroupContext) GetManagedPoliciesList(additionalPolicies []
 		managedPolicies = append(managedPolicies, fmt.Sprintf("%s/%s", awsprovider.IAMPolicyPrefix, CNIManagedPolicy))
 	}
 
-	if spec.HasWarmPool() {
-		managedPolicies = append(managedPolicies, fmt.Sprintf("%s/%s", awsprovider.IAMPolicyPrefix, AutoscalingReadOnlyPolicy))
-	}
-
 	return managedPolicies
 }
 

diff --git a/controllers/provisioners/eks/helpers_test.go b/controllers/provisioners/eks/helpers_test.go
@@ -155,7 +155,7 @@ if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
 	TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
 	INSTANCE_ID=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
 	REGION=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region)
-	LIFECYCLE=$(aws autoscaling describe-auto-scaling-instances --region $REGION --instance-id $INSTANCE_ID | jq ".AutoScalingInstances[].LifecycleState" || true)
+	LIFECYCLE=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
 	if [[ $LIFECYCLE == *"Warmed"* ]]; then
 		rm /var/lib/cloud/instances/$INSTANCE_ID/sem/config_scripts_user
 		exit 0
@@ -224,7 +224,7 @@ func TestGetBasicUserDataWindows(t *testing.T) {
   [string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
   [string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
   [string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
-  [string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
+  [string]$Lifecycle=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
   if ($Lifecycle -like "*Warmed*") {
     Echo "Not starting Kubelet due to warmed state."
     & C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule
@@ -300,7 +300,7 @@ func TestGetBasicUserDataWindowsWithInjectionDisabled(t *testing.T) {
   [string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
   [string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
   [string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
-  [string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
+  [string]$Lifecycle=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
   if ($Lifecycle -like "*Warmed*") {
     Echo "Not starting Kubelet due to warmed state."
     & C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule

diff --git a/controllers/provisioners/eks/update_test.go b/controllers/provisioners/eks/update_test.go
@@ -632,7 +632,7 @@ func TestUpdateManagedPolicies(t *testing.T) {
 
 	defaultPolicies := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly"}
 	defaultPoliciesIrsa := []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
-	defaultPoliciesWarmPool := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly", "AutoScalingReadOnlyAccess"}
+	defaultPoliciesWarmPool := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly"}
 
 	w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock)
 	ctx := MockContext(ig, k, w)
@@ -652,9 +652,9 @@ func TestUpdateManagedPolicies(t *testing.T) {
 		// when IRSA is disabled, cni policy needs to be attached
 		{attachedPolicies: MockAttachedPolicies(defaultPoliciesIrsa...), additionalPolicies: []string{}, irsaEnabled: false, expectedAttached: 1, expectedDetached: 0},
 		// when warm pool is enabled, managed role is added
-		{attachedPolicies: MockAttachedPolicies(defaultPolicies...), additionalPolicies: []string{}, hasWarmPool: true, expectedAttached: 1, expectedDetached: 0},
+		{attachedPolicies: MockAttachedPolicies(defaultPolicies...), additionalPolicies: []string{}, hasWarmPool: true, expectedAttached: 0, expectedDetached: 0},
 		// when warm pool is disabled, managed role is removed
-		{attachedPolicies: MockAttachedPolicies(defaultPoliciesWarmPool...), additionalPolicies: []string{}, hasWarmPool: false, expectedAttached: 0, expectedDetached: 1},
+		{attachedPolicies: MockAttachedPolicies(defaultPoliciesWarmPool...), additionalPolicies: []string{}, hasWarmPool: false, expectedAttached: 0, expectedDetached: 0},
 		// default policies not attached
 		{attachedPolicies: MockAttachedPolicies(), additionalPolicies: []string{}, expectedAttached: 3, expectedDetached: 0},
 		// additional policies need to be attached

diff --git a/docs/EKS.md b/docs/EKS.md
@@ -419,7 +419,6 @@ You can configure your scaling group to use [AWS Warm Pools for Auto Scaling](ht
 Warm Pools is not officially supported with EKS, hence the following requirements exist in order to use it with EKS:
 - For Amazon Linux 2 instances, your AMI must have `awscli` & `jq`, you can also install it in pre-bootstrap userdata.
 - For Windows instances, your AMI must have the `Get-ASAutoScalingInstance` cmdlet installed.
-- If you use your own IAM role for the instance group, you must make sure it has access to `DescribeAutoScalingInstances`, this is required in order to figure out the current lifecycle state within userdata. If you are provisioning your IAM role through the controller, simply be aware that the controller will add the managed policy `AutoScalingReadOnlyAccess` to the role it creates.
 - This is currently only supported for AmazonLinux2 and Windows based AMIs.
 
 We hope to get rid of these requirements in the future once AWS offers native support for Warm Pools in EKS, these are currently required in order to avoid warming instances to join the cluster briefly while they are provisioned, we are able to avoid this problem by checking if the instances is in the Warmed* lifecycle, and skipping bootstrapping in that case. Skipping on these requirements and using warm pools might mean having nodes join the cluster when the are being warmed, and cause unneeded scheduling of pods on nodes that are about to shutdown.