eks*: add S3 access to worker node roles

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>

CHANGELOG/CHANGELOG-1.4.md

@@ -34,10 +34,12 @@ See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.3.9...v1.4.0
 - Add [`eks/cluster-loader.MergePodStartupLatency`](https://github.com/aws/aws-k8s-tester/commit/322cd88e94e879157f6b409f9c604fdbbc95e465).
 - Record [`PodStartupLatency` in `eksconfig.AddOnClusterLoaderLocal` via `eks/cluster-loader/local`](https://github.com/aws/aws-k8s-tester/commit/8d4cb87b7bd798ad7f1b5d2de22d0deb26c4c75e).
 - Record [`PodStartupLatency` in `eksconfig.AddOnClusterLoaderRemote` via `eks/cluster-loader/remote`](https://github.com/aws/aws-k8s-tester/commit/8d4cb87b7bd798ad7f1b5d2de22d0deb26c4c75e).
+- Fix [`eks/cluster-loader` error handling](https://github.com/aws/aws-k8s-tester/commit/4a9d982929c32efbdfad820e0cece67721d53034).
+- Improve [`eks/stresser/remote` results fetch](https://github.com/aws/aws-k8s-tester/commit/).
 
 ### `eksconfig`
 
- - [Add ClusterAutoScaler add-on per node-group using `AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS={"GetRef.Name-...":{..."cluster-autoscaler":{"enable":false}...}}`](https://github.com/aws/aws-k8s-tester/pull/99).
+ - [Add ClusterAutoscaler add-on per node-group using `AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS={"GetRef.Name-...":{..."cluster-autoscaler":{"enable":false}...}}`](https://github.com/aws/aws-k8s-tester/pull/99).
 - Fix [typo in `eksconfig.AddOnManagedNodeGroups.LogsTarGzPath`](https://github.com/aws/aws-k8s-tester/commit/7b60047ca4d6fad281db512d4de905a27b80303a).
 - Add [`Status.PrivateDNSToSSHConfig` for node SSH access](https://github.com/aws/aws-k8s-tester/commit/a3c9d7d5e3382c378de686fe0faec6bdeb47f027).
 - Add [`Status.ClusterMetricsRawOutputDir` for `kube-apiserver` `/metrics`](https://github.com/aws/aws-k8s-tester/commit/3ee7554e14f53feae7c5b8ebb1ee4d50b71e0bd7).

eks/mng/role.go

@@ -1,11 +1,13 @@
 package mng
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/aws/aws-k8s-tester/pkg/aws/cfn"
@@ -196,6 +198,14 @@ Resources:
             - route53:ChangeResourceRecordSets
             - route53:DeleteHealthCheck
             Resource: "*"
+{{ if ne .S3BucketName "" }}          - Effect: Allow
+            Action:
+            - s3:ListBucket
+            - s3:GetObject
+            - s3:PutObject
+            Resource:
+            - !Join ['', [!Sub 'arn:${AWS::Partition}:s3:::', '{{.S3BucketName}}']]
+            - !Join ['', [!Sub 'arn:${AWS::Partition}:s3:::', '{{.S3BucketName}}', '/', '{{.ClusterName}}', '/*']]{{ end }}
 
 Outputs:
 
@@ -205,6 +215,11 @@ Outputs:
 
 `
 
+type templateRole struct {
+	S3BucketName string
+	ClusterName  string
+}
+
 func (ts *tester) createRole() error {
 	if !ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleCreate {
 		policyARNs := []string{
@@ -239,9 +254,19 @@ func (ts *tester) createRole() error {
 		return errors.New("cannot create a cluster role with an empty AddOnManagedNodeGroups.RoleName")
 	}
 
-	if err := ioutil.WriteFile(ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleCFNStackYAMLFilePath, []byte(TemplateRole), 0400); err != nil {
+	tr := templateRole{
+		S3BucketName: ts.cfg.EKSConfig.S3BucketName,
+		ClusterName:  ts.cfg.EKSConfig.Name,
+	}
+	tpl := template.Must(template.New("TemplateRole").Parse(TemplateRole))
+	buf := bytes.NewBuffer(nil)
+	if err := tpl.Execute(buf, tr); err != nil {
 		return err
 	}
+	if err := ioutil.WriteFile(ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleCFNStackYAMLFilePath, buf.Bytes(), 0400); err != nil {
+		return err
+	}
+
 	ts.cfg.Logger.Info("creating a new node group role using CFN",
 		zap.String("role-name", ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleName),
 		zap.String("role-cfn-file-path", ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleCFNStackYAMLFilePath),

eks/mng/role_test.go

@@ -0,0 +1,26 @@
+package mng
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"text/template"
+)
+
+func TestTemplateRole(t *testing.T) {
+	tpl := template.Must(template.New("TemplateRole").Parse(TemplateRole))
+	buf := bytes.NewBuffer(nil)
+	if err := tpl.Execute(buf, templateRole{}); err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(buf.String())
+
+	buf.Reset()
+	if err := tpl.Execute(buf, templateRole{
+		S3BucketName: "hello",
+		ClusterName:  "test-cluster",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(buf.String())
+}

eks/ng/autoscaler/autoscaler.go

@@ -0,0 +1,3 @@
+// Package autoscaler implements various auto-scaler.
+// ref. https://github.com/kubernetes/autoscaler
+package autoscaler

eks/ng/autoscaler/cluster-autoscaler.go

@@ -22,7 +22,7 @@ import (
 
 // auto discover
 // ref. https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml
-const ClusterAutoscalerYAML = `
+const clusterAutoscalerYAML = `
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -204,7 +204,7 @@ var caImages = map[string]string{
 
 const (
 	nodeGroupAuotDiscoveryData      = `            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/`
-	clusterAutoScalerDeploymentName = "cluster-autoscaler"
+	clusterAutoscalerDeploymentName = "cluster-autoscaler"
 )
 
 type caSpecData struct {
@@ -220,7 +220,7 @@ type Config struct {
 	K8SClient k8s_client.EKS
 }
 
-// ClusterAutoScaler defines cluster autoscaler operation.
+// ClusterAutoscaler defines cluster autoscaler operation.
 type ClusterAutoscaler interface {
 	Create() error
 }
@@ -238,7 +238,7 @@ type tester struct {
 func (ts *tester) Create() error {
 	needInstall := false
 	for _, cur := range ts.cfg.EKSConfig.AddOnNodeGroups.ASGs {
-		if cur.ClusterAutoScaler != nil && cur.ClusterAutoScaler.Enable {
+		if cur.ClusterAutoscaler != nil && cur.ClusterAutoscaler.Enable {
 			needInstall = true
 			break
 		}
@@ -259,7 +259,7 @@ func (ts *tester) installCA() error {
 		return fmt.Errorf("no CA found for %q", ts.cfg.EKSConfig.Parameters.Version)
 	}
 	caData.NodeGroupAutoDiscovery = nodeGroupAuotDiscoveryData + ts.cfg.EKSConfig.Name
-	tpl := template.Must(template.New("TemplateCA").Parse(ClusterAutoscalerYAML))
+	tpl := template.Must(template.New("TemplateCA").Parse(clusterAutoscalerYAML))
 	buf := bytes.NewBuffer(nil)
 	if err := tpl.Execute(buf, caData); err != nil {
 		return err
@@ -324,7 +324,7 @@ func (ts *tester) waitDeploymentCA() error {
 		"--namespace=kube-system",
 		"describe",
 		"deployment",
-		clusterAutoScalerDeploymentName,
+		clusterAutoscalerDeploymentName,
 	).CombinedOutput()
 	cancel()
 	if err != nil {

eks/ng/ng.go

@@ -72,7 +72,7 @@ func New(cfg Config) Tester {
 		}),
 		logsMu:     new(sync.RWMutex),
 		failedOnce: false,
-		clusterAutoScaler: autoscaler.New(autoscaler.Config{
+		clusterAutoscaler: autoscaler.New(autoscaler.Config{
 			Logger:    cfg.Logger,
 			Stopc:     cfg.Stopc,
 			EKSConfig: cfg.EKSConfig,
@@ -86,7 +86,7 @@ type tester struct {
 	nodeWaiter        wait.NodeWaiter
 	logsMu            *sync.RWMutex
 	failedOnce        bool
-	clusterAutoScaler autoscaler.ClusterAutoscaler
+	clusterAutoscaler autoscaler.ClusterAutoscaler
 }
 
 func (ts *tester) Create() (err error) {
@@ -125,7 +125,7 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createSSM(); err != nil {
 		return err
 	}
-	if err = ts.clusterAutoScaler.Create(); err != nil {
+	if err = ts.clusterAutoscaler.Create(); err != nil {
 		return err
 	}
 

eks/ng/nodes.go

@@ -448,7 +448,7 @@ func (ts *tester) createASGs() error {
 			tg.UserData += `              /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource ASG --region ${AWS::Region}`
 		}
 		tg.AsgTagData = ""
-		if cur.ClusterAutoScaler != nil && cur.ClusterAutoScaler.Enable {
+		if cur.ClusterAutoscaler != nil && cur.ClusterAutoscaler.Enable {
 			tg.AsgTagData = asgTagDataNG
 		}
 		tpl := template.Must(template.New("TemplateASG").Parse(TemplateASG))

eks/ng/role.go

@@ -198,7 +198,15 @@ Resources:
             - route53:ChangeResourceRecordSets
             - route53:DeleteHealthCheck
             Resource: "*"
-{{ if ne .ASGPolicy "" }}{{.ASGPolicy}}{{ end }}
+{{ if ne .S3BucketName "" }}          - Effect: Allow
+            Action:
+            - s3:ListBucket
+            - s3:GetObject
+            - s3:PutObject
+            Resource:
+            - !Join ['', [!Sub 'arn:${AWS::Partition}:s3:::', '{{.S3BucketName}}']]
+            - !Join ['', [!Sub 'arn:${AWS::Partition}:s3:::', '{{.S3BucketName}}', '/', '{{.ClusterName}}', '/*']]{{ end }}
+{{ if ne .ASGPolicyData "" }}{{.ASGPolicyData}}{{ end }}
 
 Outputs:
 
@@ -209,7 +217,9 @@ Outputs:
 `
 
 type templateRole struct {
-	ASGPolicy string
+	S3BucketName  string
+	ClusterName   string
+	ASGPolicyData string
 }
 
 const asgPolicyData = `          - Effect: Allow
@@ -221,8 +231,7 @@ const asgPolicyData = `          - Effect: Allow
             - autoscaling:SetDesiredCapacity
             - autoscaling:TerminateInstanceInAutoScalingGroup
             - ec2:DescribeLaunchTemplateVersions
-            Resource: "*"
-`
+            Resource: "*"`
 
 func (ts *tester) createRole() error {
 	if !ts.cfg.EKSConfig.AddOnNodeGroups.RoleCreate {
@@ -257,8 +266,15 @@ func (ts *tester) createRole() error {
 	if ts.cfg.EKSConfig.AddOnNodeGroups.RoleName == "" {
 		return errors.New("cannot create a cluster role with an empty AddOnNodeGroups.RoleName")
 	}
-	tr := templateRole{}
-	tr.ASGPolicy = asgPolicyData
+
+	tr := templateRole{
+		S3BucketName: ts.cfg.EKSConfig.S3BucketName,
+		ClusterName:  ts.cfg.EKSConfig.Name,
+	}
+	if ts.cfg.EKSConfig.AddOnNodeGroups.IsEnabledClusterAutoscaler() {
+		ts.cfg.Logger.Info("adding autoscaling policy for cluster autoscaler")
+		tr.ASGPolicyData = asgPolicyData
+	}
 	tpl := template.Must(template.New("TemplateRole").Parse(TemplateRole))
 	buf := bytes.NewBuffer(nil)
 	if err := tpl.Execute(buf, tr); err != nil {
@@ -267,6 +283,7 @@ func (ts *tester) createRole() error {
 	if err := ioutil.WriteFile(ts.cfg.EKSConfig.AddOnNodeGroups.RoleCFNStackYAMLFilePath, buf.Bytes(), 0400); err != nil {
 		return err
 	}
+
 	ts.cfg.Logger.Info("creating a new NG role using CFN",
 		zap.String("role-name", ts.cfg.EKSConfig.AddOnNodeGroups.RoleName),
 		zap.String("role-cfn-file-path", ts.cfg.EKSConfig.AddOnNodeGroups.RoleCFNStackYAMLFilePath),

eks/ng/role_test.go

@@ -0,0 +1,27 @@
+package ng
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+	"text/template"
+)
+
+func TestTemplateRole(t *testing.T) {
+	tpl := template.Must(template.New("TemplateRole").Parse(TemplateRole))
+	buf := bytes.NewBuffer(nil)
+	if err := tpl.Execute(buf, templateRole{}); err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(buf.String())
+
+	buf.Reset()
+	if err := tpl.Execute(buf, templateRole{
+		S3BucketName:  "hello",
+		ClusterName:   "test-cluster",
+		ASGPolicyData: asgPolicyData,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(buf.String())
+}

eks/stresser/stresser.go

@@ -280,7 +280,7 @@ func startWrites(
 	donec chan struct{},
 	writeLatencies chan<- metrics.Durations,
 ) {
-	lg.Info("starting startWrites")
+	lg.Info("starting writes")
 	ds := make(metrics.Durations, 0, 20000)
 	defer func() {
 		lg.Info("sending write latency results", zap.Int("total-results", len(ds)))
@@ -416,7 +416,7 @@ func startReads(
 	donec chan struct{},
 	readLatencies chan<- metrics.Durations,
 ) {
-	lg.Info("starting startReads", zap.Strings("namespaces", ns))
+	lg.Info("starting reads", zap.Strings("namespaces", ns))
 	ds := make(metrics.Durations, 0, 20000)
 	defer func() {
 		lg.Info("sending read latency results", zap.Int("total-results", len(ds)))

eksconfig/add-on-irsa.go

@@ -85,6 +85,10 @@ func (cfg *Config) validateAddOnIRSA() error {
 	if !cfg.IsEnabledAddOnIRSA() {
 		return nil
 	}
+	if cfg.S3BucketName == "" {
+		return errors.New("AddOnIRSA requires S3 bucket but S3BucketName empty")
+	}
+
 	if !cfg.IsEnabledAddOnNodeGroups() && !cfg.IsEnabledAddOnManagedNodeGroups() {
 		return errors.New("AddOnIRSA.Enable true but no node group is enabled")
 	}
@@ -106,10 +110,6 @@ func (cfg *Config) validateAddOnIRSA() error {
 		return errors.New("AddOnIRSA.RepositoryImageTag empty")
 	}
 
-	if cfg.S3BucketName == "" {
-		return errors.New("AddOnIRSA requires S3 bucket but S3BucketName empty")
-	}
-
 	if cfg.AddOnIRSA.RoleName == "" {
 		cfg.AddOnIRSA.RoleName = cfg.Name + "-add-on-irsa-role"
 	}

eksconfig/add-on-managed-node-groups.go

@@ -283,6 +283,9 @@ func (cfg *Config) validateAddOnManagedNodeGroups() error {
 		if len(cfg.AddOnManagedNodeGroups.RoleServicePrincipals) > 0 {
 			return fmt.Errorf("AddOnManagedNodeGroups.RoleCreate false; expect empty RoleServicePrincipals but got %q", cfg.AddOnManagedNodeGroups.RoleServicePrincipals)
 		}
+		if cfg.IsEnabledAddOnStresserRemote() {
+			return errors.New("'AddOnStresserRemote.Enable == true' requires 'AddOnManagedNodeGroups.RoleCreate == true' but got 'false'")
+		}
 	}
 
 	names, processed := make(map[string]struct{}), make(map[string]MNG)
@@ -439,6 +442,9 @@ func (cfg *Config) validateAddOnManagedNodeGroups() error {
 		if cfg.IsEnabledAddOnNLBHelloWorld() && cfg.AddOnNLBHelloWorld.DeploymentReplicas < int32(cur.ASGDesiredCapacity) {
 			cfg.AddOnNLBHelloWorld.DeploymentReplicas = int32(cur.ASGDesiredCapacity)
 		}
+		if cfg.IsEnabledAddOnNLBGuestbook() && cfg.AddOnNLBGuestbook.DeploymentReplicas < int32(cur.ASGDesiredCapacity) {
+			cfg.AddOnNLBGuestbook.DeploymentReplicas = int32(cur.ASGDesiredCapacity)
+		}
 		if cfg.IsEnabledAddOnALB2048() && cfg.AddOnALB2048.DeploymentReplicasALB < int32(cur.ASGDesiredCapacity) {
 			cfg.AddOnALB2048.DeploymentReplicasALB = int32(cur.ASGDesiredCapacity)
 		}