*: clean up polling function with contexts

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>

eks/cluster-loader/remote/cluster-loader.go

@@ -114,18 +114,21 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createJob(); err != nil {
 		return err
 	}
+	timeout := 5*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnClusterLoaderRemote.Runs)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	var pods []v1.Pod
 	_, pods, err = k8s_client.WaitForJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
 		2*time.Minute,
 		10*time.Second,
-		3*time.Minute+time.Duration(ts.cfg.EKSConfig.AddOnClusterLoaderRemote.Runs)*2*time.Minute,
 		ts.cfg.EKSConfig.AddOnClusterLoaderRemote.Namespace,
 		clusterLoaderJobName,
 		1,
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

eks/cluster/wait/poll.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-k8s-tester/eksconfig"
+	"github.com/aws/aws-k8s-tester/pkg/ctxutil"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/eks"
@@ -60,6 +61,7 @@ func Poll(
 		zap.String("desired-status", desiredClusterStatus),
 		zap.String("initial-wait", initialWait.String()),
 		zap.String("poll-interval", pollInterval.String()),
+		zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 	)
 
 	now := time.Now()
@@ -128,6 +130,7 @@ func Poll(
 				zap.String("cluster-name", clusterName),
 				zap.String("status", currentStatus),
 				zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
+				zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 			)
 			switch currentStatus {
 			case desiredClusterStatus:
@@ -208,7 +211,7 @@ func PollUpdate(
 	requestID string,
 	desiredUpdateStatus string,
 	initialWait time.Duration,
-	interval time.Duration,
+	pollInterval time.Duration,
 	opts ...OpOption) <-chan UpdateStatus {
 
 	ret := Op{}
@@ -219,7 +222,8 @@ func PollUpdate(
 		zap.String("request-id", requestID),
 		zap.String("desired-update-status", desiredUpdateStatus),
 		zap.String("initial-wait", initialWait.String()),
-		zap.String("interval", interval.String()),
+		zap.String("poll-interval", pollInterval.String()),
+		zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 	)
 
 	now := time.Now()
@@ -251,7 +255,7 @@ func PollUpdate(
 				// in case stack has already reached desired status
 				// wait from second interation
 				if waitDur == time.Duration(0) {
-					waitDur = interval
+					waitDur = pollInterval
 				}
 			}
 
@@ -286,6 +290,7 @@ func PollUpdate(
 				zap.String("status", currentStatus),
 				zap.String("update-type", updateType),
 				zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
+				zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 			)
 			switch currentStatus {
 			case desiredUpdateStatus:

eks/configmaps/remote/configmaps.go

@@ -119,18 +119,21 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createJob(); err != nil {
 		return err
 	}
+	timeout := 5*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnConfigmapsRemote.Completes) + time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnConfigmapsRemote.Objects/100)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	var pods []v1.Pod
 	_, pods, err = k8s_client.WaitForJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
 		3*time.Minute,
 		10*time.Second,
-		3*time.Minute+time.Duration(ts.cfg.EKSConfig.AddOnConfigmapsRemote.Completes)*30*time.Second,
 		ts.cfg.EKSConfig.AddOnConfigmapsRemote.Namespace,
 		configmapsJobName,
 		ts.cfg.EKSConfig.AddOnConfigmapsRemote.Completes,
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

eks/cron-jobs/cron-jobs.go

@@ -79,19 +79,21 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createCronJob(); err != nil {
 		return err
 	}
-
-	// take about 4-min for 10 cron jobs to trigger
-	_, pods, err := k8s_client.WaitForCronJobCompletes(
+	timeout := 10*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnCronJobs.Completes)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	var pods []v1.Pod
+	_, pods, err = k8s_client.WaitForCronJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
 		3*time.Minute,
 		5*time.Second,
-		10*time.Minute+10*time.Duration(ts.cfg.EKSConfig.AddOnCronJobs.Completes)*time.Second,
 		ts.cfg.EKSConfig.AddOnCronJobs.Namespace,
 		cronJobName,
 		int(ts.cfg.EKSConfig.AddOnCronJobs.Completes),
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

eks/csrs/remote/csrs.go

@@ -119,18 +119,21 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createJob(); err != nil {
 		return err
 	}
+	timeout := 5*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnCSRsRemote.Completes) + time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnCSRsRemote.Objects/100)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	var pods []v1.Pod
 	_, pods, err = k8s_client.WaitForJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
 		3*time.Minute,
 		10*time.Second,
-		3*time.Minute+time.Duration(ts.cfg.EKSConfig.AddOnCSRsRemote.Completes)*30*time.Second,
 		ts.cfg.EKSConfig.AddOnCSRsRemote.Namespace,
 		csrsJobName,
 		ts.cfg.EKSConfig.AddOnCSRsRemote.Completes,
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

eks/fargate/wait/poll.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-k8s-tester/pkg/ctxutil"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/eks"
@@ -53,6 +54,7 @@ func Poll(
 		zap.String("desired-status", desiredStatus),
 		zap.String("initial-wait", initialWait.String()),
 		zap.String("poll-interval", pollInterval.String()),
+		zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 	)
 
 	now := time.Now()
@@ -124,6 +126,7 @@ func Poll(
 				zap.String("fargate-name", profileName),
 				zap.String("status", currentStatus),
 				zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
+				zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 			)
 			switch currentStatus {
 			case desiredStatus:

eks/jobs-echo/jobs-echo.go

@@ -78,18 +78,21 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createJob(); err != nil {
 		return err
 	}
-
-	_, pods, err := k8s_client.WaitForJobCompletes(
+	timeout := 5*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnJobsEcho.Completes)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	var pods []v1.Pod
+	_, pods, err = k8s_client.WaitForJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
 		2*time.Minute,
 		5*time.Second,
-		3*time.Minute+10*time.Duration(ts.cfg.EKSConfig.AddOnJobsEcho.Completes)*time.Second,
 		ts.cfg.EKSConfig.AddOnJobsEcho.Namespace,
 		jobName,
 		int(ts.cfg.EKSConfig.AddOnJobsEcho.Completes),
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

eks/jobs-pi/jobs-pi.go

@@ -77,18 +77,21 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createJob(); err != nil {
 		return err
 	}
-
-	_, pods, err := k8s_client.WaitForJobCompletes(
+	timeout := 5*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnJobsPi.Completes)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	var pods []v1.Pod
+	_, pods, err = k8s_client.WaitForJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
 		2*time.Minute,
 		5*time.Second,
-		3*time.Minute+10*time.Duration(ts.cfg.EKSConfig.AddOnJobsPi.Completes)*time.Second,
 		ts.cfg.EKSConfig.AddOnJobsPi.Namespace,
 		jobName,
 		int(ts.cfg.EKSConfig.AddOnJobsPi.Completes),
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

eks/mng/scale/scale.go

@@ -135,7 +135,7 @@ func (ts *tester) scaleMNG(mngName string, update eksconfig.MNGScaleUpdate) (err
 		return fmt.Errorf("MNGs[%q] Update.Id empty", mngName)
 	}
 	initialWait := 3 * time.Minute
-	totalWait := 5*time.Minute + 20*time.Second*time.Duration(cur.ASGDesiredCapacity)
+	totalWait := 10*time.Minute + 3*time.Minute*time.Duration(cur.ASGDesiredCapacity)
 	ts.cfg.Logger.Info("sent MNG scaler request; polling",
 		zap.String("cluster-name", ts.cfg.EKSConfig.Name),
 		zap.String("mng-name", mngName),

eks/mng/version-upgrade/version-upgrade.go

@@ -119,7 +119,7 @@ func (ts *tester) Upgrade(mngName string) (err error) {
 
 	// takes TODO
 	initialWait := 3 * time.Minute
-	totalWait := 30*time.Minute + 3*time.Minute*time.Duration(cur.ASGDesiredCapacity)
+	totalWait := time.Hour + 3*time.Minute*time.Duration(cur.ASGDesiredCapacity)
 
 	ts.cfg.Logger.Info("sent MNG upgrade request; polling",
 		zap.String("cluster-name", ts.cfg.EKSConfig.Name),

eks/mng/wait/poll.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-k8s-tester/pkg/ctxutil"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/eks"
@@ -58,6 +59,7 @@ func Poll(
 		zap.String("desired-status", desiredNodeGroupStatus),
 		zap.String("initial-wait", initialWait.String()),
 		zap.String("poll-interval", pollInterval.String()),
+		zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 	)
 
 	now := time.Now()
@@ -128,6 +130,7 @@ func Poll(
 				zap.String("mng-name", mngName),
 				zap.String("status", currentStatus),
 				zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
+				zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 			)
 			switch currentStatus {
 			case desiredNodeGroupStatus:
@@ -229,7 +232,7 @@ func PollUpdate(
 	requestID string,
 	desiredUpdateStatus string,
 	initialWait time.Duration,
-	interval time.Duration,
+	pollInterval time.Duration,
 	opts ...OpOption) <-chan UpdateStatus {
 
 	ret := Op{}
@@ -241,7 +244,8 @@ func PollUpdate(
 		zap.String("request-id", requestID),
 		zap.String("desired-update-status", desiredUpdateStatus),
 		zap.String("initial-wait", initialWait.String()),
-		zap.String("interval", interval.String()),
+		zap.String("poll-interval", pollInterval.String()),
+		zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 	)
 
 	now := time.Now()
@@ -273,7 +277,7 @@ func PollUpdate(
 				// in case stack has already reached desired status
 				// wait from second interation
 				if waitDur == time.Duration(0) {
-					waitDur = interval
+					waitDur = pollInterval
 				}
 			}
 
@@ -310,6 +314,7 @@ func PollUpdate(
 				zap.String("status", currentStatus),
 				zap.String("update-type", updateType),
 				zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
+				zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 			)
 			switch currentStatus {
 			case desiredUpdateStatus:

eks/secrets/remote/secrets.go

@@ -119,18 +119,21 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createJob(); err != nil {
 		return err
 	}
+	timeout := 5*time.Minute + 5*time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnSecretsRemote.Completes) + time.Minute*time.Duration(ts.cfg.EKSConfig.AddOnSecretsRemote.Objects/100)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	var pods []v1.Pod
 	_, pods, err = k8s_client.WaitForJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
 		3*time.Minute,
 		10*time.Second,
-		3*time.Minute+time.Duration(ts.cfg.EKSConfig.AddOnSecretsRemote.Completes)*30*time.Second,
 		ts.cfg.EKSConfig.AddOnSecretsRemote.Namespace,
 		secretsJobName,
 		ts.cfg.EKSConfig.AddOnSecretsRemote.Completes,
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

eks/stresser/remote/stresser.go

@@ -127,14 +127,16 @@ func (ts *tester) Create() (err error) {
 	if err = ts.createJob(); err != nil {
 		return err
 	}
+	timeout := 15*time.Minute + ts.cfg.EKSConfig.AddOnStresserRemote.Duration
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	var pods []v1.Pod
 	_, pods, err = k8s_client.WaitForJobCompletes(
+		ctx,
 		ts.cfg.Logger,
 		ts.cfg.Stopc,
 		ts.cfg.K8SClient,
-		3*time.Minute,
-		10*time.Second,
-		ts.cfg.EKSConfig.AddOnStresserRemote.Duration+10*time.Minute,
+		time.Minute+ts.cfg.EKSConfig.AddOnStresserRemote.Duration/2,
+		15*time.Second,
 		ts.cfg.EKSConfig.AddOnStresserRemote.Namespace,
 		stresserJobName,
 		ts.cfg.EKSConfig.AddOnStresserRemote.Completes,
@@ -161,6 +163,7 @@ func (ts *tester) Create() (err error) {
 			fmt.Fprintf(ts.cfg.LogWriter, "\n\"%s\":\n%s\n", cmdLogs, out)
 		}),
 	)
+	cancel()
 	if err != nil {
 		return err
 	}

pkg/aws/cfn/cfn.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-k8s-tester/pkg/ctxutil"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
 	"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
@@ -40,6 +41,7 @@ func Poll(
 		zap.String("want", desiredStackStatus),
 		zap.String("initial-wait", initialWait.String()),
 		zap.String("poll-interval", pollInterval.String()),
+		zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 	)
 	ch := make(chan StackStatus, 10)
 	go func() {
@@ -110,12 +112,13 @@ func Poll(
 				prevStatusReason = currentStatusReason
 			}
 
-			lg.Info("polling",
+			lg.Info("poll",
 				zap.String("name", aws.StringValue(stack.StackName)),
 				zap.String("desired", desiredStackStatus),
 				zap.String("current", currentStatus),
 				zap.String("current-reason", currentStatusReason),
 				zap.String("started", humanize.RelTime(now, time.Now(), "ago", "from now")),
+				zap.String("ctx-time-left", ctxutil.TimeLeftTillDeadline(ctx)),
 			)
 
 			if desiredStackStatus != cloudformation.ResourceStatusDeleteComplete &&