Skip to content

Commit

Permalink
eks*: add "WithPodFunc"
Browse files Browse the repository at this point in the history 
Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
  • Loading branch information
@gyuho
gyuho committed Jul 12, 2020
1 parent 0c569bf commit f245f77
Show file tree
Hide file tree
Showing 9 changed files with 145 additions and 45 deletions.
1 change: 1 addition & 0 deletions CHANGELOG/CHANGELOG-1.4.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.4.4...v1.4.5
## `pkg`

- Add [`pkg/k8s-client.WaitForDeploymentCompletes`](https://github.com/aws/aws-k8s-tester/commit/a8a69c5e092abf88ff7e0ddb636c4ce8400cf2f1).
- Add [`pkg/k8s-client.WithPodFunc` to debug job pod failures](https://github.com/aws/aws-k8s-tester/commit/).



Expand Down
3 changes: 1 addition & 2 deletions eks/app-mesh/app-mesh.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,8 @@ func (ts *tester) Delete() error {
out := strings.TrimSpace(string(output))
if err != nil {
ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
} else {
fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
}
fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
}),
k8s_client.WithForceDelete(true),
); err != nil {
Expand Down
28 changes: 27 additions & 1 deletion eks/configmaps/remote/configmaps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,6 @@ func (ts *tester) Create() (err error) {
configmapsJobName,
}
descCmd := strings.Join(descArgs, " ")
ts.cfg.Logger.Info("describing job", zap.String("describe-command", descCmd))
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
Expand All @@ -158,6 +157,33 @@ func (ts *tester) Create() (err error) {
out := string(descOutput)
fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", descCmd, out)
}),
k8s_client.WithPodFunc(func(pod v1.Pod) {
switch pod.Status.Phase {
case v1.PodFailed:
ts.cfg.Logger.Warn("pod failed",
zap.String("namespace", pod.Namespace),
zap.String("pod-name", pod.Name),
zap.String("pod-status-phase", fmt.Sprintf("%v", pod.Status.Phase)),
)
descArgs := []string{
ts.cfg.EKSConfig.KubectlPath,
"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
"--namespace=" + pod.Namespace,
"describe",
"pod",
pod.Name,
}
descCmd := strings.Join(descArgs, " ")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
if err != nil {
ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
}
out := string(descOutput)
fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", descCmd, out)
}
}),
)
cancel()
if err != nil {
Expand Down
28 changes: 27 additions & 1 deletion eks/csrs/remote/csrs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,6 @@ func (ts *tester) Create() (err error) {
csrsJobName,
}
descCmd := strings.Join(descArgs, " ")
ts.cfg.Logger.Info("describing job", zap.String("describe-command", descCmd))
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
Expand All @@ -158,6 +157,33 @@ func (ts *tester) Create() (err error) {
out := string(descOutput)
fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", descCmd, out)
}),
k8s_client.WithPodFunc(func(pod v1.Pod) {
switch pod.Status.Phase {
case v1.PodFailed:
ts.cfg.Logger.Warn("pod failed",
zap.String("namespace", pod.Namespace),
zap.String("pod-name", pod.Name),
zap.String("pod-status-phase", fmt.Sprintf("%v", pod.Status.Phase)),
)
descArgs := []string{
ts.cfg.EKSConfig.KubectlPath,
"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
"--namespace=" + pod.Namespace,
"describe",
"pod",
pod.Name,
}
descCmd := strings.Join(descArgs, " ")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
if err != nil {
ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
}
out := string(descOutput)
fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", descCmd, out)
}
}),
)
cancel()
if err != nil {
Expand Down
11 changes: 3 additions & 8 deletions eks/irsa/irsa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1014,11 +1014,6 @@ func (ts *tester) checkLogs() error {
"pod/" + podName,
}
logsCmd := strings.Join(logsArgs, " ")
ts.cfg.Logger.Info("checking pod logs",
zap.String("pod-name", podName),
zap.String("node-name", nodeName),
zap.String("logs-command", logsCmd),
)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
runOutput, err := exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
cancel()
Expand All @@ -1030,7 +1025,7 @@ func (ts *tester) checkLogs() error {
if !strings.Contains(output, ts.sleepMessage) {
continue
}
if _, err = f.WriteString(fmt.Sprintf("'%s' from %q:\n\n%s\n\n", logsCmd, nodeName, output)); err != nil {
if _, err = f.WriteString(fmt.Sprintf("'%s' from node %q:\n\n%s\n\n", logsCmd, nodeName, output)); err != nil {
ts.cfg.Logger.Warn("failed to write", zap.Error(err))
continue
}
Expand Down Expand Up @@ -1076,12 +1071,12 @@ func (ts *tester) checkLogs() error {
if !strings.Contains(output, ts.testBody) {
continue
}
if _, err = f.WriteString(fmt.Sprintf("'%s' from %q:\n\n%s\n\n", catCmd, nodeName, output)); err != nil {
if _, err = f.WriteString(fmt.Sprintf("'%s' from node %q:\n\n%s\n\n", catCmd, nodeName, output)); err != nil {
ts.cfg.Logger.Warn("failed to write", zap.Error(err))
continue
}
ts.cfg.Logger.Info("checked pod output file from node, found matching text body", zap.String("pod-name", podName))

ts.cfg.Logger.Info("checked pod output file from node, found matching text body", zap.String("pod-name", podName))
success++
}
if success < expects {
Expand Down
3 changes: 0 additions & 3 deletions eks/nlb-guestbook/nlb-guestbook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,6 @@ func (ts *tester) waitDeploymentRedisLeader() (err error) {
"--selector=app.kubernetes.io/name=" + redisLabelName + ",role=" + redisLeaderRoleName,
}
logsCmd := strings.Join(logsArgs, " ")
ts.cfg.Logger.Info("fetching pod logs", zap.String("logs-command", logsCmd))
ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
logsOutput, err := exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
cancel()
Expand Down Expand Up @@ -641,7 +640,6 @@ func (ts *tester) waitDeploymentRedisFollower() (err error) {
"--selector=app.kubernetes.io/name=" + redisLabelName + ",role=" + redisFollowerRoleName,
}
logsCmd := strings.Join(logsArgs, " ")
ts.cfg.Logger.Info("fetching pod logs", zap.String("logs-command", logsCmd))
ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
logsOutput, err := exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
cancel()
Expand Down Expand Up @@ -910,7 +908,6 @@ func (ts *tester) waitDeploymentGuestbook() (err error) {
"--selector=app.kubernetes.io/name=" + nlbGuestbookAppName,
}
logsCmd := strings.Join(logsArgs, " ")
ts.cfg.Logger.Info("fetching pod logs", zap.String("logs-command", logsCmd))
ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
logsOutput, err := exec.New().CommandContext(ctx, logsArgs[0], logsArgs[1:]...).CombinedOutput()
cancel()
Expand Down
28 changes: 27 additions & 1 deletion eks/secrets/remote/secrets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,6 @@ func (ts *tester) Create() (err error) {
secretsJobName,
}
descCmd := strings.Join(descArgs, " ")
ts.cfg.Logger.Info("describing job", zap.String("describe-command", descCmd))
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
Expand All @@ -158,6 +157,33 @@ func (ts *tester) Create() (err error) {
out := string(descOutput)
fmt.Fprintf(ts.cfg.LogWriter, "\n\n\n\"%s\" output:\n\n%s\n\n", descCmd, out)
}),
k8s_client.WithPodFunc(func(pod v1.Pod) {
switch pod.Status.Phase {
case v1.PodFailed:
ts.cfg.Logger.Warn("pod failed",
zap.String("namespace", pod.Namespace),
zap.String("pod-name", pod.Name),
zap.String("pod-status-phase", fmt.Sprintf("%v", pod.Status.Phase)),
)
descArgs := []string{
ts.cfg.EKSConfig.KubectlPath,
"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
"--namespace=" + pod.Namespace,
"describe",
"pod",
pod.Name,
}
descCmd := strings.Join(descArgs, " ")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
if err != nil {
ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
}
out := string(descOutput)
fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", descCmd, out)
}
}),
)
cancel()
if err != nil {
Expand Down
36 changes: 29 additions & 7 deletions eks/stresser/remote/stresser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,6 @@ func (ts *tester) Create() (err error) {
stresserJobName,
}
descCmd := strings.Join(descArgs, " ")
ts.cfg.Logger.Info("describing job", zap.String("describe-command", descCmd))
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
Expand All @@ -174,17 +173,41 @@ func (ts *tester) Create() (err error) {
"--tail=10",
}
cmdLogs := strings.Join(argsLogs, " ")

fmt.Fprintf(ts.cfg.LogWriter, "\n")
ts.cfg.Logger.Info("running query function for remote stresser")
ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
logsOutput, err := exec.New().CommandContext(ctx, argsLogs[0], argsLogs[1:]...).CombinedOutput()
cancel()
out = string(logsOutput)
if err != nil {
ts.cfg.Logger.Warn("'kubectl logs' failed", zap.Error(err))
}
fmt.Fprintf(ts.cfg.LogWriter, "\n\"%s\":\n%s\n", cmdLogs, out)
fmt.Fprintf(ts.cfg.LogWriter, "\n\n\"%s\":\n%s\n", cmdLogs, out)
}),
k8s_client.WithPodFunc(func(pod v1.Pod) {
switch pod.Status.Phase {
case v1.PodFailed:
ts.cfg.Logger.Warn("pod failed",
zap.String("namespace", pod.Namespace),
zap.String("pod-name", pod.Name),
zap.String("pod-status-phase", fmt.Sprintf("%v", pod.Status.Phase)),
)
descArgs := []string{
ts.cfg.EKSConfig.KubectlPath,
"--kubeconfig=" + ts.cfg.EKSConfig.KubeConfigPath,
"--namespace=" + pod.Namespace,
"describe",
"pod",
pod.Name,
}
descCmd := strings.Join(descArgs, " ")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
descOutput, err := exec.New().CommandContext(ctx, descArgs[0], descArgs[1:]...).CombinedOutput()
cancel()
if err != nil {
ts.cfg.Logger.Warn("'kubectl describe job' failed", zap.Error(err))
}
out := string(descOutput)
fmt.Fprintf(ts.cfg.LogWriter, "\"%s\" output:\n\n%s\n\n", descCmd, out)
}
}),
)
cancel()
Expand Down Expand Up @@ -268,9 +291,8 @@ func (ts *tester) Delete() error {
out := strings.TrimSpace(string(output))
if err != nil {
ts.cfg.Logger.Warn("'kubectl get all' failed", zap.Error(err))
} else {
fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
}
fmt.Fprintf(ts.cfg.LogWriter, "\n\n'%s' output:\n\n%s\n\n", getAllCmd, out)
}),
k8s_client.WithForceDelete(true),
); err != nil {
Expand Down
Loading

0 comments on commit f245f77

Please sign in to comment.