When CreatePod fails, dump pod options (#2467)

* When CreatePod fails, dump pod options * Dump redacted Pod and PodOptions * Improve function description * Add unit tests
kanisterio · Nov 14, 2023 · b594d34 · b594d34
1 parent bb883c9
commit b594d34
Show file tree

Hide file tree

Showing 2 changed files with 134 additions and 0 deletions.
diff --git a/pkg/kube/pod.go b/pkg/kube/pod.go
@@ -221,6 +221,7 @@ func CreatePod(ctx context.Context, cli kubernetes.Interface, opts *PodOptions)
 
 	pod, err = cli.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
 	if err != nil {
+		log.Error().WithContext(ctx).WithError(err).Print("Failed to create pod.", field.M{"pod": getRedactedPod(pod), "options": getRedactedOptions(opts)})
 		return nil, errors.Wrapf(err, "Failed to create pod. Namespace: %s, NameFmt: %s", opts.Namespace, opts.GenerateName)
 	}
 	return pod, nil
@@ -488,3 +489,60 @@ func GetPodReadyWaitTimeout() time.Duration {
 
 	return DefaultPodReadyWaitTimeout
 }
+
+// getRedactedEnvVariables returns array of variables with removed values
+// This function should be used every time when env variables are logged
+func getRedactedEnvVariables(env []v1.EnvVar) []v1.EnvVar {
+	if len(env) == 0 {
+		return nil
+	}
+
+	result := make([]v1.EnvVar, len(env))
+	for i, ev := range env {
+		result[i] = v1.EnvVar{
+			Name:  ev.Name,
+			Value: "XXXXX",
+		}
+	}
+
+	return result
+}
+
+// getRedactedPod hides all values of env variables from pod, so that it should be safely logged
+func getRedactedPod(pod *v1.Pod) *v1.Pod {
+	if pod == nil {
+		return nil
+	}
+
+	result := *pod // Make shallow copy
+
+	getSanitizedContainers := func(containers []v1.Container) []v1.Container {
+		if len(containers) == 0 {
+			return nil
+		}
+
+		result := make([]v1.Container, len(containers))
+		for i, c := range containers {
+			result[i] = c
+			result[i].Env = getRedactedEnvVariables(c.Env)
+		}
+		return result
+	}
+
+	result.Spec.Containers = getSanitizedContainers(result.Spec.Containers)
+	result.Spec.InitContainers = getSanitizedContainers(result.Spec.InitContainers)
+
+	return &result
+}
+
+// getRedactedOptions hides all values of env variables from pod options, so that they should be safely logged
+func getRedactedOptions(opts *PodOptions) *PodOptions {
+	if opts == nil {
+		return nil
+	}
+
+	result := *opts // Make shallow copy
+
+	result.EnvironmentVariables = getRedactedEnvVariables(result.EnvironmentVariables)
+	return &result
+}
diff --git a/pkg/kube/pod_test.go b/pkg/kube/pod_test.go
@@ -931,6 +931,82 @@ func (s *PodSuite) TestSetLifecycleHook(c *C) {
 	c.Assert(pod.Spec.Containers[0].Lifecycle, DeepEquals, lch)
 }
 
+func (s *PodSuite) TestGetRedactedOptions(c *C) {
+	opts := &PodOptions{
+		Namespace:    s.namespace,
+		GenerateName: "test-",
+		Image:        consts.LatestKanisterToolsImage,
+		Command:      []string{"sh", "-c", "tail -f /dev/null"},
+		EnvironmentVariables: []corev1.EnvVar{
+			{Name: "abc", Value: "def", ValueFrom: &corev1.EnvVarSource{}},
+			{Name: "ooo", Value: "aaa", ValueFrom: &corev1.EnvVarSource{}},
+		},
+	}
+
+	po1 := getRedactedOptions(opts)
+
+	c.Assert(po1.Namespace, Equals, opts.Namespace)
+	c.Assert(po1.GenerateName, Equals, opts.GenerateName)
+	c.Assert(po1.Image, Equals, opts.Image)
+	c.Assert(po1.Command, DeepEquals, opts.Command)
+	c.Assert(po1.EnvironmentVariables, DeepEquals, []corev1.EnvVar{
+		{Name: "abc", Value: "XXXXX"},
+		{Name: "ooo", Value: "XXXXX"},
+	})
+}
+
+func (s *PodSuite) TestGetRedactedPod(c *C) {
+	pod := &corev1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Some kind",
+			APIVersion: "FakeAPI-1.0",
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:  "c1",
+					Image: "img1",
+					Env: []corev1.EnvVar{
+						{Name: "ev1", Value: "23", ValueFrom: &corev1.EnvVarSource{}},
+						{Name: "ev2", Value: "dd", ValueFrom: &corev1.EnvVarSource{}},
+					},
+				},
+				{
+					Name:  "c2",
+					Image: "img2",
+					Env: []corev1.EnvVar{
+						{Name: "a1", Value: "v1", ValueFrom: &corev1.EnvVarSource{}},
+						{Name: "a2", Value: "v2", ValueFrom: &corev1.EnvVarSource{}},
+					},
+				},
+			},
+		},
+	}
+
+	p1 := getRedactedPod(pod)
+
+	c.Assert(p1.TypeMeta, DeepEquals, pod.TypeMeta)
+	c.Assert(len(p1.Spec.Containers), Equals, len(pod.Spec.Containers))
+	c.Assert(p1.Spec.Containers, DeepEquals, []corev1.Container{
+		{
+			Name:  "c1",
+			Image: "img1",
+			Env: []corev1.EnvVar{
+				{Name: "ev1", Value: "XXXXX"},
+				{Name: "ev2", Value: "XXXXX"},
+			},
+		},
+		{
+			Name:  "c2",
+			Image: "img2",
+			Env: []corev1.EnvVar{
+				{Name: "a1", Value: "XXXXX"},
+				{Name: "a2", Value: "XXXXX"},
+			},
+		},
+	})
+}
+
 func (s *PodControllerTestSuite) TestContainerNameFromPodOptsOrDefault(c *C) {
 	for _, tc := range []struct {
 		podOptsContainerName  string