forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 102
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
UPSTREAM: <carry>: provide events, messages, and bodies for probe fai…
…lures of important pods UPSTREAM: <carry>: provide unique reason for pod probe event during termination OpenShift-Rebase-Source: 01542fc
- Loading branch information
1 parent
b6cc9bf
commit 7d67e9f
Showing
4 changed files
with
91 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
package prober | ||
|
||
import ( | ||
"net/http" | ||
"strings" | ||
"time" | ||
|
||
v1 "k8s.io/api/core/v1" | ||
"k8s.io/klog/v2" | ||
"k8s.io/kubernetes/pkg/probe" | ||
httpprobe "k8s.io/kubernetes/pkg/probe/http" | ||
) | ||
|
||
func (pb *prober) maybeProbeForBody(prober httpprobe.Prober, req *http.Request, timeout time.Duration, pod *v1.Pod, container v1.Container, probeType probeType) (probe.Result, string, error) { | ||
if !isInterestingPod(pod) { | ||
return prober.Probe(req, timeout) | ||
} | ||
bodyProber, ok := prober.(httpprobe.DetailedProber) | ||
if !ok { | ||
return prober.Probe(req, timeout) | ||
} | ||
result, output, body, probeError := bodyProber.ProbeForBody(req, timeout) | ||
switch result { | ||
case probe.Success: | ||
return result, output, probeError | ||
case probe.Warning, probe.Failure, probe.Unknown: | ||
// these pods are interesting enough to show the body content | ||
klog.Infof("interesting pod/%s container/%s namespace/%s: %s probe status=%v output=%q start-of-body=%s", | ||
pod.Name, container.Name, pod.Namespace, probeType, result, output, body) | ||
|
||
reason := "ProbeError" // this is the normal value | ||
if pod.DeletionTimestamp != nil { | ||
// If the container was sent a sig-term, we want to have a different reason so we can distinguish this in our | ||
// monitoring and watching code. | ||
// Pod delete does this, but there are other possible reasons as well. We'll start with pod delete to improve the state of the world. | ||
reason = "TerminatingPodProbeError" | ||
} | ||
|
||
// in fact, they are so interesting we'll try to send events for them | ||
pb.recordContainerEvent(pod, &container, v1.EventTypeWarning, reason, "%s probe error: %s\nbody: %s\n", probeType, output, body) | ||
return result, output, probeError | ||
default: | ||
return result, output, probeError | ||
} | ||
} | ||
|
||
func isInterestingPod(pod *v1.Pod) bool { | ||
if pod == nil { | ||
return false | ||
} | ||
if strings.HasPrefix(pod.Namespace, "openshift-") { | ||
return true | ||
} | ||
|
||
return false | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
package http | ||
|
||
import ( | ||
"net/http" | ||
"time" | ||
|
||
"k8s.io/kubernetes/pkg/probe" | ||
) | ||
|
||
// Prober is an interface that defines the Probe function for doing HTTP readiness/liveness checks. | ||
type DetailedProber interface { | ||
ProbeForBody(req *http.Request, timeout time.Duration) (probe.Result, string, string, error) | ||
} | ||
|
||
// ProbeForBody returns a ProbeRunner capable of running an HTTP check. | ||
// returns result, details, body, error | ||
func (pr httpProber) ProbeForBody(req *http.Request, timeout time.Duration) (probe.Result, string, string, error) { | ||
pr.transport.DisableCompression = true // removes Accept-Encoding header | ||
client := &http.Client{ | ||
Timeout: timeout, | ||
Transport: pr.transport, | ||
CheckRedirect: RedirectChecker(pr.followNonLocalRedirects), | ||
} | ||
return DoHTTPProbe(req, client) | ||
} |