Do not count requests with K-Kubelet-Probe header (#5987)

* Do not count requests with K-Kubelet-Probe header

When livenessProbe.httpGet is configured in Knative app, the probe
request routes through the queue-proxy and the request count is
incremented. Due to this behavior, pod cannot be scaled down.

To fix it, this patch changes to stop counting the request with
K-Kubelet-Probe header.

* Add scaledown check to e2e test

cmd/queue/main.go

@@ -157,6 +157,11 @@ type config struct {
 func handler(reqChan chan queue.ReqEvent, breaker *queue.Breaker, handler http.Handler,
 	healthState *health.State, prober func() bool, isAggressive bool) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
+		if network.IsKubeletProbe(r) {
+			handler.ServeHTTP(w, r)
+			return
+		}
+
 		// TODO: Move probe part to network.NewProbeHandler if possible or another handler.
 		if ph := network.KnativeProbeHeader(r); ph != "" {
 			handleKnativeProbe(w, r, ph, healthState, prober, isAggressive)

test/conformance/runtime/readiness_probe_test.go

@@ -23,8 +23,10 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"knative.dev/pkg/test/logstream"
+	revisionresourcenames "knative.dev/serving/pkg/reconciler/revision/resources/names"
 	v1a1opts "knative.dev/serving/pkg/testing/v1alpha1"
 	"knative.dev/serving/test"
+	"knative.dev/serving/test/e2e"
 	v1a1test "knative.dev/serving/test/v1alpha1"
 )
 
@@ -44,17 +46,22 @@ func TestProbeRuntime(t *testing.T) {
 	defer test.TearDown(clients, names)
 
 	t.Log("Creating a new Service")
-	_, _, err := v1a1test.CreateRunLatestServiceReady(t, clients, &names,
-		false /* https TODO(taragu) turn this on after helloworld test running with https */,
+	resources, _, err := v1a1test.CreateRunLatestServiceReady(t, clients, &names,
+		false, /* https TODO(taragu) turn this on after helloworld test running with https */
 		v1a1opts.WithReadinessProbe(
-		&corev1.Probe{
-			Handler: corev1.Handler{
-				HTTPGet: &corev1.HTTPGetAction{
-					Path: "/healthz",
+			&corev1.Probe{
+				Handler: corev1.Handler{
+					HTTPGet: &corev1.HTTPGetAction{
+						Path: "/healthz",
+					},
 				},
-			},
-		}))
+			}))
 	if err != nil {
 		t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err)
 	}
+
+	// Check if scaling down works even if access from liveness probe exists.
+	if err := e2e.WaitForScaleToZero(t, revisionresourcenames.Deployment(resources.Revision), clients); err != nil {
+		t.Fatalf("Could not scale to zero: %v", err)
+	}
 }