Merge pull request #121120 from enj/enj/i/h2_dos

Prevent rapid reset http2 DOS on API server

Kubernetes-commit: cb713c15e99d59cb5b2f9015d1d978fee8142965

pkg/util/runtime/runtime.go

@@ -126,14 +126,17 @@ type rudimentaryErrorBackoff struct {
 // OnError will block if it is called more often than the embedded period time.
 // This will prevent overly tight hot error loops.
 func (r *rudimentaryErrorBackoff) OnError(error) {
+	now := time.Now() // start the timer before acquiring the lock
 	r.lastErrorTimeLock.Lock()
-	defer r.lastErrorTimeLock.Unlock()
-	d := time.Since(r.lastErrorTime)
-	if d < r.minPeriod {
-		// If the time moves backwards for any reason, do nothing
-		time.Sleep(r.minPeriod - d)
-	}
+	d := now.Sub(r.lastErrorTime)
 	r.lastErrorTime = time.Now()
+	r.lastErrorTimeLock.Unlock()
+
+	// Do not sleep with the lock held because that causes all callers of HandleError to block.
+	// We only want the current goroutine to block.
+	// A negative or zero duration causes time.Sleep to return immediately.
+	// If the time moves backwards for any reason, do nothing.
+	time.Sleep(r.minPeriod - d)
 }
 
 // GetCaller returns the caller of the function that calls it.

pkg/util/runtime/runtime_test.go

@@ -24,7 +24,9 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
+	"time"
 )
 
 func TestHandleCrash(t *testing.T) {
@@ -156,3 +158,27 @@ func captureStderr(f func()) (string, error) {
 
 	return <-resultCh, nil
 }
+
+func Test_rudimentaryErrorBackoff_OnError_ParallelSleep(t *testing.T) {
+	r := &rudimentaryErrorBackoff{
+		minPeriod: time.Second,
+	}
+
+	start := make(chan struct{})
+	var wg sync.WaitGroup
+	for i := 0; i < 30; i++ {
+		wg.Add(1)
+		go func() {
+			<-start
+			r.OnError(nil) // input error is ignored
+			wg.Done()
+		}()
+	}
+	st := time.Now()
+	close(start)
+	wg.Wait()
+
+	if since := time.Since(st); since > 5*time.Second {
+		t.Errorf("OnError slept for too long: %s", since)
+	}
+}