Skip to content

Commit

Permalink
Prevent rapid reset http2 DOS on API server
Browse files Browse the repository at this point in the history
This change fully addresses CVE-2023-44487 and CVE-2023-39325 for
the API server when the client is unauthenticated.

The changes to util/runtime are required because otherwise a large
number of requests can get blocked on the time.Sleep calls.

For unauthenticated clients (either via 401 or the anonymous user),
we simply no longer allow such clients to hold open http2
connections.  They can use http2, but with the performance of http1
(with keep-alive disabled).

Since this change has the potential to cause issues, the
UnauthenticatedHTTP2DOSMitigation feature gate can be disabled to
remove this protection (it is enabled by default).  For example,
when the API server is fronted by an L7 load balancer that is set up
to mitigate http2 attacks, unauthenticated clients could force
disable connection reuse between the load balancer and the API
server (many incoming connections could share the same backend
connection).  An API server that is on a private network may opt to
disable this protection to prevent performance regressions for
unauthenticated clients.

For all other clients, we rely on the golang.org/x/net fix in
golang/net@b225e7c
That change is not sufficient to adequately protect against a
motivated client - future changes to Kube and/or golang.org/x/net
will be explored to address this gap.

The Kube API server now uses a max stream of 100 instead of 250
(this matches the Go http2 client default).  This lowers the abuse
limit from 1000 to 400.

Signed-off-by: Monis Khan <mok@microsoft.com>

Kubernetes-commit: 238d89c9a068dcd7ab994be1b3e646ce8d296ef8
  • Loading branch information
enj authored and k8s-publishing-bot committed Oct 8, 2023
1 parent e874526 commit 850deeb
Show file tree
Hide file tree
Showing 4 changed files with 261 additions and 1 deletion.
38 changes: 38 additions & 0 deletions pkg/endpoints/filters/authentication.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,11 @@ import (
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
genericfeatures "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
)

Expand Down Expand Up @@ -101,13 +104,36 @@ func withAuthentication(handler http.Handler, auth authenticator.Request, failed
)
}

// http2 is an expensive protocol that is prone to abuse,
// see CVE-2023-44487 and CVE-2023-39325 for an example.
// Do not allow unauthenticated clients to keep these
// connections open (i.e. basically degrade them to the
// performance of http1 with keep-alive disabled).
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.UnauthenticatedHTTP2DOSMitigation) && req.ProtoMajor == 2 && isAnonymousUser(resp.User) {
// limit this connection to just this request,
// and then send a GOAWAY and tear down the TCP connection
// https://github.com/golang/net/commit/97aa3a539ec716117a9d15a4659a911f50d13c3c
w.Header().Set("Connection", "close")
}

req = req.WithContext(genericapirequest.WithUser(req.Context(), resp.User))
handler.ServeHTTP(w, req)
})
}

func Unauthorized(s runtime.NegotiatedSerializer) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// http2 is an expensive protocol that is prone to abuse,
// see CVE-2023-44487 and CVE-2023-39325 for an example.
// Do not allow unauthenticated clients to keep these
// connections open (i.e. basically degrade them to the
// performance of http1 with keep-alive disabled).
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.UnauthenticatedHTTP2DOSMitigation) && req.ProtoMajor == 2 {
// limit this connection to just this request,
// and then send a GOAWAY and tear down the TCP connection
// https://github.com/golang/net/commit/97aa3a539ec716117a9d15a4659a911f50d13c3c
w.Header().Set("Connection", "close")
}
ctx := req.Context()
requestInfo, found := genericapirequest.RequestInfoFrom(ctx)
if !found {
Expand All @@ -127,3 +153,15 @@ func audiencesAreAcceptable(apiAuds, responseAudiences authenticator.Audiences)

return len(apiAuds.Intersect(responseAudiences)) > 0
}

func isAnonymousUser(u user.Info) bool {
if u.GetName() == user.Anonymous {
return true
}
for _, group := range u.GetGroups() {
if group == user.AllUnauthenticated {
return true
}
}
return false
}
199 changes: 199 additions & 0 deletions pkg/endpoints/filters/authentication_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,31 @@ package filters

import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"

"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"golang.org/x/net/http2"

"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/anonymous"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
"k8s.io/apiserver/pkg/authentication/user"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/scheme"
featuregatetesting "k8s.io/component-base/featuregate/testing"
)

func TestAuthenticateRequestWithAud(t *testing.T) {
Expand Down Expand Up @@ -465,3 +476,191 @@ func TestAuthenticateRequestClearHeaders(t *testing.T) {
})
}
}

func TestUnauthenticatedHTTP2ClientConnectionClose(t *testing.T) {
s := httptest.NewUnstartedServer(WithAuthentication(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ok")) }),
authenticator.RequestFunc(func(r *http.Request) (*authenticator.Response, bool, error) {
switch r.Header.Get("Authorization") {
case "known":
return &authenticator.Response{User: &user.DefaultInfo{Name: "panda"}}, true, nil
case "error":
return nil, false, errors.New("authn err")
case "anonymous":
return anonymous.NewAuthenticator().AuthenticateRequest(r)
case "anonymous_group":
return &authenticator.Response{User: &user.DefaultInfo{Groups: []string{user.AllUnauthenticated}}}, true, nil
default:
return nil, false, nil
}
}),
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(genericapirequest.WithRequestInfo(r.Context(), &genericapirequest.RequestInfo{}))
Unauthorized(scheme.Codecs).ServeHTTP(w, r)
}),
nil,
nil,
))

http2Options := &http2.Server{}

if err := http2.ConfigureServer(s.Config, http2Options); err != nil {
t.Fatal(err)
}

s.TLS = s.Config.TLSConfig

s.StartTLS()
t.Cleanup(s.Close)

const reqs = 4

cases := []struct {
name string
authorizationHeader string
skipHTTP2DOSMitigation bool
expectConnections uint64
}{
{
name: "known",
authorizationHeader: "known",
skipHTTP2DOSMitigation: false,
expectConnections: 1,
},
{
name: "error",
authorizationHeader: "error",
skipHTTP2DOSMitigation: false,
expectConnections: reqs,
},
{
name: "anonymous",
authorizationHeader: "anonymous",
skipHTTP2DOSMitigation: false,
expectConnections: reqs,
},
{
name: "anonymous_group",
authorizationHeader: "anonymous_group",
skipHTTP2DOSMitigation: false,
expectConnections: reqs,
},
{
name: "other",
authorizationHeader: "other",
skipHTTP2DOSMitigation: false,
expectConnections: reqs,
},

{
name: "known skip=true",
authorizationHeader: "known",
skipHTTP2DOSMitigation: true,
expectConnections: 1,
},
{
name: "error skip=true",
authorizationHeader: "error",
skipHTTP2DOSMitigation: true,
expectConnections: 1,
},
{
name: "anonymous skip=true",
authorizationHeader: "anonymous",
skipHTTP2DOSMitigation: true,
expectConnections: 1,
},
{
name: "anonymous_group skip=true",
authorizationHeader: "anonymous_group",
skipHTTP2DOSMitigation: true,
expectConnections: 1,
},
{
name: "other skip=true",
authorizationHeader: "other",
skipHTTP2DOSMitigation: true,
expectConnections: 1,
},
}

rootCAs := x509.NewCertPool()
rootCAs.AddCert(s.Certificate())

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
f := func(t *testing.T, nextProto string, expectConnections uint64) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.UnauthenticatedHTTP2DOSMitigation, !tc.skipHTTP2DOSMitigation)()

var localAddrs atomic.Uint64 // indicates how many TCP connection set up

tlsConfig := &tls.Config{
RootCAs: rootCAs,
NextProtos: []string{nextProto},
}

dailer := tls.Dialer{
Config: tlsConfig,
}

tr := &http.Transport{
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: tlsConfig,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := dailer.DialContext(ctx, network, addr)
if err != nil {
return nil, err
}

localAddrs.Add(1)

return conn, nil
},
}

tr.MaxIdleConnsPerHost = 1 // allow http1 to have keep alive connections open
if nextProto == http2.NextProtoTLS {
// Disable connection pooling to avoid additional connections
// that cause the test to flake
tr.MaxIdleConnsPerHost = -1
if err := http2.ConfigureTransport(tr); err != nil {
t.Fatal(err)
}
}

client := &http.Client{
Transport: tr,
}

for i := 0; i < reqs; i++ {
req, err := http.NewRequest(http.MethodGet, s.URL, nil)
if err != nil {
t.Fatal(err)
}
if len(tc.authorizationHeader) > 0 {
req.Header.Set("Authorization", tc.authorizationHeader)
}

resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}

if expectConnections != localAddrs.Load() {
t.Fatalf("expect TCP connection: %d, actual: %d", expectConnections, localAddrs.Load())
}
}

t.Run(http2.NextProtoTLS, func(t *testing.T) {
f(t, http2.NextProtoTLS, tc.expectConnections)
})

t.Run("http/1.1", func(t *testing.T) {
f(t, "http/1.1", 1)
})
})
}
}
20 changes: 20 additions & 0 deletions pkg/features/kube_features.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,24 @@ const (
// Enables server-side field validation.
ServerSideFieldValidation featuregate.Feature = "ServerSideFieldValidation"

// owner: @enj
// beta: v1.29
//
// Enables http2 DOS mitigations for unauthenticated clients.
//
// Some known reasons to disable these mitigations:
//
// An API server that is fronted by an L7 load balancer that is set up
// to mitigate http2 attacks may opt to disable this protection to prevent
// unauthenticated clients from disabling connection reuse between the load
// balancer and the API server (many incoming connections could share the
// same backend connection).
//
// An API server that is on a private network may opt to disable this
// protection to prevent performance regressions for unauthenticated
// clients.
UnauthenticatedHTTP2DOSMitigation featuregate.Feature = "UnauthenticatedHTTP2DOSMitigation"

// owner: @caesarxuchao @roycaihw
// alpha: v1.20
//
Expand Down Expand Up @@ -276,6 +294,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS

StorageVersionHash: {Default: true, PreRelease: featuregate.Beta},

UnauthenticatedHTTP2DOSMitigation: {Default: true, PreRelease: featuregate.Beta},

WatchBookmark: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},

InPlacePodVerticalScaling: {Default: false, PreRelease: featuregate.Alpha},
Expand Down
5 changes: 4 additions & 1 deletion pkg/server/secure_serving.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,10 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
if s.HTTP2MaxStreamsPerConnection > 0 {
http2Options.MaxConcurrentStreams = uint32(s.HTTP2MaxStreamsPerConnection)
} else {
http2Options.MaxConcurrentStreams = 250
// match http2.initialMaxConcurrentStreams used by clients
// this makes it so that a malicious client can only open 400 streams before we forcibly close the connection
// https://github.com/golang/net/commit/b225e7ca6dde1ef5a5ae5ce922861bda011cfabd
http2Options.MaxConcurrentStreams = 100
}

// increase the connection buffer size from the 1MB default to handle the specified number of concurrent streams
Expand Down

0 comments on commit 850deeb

Please sign in to comment.