Skip to content

Commit

Permalink
[v14] Limit gRPC Active streams (#33985)
Browse files Browse the repository at this point in the history 
* Limit GRPC Active streams (#33936)

Originally there was a default limit of 100 max concurrent streams, however in 2017 the GRPC team removed this default: grpc/grpc-go#1624

With the recent HTTP/2 Rapid Reset DoS, it is now being encouraged to re-introduce a limit.  The fix requires this value to be configured in fact: grpc/grpc-go#6703

* Update gRPC to 1.58.3 to address GHSA-m425-mq94-257g
  • Loading branch information
@jentfoo
jentfoo authored Oct 27, 2023
1 parent 019a5ff commit 34635b2
Show file tree
Hide file tree
Showing 14 changed files with 34 additions and 20 deletions.
2 changes: 1 addition & 1 deletion api/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
golang.org/x/net v0.17.0
golang.org/x/term v0.13.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98
google.golang.org/grpc v1.58.2
google.golang.org/grpc v1.58.3
google.golang.org/protobuf v1.31.0
gopkg.in/yaml.v2 v2.4.0
)
Expand Down
4 changes: 2 additions & 2 deletions api/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,8 +246,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I=
google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
6 changes: 3 additions & 3 deletions examples/go-client/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ require (
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
google.golang.org/grpc v1.57.1 // indirect
google.golang.org/protobuf v1.30.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/grpc v1.58.3 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
12 changes: 6 additions & 6 deletions examples/go-client/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,17 +154,17 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg=
google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand All @@ -176,8 +176,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ require (
golang.org/x/time v0.3.0
google.golang.org/api v0.138.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577
google.golang.org/grpc v1.58.2
google.golang.org/grpc v1.58.3
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
google.golang.org/protobuf v1.31.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2214,8 +2214,8 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I=
google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y=
Expand Down
2 changes: 1 addition & 1 deletion integrations/kube-agent-updater/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ require (
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.57.1 // indirect
google.golang.org/grpc v1.58.3 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions integrations/kube-agent-updater/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -971,8 +971,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg=
google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
2 changes: 2 additions & 0 deletions lib/auth/grpcserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ import (
wanlib "github.com/gravitational/teleport/lib/auth/webauthn"
"github.com/gravitational/teleport/lib/authz"
"github.com/gravitational/teleport/lib/backend"
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/events"
"github.com/gravitational/teleport/lib/httplib"
"github.com/gravitational/teleport/lib/joinserver"
Expand Down Expand Up @@ -5308,6 +5309,7 @@ func NewGRPCServer(cfg GRPCServerConfig) (*GRPCServer, error) {
PermitWithoutStream: true,
},
),
grpc.MaxConcurrentStreams(defaults.GRPCMaxConcurrentStreams),
)
if err != nil {
return nil, trace.Wrap(err)
Expand Down
4 changes: 4 additions & 0 deletions lib/defaults/defaults.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ const (
// By default all users use /bin/bash
DefaultShell = "/bin/bash"

// GRPCMaxConcurrentStreams is the max GRPC streams that can be active at a time. Once the limit is reached new
// RPC calls will queue until capacity is available.
GRPCMaxConcurrentStreams = 1000

// HTTPMaxIdleConns is the max idle connections across all hosts.
HTTPMaxIdleConns = 2000

Expand Down
2 changes: 1 addition & 1 deletion lib/observability/tracing/collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func NewCollector(cfg CollectorConfig) (*Collector, error) {
c := &Collector{
grpcLn: grpcLn,
httpLn: httpLn,
grpcServer: grpc.NewServer(grpc.Creds(creds)),
grpcServer: grpc.NewServer(grpc.Creds(creds), grpc.MaxConcurrentStreams(defaults.GRPCMaxConcurrentStreams)),
tlsConfing: tlsConfig,
exportedC: make(chan struct{}, 1),
}
Expand Down
2 changes: 2 additions & 0 deletions lib/proxy/peer/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/gravitational/teleport/api/metadata"
"github.com/gravitational/teleport/api/utils/grpc/interceptors"
"github.com/gravitational/teleport/lib/auth"
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/utils"
)

Expand Down Expand Up @@ -141,6 +142,7 @@ func NewServer(config ServerConfig) (*Server, error) {
MinTime: peerKeepAlive,
PermitWithoutStream: true,
}),
grpc.MaxConcurrentStreams(defaults.GRPCMaxConcurrentStreams),
)

proto.RegisterProxyServiceServer(server, config.service)
Expand Down
3 changes: 3 additions & 0 deletions lib/service/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4268,6 +4268,7 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error {
otelgrpc.StreamServerInterceptor(),
),
grpc.Creds(creds),
grpc.MaxConcurrentStreams(defaults.GRPCMaxConcurrentStreams),
)

connMonitor, err := srv.NewConnectionMonitor(srv.ConnectionMonitorConfig{
Expand Down Expand Up @@ -5930,6 +5931,7 @@ func (process *TeleportProcess) initPublicGRPCServer(
// available for some time.
MaxConnectionIdle: 10 * time.Second,
}),
grpc.MaxConcurrentStreams(defaults.GRPCMaxConcurrentStreams),
)
joinServiceServer := joinserver.NewJoinServiceGRPCServer(conn.Client)
proto.RegisterJoinServiceServer(server, joinServiceServer)
Expand Down Expand Up @@ -5989,6 +5991,7 @@ func (process *TeleportProcess) initSecureGRPCServer(cfg initSecureGRPCServerCfg
grpc.ChainUnaryInterceptor(authMiddleware.UnaryInterceptors()...),
grpc.ChainStreamInterceptor(authMiddleware.StreamInterceptors()...),
grpc.Creds(creds),
grpc.MaxConcurrentStreams(defaults.GRPCMaxConcurrentStreams),
)

kubeServer, err := kubegrpc.New(kubegrpc.Config{
Expand Down
5 changes: 4 additions & 1 deletion lib/teleterm/apiserver/apiserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"google.golang.org/grpc"

api "github.com/gravitational/teleport/gen/proto/go/teleport/lib/teleterm/v1"
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/teleterm/apiserver/handler"
"github.com/gravitational/teleport/lib/utils"
)
Expand All @@ -41,7 +42,9 @@ func New(cfg Config) (*APIServer, error) {
}

grpcServer := grpc.NewServer(cfg.TshdServerCreds,
grpc.ChainUnaryInterceptor(withErrorHandling(cfg.Log)))
grpc.ChainUnaryInterceptor(withErrorHandling(cfg.Log)),
grpc.MaxConcurrentStreams(defaults.GRPCMaxConcurrentStreams),
)

// Create Terminal service.

Expand Down

0 comments on commit 34635b2

Please sign in to comment.