diff --git a/integration/proxy/teleterm_test.go b/integration/proxy/teleterm_test.go index 18b0efd4884c7..4c73638c3d897 100644 --- a/integration/proxy/teleterm_test.go +++ b/integration/proxy/teleterm_test.go @@ -198,6 +198,8 @@ type gatewayCertRenewalParams struct { webauthnLogin libclient.WebauthnLoginFunc generateAndSetupUserCreds generateAndSetupUserCredsFunc wantPromptMFACallCount int + customCertsExpireFunc func(gateway.Gateway) + expectNoRelogin bool } func testGatewayCertRenewal(ctx context.Context, t *testing.T, params gatewayCertRenewalParams) { @@ -282,11 +284,15 @@ func testGatewayCertRenewal(ctx context.Context, t *testing.T, params gatewayCer params.testGatewayConnection(ctx, t, daemonService, gateway) - // Advance the fake clock to simulate the db cert expiry inside the middleware. - fakeClock.Advance(time.Hour * 48) + if params.customCertsExpireFunc != nil { + params.customCertsExpireFunc(gateway) + } else { + // Advance the fake clock to simulate the db cert expiry inside the middleware. + fakeClock.Advance(time.Hour * 48) - // Overwrite user certs with expired ones to simulate the user cert expiry. - params.generateAndSetupUserCreds(t, tc, -time.Hour) + // Overwrite user certs with expired ones to simulate the user cert expiry. + params.generateAndSetupUserCreds(t, tc, -time.Hour) + } // Open a new connection. // This should trigger the relogin flow. The middleware will notice that the cert has expired @@ -295,7 +301,11 @@ func testGatewayCertRenewal(ctx context.Context, t *testing.T, params gatewayCer // will let the connection through. params.testGatewayConnection(ctx, t, daemonService, gateway) - require.Equal(t, uint32(1), tshdEventsService.reloginCallCount.Load(), + expectedReloginCalls := uint32(1) + if params.expectNoRelogin { + expectedReloginCalls = uint32(0) + } + require.Equal(t, expectedReloginCalls, tshdEventsService.reloginCallCount.Load(), "Unexpected number of calls to TSHDEventsClient.Relogin") require.Equal(t, uint32(0), tshdEventsService.sendNotificationCallCount.Load(), "Unexpected number of calls to TSHDEventsClient.SendNotification") @@ -497,13 +507,51 @@ func TestTeletermKubeGateway(t *testing.T) { webauthnLogin: webauthnLogin, }) }) + t.Run("reissue cert after clearing it for root kube", func(t *testing.T) { + profileName := mustGetProfileName(t, suite.root.Web) + kubeURI := uri.NewClusterURI(profileName).AppendKube(kubeClusterName) + // The test can potentially hang forever if something is wrong with the MFA prompt, add a timeout. + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + t.Cleanup(cancel) + testKubeGatewayCertRenewal(ctx, t, kubeGatewayCertRenewalParams{ + suite: suite, + kubeURI: kubeURI, + webauthnLogin: webauthnLogin, + customCertsExpireFunc: func(gw gateway.Gateway) { + kubeGw, err := gateway.AsKube(gw) + require.NoError(t, err) + kubeGw.ClearCerts() + }, + expectNoRelogin: true, + }) + }) + t.Run("reissue cert after clearing it for leaf kube", func(t *testing.T) { + profileName := mustGetProfileName(t, suite.root.Web) + kubeURI := uri.NewClusterURI(profileName).AppendLeafCluster(suite.leaf.Secrets.SiteName).AppendKube(kubeClusterName) + // The test can potentially hang forever if something is wrong with the MFA prompt, add a timeout. + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + t.Cleanup(cancel) + testKubeGatewayCertRenewal(ctx, t, kubeGatewayCertRenewalParams{ + suite: suite, + kubeURI: kubeURI, + webauthnLogin: webauthnLogin, + customCertsExpireFunc: func(gw gateway.Gateway) { + kubeGw, err := gateway.AsKube(gw) + require.NoError(t, err) + kubeGw.ClearCerts() + }, + expectNoRelogin: true, + }) + }) } type kubeGatewayCertRenewalParams struct { - suite *Suite - kubeURI uri.ResourceURI - albAddr string - webauthnLogin libclient.WebauthnLoginFunc + suite *Suite + kubeURI uri.ResourceURI + albAddr string + webauthnLogin libclient.WebauthnLoginFunc + customCertsExpireFunc func(gateway.Gateway) + expectNoRelogin bool } func testKubeGatewayCertRenewal(ctx context.Context, t *testing.T, params kubeGatewayCertRenewalParams) { @@ -552,6 +600,8 @@ func testKubeGatewayCertRenewal(ctx context.Context, t *testing.T, params kubeGa }, testGatewayConnection: testKubeConnection, webauthnLogin: params.webauthnLogin, + customCertsExpireFunc: params.customCertsExpireFunc, + expectNoRelogin: params.expectNoRelogin, generateAndSetupUserCreds: func(t *testing.T, tc *libclient.TeleportClient, ttl time.Duration) { creds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{ Process: params.suite.root.Process, diff --git a/lib/srv/alpnproxy/common/kube.go b/lib/srv/alpnproxy/common/kube.go index ae7f27efd2b2b..a4dc3710b439d 100644 --- a/lib/srv/alpnproxy/common/kube.go +++ b/lib/srv/alpnproxy/common/kube.go @@ -22,6 +22,8 @@ import ( "encoding/hex" "fmt" "strings" + + "github.com/gravitational/trace" ) // KubeLocalProxySNI generates the SNI used for Kube local proxy. @@ -37,6 +39,16 @@ func TeleportClusterFromKubeLocalProxySNI(serverName string) string { return teleportCluster } +// KubeClusterFromKubeLocalProxySNI returns Kubernetes cluster name from SNI. +func KubeClusterFromKubeLocalProxySNI(serverName string) (string, error) { + kubeCluster, _, _ := strings.Cut(serverName, ".") + str, err := hex.DecodeString(kubeCluster) + if err != nil { + return "", trace.Wrap(err) + } + return string(str), nil +} + // KubeLocalProxyWildcardDomain returns the wildcard domain used to generate // local self-signed CA for provided Teleport cluster. func KubeLocalProxyWildcardDomain(teleportCluster string) string { diff --git a/lib/srv/alpnproxy/kube.go b/lib/srv/alpnproxy/kube.go index 36ee94ce7fb93..fe9635763a0ff 100644 --- a/lib/srv/alpnproxy/kube.go +++ b/lib/srv/alpnproxy/kube.go @@ -157,12 +157,23 @@ func writeKubeError(ctx context.Context, rw http.ResponseWriter, kubeError *apie } } +// ClearCerts clears the middleware certs. +// It will try to reissue them when a new request comes in. +func (m *KubeMiddleware) ClearCerts() { + m.certsMu.Lock() + defer m.certsMu.Unlock() + clear(m.certs) +} + // HandleRequest checks if middleware has valid certificate for this request and // reissues it if needed. In case of reissuing error we write directly to the response and return true, // so caller won't continue processing the request. func (m *KubeMiddleware) HandleRequest(rw http.ResponseWriter, req *http.Request) bool { cert, err := m.getCertForRequest(req) - if err != nil { + // If the cert is cleared using m.ClearCerts(), it won't be found. + // This forces the middleware to issue a new cert on a new request. + // This is used in access requests in Connect where we want to refresh certs without closing the proxy. + if err != nil && !trace.IsNotFound(err) { return false } @@ -220,23 +231,38 @@ func (m *KubeMiddleware) OverwriteClientCerts(req *http.Request) ([]tls.Certific var ErrUserInputRequired = errors.New("user input required") // reissueCertIfExpired checks if provided certificate has expired and reissues it if needed and replaces in the middleware certs. +// serverName has a form of .. func (m *KubeMiddleware) reissueCertIfExpired(ctx context.Context, cert tls.Certificate, serverName string) error { - x509Cert, err := utils.TLSCertLeaf(cert) - if err != nil { - return trace.Wrap(err) + needsReissue := false + if len(cert.Certificate) == 0 { + m.logger.InfoContext(ctx, "missing TLS certificate, attempting to reissue a new one") + needsReissue = true + } else { + x509Cert, err := utils.TLSCertLeaf(cert) + if err != nil { + return trace.Wrap(err) + } + if err := utils.VerifyCertificateExpiry(x509Cert, m.clock); err != nil { + needsReissue = true + } } - if err := utils.VerifyCertificateExpiry(x509Cert, m.clock); err == nil { + if !needsReissue { return nil } if m.certReissuer == nil { - return trace.BadParameter("can't reissue expired proxy certificate - reissuer is not available") + return trace.BadParameter("can't reissue proxy certificate - reissuer is not available") } - - // If certificate has expired we try to reissue it. - identity, err := tlsca.FromSubject(x509Cert.Subject, x509Cert.NotAfter) + teleportCluster := common.TeleportClusterFromKubeLocalProxySNI(serverName) + if teleportCluster == "" { + return trace.BadParameter("can't reissue proxy certificate - teleport cluster is empty") + } + kubeCluster, err := common.KubeClusterFromKubeLocalProxySNI(serverName) if err != nil { - return trace.Wrap(err) + return trace.Wrap(err, "can't reissue proxy certificate - kube cluster name is invalid") + } + if kubeCluster == "" { + return trace.BadParameter("can't reissue proxy certificate - kube cluster is empty") } errCh := make(chan error, 1) @@ -247,11 +273,7 @@ func (m *KubeMiddleware) reissueCertIfExpired(ctx context.Context, cert tls.Cert go func() { defer m.isCertReissuingRunning.Store(false) - cluster := identity.TeleportCluster - if identity.RouteToCluster != "" { - cluster = identity.RouteToCluster - } - newCert, err := m.certReissuer(m.closeContext, cluster, identity.KubernetesCluster) + newCert, err := m.certReissuer(m.closeContext, teleportCluster, kubeCluster) if err == nil { m.certsMu.Lock() m.certs[serverName] = newCert diff --git a/lib/srv/alpnproxy/local_proxy_http_middleware.go b/lib/srv/alpnproxy/local_proxy_http_middleware.go index a2d619f6bd38b..b0aab1c539b65 100644 --- a/lib/srv/alpnproxy/local_proxy_http_middleware.go +++ b/lib/srv/alpnproxy/local_proxy_http_middleware.go @@ -38,6 +38,10 @@ type LocalProxyHTTPMiddleware interface { // OverwriteClientCerts overwrites the client certs used for upstream connection. OverwriteClientCerts(req *http.Request) ([]tls.Certificate, error) + + // ClearCerts clears the middleware certs. + // It will try to reissue them when a new request comes in. + ClearCerts() } // DefaultLocalProxyHTTPMiddleware provides default implementations for LocalProxyHTTPMiddleware. @@ -56,3 +60,4 @@ func (m *DefaultLocalProxyHTTPMiddleware) HandleResponse(resp *http.Response) er func (m *DefaultLocalProxyHTTPMiddleware) OverwriteClientCerts(req *http.Request) ([]tls.Certificate, error) { return nil, trace.NotImplemented("not implemented") } +func (m *DefaultLocalProxyHTTPMiddleware) ClearCerts() {} diff --git a/lib/srv/alpnproxy/local_proxy_test.go b/lib/srv/alpnproxy/local_proxy_test.go index 063992f56f962..ed010a9e4faae 100644 --- a/lib/srv/alpnproxy/local_proxy_test.go +++ b/lib/srv/alpnproxy/local_proxy_test.go @@ -50,6 +50,7 @@ import ( "github.com/gravitational/teleport/lib/kube/proxy/responsewriters" "github.com/gravitational/teleport/lib/srv/alpnproxy/common" "github.com/gravitational/teleport/lib/tlsca" + "github.com/gravitational/teleport/lib/utils" ) // TestHandleAWSAccessSigVerification tests if LocalProxy verifies the AWS SigV4 signature of incoming request. @@ -471,7 +472,7 @@ func TestKubeMiddleware(t *testing.T) { now := time.Now() clock := clockwork.NewFakeClockAt(now) - var certReissuer KubeCertReissuer + teleportCluster := "localhost" ca := mustGenSelfSignedCert(t) kube1Cert := mustGenCertSignedWithCA(t, ca, @@ -499,7 +500,7 @@ func TestKubeMiddleware(t *testing.T) { withClock(clock), ) - certReissuer = func(ctx context.Context, teleportCluster, kubeCluster string) (tls.Certificate, error) { + certReissuer := func(ctx context.Context, teleportCluster, kubeCluster string) (tls.Certificate, error) { select { case <-ctx.Done(): return tls.Certificate{}, ctx.Err() @@ -511,7 +512,7 @@ func TestKubeMiddleware(t *testing.T) { t.Run("expired certificate is still reissued if request context expires", func(t *testing.T) { req := &http.Request{ TLS: &tls.ConnectionState{ - ServerName: "kube1", + ServerName: common.KubeLocalProxySNI(teleportCluster, "kube1"), }, } // we set request context to a context that is already canceled, so handler function will start reissuing @@ -520,8 +521,10 @@ func TestKubeMiddleware(t *testing.T) { cancel() req = req.WithContext(reqCtx) + startCerts := KubeClientCerts{} + startCerts.Add(teleportCluster, "kube1", kube1Cert) km := NewKubeMiddleware(KubeMiddlewareConfig{ - Certs: KubeClientCerts{"kube1": kube1Cert}, + Certs: startCerts, CertReissuer: certReissuer, Clock: clockwork.NewFakeClockAt(now.Add(time.Hour * 2)), CloseContext: context.Background(), @@ -553,6 +556,12 @@ func TestKubeMiddleware(t *testing.T) { require.Equal(t, newCert, certs[0], "certificate was not reissued") }) + getStartCerts := func() KubeClientCerts { + certs := KubeClientCerts{} + certs.Add(teleportCluster, "kube1", kube1Cert) + certs.Add(teleportCluster, "kube2", kube2Cert) + return certs + } testCases := []struct { name string reqClusterName string @@ -562,32 +571,33 @@ func TestKubeMiddleware(t *testing.T) { wantErr string }{ { - name: "kube cluster not found", - reqClusterName: "kube3", - startCerts: KubeClientCerts{"kube1": kube1Cert, "kube2": kube2Cert}, - clock: clockwork.NewFakeClockAt(now), - wantErr: "no client cert found for kube3", + name: "reissue cert when not found", + reqClusterName: "kube3", + startCerts: getStartCerts(), + clock: clockwork.NewFakeClockAt(now), + overwrittenCert: newCert, + wantErr: "", }, { - name: "expired cert reissued", + name: "expired cert is reissued", reqClusterName: "kube1", - startCerts: KubeClientCerts{"kube1": kube1Cert, "kube2": kube2Cert}, + startCerts: getStartCerts(), clock: clockwork.NewFakeClockAt(now.Add(time.Hour * 2)), overwrittenCert: newCert, wantErr: "", }, { - name: "success kube1", + name: "valid cert for kube1 is returned", reqClusterName: "kube1", - startCerts: KubeClientCerts{"kube1": kube1Cert, "kube2": kube2Cert}, + startCerts: getStartCerts(), clock: clockwork.NewFakeClockAt(now), overwrittenCert: kube1Cert, wantErr: "", }, { - name: "success kube2", + name: "valid cert for kube2 is returned", reqClusterName: "kube2", - startCerts: KubeClientCerts{"kube1": kube1Cert, "kube2": kube2Cert}, + startCerts: getStartCerts(), clock: clockwork.NewFakeClockAt(now), overwrittenCert: kube2Cert, wantErr: "", @@ -598,12 +608,13 @@ func TestKubeMiddleware(t *testing.T) { t.Run(tt.name, func(t *testing.T) { req := http.Request{ TLS: &tls.ConnectionState{ - ServerName: tt.reqClusterName, + ServerName: common.KubeLocalProxySNI(teleportCluster, tt.reqClusterName), }, } km := NewKubeMiddleware(KubeMiddlewareConfig{ Certs: tt.startCerts, CertReissuer: certReissuer, + Logger: utils.NewSlogLoggerForTests(), Clock: tt.clock, CloseContext: context.Background(), }) diff --git a/lib/teleterm/daemon/daemon.go b/lib/teleterm/daemon/daemon.go index b27ded1ba205c..e7b1f9eea7fe3 100644 --- a/lib/teleterm/daemon/daemon.go +++ b/lib/teleterm/daemon/daemon.go @@ -811,6 +811,28 @@ func (s *Service) AssumeRole(ctx context.Context, req *api.AssumeRoleRequest) er return trace.Wrap(err) } + // Clear certs in kube gateways. + // Access requests may grant elevated permissions for accessing a kube cluster. + // To allow the user to use these permissions, we clear the existing certs. + // When a kube proxy receives a new request, it will issue new certs, + // similarly to the process when certs expire. + // + // We don't know which gateways are affected by the access request, + // so we need to clear certs for all of them. + s.mu.RLock() + defer s.mu.RUnlock() + for _, gw := range s.gateways { + targetURI := gw.TargetURI() + if !(targetURI.IsKube() && targetURI.GetRootClusterURI() == cluster.URI) { + continue + } + kubeGw, err := gateway.AsKube(gw) + if err != nil { + s.cfg.Logger.ErrorContext(ctx, "Could not clear certs for kube when assuming request", "error", err, "target_uri", targetURI) + } + kubeGw.ClearCerts() + } + // We have to reconnect using the updated cert. return trace.Wrap(s.ClearCachedClientsForRoot(cluster.URI)) } diff --git a/lib/teleterm/daemon/daemon_test.go b/lib/teleterm/daemon/daemon_test.go index 6f5670d61fe57..88d8f912f08a6 100644 --- a/lib/teleterm/daemon/daemon_test.go +++ b/lib/teleterm/daemon/daemon_test.go @@ -694,7 +694,7 @@ func TestGetGatewayCLICommand(t *testing.T) { }, { name: "kube gateway", - inputGateway: fakeGateway{ + inputGateway: fakeKubeGateway{ targetURI: uri.NewClusterURI("profile").AppendKube("kube"), }, checkError: require.NoError, @@ -722,7 +722,7 @@ type fakeGateway struct { } func (m fakeGateway) TargetURI() uri.ResourceURI { return m.targetURI } -func (m fakeGateway) TargetName() string { return m.targetURI.GetDbName() + m.targetURI.GetKubeName() } +func (m fakeGateway) TargetName() string { return m.targetURI.GetDbName() } func (m fakeGateway) TargetUser() string { return "alice" } func (m fakeGateway) TargetSubresourceName() string { return m.subresourceName } func (m fakeGateway) Protocol() string { return defaults.ProtocolSQLServer } @@ -730,7 +730,23 @@ func (m fakeGateway) Log() *slog.Logger { return nil } func (m fakeGateway) LocalAddress() string { return "localhost" } func (m fakeGateway) LocalPortInt() int { return 8888 } func (m fakeGateway) LocalPort() string { return "8888" } -func (m fakeGateway) KubeconfigPath() string { return "test.kubeconfig" } + +type fakeKubeGateway struct { + gateway.Kube + targetURI uri.ResourceURI + subresourceName string +} + +func (m fakeKubeGateway) TargetURI() uri.ResourceURI { return m.targetURI } +func (m fakeKubeGateway) TargetName() string { return m.targetURI.GetKubeName() } +func (m fakeKubeGateway) TargetUser() string { return "alice" } +func (m fakeKubeGateway) TargetSubresourceName() string { return m.subresourceName } +func (m fakeKubeGateway) Protocol() string { return "" } +func (m fakeKubeGateway) Log() *slog.Logger { return nil } +func (m fakeKubeGateway) LocalAddress() string { return "localhost" } +func (m fakeKubeGateway) LocalPortInt() int { return 8888 } +func (m fakeKubeGateway) LocalPort() string { return "8888" } +func (m fakeKubeGateway) KubeconfigPath() string { return "test.kubeconfig" } type fakeStorage struct { Storage diff --git a/lib/teleterm/gateway/interfaces.go b/lib/teleterm/gateway/interfaces.go index 9d102d788f041..87560f2482659 100644 --- a/lib/teleterm/gateway/interfaces.go +++ b/lib/teleterm/gateway/interfaces.go @@ -92,6 +92,9 @@ type Kube interface { // KubeconfigPath returns the path to the kubeconfig used to connect the // local proxy. KubeconfigPath() string + // ClearCerts clears the local proxy middleware certs. + // It will try to reissue them when a new request comes in. + ClearCerts() } // App defines an app gateway. diff --git a/lib/teleterm/gateway/kube.go b/lib/teleterm/gateway/kube.go index d39f925bd75bf..727b7259e581e 100644 --- a/lib/teleterm/gateway/kube.go +++ b/lib/teleterm/gateway/kube.go @@ -42,6 +42,19 @@ import ( type kube struct { *base + middleware kubeMiddleware +} + +type kubeMiddleware interface { + ClearCerts() +} + +// ClearCerts clears the local proxy middleware certs. +// It will try to reissue them when a new request comes in. +func (k *kube) ClearCerts() { + if k.middleware != nil { + k.middleware.ClearCerts() + } } // KubeconfigPath returns the kubeconfig path that can be used for clients to @@ -62,7 +75,7 @@ func makeKubeGateway(cfg Config) (Kube, error) { return nil, trace.Wrap(err) } - k := &kube{base} + k := &kube{base: base} // Generate a new private key for the proxy. The client's existing private key may be // a hardware-backed private key, which cannot be added to the local proxy kube config. @@ -128,6 +141,7 @@ func (k *kube) makeALPNLocalProxyForKube(cas map[string]tls.Certificate) error { if err != nil { return trace.NewAggregate(err, listener.Close()) } + k.middleware = middleware webProxyHost, err := utils.Host(k.cfg.WebProxyAddr) if err != nil { @@ -155,7 +169,7 @@ func (k *kube) makeALPNLocalProxyForKube(cas map[string]tls.Certificate) error { func (k *kube) makeKubeMiddleware() (alpnproxy.LocalProxyHTTPMiddleware, error) { certs := make(alpnproxy.KubeClientCerts) certs.Add(k.cfg.ClusterName, k.cfg.TargetName, k.cfg.Cert) - return alpnproxy.NewKubeMiddleware(alpnproxy.KubeMiddlewareConfig{ + middleware := alpnproxy.NewKubeMiddleware(alpnproxy.KubeMiddlewareConfig{ Certs: certs, CertReissuer: func(ctx context.Context, teleportCluster, kubeCluster string) (tls.Certificate, error) { cert, err := k.cfg.OnExpiredCert(ctx, k) @@ -165,7 +179,9 @@ func (k *kube) makeKubeMiddleware() (alpnproxy.LocalProxyHTTPMiddleware, error) // TODO(tross): update this when kube is converted to use slog. Logger: slog.Default(), CloseContext: k.closeContext, - }), nil + }) + + return middleware, nil } func (k *kube) makeForwardProxyForKube() error {