auth: fix cluster init deadlock (#49638)

* auth: fix cluster init deadlock

If the bootstrapped resources are created but a failure occurs later
in the cluster init process subsequent runs of the cluster initialization
will fail due to the existing bootstrap resources.

* add test for new behavior

* add test for cluster init failures recovery

* Update lib/services/local/resource_test.go

Co-authored-by: Zac Bergquist <zac.bergquist@goteleport.com>

* reorder assertion values

---------

Co-authored-by: Zac Bergquist <zac.bergquist@goteleport.com>

lib/auth/init_test.go

@@ -493,6 +493,40 @@ func keysIn[K comparable, V any](m map[K]V) []K {
 	return result
 }
 
+type failingTrustInternal struct {
+	services.TrustInternal
+}
+
+func (t *failingTrustInternal) CreateCertAuthority(ctx context.Context, ca types.CertAuthority) error {
+	return trace.Errorf("error")
+}
+
+// TestInitCertFailureRecovery ensures the auth server is able to recover from
+// a failure in the cert creation process.
+func TestInitCertFailureRecovery(t *testing.T) {
+	ctx := context.Background()
+	cap, err := types.NewAuthPreference(types.AuthPreferenceSpecV2{
+		Type: constants.SAML,
+	})
+	require.NoError(t, err)
+
+	conf := setupConfig(t)
+
+	// BootstrapResources have lead to an unrecoverable state in the past.
+	// See https://github.com/gravitational/teleport/pull/49638.
+	conf.BootstrapResources = []types.Resource{cap}
+	_, err = Init(ctx, conf, func(s *Server) error {
+		s.TrustInternal = &failingTrustInternal{
+			TrustInternal: s.TrustInternal,
+		}
+		return nil
+	})
+	require.Error(t, err)
+
+	_, err = Init(ctx, conf)
+	require.NoError(t, err)
+}
+
 // TestPresets tests behavior of presets
 func TestPresets(t *testing.T) {
 	ctx := context.Background()

lib/services/local/resource.go

@@ -32,8 +32,8 @@ import (
 )
 
 // CreateResources attempts to dynamically create the supplied resources.
-// This function returns `trace.AlreadyExistsError` if one or more resources
-// would be overwritten, and `trace.NotImplementedError` if any resources
+// If any resources already exist they are skipped and not overwritten.
+// This function returns a `trace.NotImplementedError` if any resources
 // are of an unsupported type (see `itemsFromResources(...)`).
 //
 // NOTE: This function is non-atomic and performs no internal synchronization;
@@ -43,20 +43,9 @@ func CreateResources(ctx context.Context, b backend.Backend, resources ...types.
 	if err != nil {
 		return trace.Wrap(err)
 	}
-	// ensure all items do not exist before continuing.
-	for _, item := range items {
-		_, err = b.Get(ctx, item.Key)
-		if !trace.IsNotFound(err) {
-			if err != nil {
-				return trace.Wrap(err)
-			}
-			return trace.AlreadyExists("resource %q already exists", string(item.Key))
-		}
-	}
-	// create all items.
 	for _, item := range items {
 		_, err := b.Create(ctx, item)
-		if err != nil {
+		if !trace.IsAlreadyExists(err) && err != nil {
 			return trace.Wrap(err)
 		}
 	}

lib/services/local/resource_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/bcrypt"
 
+	"github.com/gravitational/teleport/api/constants"
 	apidefaults "github.com/gravitational/teleport/api/defaults"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/lib/services"
@@ -55,6 +56,34 @@ func TestCreateResourcesProvisionToken(t *testing.T) {
 	require.Empty(t, cmp.Diff(token, fetchedToken, cmpopts.IgnoreFields(types.Metadata{}, "Revision")))
 }
 
+func TestCreateResource(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	tt := setupServicesContext(ctx, t)
+	cap, err := types.NewAuthPreference(types.AuthPreferenceSpecV2{
+		Type: constants.Local,
+	})
+	require.NoError(t, err)
+
+	// Check that the initial call to CreateResources creates the given resources.
+	s, err := NewClusterConfigurationService(tt.bk)
+	require.NoError(t, err)
+	err = CreateResources(ctx, tt.bk, cap)
+	require.NoError(t, err)
+	got, err := s.GetAuthPreference(ctx)
+	require.NoError(t, err)
+	require.Equal(t, cap.GetType(), got.GetType())
+
+	// Check that already exists errors are ignored and the resource is not
+	// updated.
+	cap.SetType(constants.SAML)
+	err = CreateResources(ctx, tt.bk, cap)
+	require.NoError(t, err)
+	got, err = s.GetAuthPreference(ctx)
+	require.NoError(t, err)
+	require.NotEqual(t, cap.GetType(), got.GetType())
+}
+
 func TestUserResource(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()