chore: make identity schema provider a proper service (#3908)

.docker/Dockerfile-build

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1-experimental
-FROM golang:1.21-bullseye AS builder
+FROM golang:1.22-bullseye AS builder
 
 RUN apt-get update && apt-get upgrade -y &&\
   mkdir -p /var/lib/sqlite

.docker/Dockerfile-debug

@@ -1,4 +1,4 @@
-FROM golang:1.21-bullseye
+FROM golang:1.22-bullseye
 ENV CGO_ENABLED 1
 
 RUN apt-get update && apt-get install -y --no-install-recommends inotify-tools psmisc

.github/workflows/ci.yaml

@@ -79,7 +79,7 @@ jobs:
           fetch-depth: 2
       - uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - run: go list -json > go.list
       - name: Run nancy
         uses: sonatype-nexus-community/nancy-github-action@v1.0.2
@@ -170,7 +170,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22"
 
       - name: Install selfservice-ui-react-native
         uses: actions/checkout@v3
@@ -274,7 +274,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - run: go build -tags sqlite,json1 .
 
       - name: Install selfservice-ui-react-native

.github/workflows/format.yml

@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - run: make format
       - name: Indicate formatting issues
         run: git diff HEAD --exit-code --color

.github/workflows/licenses.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - uses: actions/setup-node@v2
         with:
           node-version: "18"

driver/registry.go

@@ -106,7 +106,7 @@ type Registry interface {
 	courier.PersistenceProvider
 
 	schema.HandlerProvider
-	schema.IdentityTraitsProvider
+	schema.IdentitySchemaProvider
 
 	password2.ValidationProvider
 
@@ -180,15 +180,16 @@ func NewRegistryFromDSN(ctx context.Context, c *config.Config, l *logrusx.Logger
 }
 
 type options struct {
-	skipNetworkInit         bool
-	config                  *config.Config
-	replaceTracer           func(*otelx.Tracer) *otelx.Tracer
-	inspect                 func(Registry) error
-	extraMigrations         []fs.FS
-	replacementStrategies   []NewStrategy
-	extraHooks              map[string]func(config.SelfServiceHook) any
-	disableMigrationLogging bool
-	jsonnetPool             jsonnetsecure.Pool
+	skipNetworkInit               bool
+	config                        *config.Config
+	replaceTracer                 func(*otelx.Tracer) *otelx.Tracer
+	replaceIdentitySchemaProvider func(Registry) schema.IdentitySchemaProvider
+	inspect                       func(Registry) error
+	extraMigrations               []fs.FS
+	replacementStrategies         []NewStrategy
+	extraHooks                    map[string]func(config.SelfServiceHook) any
+	disableMigrationLogging       bool
+	jsonnetPool                   jsonnetsecure.Pool
 }
 
 type RegistryOption func(*options)
@@ -209,6 +210,12 @@ func WithConfig(config *config.Config) RegistryOption {
 	}
 }
 
+func WithIdentitySchemaProvider(f func(r Registry) schema.IdentitySchemaProvider) RegistryOption {
+	return func(o *options) {
+		o.replaceIdentitySchemaProvider = f
+	}
+}
+
 func ReplaceTracer(f func(*otelx.Tracer) *otelx.Tracer) RegistryOption {
 	return func(o *options) {
 		o.replaceTracer = f

driver/registry_default.go

@@ -93,9 +93,10 @@ type RegistryDefault struct {
 	hookCodeAddressVerifier *hook.CodeAddressVerifier
 	hookTwoStepRegistration *hook.TwoStepRegistration
 
-	identityHandler   *identity.Handler
-	identityValidator *identity.Validator
-	identityManager   *identity.Manager
+	identityHandler        *identity.Handler
+	identityValidator      *identity.Validator
+	identityManager        *identity.Manager
+	identitySchemaProvider schema.IdentitySchemaProvider
 
 	courierHandler *courier.Handler
 
@@ -621,6 +622,7 @@ func (m *RegistryDefault) Init(ctx context.Context, ctxer contextx.Contextualize
 			instrumentedsql.WithOmitArgs(), // don't risk leaking PII or secrets
 		}
 	}
+
 	if o.replaceTracer != nil {
 		m.trc = o.replaceTracer(m.trc)
 	}
@@ -633,6 +635,10 @@ func (m *RegistryDefault) Init(ctx context.Context, ctxer contextx.Contextualize
 		m.WithHooks(o.extraHooks)
 	}
 
+	if o.replaceIdentitySchemaProvider != nil {
+		m.identitySchemaProvider = o.replaceIdentitySchemaProvider(m)
+	}
+
 	bc := backoff.NewExponentialBackOff()
 	bc.MaxElapsedTime = time.Minute * 5
 	bc.Reset()

driver/registry_default_schemas.go

@@ -5,32 +5,13 @@ package driver
 
 import (
 	"context"
-	"net/url"
-
-	"github.com/pkg/errors"
 
 	"github.com/ory/kratos/schema"
 )
 
-func (m *RegistryDefault) IdentityTraitsSchemas(ctx context.Context) (schema.Schemas, error) {
-	ms, err := m.Config().IdentityTraitsSchemas(ctx)
-	if err != nil {
-		return nil, err
+func (m *RegistryDefault) IdentityTraitsSchemas(ctx context.Context) (schema.IdentitySchemaList, error) {
+	if m.identitySchemaProvider == nil {
+		m.identitySchemaProvider = schema.NewDefaultIdentityTraitsProvider(m)
 	}
-
-	var ss schema.Schemas
-	for _, s := range ms {
-		surl, err := url.Parse(s.URL)
-		if err != nil {
-			return nil, errors.WithStack(err)
-		}
-
-		ss = append(ss, schema.Schema{
-			ID:     s.ID,
-			URL:    surl,
-			RawURL: s.URL,
-		})
-	}
-
-	return ss, nil
+	return m.identitySchemaProvider.IdentityTraitsSchemas(ctx)
 }

driver/registry_default_schemas_test.go

@@ -39,7 +39,7 @@ func TestRegistryDefault_IdentityTraitsSchemas(t *testing.T) {
 
 	ss, err := reg.IdentityTraitsSchemas(context.Background())
 	require.NoError(t, err)
-	assert.Equal(t, 2, len(ss))
+	assert.Equal(t, 2, ss.Total())
 	assert.Contains(t, ss, defaultSchema)
 	assert.Contains(t, ss, altSchema)
 }

embedx/config.schema.json

@@ -2861,6 +2861,19 @@
       "description": "Secifies which organizations are available. Only effective in the Ory Network.",
       "type": "array",
       "default": []
+    },
+    "enterprise": {
+      "title": "Enterprise features",
+      "description": "Specifies enterprise features. Only effective in the Ory Network or with a valid license.",
+      "type": "object",
+      "properties": {
+        "identity_schema_fallback_url_template": {
+          "type": "string",
+          "title": "Fallback URL template for identity schemas",
+          "description": "A fallback URL template used when looking up identity schemas."
+        }
+      },
+      "additionalProperties": false
     }
   },
   "allOf": [

identity/validator.go

@@ -19,7 +19,7 @@ import (
 
 type (
 	validatorDependencies interface {
-		IdentityTraitsSchemas(ctx context.Context) (schema.Schemas, error)
+		schema.IdentitySchemaProvider
 		config.Provider
 	}
 	Validator struct {

internal/client-go/go.sum

@@ -4,6 +4,7 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

persistence/sql/identity/persister_identity.go

@@ -47,7 +47,7 @@ var (
 )
 
 type dependencies interface {
-	schema.IdentityTraitsProvider
+	schema.IdentitySchemaProvider
 	identity.ValidationProvider
 	x.LoggingProvider
 	config.Provider

persistence/sql/persister.go

@@ -40,7 +40,7 @@ type (
 		config.Provider
 		contextx.Provider
 		x.TracingProvider
-		schema.IdentityTraitsProvider
+		schema.IdentitySchemaProvider
 		identity.ValidationProvider
 	}
 	Persister struct {

persistence/sql/persister_hmac_test.go

@@ -52,7 +52,7 @@ func (l *logRegistryOnly) Audit() *logrusx.Logger {
 func (l *logRegistryOnly) Tracer(ctx context.Context) *otelx.Tracer {
 	return otelx.NewNoop(l.l, new(otelx.Config))
 }
-func (l *logRegistryOnly) IdentityTraitsSchemas(ctx context.Context) (schema.Schemas, error) {
+func (l *logRegistryOnly) IdentityTraitsSchemas(ctx context.Context) (schema.IdentitySchemaList, error) {
 	panic("implement me")
 }
 

schema/handler.go

@@ -31,7 +31,7 @@ type (
 	handlerDependencies interface {
 		x.WriterProvider
 		x.LoggingProvider
-		IdentityTraitsProvider
+		IdentitySchemaProvider
 		x.CSRFProvider
 		config.Provider
 		x.TracingProvider

schema/schema.go

@@ -4,6 +4,7 @@
 package schema
 
 import (
+	"cmp"
 	"context"
 	"encoding/base64"
 	"io"
@@ -21,16 +22,58 @@ import (
 	"github.com/ory/x/urlx"
 )
 
+var _ IdentitySchemaList = (*Schemas)(nil)
+
 type Schemas []Schema
-type IdentityTraitsProvider interface {
-	IdentityTraitsSchemas(ctx context.Context) (Schemas, error)
+
+type IdentitySchemaProvider interface {
+	IdentityTraitsSchemas(ctx context.Context) (IdentitySchemaList, error)
 }
 
-func (s Schemas) GetByID(id string) (*Schema, error) {
-	if id == "" {
-		id = config.DefaultIdentityTraitsSchemaID
+type deps interface {
+	config.Provider
+}
+
+type DefaultIdentitySchemaProvider struct {
+	d deps
+}
+
+func NewDefaultIdentityTraitsProvider(d deps) *DefaultIdentitySchemaProvider {
+	return &DefaultIdentitySchemaProvider{d: d}
+}
+
+func (d *DefaultIdentitySchemaProvider) IdentityTraitsSchemas(ctx context.Context) (IdentitySchemaList, error) {
+	ms, err := d.d.Config().IdentityTraitsSchemas(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var ss Schemas
+	for _, s := range ms {
+		surl, err := url.Parse(s.URL)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		ss = append(ss, Schema{
+			ID:     s.ID,
+			URL:    surl,
+			RawURL: s.URL,
+		})
 	}
 
+	return ss, nil
+}
+
+type IdentitySchemaList interface {
+	GetByID(id string) (*Schema, error)
+	Total() int
+	List(page, perPage int) Schemas
+}
+
+func (s Schemas) GetByID(id string) (*Schema, error) {
+	id = cmp.Or(id, config.DefaultIdentityTraitsSchemaID)
+
 	for _, ss := range s {
 		if ss.ID == id {
 			return &ss, nil
@@ -98,11 +141,16 @@ func GetKeysInOrder(ctx context.Context, schemaRef string) ([]string, error) {
 }
 
 type Schema struct {
-	ID     string   `json:"id"`
-	URL    *url.URL `json:"-"`
-	RawURL string   `json:"url"`
+	ID  string   `json:"id"`
+	URL *url.URL `json:"-"`
+	// RawURL contains the raw URL value as it was passed in the configuration. URL parsing can break base64 encoded URLs.
+	RawURL string `json:"url"`
 }
 
 func (s *Schema) SchemaURL(host *url.URL) *url.URL {
-	return urlx.AppendPaths(host, SchemasPath, base64.RawURLEncoding.EncodeToString([]byte(s.ID)))
+	return IDToURL(host, s.ID)
+}
+
+func IDToURL(host *url.URL, id string) *url.URL {
+	return urlx.AppendPaths(host, SchemasPath, base64.RawURLEncoding.EncodeToString([]byte(id)))
 }

selfservice/flow/settings/error.go

@@ -4,7 +4,6 @@
 package settings
 
 import (
-	"context"
 	"net/http"
 	"net/url"
 
@@ -43,7 +42,7 @@ type (
 
 		HandlerProvider
 		FlowPersistenceProvider
-		IdentityTraitsSchemas(ctx context.Context) (schema.Schemas, error)
+		schema.IdentitySchemaProvider
 	}
 
 	ErrorHandlerProvider interface{ SettingsFlowErrorHandler() *ErrorHandler }

selfservice/flow/settings/handler.go

@@ -68,7 +68,7 @@ type (
 		HookExecutorProvider
 		x.CSRFTokenGeneratorProvider
 
-		schema.IdentityTraitsProvider
+		schema.IdentitySchemaProvider
 
 		login.HandlerProvider
 	}

selfservice/strategy/code/strategy.go

@@ -103,7 +103,7 @@ type (
 		RegistrationCodePersistenceProvider
 		LoginCodePersistenceProvider
 
-		schema.IdentityTraitsProvider
+		schema.IdentitySchemaProvider
 		session.PersistenceProvider
 
 		sessiontokenexchange.PersistenceProvider

selfservice/strategy/link/strategy.go

@@ -75,7 +75,7 @@ type (
 		VerificationTokenPersistenceProvider
 		SenderProvider
 
-		schema.IdentityTraitsProvider
+		schema.IdentitySchemaProvider
 	}
 
 	Strategy struct {