diff --git a/docs/tempo/website/configuration/ingestion-limit.md b/docs/tempo/website/configuration/ingestion-limit.md index 601b5f3489f..d6797fcd746 100644 --- a/docs/tempo/website/configuration/ingestion-limit.md +++ b/docs/tempo/website/configuration/ingestion-limit.md @@ -81,6 +81,16 @@ Sometimes you don't want all tenants within the cluster to have the same setting This overrides file is dynamically loaded. It can be changed at runtime and will be reloaded by Tempo without restarting the application. +1. Additionally a "wildcard" override can be used that will apply to all tenants if a match is not found otherwise. +``` + overrides: + "*": + ingestion_max_batch_size: 5000 + ingestion_rate_limit: 400000 + max_spans_per_trace: 250000 + max_traces_per_user: 100000 +``` + ## Override strategies The trace limits specified by the various parameters are, by default, applied as per-distributor limits. For example, a `max_traces_per_user` setting of 10000 means that each distributor within the cluster has a limit of 10000 traces per user. This is known as a `local` strategy in that the specified trace limits are local to each distributor. diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index 62f0116f676..c3ef218eb9a 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -16,6 +16,8 @@ import ( // nil, if there are no tenant-specific limits. type TenantLimits func(userID string) *Limits +const wildcardTenant = "*" + // perTenantOverrides represents the overrides config file type perTenantOverrides struct { TenantLimits map[string]*Limits `yaml:"overrides"` @@ -165,6 +167,11 @@ func (o *Overrides) getOverridesForUser(userID string) *Limits { if l != nil { return l } + + l = o.tenantLimits(wildcardTenant) + if l != nil { + return l + } } return o.defaultLimits } diff --git a/modules/overrides/overrides_test.go b/modules/overrides/overrides_test.go index d8cd4c4f5d4..1a47fbca45f 100644 --- a/modules/overrides/overrides_test.go +++ b/modules/overrides/overrides_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" @@ -42,7 +43,7 @@ func TestOverrides(t *testing.T) { expectedIngestionRateSpans: map[string]int{"user1": 5, "user2": 5}, }, { - name: "basic override", + name: "basic overrides", limits: Limits{ MaxGlobalTracesPerUser: 1, MaxLocalTracesPerUser: 2, @@ -67,6 +68,39 @@ func TestOverrides(t *testing.T) { expectedIngestionBurstSpans: map[string]int{"user1": 9, "user2": 4}, expectedIngestionRateSpans: map[string]int{"user1": 10, "user2": 5}, }, + { + name: "wildcard override", + limits: Limits{ + MaxGlobalTracesPerUser: 1, + MaxLocalTracesPerUser: 2, + MaxSpansPerTrace: 3, + IngestionBurstSize: 4, + IngestionRateSpans: 5, + }, + overrides: &perTenantOverrides{ + TenantLimits: map[string]*Limits{ + "user1": { + MaxGlobalTracesPerUser: 6, + MaxLocalTracesPerUser: 7, + MaxSpansPerTrace: 8, + IngestionBurstSize: 9, + IngestionRateSpans: 10, + }, + "*": { + MaxGlobalTracesPerUser: 11, + MaxLocalTracesPerUser: 12, + MaxSpansPerTrace: 13, + IngestionBurstSize: 14, + IngestionRateSpans: 15, + }, + }, + }, + expectedMaxGlobalTraces: map[string]int{"user1": 6, "user2": 11}, + expectedMaxLocalTraces: map[string]int{"user1": 7, "user2": 12}, + expectedMaxSpansPerTrace: map[string]int{"user1": 8, "user2": 13}, + expectedIngestionBurstSpans: map[string]int{"user1": 9, "user2": 14}, + expectedIngestionRateSpans: map[string]int{"user1": 10, "user2": 15}, + }, } for _, tt := range tests { @@ -84,6 +118,7 @@ func TestOverrides(t *testing.T) { tt.limits.PerTenantOverridePeriod = time.Hour } + prometheus.DefaultRegisterer = prometheus.NewRegistry() // have to overwrite the registry or test panics with multiple metric reg overrides, err := NewOverrides(tt.limits) require.NoError(t, err) err = services.StartAndAwaitRunning(context.TODO(), overrides)