From 38c9481da88e9cb8a62f4c6cfd0c85fea3fdcd81 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 11 Mar 2021 16:10:51 -0500 Subject: [PATCH 1/3] Added wildcard tenant Signed-off-by: Joe Elliott --- modules/overrides/overrides.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/overrides/overrides.go b/modules/overrides/overrides.go index 62f0116f676..c3ef218eb9a 100644 --- a/modules/overrides/overrides.go +++ b/modules/overrides/overrides.go @@ -16,6 +16,8 @@ import ( // nil, if there are no tenant-specific limits. type TenantLimits func(userID string) *Limits +const wildcardTenant = "*" + // perTenantOverrides represents the overrides config file type perTenantOverrides struct { TenantLimits map[string]*Limits `yaml:"overrides"` @@ -165,6 +167,11 @@ func (o *Overrides) getOverridesForUser(userID string) *Limits { if l != nil { return l } + + l = o.tenantLimits(wildcardTenant) + if l != nil { + return l + } } return o.defaultLimits } From f9b978a9f72f4392fedc2f1f0b464f0cf9ea65ad Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 11 Mar 2021 16:18:40 -0500 Subject: [PATCH 2/3] added test Signed-off-by: Joe Elliott --- modules/overrides/overrides_test.go | 37 ++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/modules/overrides/overrides_test.go b/modules/overrides/overrides_test.go index d8cd4c4f5d4..1a47fbca45f 100644 --- a/modules/overrides/overrides_test.go +++ b/modules/overrides/overrides_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" @@ -42,7 +43,7 @@ func TestOverrides(t *testing.T) { expectedIngestionRateSpans: map[string]int{"user1": 5, "user2": 5}, }, { - name: "basic override", + name: "basic overrides", limits: Limits{ MaxGlobalTracesPerUser: 1, MaxLocalTracesPerUser: 2, @@ -67,6 +68,39 @@ func TestOverrides(t *testing.T) { expectedIngestionBurstSpans: map[string]int{"user1": 9, "user2": 4}, expectedIngestionRateSpans: map[string]int{"user1": 10, "user2": 5}, }, + { + name: "wildcard override", + limits: Limits{ + MaxGlobalTracesPerUser: 1, + MaxLocalTracesPerUser: 2, + MaxSpansPerTrace: 3, + IngestionBurstSize: 4, + IngestionRateSpans: 5, + }, + overrides: &perTenantOverrides{ + TenantLimits: map[string]*Limits{ + "user1": { + MaxGlobalTracesPerUser: 6, + MaxLocalTracesPerUser: 7, + MaxSpansPerTrace: 8, + IngestionBurstSize: 9, + IngestionRateSpans: 10, + }, + "*": { + MaxGlobalTracesPerUser: 11, + MaxLocalTracesPerUser: 12, + MaxSpansPerTrace: 13, + IngestionBurstSize: 14, + IngestionRateSpans: 15, + }, + }, + }, + expectedMaxGlobalTraces: map[string]int{"user1": 6, "user2": 11}, + expectedMaxLocalTraces: map[string]int{"user1": 7, "user2": 12}, + expectedMaxSpansPerTrace: map[string]int{"user1": 8, "user2": 13}, + expectedIngestionBurstSpans: map[string]int{"user1": 9, "user2": 14}, + expectedIngestionRateSpans: map[string]int{"user1": 10, "user2": 15}, + }, } for _, tt := range tests { @@ -84,6 +118,7 @@ func TestOverrides(t *testing.T) { tt.limits.PerTenantOverridePeriod = time.Hour } + prometheus.DefaultRegisterer = prometheus.NewRegistry() // have to overwrite the registry or test panics with multiple metric reg overrides, err := NewOverrides(tt.limits) require.NoError(t, err) err = services.StartAndAwaitRunning(context.TODO(), overrides) From 64296b0bc18cdff4b908816c0984a94b03c954f8 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 12 Mar 2021 13:52:55 -0500 Subject: [PATCH 3/3] Added wildcard tenant to the docs Signed-off-by: Joe Elliott --- docs/tempo/website/configuration/ingestion-limit.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/tempo/website/configuration/ingestion-limit.md b/docs/tempo/website/configuration/ingestion-limit.md index 601b5f3489f..d6797fcd746 100644 --- a/docs/tempo/website/configuration/ingestion-limit.md +++ b/docs/tempo/website/configuration/ingestion-limit.md @@ -81,6 +81,16 @@ Sometimes you don't want all tenants within the cluster to have the same setting This overrides file is dynamically loaded. It can be changed at runtime and will be reloaded by Tempo without restarting the application. +1. Additionally a "wildcard" override can be used that will apply to all tenants if a match is not found otherwise. +``` + overrides: + "*": + ingestion_max_batch_size: 5000 + ingestion_rate_limit: 400000 + max_spans_per_trace: 250000 + max_traces_per_user: 100000 +``` + ## Override strategies The trace limits specified by the various parameters are, by default, applied as per-distributor limits. For example, a `max_traces_per_user` setting of 10000 means that each distributor within the cluster has a limit of 10000 traces per user. This is known as a `local` strategy in that the specified trace limits are local to each distributor.