-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Playing around with a POC based on the design doc. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Getting series ID for label matchers is now working. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes tests. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * wip/ Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * We're parsing the index label now. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Starting to extract interfaces to make the code testable. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Work in progress will try to add labels to chunk ref iterator Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Iterator for chunks ref with all labels !!!. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Optimize code away. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * with delete into the mix Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Cleaner but not yet working for v10 and v11. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes series cleaner. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * hooking into the compactor. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Hooking limit retention config. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Finishing off the marker processor. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Working on sweeper and fixing tests. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Adding more tests and founding more bugs along the way. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Bug with path once boltdb is closed. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixing more bug and more robust test. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * More test and cleanup getting close. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes moar bugs with regards to period schema Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fix a flaky tests because of boltdb still open. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add more metrics. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Adding metrics. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Improve benchmark. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes issue. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * lint code. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * more logs. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Save files without using table key Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Improve logging and ability to use more goroutines. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Removes duplicate metrics since histogram contains total too. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add more logs. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes a deadlock bug when too many workers are trying to update the mark file. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes a deadlock when reading and updating db at the same time. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes default config test of boltdb. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * PR Review feedbacks. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Remove the user of assert to not fail a test if it's incorrect on the first shot. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add experimental notice to the flag documentation Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes empty index detection and table deletion. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Do not delete folder it's not necessary with object store. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Better working path cleanup Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * got linted. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
- Loading branch information
1 parent
771d33b
commit 806d6a5
Showing
27 changed files
with
2,825 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
package loki | ||
|
||
import ( | ||
"context" | ||
"flag" | ||
"io" | ||
"io/ioutil" | ||
"strings" | ||
"testing" | ||
"time" | ||
|
||
"github.com/cortexproject/cortex/pkg/util/runtimeconfig" | ||
"github.com/prometheus/client_golang/prometheus" | ||
"github.com/prometheus/prometheus/pkg/labels" | ||
"github.com/stretchr/testify/require" | ||
|
||
"github.com/grafana/loki/pkg/validation" | ||
) | ||
|
||
func Test_LoadRetentionRules(t *testing.T) { | ||
overrides := newTestOverrides(t, | ||
` | ||
overrides: | ||
"1": | ||
creation_grace_period: 48h | ||
"29": | ||
creation_grace_period: 48h | ||
ingestion_burst_size_mb: 140 | ||
ingestion_rate_mb: 120 | ||
max_concurrent_tail_requests: 1000 | ||
max_global_streams_per_user: 100000 | ||
max_label_names_per_series: 30 | ||
max_query_parallelism: 256 | ||
split_queries_by_interval: 15m | ||
retention_period: 1440h | ||
retention_stream: | ||
- selector: '{app="foo"}' | ||
period: 48h | ||
priority: 10 | ||
- selector: '{namespace="bar", cluster=~"fo.*|b.+|[1-2]"}' | ||
period: 24h | ||
priority: 5 | ||
`) | ||
require.Equal(t, 31*24*time.Hour, overrides.RetentionPeriod("1")) // default | ||
require.Equal(t, 2*30*24*time.Hour, overrides.RetentionPeriod("29")) // overrides | ||
require.Equal(t, []validation.StreamRetention(nil), overrides.StreamRetention("1")) | ||
require.Equal(t, []validation.StreamRetention{ | ||
{Period: 48 * time.Hour, Priority: 10, Selector: `{app="foo"}`, Matchers: []*labels.Matcher{ | ||
labels.MustNewMatcher(labels.MatchEqual, "app", "foo"), | ||
}}, | ||
{Period: 24 * time.Hour, Priority: 5, Selector: `{namespace="bar", cluster=~"fo.*|b.+|[1-2]"}`, Matchers: []*labels.Matcher{ | ||
labels.MustNewMatcher(labels.MatchEqual, "namespace", "bar"), | ||
labels.MustNewMatcher(labels.MatchRegexp, "cluster", "fo.*|b.+|[1-2]"), | ||
}}, | ||
}, overrides.StreamRetention("29")) | ||
} | ||
|
||
func Test_ValidateRules(t *testing.T) { | ||
_, err := loadRuntimeConfig(strings.NewReader( | ||
` | ||
overrides: | ||
"29": | ||
retention_stream: | ||
- selector: '{app=foo"}' | ||
period: 48h | ||
priority: 10 | ||
- selector: '{namespace="bar", cluster=~"fo.*|b.+|[1-2]"}' | ||
period: 24h | ||
priority: 10 | ||
`)) | ||
require.Equal(t, "invalid override for tenant 29: invalid labels matchers: parse error at line 1, col 6: syntax error: unexpected IDENTIFIER, expecting STRING", err.Error()) | ||
_, err = loadRuntimeConfig(strings.NewReader( | ||
` | ||
overrides: | ||
"29": | ||
retention_stream: | ||
- selector: '{app="foo"}' | ||
period: 5h | ||
priority: 10 | ||
`)) | ||
require.Equal(t, "invalid override for tenant 29: retention period must be >= 24h was 5h0m0s", err.Error()) | ||
} | ||
|
||
func newTestOverrides(t *testing.T, yaml string) *validation.Overrides { | ||
t.Helper() | ||
f, err := ioutil.TempFile(t.TempDir(), "bar") | ||
require.NoError(t, err) | ||
path := f.Name() | ||
// fake loader to load from string instead of file. | ||
loader := func(_ io.Reader) (interface{}, error) { | ||
return loadRuntimeConfig(strings.NewReader(yaml)) | ||
} | ||
cfg := runtimeconfig.ManagerConfig{ | ||
ReloadPeriod: 1 * time.Second, | ||
Loader: loader, | ||
LoadPath: path, | ||
} | ||
flagset := flag.NewFlagSet("", flag.PanicOnError) | ||
var defaults validation.Limits | ||
defaults.RegisterFlags(flagset) | ||
require.NoError(t, flagset.Parse(nil)) | ||
validation.SetDefaultLimitsForYAMLUnmarshalling(defaults) | ||
|
||
runtimeConfig, err := runtimeconfig.NewRuntimeConfigManager(cfg, prometheus.DefaultRegisterer) | ||
require.NoError(t, err) | ||
|
||
require.NoError(t, runtimeConfig.StartAsync(context.Background())) | ||
require.NoError(t, runtimeConfig.AwaitRunning(context.Background())) | ||
defer func() { | ||
runtimeConfig.StopAsync() | ||
require.NoError(t, runtimeConfig.AwaitTerminated(context.Background())) | ||
}() | ||
|
||
overrides, err := validation.NewOverrides(defaults, tenantLimitsFromRuntimeConfig(runtimeConfig)) | ||
require.NoError(t, err) | ||
return overrides | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.