Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Loki: Do not store exact log line duplicates (now including unordered inserts) #6642

Merged
merged 3 commits into from
Jul 18, 2022
Merged

Loki: Do not store exact log line duplicates (now including unordered inserts) #6642

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0c22ff4
do not insert log lines which are an exact duplicate in both timestam…
slim-bean Jul 8, 2022
232fb42
update test to now ignore dupes pushed out of order
slim-bean Jul 18, 2022
0dab1ff
make the test unordered to be more thorough
slim-bean Jul 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions pkg/chunkenc/unordered.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,15 @@ func (hb *unorderedHeadBlock) Append(ts int64, line string) error {
}
displaced := hb.rt.Add(e)
if displaced[0] != nil {
// While we support multiple entries at the same timestamp, we _do_ de-duplicate
// entries at the same time with the same content, iterate through any existing
// entries and ignore the line if we already have an entry with the same content
for _, et := range displaced[0].(*nsEntries).entries {
slim-bean marked this conversation as resolved.
Show resolved Hide resolved
if et == line {
e.entries = displaced[0].(*nsEntries).entries
return nil
}
}
e.entries = append(displaced[0].(*nsEntries).entries, line)
} else {
e.entries = []string{line}
Expand Down
20 changes: 20 additions & 0 deletions pkg/chunkenc/unordered_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,26 @@ func Test_Unordered_InsertRetrieval(t *testing.T) {
},
dir: logproto.BACKWARD,
},
{
desc: "ts remove exact dupe forward",
input: []entry{
{0, "a"}, {0, "b"}, {0, "b"}, {1, "c"},
slim-bean marked this conversation as resolved.
Show resolved Hide resolved
},
exp: []entry{
{0, "a"}, {0, "b"}, {1, "c"},
},
dir: logproto.FORWARD,
},
{
desc: "ts remove exact dupe backward",
input: []entry{
{0, "a"}, {0, "b"}, {0, "b"}, {1, "c"},
},
exp: []entry{
{1, "c"}, {0, "b"}, {0, "a"},
},
dir: logproto.BACKWARD,
},
} {
t.Run(tc.desc, func(t *testing.T) {
hb := newUnorderedHeadBlock()
Expand Down