Skip to content

Commit

Permalink
Improve LogQL format stages requireLabel (#4769)
Browse files Browse the repository at this point in the history
* Improve LogQL format stages.

This only includes required labels.

```
➜ go test -benchmem -run=^$  -bench ^Benchmark_Pipeline ./pkg/logql/log -v -count 5  > after.txt && benchstat before.txt after.txt
name                                 old time/op    new time/op    delta
_Pipeline/pipeline_bytes-16            13.8µs ± 6%    11.5µs ± 1%  -16.42%  (p=0.008 n=5+5)
_Pipeline/pipeline_string-16           13.9µs ± 2%    11.5µs ± 2%  -17.27%  (p=0.008 n=5+5)
_Pipeline/line_extractor_bytes-16      14.0µs ± 4%    11.6µs ± 0%  -17.45%  (p=0.016 n=5+4)
_Pipeline/line_extractor_string-16     13.5µs ± 0%    11.5µs ± 0%  -14.67%  (p=0.008 n=5+5)
_Pipeline/label_extractor_bytes-16     13.7µs ± 1%    11.7µs ± 1%  -14.69%  (p=0.008 n=5+5)
_Pipeline/label_extractor_string-16    14.1µs ±11%    11.6µs ± 0%  -17.89%  (p=0.008 n=5+5)

name                                 old alloc/op   new alloc/op   delta
_Pipeline/pipeline_bytes-16            9.51kB ± 0%    7.50kB ± 0%  -21.19%  (p=0.008 n=5+5)
_Pipeline/pipeline_string-16           9.51kB ± 0%    7.50kB ± 0%  -21.19%  (p=0.008 n=5+5)
_Pipeline/line_extractor_bytes-16      9.51kB ± 0%    7.50kB ± 0%  -21.19%  (p=0.008 n=5+5)
_Pipeline/line_extractor_string-16     9.51kB ± 0%    7.50kB ± 0%     ~     (p=0.079 n=4+5)
_Pipeline/label_extractor_bytes-16     9.51kB ± 0%    7.50kB ± 0%  -21.18%  (p=0.008 n=5+5)
_Pipeline/label_extractor_string-16    9.51kB ± 0%    7.50kB ± 0%  -21.18%  (p=0.008 n=5+5)

name                                 old allocs/op  new allocs/op  delta
_Pipeline/pipeline_bytes-16              46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/pipeline_string-16             46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/line_extractor_bytes-16        46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/line_extractor_string-16       46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/label_extractor_bytes-16       46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/label_extractor_string-16      46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
```

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* rollback model change

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* remove unused variable

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
  • Loading branch information
cyriltovena authored Nov 26, 2021
1 parent 347e3e3 commit 0a94467
Showing 1 changed file with 23 additions and 18 deletions.
41 changes: 23 additions & 18 deletions pkg/logql/log/fmt.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ type LineFormatter struct {
func NewFormatter(tmpl string) (*LineFormatter, error) {
t, err := template.New("line").Option("missingkey=zero").Funcs(functionMap).Parse(tmpl)
if err != nil {
return nil, fmt.Errorf("invalid line template: %s", err)
return nil, fmt.Errorf("invalid line template: %w", err)
}
return &LineFormatter{
Template: t,
Expand All @@ -111,6 +111,7 @@ func NewFormatter(tmpl string) (*LineFormatter, error) {

func (lf *LineFormatter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) {
lf.buf.Reset()

if err := lf.Template.Execute(lf.buf, lbs.Labels().Map()); err != nil {
lbs.SetErr(errTemplateFormat)
return line, true
Expand All @@ -122,18 +123,25 @@ func (lf *LineFormatter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool)
}

func (lf *LineFormatter) RequiredLabelNames() []string {
return uniqueString(listNodeFields(lf.Root))
return uniqueString(listNodeFields([]parse.Node{lf.Root}))
}

func listNodeFields(node parse.Node) []string {
func listNodeFields(nodes []parse.Node) []string {
var res []string
if node.Type() == parse.NodeAction {
res = append(res, listNodeFieldsFromPipe(node.(*parse.ActionNode).Pipe)...)
}
res = append(res, listNodeFieldsFromBranch(node)...)
if ln, ok := node.(*parse.ListNode); ok {
for _, n := range ln.Nodes {
res = append(res, listNodeFields(n)...)
for _, node := range nodes {
switch node.Type() {
case parse.NodePipe:
res = append(res, listNodeFieldsFromPipe(node.(*parse.PipeNode))...)
case parse.NodeAction:
res = append(res, listNodeFieldsFromPipe(node.(*parse.ActionNode).Pipe)...)
case parse.NodeList:
res = append(res, listNodeFields(node.(*parse.ListNode).Nodes)...)
case parse.NodeCommand:
res = append(res, listNodeFields(node.(*parse.CommandNode).Args)...)
case parse.NodeIf, parse.NodeWith, parse.NodeRange:
res = append(res, listNodeFieldsFromBranch(node)...)
case parse.NodeField:
res = append(res, node.(*parse.FieldNode).Ident...)
}
}
return res
Expand All @@ -156,22 +164,18 @@ func listNodeFieldsFromBranch(node parse.Node) []string {
res = append(res, listNodeFieldsFromPipe(b.Pipe)...)
}
if b.List != nil {
res = append(res, listNodeFields(b.List)...)
res = append(res, listNodeFields(b.List.Nodes)...)
}
if b.ElseList != nil {
res = append(res, listNodeFields(b.ElseList)...)
res = append(res, listNodeFields(b.ElseList.Nodes)...)
}
return res
}

func listNodeFieldsFromPipe(p *parse.PipeNode) []string {
var res []string
for _, c := range p.Cmds {
for _, a := range c.Args {
if f, ok := a.(*parse.FieldNode); ok {
res = append(res, f.Ident...)
}
}
res = append(res, listNodeFields(c.Args)...)
}
return res
}
Expand Down Expand Up @@ -220,6 +224,7 @@ func NewLabelsFormatter(fmts []LabelFmt) (*LabelsFormatter, error) {
return nil, err
}
formats := make([]labelFormatter, 0, len(fmts))

for _, fm := range fmts {
toAdd := labelFormatter{LabelFmt: fm}
if !fm.Rename {
Expand Down Expand Up @@ -284,7 +289,7 @@ func (lf *LabelsFormatter) RequiredLabelNames() []string {
names = append(names, fm.Value)
continue
}
names = append(names, listNodeFields(fm.tmpl.Root)...)
names = append(names, listNodeFields([]parse.Node{fm.tmpl.Root})...)
}
return uniqueString(names)
}
Expand Down

0 comments on commit 0a94467

Please sign in to comment.