diff --git a/CHANGELOG.md b/CHANGELOG.md index 773f4e0358bf..26938816d84e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ #### Loki ##### Enhancements +* [6983](https://github.com/grafana/loki/pull/6983) **slim-bean**: `__timestamp__` and `__line__` are now available in the logql `label_format` query stage. * [6821](https://github.com/grafana/loki/pull/6821) **kavirajk**: Introduce new cache type `embedded-cache` which is an in-process cache system that runs loki without the need for an external cache (like memcached, redis, etc). It can be run in two modes `distributed: false` (default, and same as old `fifocache`) and `distributed: true` which runs cache in distributed fashion sharding keys across peers if Loki is run in microservices or SSD mode. * [6691](https://github.com/grafana/loki/pull/6691) **dannykopping**: Update production-ready Loki cluster in docker-compose * [6317](https://github.com/grafana/loki/pull/6317) **dannykoping**: General: add cache usage statistics diff --git a/docs/sources/logql/template_functions.md b/docs/sources/logql/template_functions.md index d56c44733e38..3f3e52c62302 100644 --- a/docs/sources/logql/template_functions.md +++ b/docs/sources/logql/template_functions.md @@ -13,7 +13,7 @@ All labels are added as variables in the template engine. They can be referenced {{ .path }} ``` -Additionally you can also access the log line using the [`__line__`](#__line__) function. +Additionally you can also access the log line using the [`__line__`](#__line__) function and the timestamp using the [`__timestamp__`](#__timestamp__) function. You can take advantage of [pipeline](https://golang.org/pkg/text/template/#hdr-Pipelines) to join together multiple functions. In a chained pipeline, the result of each command is passed as the last argument of the following command. diff --git a/pkg/logql/log/fmt.go b/pkg/logql/log/fmt.go index 04d5f896cd0e..e13f4e865e3c 100644 --- a/pkg/logql/log/fmt.go +++ b/pkg/logql/log/fmt.go @@ -90,6 +90,20 @@ var ( } ) +func addLineAndTimestampFunctions(currLine func() string, currTimestamp func() int64) map[string]interface{} { + functions := make(map[string]interface{}, len(functionMap)+2) + for k, v := range functionMap { + functions[k] = v + } + functions[functionLineName] = func() string { + return currLine() + } + functions[functionTimestampName] = func() time.Time { + return time.Unix(0, currTimestamp()) + } + return functions +} + func init() { sprigFuncMap := sprig.GenericFuncMap() for _, v := range templateFunctions { @@ -112,16 +126,13 @@ func NewFormatter(tmpl string) (*LineFormatter, error) { lf := &LineFormatter{ buf: bytes.NewBuffer(make([]byte, 4096)), } - functions := make(map[string]interface{}, len(functionMap)+1) - for k, v := range functionMap { - functions[k] = v - } - functions[functionLineName] = func() string { + + functions := addLineAndTimestampFunctions(func() string { return unsafeGetString(lf.currentLine) - } - functions[functionTimestampName] = func() time.Time { - return time.Unix(0, lf.currentTs) - } + }, func() int64 { + return lf.currentTs + }) + t, err := template.New("line").Option("missingkey=zero").Funcs(functions).Parse(tmpl) if err != nil { return nil, fmt.Errorf("invalid line template: %w", err) @@ -235,6 +246,9 @@ type labelFormatter struct { type LabelsFormatter struct { formats []labelFormatter buf *bytes.Buffer + + currentLine []byte + currentTs int64 } // NewLabelsFormatter creates a new formatter that can format multiple labels at once. @@ -246,10 +260,20 @@ func NewLabelsFormatter(fmts []LabelFmt) (*LabelsFormatter, error) { } formats := make([]labelFormatter, 0, len(fmts)) + lf := &LabelsFormatter{ + buf: bytes.NewBuffer(make([]byte, 1024)), + } + + functions := addLineAndTimestampFunctions(func() string { + return unsafeGetString(lf.currentLine) + }, func() int64 { + return lf.currentTs + }) + for _, fm := range fmts { toAdd := labelFormatter{LabelFmt: fm} if !fm.Rename { - t, err := template.New("label").Option("missingkey=zero").Funcs(functionMap).Parse(fm.Value) + t, err := template.New("label").Option("missingkey=zero").Funcs(functions).Parse(fm.Value) if err != nil { return nil, fmt.Errorf("invalid template for label '%s': %s", fm.Name, err) } @@ -257,10 +281,8 @@ func NewLabelsFormatter(fmts []LabelFmt) (*LabelsFormatter, error) { } formats = append(formats, toAdd) } - return &LabelsFormatter{ - formats: formats, - buf: bytes.NewBuffer(make([]byte, 1024)), - }, nil + lf.formats = formats + return lf, nil } func validate(fmts []LabelFmt) error { @@ -279,7 +301,10 @@ func validate(fmts []LabelFmt) error { return nil } -func (lf *LabelsFormatter) Process(_ int64, l []byte, lbs *LabelsBuilder) ([]byte, bool) { +func (lf *LabelsFormatter) Process(ts int64, l []byte, lbs *LabelsBuilder) ([]byte, bool) { + lf.currentLine = l + lf.currentTs = ts + var data interface{} for _, f := range lf.formats { if f.Rename { diff --git a/pkg/logql/log/fmt_test.go b/pkg/logql/log/fmt_test.go index f7069833354f..55d4fe7c7c6f 100644 --- a/pkg/logql/log/fmt_test.go +++ b/pkg/logql/log/fmt_test.go @@ -420,13 +420,43 @@ func Test_labelsFormatter_Format(t *testing.T) { {Name: "__error_details__", Value: "template: label:1:2: executing \"label\" at : wrong number of args for replace: want 3 got 2"}, }, }, + { + "line", + mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("line", "{{ __line__ }}")}), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + labels.Labels{ + {Name: "foo", Value: "blip"}, + {Name: "bar", Value: "blop"}, + {Name: "line", Value: "test line"}, + }, + }, + { + "timestamp", + mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("ts", "{{ __timestamp__ | date \"2006-01-02\" }}")}), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + labels.Labels{ + {Name: "foo", Value: "blip"}, + {Name: "bar", Value: "blop"}, + {Name: "ts", Value: "2022-08-26"}, + }, + }, + { + "timestamp_unix", + mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("ts", "{{ __timestamp__ | unixEpoch }}")}), + labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}}, + labels.Labels{ + {Name: "foo", Value: "blip"}, + {Name: "bar", Value: "blop"}, + {Name: "ts", Value: "1661518453"}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { builder := NewBaseLabelsBuilder().ForLabels(tt.in, tt.in.Hash()) builder.Reset() - _, _ = tt.fmter.Process(0, nil, builder) + _, _ = tt.fmter.Process(1661518453244672570, []byte("test line"), builder) sort.Sort(tt.want) require.Equal(t, tt.want, builder.LabelsResult().Labels()) })