From f3f01c3c3b1b7a869080ab40d248ac91e908c656 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 25 Jan 2024 10:30:13 +0100 Subject: [PATCH 1/5] Fix detection of abbreviated locations for parseDate --- x-pack/filebeat/input/httpjson/value_tpl.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/httpjson/value_tpl.go b/x-pack/filebeat/input/httpjson/value_tpl.go index 97bc75a62d9..a47753d23b2 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/value_tpl.go @@ -191,7 +191,17 @@ func parseDate(date string, layout ...string) time.Time { return time.Time{} } - return t.UTC() + loc, err := time.LoadLocation(t.Location().String()) + if err != nil { + return t.UTC() + } + + tInLoc, err := time.ParseInLocation(ly, date, loc) + if err != nil { + return t.UTC() + } + + return tInLoc.UTC() } func formatDate(date time.Time, layouttz ...string) string { From e5d9cb8788860f48f9bf4e1e8d1920c2b411704d Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 25 Jan 2024 10:30:41 +0100 Subject: [PATCH 2/5] Add test cases --- .../filebeat/input/httpjson/value_tpl_test.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/x-pack/filebeat/input/httpjson/value_tpl_test.go b/x-pack/filebeat/input/httpjson/value_tpl_test.go index 487451099ad..8944d590765 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/value_tpl_test.go @@ -142,6 +142,27 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: "2020-11-05 12:25:32 +0000 UTC", }, + { + name: "func parseDate MST default layout", + value: `[[ parseDate "Mon, 02 Jan 2006 15:04:05 MST" "Mon, 02 Jan 2006 15:04:05 MST" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2006-01-02 22:04:05 +0000 UTC", + }, + { + name: "func parseDate MST RFC1123", + value: `[[ parseDate "Mon, 02 Jan 2006 15:04:05 MST" "RFC1123" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2006-01-02 22:04:05 +0000 UTC", + }, + { + name: "func parseDate EST unix date", + value: `[[ parseDate "Mon Jan 2 15:04:05 EST 2006" "UnixDate" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2006-01-02 20:04:05 +0000 UTC", + }, { name: "func formatDate", setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, From 492c9ea599bc34e20839f6cfef518bb82827f540 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 25 Jan 2024 10:45:54 +0100 Subject: [PATCH 3/5] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a1c8df5256c..081a2966cae 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -95,6 +95,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] - [threatintel] MISP pagination fixes {pull}37898[37898] - Fix file handle leak when handling errors in filestream {pull}37973[37973] +- Fix location detection for parseDate value template in the HTTPJSON input {pull}37738[37738] *Heartbeat* From 741542b71cf7807b70ed2fdb7359c8644ac54a24 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Tue, 20 Feb 2024 19:19:39 +0100 Subject: [PATCH 4/5] Create parseDateInTZ value template --- .../docs/inputs/input-httpjson.asciidoc | 3 +- x-pack/filebeat/input/httpjson/value_tpl.go | 55 +++++++++++++++++-- .../filebeat/input/httpjson/value_tpl_test.go | 32 ++++++++--- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index cc3594780e4..5fbd5dc15a5 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -220,7 +220,8 @@ Some built-in helper functions are provided to work with the input state inside - `min`: returns the minimum of two values. - `mul`: multiplies two integers. - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now. -- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`. +- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Note: Parsing timezone abbreviations may cause ambiguities. Prefer `parseDateInTZ` for explicit timezone handling. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`. +- `parseDateInTZ`: parses a date string within a specified timezone (TZ), returning a `time.Time` in UTC. Specified timezone overwrites implicit timezone from the input date. Accepts timezone offsets ("-07:00", "-0700", "-07") or IANA Time Zone names ("America/New_York"). If TZ is invalid, defaults to UTC. Optional layout argument as in parseDate. Example: `[[ parseDateInTZ "2020-11-05T12:25:32" "America/New_York" ]]`, `[[ parseDateInTZ "2020-11-05T12:25:32" "-07:00" "RFC3339" ]]`. - `parseDuration`: parses duration strings and returns `time.Duration`. Example: `[[parseDuration "1h"]]`. - `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000]]` returns `2020-11-05 13:25:32 +0000 UTC`. - `parseTimestampNano`: parses a timestamp in nanoseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000000000]]` returns `2020-11-05 13:25:32 +0000 UTC`. diff --git a/x-pack/filebeat/input/httpjson/value_tpl.go b/x-pack/filebeat/input/httpjson/value_tpl.go index a47753d23b2..cf7e43cf8e4 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/value_tpl.go @@ -71,6 +71,7 @@ func (t *valueTpl) Unpack(in string) error { "mul": mul, "now": now, "parseDate": parseDate, + "parseDateInTZ": parseDateInTZ, "parseDuration": parseDuration, "parseTimestamp": parseTimestamp, "parseTimestampMilli": parseTimestampMilli, @@ -191,17 +192,59 @@ func parseDate(date string, layout ...string) time.Time { return time.Time{} } - loc, err := time.LoadLocation(t.Location().String()) - if err != nil { - return t.UTC() + return t.UTC() +} + +// parseDateInTZ parses a date string within a specified timezone, returning a time.Time +// 'tz' is the timezone (offset or IANA name) for parsing +func parseDateInTZ(date string, tz string, layout ...string) time.Time { + var ly string + if len(layout) == 0 { + ly = defaultTimeLayout + } else { + ly = layout[0] + } + if found := predefinedLayouts[ly]; found != "" { + ly = found } - tInLoc, err := time.ParseInLocation(ly, date, loc) + var loc *time.Location + // Attempt to parse timezone as offset in various formats + for _, format := range []string{"-07", "-0700", "-07:00"} { + t, err := time.Parse(format, tz) + if err != nil { + continue + } + name, offset := t.Zone() + loc = time.FixedZone(name, offset) + break + } + + // If parsing tz as offset fails, try loading location by name + if loc == nil { + var err error + loc, err = time.LoadLocation(tz) + if err != nil { + loc = time.UTC // Default to UTC on error + } + } + + // Using Parse allows us not to worry about the timezone + // as the predefined timezone is applied afterwards + t, err := time.Parse(ly, date) if err != nil { - return t.UTC() + return time.Time{} } - return tInLoc.UTC() + // Manually create a new time object with the parsed date components and the desired location + // It allows interpreting the parsed time in the specified timezone + year, month, day := t.Date() + hour, min, sec := t.Clock() + nanosec := t.Nanosecond() + localTime := time.Date(year, month, day, hour, min, sec, nanosec, loc) + + // Convert the time to UTC to standardize the output + return localTime.UTC() } func formatDate(date time.Time, layouttz ...string) string { diff --git a/x-pack/filebeat/input/httpjson/value_tpl_test.go b/x-pack/filebeat/input/httpjson/value_tpl_test.go index 8944d590765..4b642a16973 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/value_tpl_test.go @@ -143,25 +143,39 @@ func TestValueTpl(t *testing.T) { expectedVal: "2020-11-05 12:25:32 +0000 UTC", }, { - name: "func parseDate MST default layout", - value: `[[ parseDate "Mon, 02 Jan 2006 15:04:05 MST" "Mon, 02 Jan 2006 15:04:05 MST" ]]`, + name: "func parseDateInTZ with RFC3339Nano and timezone offset", + value: `[[ parseDateInTZ "2020-11-05T12:25:32.1234567Z" "-0700" "RFC3339Nano" ]]`, paramCtx: emptyTransformContext(), paramTr: transformable{}, - expectedVal: "2006-01-02 22:04:05 +0000 UTC", + expectedVal: "2020-11-05 19:25:32.1234567 +0000 UTC", }, { - name: "func parseDate MST RFC1123", - value: `[[ parseDate "Mon, 02 Jan 2006 15:04:05 MST" "RFC1123" ]]`, + name: "func parseDateInTZ defaults to RFC3339 with implicit offset and timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32+04:00" "-0700" ]]`, paramCtx: emptyTransformContext(), paramTr: transformable{}, - expectedVal: "2006-01-02 22:04:05 +0000 UTC", + expectedVal: "2020-11-05 19:25:32 +0000 UTC", }, { - name: "func parseDate EST unix date", - value: `[[ parseDate "Mon Jan 2 15:04:05 EST 2006" "UnixDate" ]]`, + name: "func parseDateInTZ defaults to RFC3339 with IANA timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "America/New_York" ]]`, paramCtx: emptyTransformContext(), paramTr: transformable{}, - expectedVal: "2006-01-02 20:04:05 +0000 UTC", + expectedVal: "2020-11-05 17:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ with custom layout and timezone name", + value: `[[ parseDateInTZ "Thu Nov 5 12:25:32 2020" "Europe/Paris" "Mon Jan _2 15:04:05 2006" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 11:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ with invalid timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "Invalid/Timezone" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 12:25:32 +0000 UTC", }, { name: "func formatDate", From b767c90df43de173e278596b6c5e54f5dec02ed2 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Tue, 20 Feb 2024 19:23:20 +0100 Subject: [PATCH 5/5] Update changelog after new changes --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 081a2966cae..14a24c00aab 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -95,7 +95,6 @@ fields added to events containing the Beats version. {pull}37553[37553] - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] - [threatintel] MISP pagination fixes {pull}37898[37898] - Fix file handle leak when handling errors in filestream {pull}37973[37973] -- Fix location detection for parseDate value template in the HTTPJSON input {pull}37738[37738] *Heartbeat* @@ -188,6 +187,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add ETW input. {pull}36915[36915] - Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] - Add logging for cache processor file reads and writes. {pull}38052[38052] +- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] *Auditbeat*