x-pack/filebeat/input/httpjson: Fix parseDate location offset (#37738)

Add a new value template helper `parseDateInTZ` where users can apply the proper timezone when parsing dates, avoiding the limitations of `parseDate` with timezone abbreviations. It accepts numeric offsets and IANA time zone names.

(cherry picked from commit 8214f9f)

CHANGELOG.next.asciidoc

@@ -185,6 +185,8 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 - Add support for PEM-based Okta auth in CEL. {pull}37813[37813]
 - Add ETW input. {pull}36915[36915]
 - Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971]
+- Add logging for cache processor file reads and writes. {pull}38052[38052]
+- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738]
 
 *Auditbeat*
 

x-pack/filebeat/docs/inputs/input-httpjson.asciidoc

@@ -220,7 +220,8 @@ Some built-in helper functions are provided to work with the input state inside
 - `min`: returns the minimum of two values.
 - `mul`: multiplies two integers.
 - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now.
-- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov  5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`.
+- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Note: Parsing timezone abbreviations may cause ambiguities. Prefer `parseDateInTZ` for explicit timezone handling. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov  5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`.
+- `parseDateInTZ`: parses a date string within a specified timezone (TZ), returning a `time.Time` in UTC. Specified timezone overwrites implicit timezone from the input date. Accepts timezone offsets ("-07:00", "-0700", "-07") or IANA Time Zone names ("America/New_York"). If TZ is invalid, defaults to UTC. Optional layout argument as in parseDate. Example: `[[ parseDateInTZ "2020-11-05T12:25:32" "America/New_York" ]]`, `[[ parseDateInTZ "2020-11-05T12:25:32" "-07:00" "RFC3339" ]]`.
 - `parseDuration`: parses duration strings and returns `time.Duration`. Example: `[[parseDuration "1h"]]`.
 - `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000]]` returns `2020-11-05 13:25:32 +0000 UTC`.
 - `parseTimestampNano`: parses a timestamp in nanoseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000000000]]` returns `2020-11-05 13:25:32 +0000 UTC`.

x-pack/filebeat/input/httpjson/value_tpl.go

@@ -71,6 +71,7 @@ func (t *valueTpl) Unpack(in string) error {
 			"mul":                 mul,
 			"now":                 now,
 			"parseDate":           parseDate,
+			"parseDateInTZ":       parseDateInTZ,
 			"parseDuration":       parseDuration,
 			"parseTimestamp":      parseTimestamp,
 			"parseTimestampMilli": parseTimestampMilli,
@@ -194,6 +195,58 @@ func parseDate(date string, layout ...string) time.Time {
 	return t.UTC()
 }
 
+// parseDateInTZ parses a date string within a specified timezone, returning a time.Time
+// 'tz' is the timezone (offset or IANA name) for parsing
+func parseDateInTZ(date string, tz string, layout ...string) time.Time {
+	var ly string
+	if len(layout) == 0 {
+		ly = defaultTimeLayout
+	} else {
+		ly = layout[0]
+	}
+	if found := predefinedLayouts[ly]; found != "" {
+		ly = found
+	}
+
+	var loc *time.Location
+	// Attempt to parse timezone as offset in various formats
+	for _, format := range []string{"-07", "-0700", "-07:00"} {
+		t, err := time.Parse(format, tz)
+		if err != nil {
+			continue
+		}
+		name, offset := t.Zone()
+		loc = time.FixedZone(name, offset)
+		break
+	}
+
+	// If parsing tz as offset fails, try loading location by name
+	if loc == nil {
+		var err error
+		loc, err = time.LoadLocation(tz)
+		if err != nil {
+			loc = time.UTC // Default to UTC on error
+		}
+	}
+
+	// Using Parse allows us not to worry about the timezone
+	// as the predefined timezone is applied afterwards
+	t, err := time.Parse(ly, date)
+	if err != nil {
+		return time.Time{}
+	}
+
+	// Manually create a new time object with the parsed date components and the desired location
+	// It allows interpreting the parsed time in the specified timezone
+	year, month, day := t.Date()
+	hour, min, sec := t.Clock()
+	nanosec := t.Nanosecond()
+	localTime := time.Date(year, month, day, hour, min, sec, nanosec, loc)
+
+	// Convert the time to UTC to standardize the output
+	return localTime.UTC()
+}
+
 func formatDate(date time.Time, layouttz ...string) string {
 	var layout, tz string
 	switch {

x-pack/filebeat/input/httpjson/value_tpl_test.go

@@ -142,6 +142,41 @@ func TestValueTpl(t *testing.T) {
 			paramTr:     transformable{},
 			expectedVal: "2020-11-05 12:25:32 +0000 UTC",
 		},
+		{
+			name:        "func parseDateInTZ with RFC3339Nano and timezone offset",
+			value:       `[[ parseDateInTZ "2020-11-05T12:25:32.1234567Z" "-0700" "RFC3339Nano" ]]`,
+			paramCtx:    emptyTransformContext(),
+			paramTr:     transformable{},
+			expectedVal: "2020-11-05 19:25:32.1234567 +0000 UTC",
+		},
+		{
+			name:        "func parseDateInTZ defaults to RFC3339 with implicit offset and timezone",
+			value:       `[[ parseDateInTZ "2020-11-05T12:25:32+04:00" "-0700" ]]`,
+			paramCtx:    emptyTransformContext(),
+			paramTr:     transformable{},
+			expectedVal: "2020-11-05 19:25:32 +0000 UTC",
+		},
+		{
+			name:        "func parseDateInTZ defaults to RFC3339 with IANA timezone",
+			value:       `[[ parseDateInTZ "2020-11-05T12:25:32Z" "America/New_York" ]]`,
+			paramCtx:    emptyTransformContext(),
+			paramTr:     transformable{},
+			expectedVal: "2020-11-05 17:25:32 +0000 UTC",
+		},
+		{
+			name:        "func parseDateInTZ with custom layout and timezone name",
+			value:       `[[ parseDateInTZ "Thu Nov  5 12:25:32 2020" "Europe/Paris" "Mon Jan _2 15:04:05 2006" ]]`,
+			paramCtx:    emptyTransformContext(),
+			paramTr:     transformable{},
+			expectedVal: "2020-11-05 11:25:32 +0000 UTC",
+		},
+		{
+			name:        "func parseDateInTZ with invalid timezone",
+			value:       `[[ parseDateInTZ "2020-11-05T12:25:32Z" "Invalid/Timezone" ]]`,
+			paramCtx:    emptyTransformContext(),
+			paramTr:     transformable{},
+			expectedVal: "2020-11-05 12:25:32 +0000 UTC",
+		},
 		{
 			name:        "func formatDate",
 			setup:       func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } },