From 1fac63591e39f09ce04251b3f84d0554a6d12895 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 14:40:50 -0400 Subject: [PATCH 1/8] [filebeat] Fix long filepaths in diagnostics exceeding max path limits on Windows --- filebeat/input/v2/compat/compat.go | 1 + filebeat/input/v2/input-cursor/input.go | 2 ++ filebeat/input/v2/input.go | 4 ++++ x-pack/filebeat/input/cel/input.go | 2 +- x-pack/filebeat/input/entityanalytics/provider/jamf/jamf.go | 2 +- x-pack/filebeat/input/entityanalytics/provider/okta/okta.go | 2 +- x-pack/filebeat/input/http_endpoint/input.go | 2 +- x-pack/filebeat/input/httpjson/input.go | 2 +- 8 files changed, 12 insertions(+), 5 deletions(-) diff --git a/filebeat/input/v2/compat/compat.go b/filebeat/input/v2/compat/compat.go index d84c9e39bfd..416b6628e19 100644 --- a/filebeat/input/v2/compat/compat.go +++ b/filebeat/input/v2/compat/compat.go @@ -128,6 +128,7 @@ func (r *runner) Start() { err := r.input.Run( v2.Context{ ID: r.id, + IDWithoutName: r.id, Agent: *r.agent, Logger: log, Cancelation: r.sig, diff --git a/filebeat/input/v2/input-cursor/input.go b/filebeat/input/v2/input-cursor/input.go index c1d4cec0762..547c7721c3b 100644 --- a/filebeat/input/v2/input-cursor/input.go +++ b/filebeat/input/v2/input-cursor/input.go @@ -115,6 +115,8 @@ func (inp *managedInput) Run( grp.Go(func() (err error) { // refine per worker context inpCtx := ctx + // Preserve IDWithoutName, in case the context was constructed who knows how + inpCtx.IDWithoutName = ctx.ID inpCtx.ID = ctx.ID + "::" + source.Name() inpCtx.Logger = ctx.Logger.With("input_source", source.Name()) diff --git a/filebeat/input/v2/input.go b/filebeat/input/v2/input.go index 9b78bc427ae..f62cc149a93 100644 --- a/filebeat/input/v2/input.go +++ b/filebeat/input/v2/input.go @@ -79,6 +79,10 @@ type Context struct { // The input ID. ID string + // The input ID without name. Some inputs append sourcename, we need the id to be untouched + // https://github.com/elastic/beats/blob/43d80af2aea60b0c45711475d114e118d90c4581/filebeat/input/v2/input-cursor/input.go#L118 + IDWithoutName string + // Agent provides additional Beat info like instance ID or beat name. Agent beat.Info diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 2096383de39..8333dd1b216 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -142,7 +142,7 @@ func (i input) run(env v2.Context, src *source, cursor map[string]interface{}, p ctx := ctxtool.FromCanceller(env.Cancelation) if cfg.Resource.Tracer != nil { - id := sanitizeFileName(env.ID) + id := sanitizeFileName(env.IDWithoutName) cfg.Resource.Tracer.Filename = strings.ReplaceAll(cfg.Resource.Tracer.Filename, "*", id) } diff --git a/x-pack/filebeat/input/entityanalytics/provider/jamf/jamf.go b/x-pack/filebeat/input/entityanalytics/provider/jamf/jamf.go index 0affa5b553d..8534187f6ac 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/jamf/jamf.go +++ b/x-pack/filebeat/input/entityanalytics/provider/jamf/jamf.go @@ -109,7 +109,7 @@ func (p *jamfInput) Run(inputCtx v2.Context, store *kvstore.Store, client beat.C updateTimer := time.NewTimer(updateWaitTime) if p.cfg.Tracer != nil { - id := sanitizeFileName(inputCtx.ID) + id := sanitizeFileName(inputCtx.IDWithoutName) p.cfg.Tracer.Filename = strings.ReplaceAll(p.cfg.Tracer.Filename, "*", id) } diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 70f9a8f3a55..404f069a7ec 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -113,7 +113,7 @@ func (p *oktaInput) Run(inputCtx v2.Context, store *kvstore.Store, client beat.C p.lim = rate.NewLimiter(1, 1) if p.cfg.Tracer != nil { - id := sanitizeFileName(inputCtx.ID) + id := sanitizeFileName(inputCtx.IDWithoutName) p.cfg.Tracer.Filename = strings.ReplaceAll(p.cfg.Tracer.Filename, "*", id) } diff --git a/x-pack/filebeat/input/http_endpoint/input.go b/x-pack/filebeat/input/http_endpoint/input.go index 43adfa0b469..b4ad07e7626 100644 --- a/x-pack/filebeat/input/http_endpoint/input.go +++ b/x-pack/filebeat/input/http_endpoint/input.go @@ -108,7 +108,7 @@ func (e *httpEndpoint) Run(ctx v2.Context, pipeline beat.Pipeline) error { defer metrics.Close() if e.config.Tracer != nil { - id := sanitizeFileName(ctx.ID) + id := sanitizeFileName(ctx.IDWithoutName) e.config.Tracer.Filename = strings.ReplaceAll(e.config.Tracer.Filename, "*", id) } diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 67daa7ef849..ad61aceff89 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -123,7 +123,7 @@ func run(ctx v2.Context, cfg config, pub inputcursor.Publisher, crsr *inputcurso stdCtx := ctxtool.FromCanceller(ctx.Cancelation) if cfg.Request.Tracer != nil { - id := sanitizeFileName(ctx.ID) + id := sanitizeFileName(ctx.IDWithoutName) cfg.Request.Tracer.Filename = strings.ReplaceAll(cfg.Request.Tracer.Filename, "*", id) // Propagate tracer behaviour to all chain children. From 035398cc1001fb648793311998b12db64dcef1f0 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 14:48:43 -0400 Subject: [PATCH 2/8] Update changelog with PR number --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e36ea3396a9..d67c1ae79c8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -166,6 +166,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix publication of group data from the Okta entity analytics provider. {pull}40681[40681] - Ensure netflow custom field configuration is applied. {issue}40735[40735] {pull}40730[40730] - Fix replace processor handling of zero string replacement validation. {pull}40751[40751] +- Fix long filepaths in diagnostics exceeding max path limits on Windows. {pull}40909[40909] + *Heartbeat* From 2031667d2f3f9bc51d86eee44e5cf9799c8823ef Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 15:30:36 -0400 Subject: [PATCH 3/8] Adjust utz --- x-pack/filebeat/input/cel/input_test.go | 8 +++++--- x-pack/filebeat/input/httpjson/input_test.go | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/input/cel/input_test.go b/x-pack/filebeat/input/cel/input_test.go index 6ef9dcbc824..02daa1148d7 100644 --- a/x-pack/filebeat/input/cel/input_test.go +++ b/x-pack/filebeat/input/cel/input_test.go @@ -1690,10 +1690,12 @@ func TestInput(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() + id := "test_id:" + test.name v2Ctx := v2.Context{ - Logger: logp.NewLogger("cel_test"), - ID: "test_id:" + test.name, - Cancelation: ctx, + Logger: logp.NewLogger("cel_test"), + ID: id, + IDWithoutName: id, + Cancelation: ctx, } var client publisher client.done = func() { diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index dac538b54dd..4f09d8f057f 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -1660,9 +1660,10 @@ func newChainPaginationTestServer( func newV2Context(id string) (v2.Context, func()) { ctx, cancel := context.WithCancel(context.Background()) return v2.Context{ - Logger: logp.NewLogger("httpjson_test"), - ID: id, - Cancelation: ctx, + Logger: logp.NewLogger("httpjson_test"), + ID: id, + IDWithoutName: id, + Cancelation: ctx, }, cancel } From dbf0cffacd31311a2eeb819d1d3113abe5d7593c Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 15:44:23 -0400 Subject: [PATCH 4/8] Quiet the linter false positive --- x-pack/filebeat/input/cel/input.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 8333dd1b216..b49704d204a 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -1250,6 +1250,7 @@ func cloneMap(dst, src mapstr.M) { // walkMap walks to all ends of the provided path in m and applies fn to the // final element of each walk. Nested arrays are not handled. func walkMap(m mapstr.M, path string, fn func(parent mapstr.M, key string)) { + //nolint:typecheck // typecheck linter is buggy and thinks rest is not used. key, rest, more := strings.Cut(path, ".") v, ok := m[key] if !ok { From 07090aeca0d2d5a04c5a6bb12843a1738cb16f60 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 16:02:50 -0400 Subject: [PATCH 5/8] More linter pacification --- x-pack/filebeat/input/cel/input.go | 2 +- x-pack/filebeat/input/cel/input_test.go | 10 +++++----- x-pack/filebeat/input/httpjson/input_test.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index b49704d204a..b4ee58b4a2e 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -1250,7 +1250,7 @@ func cloneMap(dst, src mapstr.M) { // walkMap walks to all ends of the provided path in m and applies fn to the // final element of each walk. Nested arrays are not handled. func walkMap(m mapstr.M, path string, fn func(parent mapstr.M, key string)) { - //nolint:typecheck // typecheck linter is buggy and thinks rest is not used. + //nolint:typecheck,nolintlint // typecheck linter is buggy and thinks rest is not used. key, rest, more := strings.Cut(path, ".") v, ok := m[key] if !ok { diff --git a/x-pack/filebeat/input/cel/input_test.go b/x-pack/filebeat/input/cel/input_test.go index 02daa1148d7..54159b07908 100644 --- a/x-pack/filebeat/input/cel/input_test.go +++ b/x-pack/filebeat/input/cel/input_test.go @@ -448,9 +448,9 @@ var inputTests = []struct { ` - io.ReadAll(r.Body) + _, _ = io.ReadAll(r.Body) r.Body.Close() - w.Write([]byte(text)) + _, _ = w.Write([]byte(text)) }) server := httptest.NewServer(r) config["resource.url"] = server.URL @@ -582,7 +582,7 @@ var inputTests = []struct { msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) } - w.Write([]byte(msg)) + _, _ = w.Write([]byte(msg)) }, want: []map[string]interface{}{ { @@ -630,7 +630,7 @@ var inputTests = []struct { msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) } - w.Write([]byte(msg)) + _, _ = w.Write([]byte(msg)) }, want: []map[string]interface{}{ { @@ -676,7 +676,7 @@ var inputTests = []struct { msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) } - w.Write([]byte(msg)) + _, _ = w.Write([]byte(msg)) }, want: []map[string]interface{}{ { diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index 4f09d8f057f..cc2995164d4 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -1357,9 +1357,9 @@ var testCases = []struct { ` - io.ReadAll(r.Body) + _, _ = io.ReadAll(r.Body) r.Body.Close() - w.Write([]byte(text)) + _, _ = w.Write([]byte(text)) }) server := httptest.NewServer(r) config["request.url"] = server.URL From 3eaef84488b710c74d5831a9e547ec3ff9f7fea1 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 17:34:50 -0400 Subject: [PATCH 6/8] Fix linter complains for errors checks in okta.go --- x-pack/filebeat/input/entityanalytics/provider/okta/okta.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 404f069a7ec..571ff6ab346 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -478,7 +478,7 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn next, err := okta.Next(h) if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { break } p.logger.Debugf("received %d users from API", len(users)) @@ -588,7 +588,7 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS next, err := okta.Next(h) if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { break } p.logger.Debugf("received %d devices from API", len(devices)) @@ -617,7 +617,7 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS next, err := okta.Next(h) if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { break } p.logger.Debugf("received %d devices from API", len(devices)) From 4757a4fb599a31010c6c7479fb6bce08c6d5b0ee Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 17:45:53 -0400 Subject: [PATCH 7/8] Let the linter complain again --- x-pack/filebeat/input/cel/input.go | 1 - x-pack/filebeat/input/cel/input_test.go | 10 +++++----- .../input/entityanalytics/provider/okta/okta.go | 6 +++--- x-pack/filebeat/input/httpjson/input_test.go | 6 +++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index b4ee58b4a2e..8333dd1b216 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -1250,7 +1250,6 @@ func cloneMap(dst, src mapstr.M) { // walkMap walks to all ends of the provided path in m and applies fn to the // final element of each walk. Nested arrays are not handled. func walkMap(m mapstr.M, path string, fn func(parent mapstr.M, key string)) { - //nolint:typecheck,nolintlint // typecheck linter is buggy and thinks rest is not used. key, rest, more := strings.Cut(path, ".") v, ok := m[key] if !ok { diff --git a/x-pack/filebeat/input/cel/input_test.go b/x-pack/filebeat/input/cel/input_test.go index 54159b07908..02daa1148d7 100644 --- a/x-pack/filebeat/input/cel/input_test.go +++ b/x-pack/filebeat/input/cel/input_test.go @@ -448,9 +448,9 @@ var inputTests = []struct { ` - _, _ = io.ReadAll(r.Body) + io.ReadAll(r.Body) r.Body.Close() - _, _ = w.Write([]byte(text)) + w.Write([]byte(text)) }) server := httptest.NewServer(r) config["resource.url"] = server.URL @@ -582,7 +582,7 @@ var inputTests = []struct { msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) } - _, _ = w.Write([]byte(msg)) + w.Write([]byte(msg)) }, want: []map[string]interface{}{ { @@ -630,7 +630,7 @@ var inputTests = []struct { msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) } - _, _ = w.Write([]byte(msg)) + w.Write([]byte(msg)) }, want: []map[string]interface{}{ { @@ -676,7 +676,7 @@ var inputTests = []struct { msg = fmt.Sprintf(`{"error":"expected method was %#q"}`, http.MethodGet) } - _, _ = w.Write([]byte(msg)) + w.Write([]byte(msg)) }, want: []map[string]interface{}{ { diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 571ff6ab346..404f069a7ec 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -478,7 +478,7 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn next, err := okta.Next(h) if err != nil { - if errors.Is(err, io.EOF) { + if err == io.EOF { break } p.logger.Debugf("received %d users from API", len(users)) @@ -588,7 +588,7 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS next, err := okta.Next(h) if err != nil { - if errors.Is(err, io.EOF) { + if err == io.EOF { break } p.logger.Debugf("received %d devices from API", len(devices)) @@ -617,7 +617,7 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS next, err := okta.Next(h) if err != nil { - if errors.Is(err, io.EOF) { + if err == io.EOF { break } p.logger.Debugf("received %d devices from API", len(devices)) diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index cc2995164d4..7b0ba0f56f7 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -1357,9 +1357,9 @@ var testCases = []struct { ` - _, _ = io.ReadAll(r.Body) + io.ReadAll(r.Body) r.Body.Close() - _, _ = w.Write([]byte(text)) + w.Write([]byte(text)) }) server := httptest.NewServer(r) config["request.url"] = server.URL @@ -1695,7 +1695,7 @@ func defaultHandler(expectedMethod, expectedBody, msg string) http.HandlerFunc { } } - _, _ = w.Write([]byte(msg)) + w.Write([]byte(msg)) } } From a0f484acf867488a7eb6f7bbdc8157bf52cef2c1 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Thu, 19 Sep 2024 17:48:22 -0400 Subject: [PATCH 8/8] Revert revert --- x-pack/filebeat/input/httpjson/input_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index 7b0ba0f56f7..4f09d8f057f 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -1695,7 +1695,7 @@ func defaultHandler(expectedMethod, expectedBody, msg string) http.HandlerFunc { } } - w.Write([]byte(msg)) + _, _ = w.Write([]byte(msg)) } }