From 6d39e22ef19d6c28b67188a88c7aa1b6fdcb3736 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Fri, 3 May 2024 10:33:52 -0700 Subject: [PATCH] [chore] add an error log for backpressure from upstream (#32790) Adds an error log on error reported by the backend indicating that backpressure is being applied. --- exporter/splunkhecexporter/client.go | 2 +- exporter/splunkhecexporter/client_test.go | 16 ++++++++-------- exporter/splunkhecexporter/hec_worker.go | 6 ++++++ exporter/splunkhecexporter/integration_test.go | 6 +++--- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go index 90ec9289dd61..3ae5259fae1a 100644 --- a/exporter/splunkhecexporter/client.go +++ b/exporter/splunkhecexporter/client.go @@ -631,7 +631,7 @@ func (c *client) start(ctx context.Context, host component.Host) (err error) { } } url, _ := c.config.getURL() - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, c.buildInfo)} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, c.buildInfo), c.logger} c.heartbeater = newHeartbeater(c.config, c.buildInfo, getPushLogFn(c)) if c.config.Heartbeat.Startup { if err := c.heartbeater.sendHeartbeat(c.config, c.buildInfo, getPushLogFn(c)); err != nil { diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go index 8c14b8a19be8..777d8e5d142c 100644 --- a/exporter/splunkhecexporter/client_test.go +++ b/exporter/splunkhecexporter/client_test.go @@ -1560,7 +1560,7 @@ func Test_pushLogData_InvalidLog(t *testing.T) { func Test_pushLogData_PostError(t *testing.T) { c := newLogsClient(exportertest.NewNopCreateSettings(), NewFactory().CreateDefaultConfig().(*Config)) - c.hecWorker = &defaultHecWorker{url: &url.URL{Host: "in va lid"}} + c.hecWorker = &defaultHecWorker{url: &url.URL{Host: "in va lid"}, logger: zap.NewNop()} // 2000 log records -> ~371888 bytes when JSON encoded. logs := createLogData(1, 1, 2000) @@ -1605,7 +1605,7 @@ func Test_pushLogData_ShouldAddResponseTo400Error(t *testing.T) { // An HTTP client that returns status code 400 and response body responseBody. httpClient, _ := newTestClient(400, responseBody) - splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()} // Sending logs using the client. err := splunkClient.pushLogData(context.Background(), logs) require.True(t, consumererror.IsPermanent(err), "Expecting permanent error") @@ -1615,7 +1615,7 @@ func Test_pushLogData_ShouldAddResponseTo400Error(t *testing.T) { // An HTTP client that returns some other status code other than 400 and response body responseBody. httpClient, _ = newTestClient(500, responseBody) - splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()} // Sending logs using the client. err = splunkClient.pushLogData(context.Background(), logs) require.False(t, consumererror.IsPermanent(err), "Expecting non-permanent error") @@ -1638,7 +1638,7 @@ func Test_pushLogData_ShouldReturnUnsentLogsOnly(t *testing.T) { // The first record is to be sent successfully, the second one should not httpClient, _ := newTestClientWithPresetResponses([]int{200, 400}, []string{"OK", "NOK"}) - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()} err := c.pushLogData(context.Background(), logs) require.Error(t, err) @@ -1665,7 +1665,7 @@ func Test_pushLogData_ShouldAddHeadersForProfilingData(t *testing.T) { httpClient, headers := newTestClient(200, "OK") url := &url.URL{Scheme: "http", Host: "splunk"} - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()} err := c.pushLogData(context.Background(), logs) require.NoError(t, err) @@ -1948,7 +1948,7 @@ func Test_pushLogData_Small_MaxContentLength(t *testing.T) { config.DisableCompression = disable c := newLogsClient(exportertest.NewNopCreateSettings(), config) - c.hecWorker = &defaultHecWorker{&url.URL{Scheme: "http", Host: "splunk"}, http.DefaultClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{&url.URL{Scheme: "http", Host: "splunk"}, http.DefaultClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()} err := c.pushLogData(context.Background(), logs) require.Error(t, err) @@ -2060,7 +2060,7 @@ func TestPushLogsPartialSuccess(t *testing.T) { // The first request succeeds, the second fails. httpClient, _ := newTestClientWithPresetResponses([]int{200, 503}, []string{"OK", "NOK"}) url := &url.URL{Scheme: "http", Host: "splunk"} - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(cfg, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(cfg, component.NewDefaultBuildInfo()), zap.NewNop()} logs := plog.NewLogs() logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords() @@ -2081,7 +2081,7 @@ func TestPushLogsRetryableFailureMultipleResources(t *testing.T) { httpClient, _ := newTestClientWithPresetResponses([]int{503}, []string{"NOK"}) url := &url.URL{Scheme: "http", Host: "splunk"} - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, component.NewDefaultBuildInfo()), zap.NewNop()} logs := plog.NewLogs() logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty().Body().SetStr("log-1") diff --git a/exporter/splunkhecexporter/hec_worker.go b/exporter/splunkhecexporter/hec_worker.go index 3e9fad0ea8ae..64ff98738932 100644 --- a/exporter/splunkhecexporter/hec_worker.go +++ b/exporter/splunkhecexporter/hec_worker.go @@ -10,6 +10,7 @@ import ( "net/url" "go.opentelemetry.io/collector/consumer/consumererror" + "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk" ) @@ -22,6 +23,7 @@ type defaultHecWorker struct { url *url.URL client *http.Client headers map[string]string + logger *zap.Logger } func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[string]string) error { @@ -51,6 +53,10 @@ func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[s } defer resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable { + hec.logger.Error("Splunk is unable to receive data. Please investigate the health of the cluster", zap.Int("status", resp.StatusCode), zap.String("host", hec.url.String())) + } + err = splunk.HandleHTTPCode(resp) if err != nil { return err diff --git a/exporter/splunkhecexporter/integration_test.go b/exporter/splunkhecexporter/integration_test.go index f08b8e12849c..2a8af5480838 100644 --- a/exporter/splunkhecexporter/integration_test.go +++ b/exporter/splunkhecexporter/integration_test.go @@ -245,7 +245,7 @@ func logsTest(t *testing.T, config *Config, url *url.URL, test testCfg) { } httpClient := createInsecureClient() - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger} err := c.pushLogData(context.Background(), logs) require.NoError(t, err, "Must not error while sending Logs data") @@ -268,7 +268,7 @@ func metricsTest(t *testing.T, config *Config, url *url.URL, test testCfg) { metricData := prepareMetricsData(test.config.event) httpClient := createInsecureClient() - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger} err := c.pushMetricsData(context.Background(), metricData) require.NoError(t, err, "Must not error while sending Metrics data") @@ -284,7 +284,7 @@ func tracesTest(t *testing.T, config *Config, url *url.URL, test testCfg) { tracesData := prepareTracesData(test.config.index, test.config.source, test.config.sourcetype) httpClient := createInsecureClient() - c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())} + c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger} err := c.pushTraceData(context.Background(), tracesData) require.NoError(t, err, "Must not error while sending Trace data")