Skip to content

Commit

Permalink
[chore] add an error log for backpressure from upstream (#32790)
Browse files Browse the repository at this point in the history 
Adds an error log on error reported by the backend indicating that
backpressure is being applied.
  • Loading branch information
@atoulme
atoulme authored May 3, 2024
1 parent a298c5a commit 6d39e22
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 12 deletions.
2 changes: 1 addition & 1 deletion exporter/splunkhecexporter/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ func (c *client) start(ctx context.Context, host component.Host) (err error) {
}
}
url, _ := c.config.getURL()
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, c.buildInfo)}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, c.buildInfo), c.logger}
c.heartbeater = newHeartbeater(c.config, c.buildInfo, getPushLogFn(c))
if c.config.Heartbeat.Startup {
if err := c.heartbeater.sendHeartbeat(c.config, c.buildInfo, getPushLogFn(c)); err != nil {
Expand Down
16 changes: 8 additions & 8 deletions exporter/splunkhecexporter/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1560,7 +1560,7 @@ func Test_pushLogData_InvalidLog(t *testing.T) {

func Test_pushLogData_PostError(t *testing.T) {
c := newLogsClient(exportertest.NewNopCreateSettings(), NewFactory().CreateDefaultConfig().(*Config))
c.hecWorker = &defaultHecWorker{url: &url.URL{Host: "in va lid"}}
c.hecWorker = &defaultHecWorker{url: &url.URL{Host: "in va lid"}, logger: zap.NewNop()}

// 2000 log records -> ~371888 bytes when JSON encoded.
logs := createLogData(1, 1, 2000)
Expand Down Expand Up @@ -1605,7 +1605,7 @@ func Test_pushLogData_ShouldAddResponseTo400Error(t *testing.T) {

// An HTTP client that returns status code 400 and response body responseBody.
httpClient, _ := newTestClient(400, responseBody)
splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}
// Sending logs using the client.
err := splunkClient.pushLogData(context.Background(), logs)
require.True(t, consumererror.IsPermanent(err), "Expecting permanent error")
Expand All @@ -1615,7 +1615,7 @@ func Test_pushLogData_ShouldAddResponseTo400Error(t *testing.T) {

// An HTTP client that returns some other status code other than 400 and response body responseBody.
httpClient, _ = newTestClient(500, responseBody)
splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}
// Sending logs using the client.
err = splunkClient.pushLogData(context.Background(), logs)
require.False(t, consumererror.IsPermanent(err), "Expecting non-permanent error")
Expand All @@ -1638,7 +1638,7 @@ func Test_pushLogData_ShouldReturnUnsentLogsOnly(t *testing.T) {

// The first record is to be sent successfully, the second one should not
httpClient, _ := newTestClientWithPresetResponses([]int{200, 400}, []string{"OK", "NOK"})
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}

err := c.pushLogData(context.Background(), logs)
require.Error(t, err)
Expand All @@ -1665,7 +1665,7 @@ func Test_pushLogData_ShouldAddHeadersForProfilingData(t *testing.T) {

httpClient, headers := newTestClient(200, "OK")
url := &url.URL{Scheme: "http", Host: "splunk"}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}

err := c.pushLogData(context.Background(), logs)
require.NoError(t, err)
Expand Down Expand Up @@ -1948,7 +1948,7 @@ func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
config.DisableCompression = disable

c := newLogsClient(exportertest.NewNopCreateSettings(), config)
c.hecWorker = &defaultHecWorker{&url.URL{Scheme: "http", Host: "splunk"}, http.DefaultClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{&url.URL{Scheme: "http", Host: "splunk"}, http.DefaultClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}

err := c.pushLogData(context.Background(), logs)
require.Error(t, err)
Expand Down Expand Up @@ -2060,7 +2060,7 @@ func TestPushLogsPartialSuccess(t *testing.T) {
// The first request succeeds, the second fails.
httpClient, _ := newTestClientWithPresetResponses([]int{200, 503}, []string{"OK", "NOK"})
url := &url.URL{Scheme: "http", Host: "splunk"}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(cfg, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(cfg, component.NewDefaultBuildInfo()), zap.NewNop()}

logs := plog.NewLogs()
logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords()
Expand All @@ -2081,7 +2081,7 @@ func TestPushLogsRetryableFailureMultipleResources(t *testing.T) {

httpClient, _ := newTestClientWithPresetResponses([]int{503}, []string{"NOK"})
url := &url.URL{Scheme: "http", Host: "splunk"}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, component.NewDefaultBuildInfo()), zap.NewNop()}

logs := plog.NewLogs()
logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty().Body().SetStr("log-1")
Expand Down
6 changes: 6 additions & 0 deletions exporter/splunkhecexporter/hec_worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"net/url"

"go.opentelemetry.io/collector/consumer/consumererror"
"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
)
Expand All @@ -22,6 +23,7 @@ type defaultHecWorker struct {
url *url.URL
client *http.Client
headers map[string]string
logger *zap.Logger
}

func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[string]string) error {
Expand Down Expand Up @@ -51,6 +53,10 @@ func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[s
}
defer resp.Body.Close()

if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
hec.logger.Error("Splunk is unable to receive data. Please investigate the health of the cluster", zap.Int("status", resp.StatusCode), zap.String("host", hec.url.String()))
}

err = splunk.HandleHTTPCode(resp)
if err != nil {
return err
Expand Down
6 changes: 3 additions & 3 deletions exporter/splunkhecexporter/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func logsTest(t *testing.T, config *Config, url *url.URL, test testCfg) {
}

httpClient := createInsecureClient()
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger}

err := c.pushLogData(context.Background(), logs)
require.NoError(t, err, "Must not error while sending Logs data")
Expand All @@ -268,7 +268,7 @@ func metricsTest(t *testing.T, config *Config, url *url.URL, test testCfg) {
metricData := prepareMetricsData(test.config.event)

httpClient := createInsecureClient()
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger}

err := c.pushMetricsData(context.Background(), metricData)
require.NoError(t, err, "Must not error while sending Metrics data")
Expand All @@ -284,7 +284,7 @@ func tracesTest(t *testing.T, config *Config, url *url.URL, test testCfg) {
tracesData := prepareTracesData(test.config.index, test.config.source, test.config.sourcetype)

httpClient := createInsecureClient()
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger}

err := c.pushTraceData(context.Background(), tracesData)
require.NoError(t, err, "Must not error while sending Trace data")
Expand Down

0 comments on commit 6d39e22

Please sign in to comment.