Add support for idle_connection_timeout to elasticsearch output

elastic · Oct 13, 2023 · dcca9b8 · dcca9b8
1 parent 8a70712
commit dcca9b8
Show file tree

Hide file tree

Showing 22 changed files with 92 additions and 6 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -162,6 +162,7 @@ is collected by it.
 - Add support for AWS external IDs. {issue}36321[36321] {pull}36322[36322]
 - [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506]
   Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor
+- elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}99999[99999]
 
 *Auditbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -12712,11 +12712,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.3.15-0.20230913212237-dbdaf18c898b
+Version: v0.4.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.15-0.20230913212237-dbdaf18c898b/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.4.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -522,6 +522,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1618,6 +1618,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/go.mod b/go.mod
@@ -202,7 +202,7 @@ require (
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/elastic-agent-autodiscover v0.6.2
-	github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b
+	github.com/elastic/elastic-agent-libs v0.4.0
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.7.0
 	github.com/elastic/go-elasticsearch/v8 v8.10.0

diff --git a/go.sum b/go.sum
@@ -653,8 +653,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+P
 github.com/elastic/elastic-agent-autodiscover v0.6.2/go.mod h1:yXYKFAG+Py+TcE4CCR8EAbJiYb+6Dz9sCDoWgOveqtU=
 github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo=
 github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0=
-github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a2iuOokwld+D7VhyFymVtsPoqxZ8fkkOCOOjeYU9CDM=
-github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ=
+github.com/elastic/elastic-agent-libs v0.4.0 h1:P0b+xcvYK+dEwldvRXObO1dj3rjjR5qEXAl6TwRCAy0=
+github.com/elastic/elastic-agent-libs v0.4.0/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
 github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ=

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -614,6 +614,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl
@@ -81,6 +81,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go
@@ -102,6 +102,7 @@ func NewClient(
 		CompressionLevel: s.CompressionLevel,
 		EscapeHTML:       s.EscapeHTML,
 		Transport:        s.Transport,
+		IdleConnTimeout:  s.IdleConnTimeout,
 	})
 	if err != nil {
 		return nil, err

diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc
@@ -689,6 +689,12 @@ default is `1s`.
 The maximum number of seconds to wait before attempting to connect to
 Elasticsearch after a network error. The default is `60s`.
 
+===== `idle_connection_timeout`
+
+The maximum amount of time an idle connection will remain idle before closing itself.
+Zero means no limit. The format is a Go language duration (example 60s is 60 seconds).
+The default is 0.
+
 ===== `timeout`
 
 The http request timeout in seconds for the Elasticsearch request. The default is 90.

diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go
@@ -41,7 +41,9 @@ func makeES(
 ) (outputs.Group, error) {
 	log := logp.NewLogger(logSelector)
 	if !cfg.HasField("bulk_max_size") {
-		cfg.SetInt("bulk_max_size", -1, defaultBulkSize)
+		if err := cfg.SetInt("bulk_max_size", -1, defaultBulkSize); err != nil {
+			return outputs.Fail(err)
+		}
 	}
 
 	index, pipeline, err := buildSelectors(im, beat, cfg)
@@ -105,6 +107,7 @@ func makeES(
 				Observer:         observer,
 				EscapeHTML:       config.EscapeHTML,
 				Transport:        config.Transport,
+				IdleConnTimeout:  config.Transport.IdleConnTimeout,
 			},
 			Index:              index,
 			Pipeline:           pipeline,

diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -1357,6 +1357,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
@@ -988,6 +988,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
@@ -404,6 +404,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml
@@ -578,6 +578,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -3988,6 +3988,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml
@@ -646,6 +646,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml
@@ -614,6 +614,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -1918,6 +1918,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml
@@ -365,6 +365,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml
@@ -988,6 +988,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 

diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml
@@ -406,6 +406,11 @@ output.elasticsearch:
   # Elasticsearch after a network error. The default is 60s.
   #backoff.max: 60s
 
+  # The maximum amount of time an idle connection will remain idle
+  # before closing itself.  Zero means no limit. The format is a Go
+  # language duration (example 60s is 60 seconds). The default is 0.
+  #idle_connection_timeout: 0
+
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90