[Filebeat] Add Proxy config to httpjson v2 (elastic#24662)

Adds a `proxy_url` config option to the httpjson v2 input. Updates all httpjson based modules to support proxy_url. Fixes elastic#24615. Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
andrewkroh · Mar 24, 2021 · 9a5f5d0 · 9a5f5d0
1 parent 7f5a358
commit 9a5f5d0
Show file tree

Hide file tree

Showing 52 changed files with 137 additions and 7 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -804,6 +804,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Support X-Forwarder-For in IIS logs. {pull}19142[192142]
 - Updating field mappings for Cisco AMP module, fixing certain fields. {pull}24661[24661]
 - Added NTP fileset to Zeek module {pull}24224[24224]
+- Add `proxy_url` config for httpjson v2 input. {issue}24615[24615] {pull}24662[24662]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc
@@ -20,6 +20,7 @@ This input supports:
 * Pagination
 * Retries
 * Rate limiting
+* Proxying
 * Request transformations
 * Response transformations
 
@@ -383,6 +384,22 @@ This specifies SSL/TLS configuration. If the ssl section is missing, the host's
 CAs are used for HTTPS connections. See <<configuration-ssl>> for more
 information.
 
+[float]
+==== `request.proxy_url`
+
+This specifies proxy configuration in the form of `http[s]://<user>:<password>@<server name/ip>:<port>`
+
+["source","yaml",subs="attributes"]
+----
+filebeat.inputs:
+# Fetch your public IP every minute.
+- type: httpjson
+  config_version: 2
+  interval: 1m
+  request.url: https://api.ipify.org/?format=json
+  request.proxy_url: http://proxy.example:8080
+----
+
 [float]
 ==== `request.retry.max_attempts`
 

diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go
@@ -88,6 +88,7 @@ type requestConfig struct {
 	RedirectMaxRedirects   int               `config:"redirect.max_redirects"`
 	RateLimit              *rateLimitConfig  `config:"rate_limit"`
 	Transforms             transformsConfig  `config:"transforms"`
+	ProxyURL               *urlConfig        `config:"proxy_url"`
 }
 
 func (c requestConfig) getTimeout() time.Duration {

diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go
@@ -149,17 +149,22 @@ func run(
 
 func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSConfig, log *logp.Logger) (*httpClient, error) {
 	timeout := config.Request.getTimeout()
+	proxy_url := config.Request.ProxyURL
 
 	// Make retryable HTTP client
+	transport := &http.Transport{
+		DialContext: (&net.Dialer{
+			Timeout: timeout,
+		}).DialContext,
+		TLSClientConfig:   tlsConfig.ToConfig(),
+		DisableKeepAlives: true,
+	}
+	if proxy_url != nil && proxy_url.URL != nil {
+		transport.Proxy = http.ProxyURL(proxy_url.URL)
+	}
 	client := &retryablehttp.Client{
 		HTTPClient: &http.Client{
-			Transport: &http.Transport{
-				DialContext: (&net.Dialer{
-					Timeout: timeout,
-				}).DialContext,
-				TLSClientConfig:   tlsConfig.ToConfig(),
-				DisableKeepAlives: true,
-			},
+			Transport:     transport,
 			Timeout:       timeout,
 			CheckRedirect: checkRedirect(config.Request, log),
 		},

diff --git a/x-pack/filebeat/module/cisco/amp/config/config.yml b/x-pack/filebeat/module/cisco/amp/config/config.yml
@@ -17,6 +17,9 @@ request.timeout: {{ .request_timeout }}
 {{ if .ssl }}
 request.ssl: {{ .ssl | tojson }}
 {{ end }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.transforms:
 - set:
     target: url.params.start_date

diff --git a/x-pack/filebeat/module/cisco/amp/manifest.yml b/x-pack/filebeat/module/cisco/amp/manifest.yml
@@ -18,6 +18,7 @@ var:
     default: 24h
   - name: interval
     default: 60m
+  - name: proxy_url
 
 ingest_pipeline:
   - ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/google_workspace/admin/config/config.yml b/x-pack/filebeat/module/google_workspace/admin/config/config.yml
@@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user
 {{ if .http_client_timeout }}
 request.timeout: {{ .http_client_timeout }}
 {{ end }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.transforms:
   - set:
       target: url.params.startTime

diff --git a/x-pack/filebeat/module/google_workspace/admin/manifest.yml b/x-pack/filebeat/module/google_workspace/admin/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/google_workspace/drive/config/config.yml b/x-pack/filebeat/module/google_workspace/drive/config/config.yml
@@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user
 {{ if .http_client_timeout }}
 request.timeout: {{ .http_client_timeout }}
 {{ end }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.transforms:
   - set:
       target: url.params.startTime

diff --git a/x-pack/filebeat/module/google_workspace/drive/manifest.yml b/x-pack/filebeat/module/google_workspace/drive/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/google_workspace/groups/config/config.yml b/x-pack/filebeat/module/google_workspace/groups/config/config.yml
@@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user
 {{ if .http_client_timeout }}
 request.timeout: {{ .http_client_timeout }}
 {{ end }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.transforms:
   - set:
       target: url.params.startTime

diff --git a/x-pack/filebeat/module/google_workspace/groups/manifest.yml b/x-pack/filebeat/module/google_workspace/groups/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/google_workspace/login/config/config.yml b/x-pack/filebeat/module/google_workspace/login/config/config.yml
@@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user
 {{ if .http_client_timeout }}
 request.timeout: {{ .http_client_timeout }}
 {{ end }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.transforms:
   - set:
       target: url.params.startTime

diff --git a/x-pack/filebeat/module/google_workspace/login/manifest.yml b/x-pack/filebeat/module/google_workspace/login/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/google_workspace/saml/config/config.yml b/x-pack/filebeat/module/google_workspace/saml/config/config.yml
@@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user
 {{ if .http_client_timeout }}
 request.timeout: {{ .http_client_timeout }}
 {{ end }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.transforms:
   - set:
       target: url.params.startTime

diff --git a/x-pack/filebeat/module/google_workspace/saml/manifest.yml b/x-pack/filebeat/module/google_workspace/saml/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml
@@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user
 {{ if .http_client_timeout }}
 request.timeout: {{ .http_client_timeout }}
 {{ end }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.transforms:
   - set:
       target: url.params.startTime

diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/manifest.yml b/x-pack/filebeat/module/google_workspace/user_accounts/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/gsuite/admin/config/config.yml b/x-pack/filebeat/module/gsuite/admin/config/config.yml
@@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }}
 pagination.id_field: nextPageToken
 pagination.url_field: pageToken
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 type: log
 paths:

diff --git a/x-pack/filebeat/module/gsuite/admin/manifest.yml b/x-pack/filebeat/module/gsuite/admin/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/gsuite/drive/config/config.yml b/x-pack/filebeat/module/gsuite/drive/config/config.yml
@@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }}
 pagination.id_field: nextPageToken
 pagination.url_field: pageToken
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 type: log
 paths:

diff --git a/x-pack/filebeat/module/gsuite/drive/manifest.yml b/x-pack/filebeat/module/gsuite/drive/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/gsuite/groups/config/config.yml b/x-pack/filebeat/module/gsuite/groups/config/config.yml
@@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }}
 pagination.id_field: nextPageToken
 pagination.url_field: pageToken
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 type: log
 paths:

diff --git a/x-pack/filebeat/module/gsuite/groups/manifest.yml b/x-pack/filebeat/module/gsuite/groups/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/gsuite/login/config/config.yml b/x-pack/filebeat/module/gsuite/login/config/config.yml
@@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }}
 pagination.id_field: nextPageToken
 pagination.url_field: pageToken
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 type: log
 paths:

diff --git a/x-pack/filebeat/module/gsuite/login/manifest.yml b/x-pack/filebeat/module/gsuite/login/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/gsuite/saml/config/config.yml b/x-pack/filebeat/module/gsuite/saml/config/config.yml
@@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }}
 pagination.id_field: nextPageToken
 pagination.url_field: pageToken
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 type: log
 paths:

diff --git a/x-pack/filebeat/module/gsuite/saml/manifest.yml b/x-pack/filebeat/module/gsuite/saml/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml
@@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }}
 pagination.id_field: nextPageToken
 pagination.url_field: pageToken
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 type: log
 paths:

diff --git a/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml b/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml
@@ -15,6 +15,7 @@ var:
     default: 2h
   - name: tags
     default: [forwarded]
+  - name: proxy_url
 
 input: config/config.yml
 ingest_pipeline: ../ingest/common.yml

diff --git a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml
@@ -9,6 +9,10 @@ auth.oauth2: {{ .oauth2 | tojson }}
 auth.oauth2.provider: azure
 auth.oauth2.azure.resource: https://api.securitycenter.windows.com/
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 request.url: "https://api.securitycenter.windows.com/api/alerts"
 request.method: GET
 request.transforms:

diff --git a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml
@@ -8,6 +8,7 @@ var:
   - name: tags
     default: [defender-atp, forwarded]
   - name: oauth2
+  - name: proxy_url
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/atp.yml

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml
@@ -9,6 +9,10 @@ auth.oauth2: {{ .oauth2 | tojson }}
 auth.oauth2.provider: azure
 auth.oauth2.azure.resource: https://api.securitycenter.windows.com/
 
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 request.url: "https://api.security.microsoft.com/api/incidents"
 request.method: GET
 request.transforms:

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml
@@ -8,6 +8,7 @@ var:
   - name: tags
     default: [m365-defender, forwarded]
   - name: oauth2
+  - name: proxy_url
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/defender.yml

diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml
@@ -6,6 +6,9 @@ interval: {{ .interval }}
 
 request.method: POST
 request.ssl: {{ .ssl | tojson }}
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
 request.url: {{ .url }}
 request.timeout: {{ .http_client_timeout }}
 request.body: {{ .http_request_body | tojson }}

diff --git a/x-pack/filebeat/module/misp/threat/manifest.yml b/x-pack/filebeat/module/misp/threat/manifest.yml
@@ -13,6 +13,7 @@ var:
     default: "60s"
   - name: url
   - name: ssl
+  - name: proxy_url
 
 input: config/input.yml
 ingest_pipeline: ingest/pipeline.json

diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml
@@ -12,6 +12,11 @@ request.ssl: {{ .ssl | tojson }}
 request.timeout: {{ .http_client_timeout }}
 {{ end }}
 
+
+{{ if .proxy_url }}
+request.proxy_url: {{ .proxy_url }}
+{{ end }}
+
 request.method: GET
 request.url: {{ .url }}
 request.rate_limit:

diff --git a/x-pack/filebeat/module/okta/system/manifest.yml b/x-pack/filebeat/module/okta/system/manifest.yml
@@ -13,6 +13,7 @@ var:
   - name: tags
     default: [forwarded]
   - name: url
+  - name: proxy_url
   - name: initial_interval
     default: 24h