From 9a5f5d0eb047d761d33e048f8b985b4d075dd252 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Wed, 24 Mar 2021 13:50:56 -0500 Subject: [PATCH] [Filebeat] Add Proxy config to httpjson v2 (#24662) Adds a `proxy_url` config option to the httpjson v2 input. Updates all httpjson based modules to support proxy_url. Fixes #24615. Co-authored-by: Andrew Kroh --- CHANGELOG.next.asciidoc | 1 + .../docs/inputs/input-httpjson.asciidoc | 17 +++++++++++++++++ .../httpjson/internal/v2/config_request.go | 1 + .../input/httpjson/internal/v2/input.go | 19 ++++++++++++------- .../module/cisco/amp/config/config.yml | 3 +++ x-pack/filebeat/module/cisco/amp/manifest.yml | 1 + .../google_workspace/admin/config/config.yml | 3 +++ .../google_workspace/admin/manifest.yml | 1 + .../google_workspace/drive/config/config.yml | 3 +++ .../google_workspace/drive/manifest.yml | 1 + .../google_workspace/groups/config/config.yml | 3 +++ .../google_workspace/groups/manifest.yml | 1 + .../google_workspace/login/config/config.yml | 3 +++ .../google_workspace/login/manifest.yml | 1 + .../google_workspace/saml/config/config.yml | 3 +++ .../module/google_workspace/saml/manifest.yml | 1 + .../user_accounts/config/config.yml | 3 +++ .../user_accounts/manifest.yml | 1 + .../module/gsuite/admin/config/config.yml | 4 ++++ .../filebeat/module/gsuite/admin/manifest.yml | 1 + .../module/gsuite/drive/config/config.yml | 4 ++++ .../filebeat/module/gsuite/drive/manifest.yml | 1 + .../module/gsuite/groups/config/config.yml | 4 ++++ .../module/gsuite/groups/manifest.yml | 1 + .../module/gsuite/login/config/config.yml | 4 ++++ .../filebeat/module/gsuite/login/manifest.yml | 1 + .../module/gsuite/saml/config/config.yml | 4 ++++ .../filebeat/module/gsuite/saml/manifest.yml | 1 + .../gsuite/user_accounts/config/config.yml | 4 ++++ .../module/gsuite/user_accounts/manifest.yml | 1 + .../microsoft/defender_atp/config/atp.yml | 4 ++++ .../microsoft/defender_atp/manifest.yml | 1 + .../m365_defender/config/defender.yml | 4 ++++ .../microsoft/m365_defender/manifest.yml | 1 + .../module/misp/threat/config/input.yml | 3 +++ .../filebeat/module/misp/threat/manifest.yml | 1 + .../module/okta/system/config/input.yml | 5 +++++ .../filebeat/module/okta/system/manifest.yml | 1 + .../module/snyk/audit/config/config.yml | 3 +++ .../filebeat/module/snyk/audit/manifest.yml | 1 + .../snyk/vulnerabilities/config/config.yml | 3 +++ .../module/snyk/vulnerabilities/manifest.yml | 1 + .../abusemalware/config/config.yml | 3 +++ .../threatintel/abusemalware/manifest.yml | 1 + .../threatintel/abuseurl/config/config.yml | 3 +++ .../module/threatintel/abuseurl/manifest.yml | 1 + .../threatintel/anomali/config/config.yml | 3 +++ .../module/threatintel/anomali/manifest.yml | 1 + .../module/threatintel/misp/config/config.yml | 3 +++ .../module/threatintel/misp/manifest.yml | 1 + .../module/threatintel/otx/config/config.yml | 3 +++ .../module/threatintel/otx/manifest.yml | 1 + 52 files changed, 137 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ead7b537f45..a29f3f21322 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -804,6 +804,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Support X-Forwarder-For in IIS logs. {pull}19142[192142] - Updating field mappings for Cisco AMP module, fixing certain fields. {pull}24661[24661] - Added NTP fileset to Zeek module {pull}24224[24224] +- Add `proxy_url` config for httpjson v2 input. {issue}24615[24615] {pull}24662[24662] *Heartbeat* diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index be5763830ad..a0ac58f7247 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -20,6 +20,7 @@ This input supports: * Pagination * Retries * Rate limiting +* Proxying * Request transformations * Response transformations @@ -383,6 +384,22 @@ This specifies SSL/TLS configuration. If the ssl section is missing, the host's CAs are used for HTTPS connections. See <> for more information. +[float] +==== `request.proxy_url` + +This specifies proxy configuration in the form of `http[s]://:@:` + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +# Fetch your public IP every minute. +- type: httpjson + config_version: 2 + interval: 1m + request.url: https://api.ipify.org/?format=json + request.proxy_url: http://proxy.example:8080 +---- + [float] ==== `request.retry.max_attempts` diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go index d2c93363c87..f64a03d9899 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go @@ -88,6 +88,7 @@ type requestConfig struct { RedirectMaxRedirects int `config:"redirect.max_redirects"` RateLimit *rateLimitConfig `config:"rate_limit"` Transforms transformsConfig `config:"transforms"` + ProxyURL *urlConfig `config:"proxy_url"` } func (c requestConfig) getTimeout() time.Duration { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go index f9c76226818..e8c8fe51082 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input.go @@ -149,17 +149,22 @@ func run( func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSConfig, log *logp.Logger) (*httpClient, error) { timeout := config.Request.getTimeout() + proxy_url := config.Request.ProxyURL // Make retryable HTTP client + transport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: timeout, + }).DialContext, + TLSClientConfig: tlsConfig.ToConfig(), + DisableKeepAlives: true, + } + if proxy_url != nil && proxy_url.URL != nil { + transport.Proxy = http.ProxyURL(proxy_url.URL) + } client := &retryablehttp.Client{ HTTPClient: &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: timeout, - }).DialContext, - TLSClientConfig: tlsConfig.ToConfig(), - DisableKeepAlives: true, - }, + Transport: transport, Timeout: timeout, CheckRedirect: checkRedirect(config.Request, log), }, diff --git a/x-pack/filebeat/module/cisco/amp/config/config.yml b/x-pack/filebeat/module/cisco/amp/config/config.yml index 0aa38440947..bd5f93f7a90 100644 --- a/x-pack/filebeat/module/cisco/amp/config/config.yml +++ b/x-pack/filebeat/module/cisco/amp/config/config.yml @@ -17,6 +17,9 @@ request.timeout: {{ .request_timeout }} {{ if .ssl }} request.ssl: {{ .ssl | tojson }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: url.params.start_date diff --git a/x-pack/filebeat/module/cisco/amp/manifest.yml b/x-pack/filebeat/module/cisco/amp/manifest.yml index 9458f80a17d..1c3b263d34c 100644 --- a/x-pack/filebeat/module/cisco/amp/manifest.yml +++ b/x-pack/filebeat/module/cisco/amp/manifest.yml @@ -18,6 +18,7 @@ var: default: 24h - name: interval default: 60m + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/google_workspace/admin/config/config.yml b/x-pack/filebeat/module/google_workspace/admin/config/config.yml index 1b992f50d13..b2733ef6a44 100644 --- a/x-pack/filebeat/module/google_workspace/admin/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/admin/config/config.yml @@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user {{ if .http_client_timeout }} request.timeout: {{ .http_client_timeout }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: url.params.startTime diff --git a/x-pack/filebeat/module/google_workspace/admin/manifest.yml b/x-pack/filebeat/module/google_workspace/admin/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/google_workspace/admin/manifest.yml +++ b/x-pack/filebeat/module/google_workspace/admin/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/google_workspace/drive/config/config.yml b/x-pack/filebeat/module/google_workspace/drive/config/config.yml index e88aecca528..e1d39b69cdb 100644 --- a/x-pack/filebeat/module/google_workspace/drive/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/drive/config/config.yml @@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user {{ if .http_client_timeout }} request.timeout: {{ .http_client_timeout }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: url.params.startTime diff --git a/x-pack/filebeat/module/google_workspace/drive/manifest.yml b/x-pack/filebeat/module/google_workspace/drive/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/google_workspace/drive/manifest.yml +++ b/x-pack/filebeat/module/google_workspace/drive/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/google_workspace/groups/config/config.yml b/x-pack/filebeat/module/google_workspace/groups/config/config.yml index d40347b89fb..5b28babd068 100644 --- a/x-pack/filebeat/module/google_workspace/groups/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/groups/config/config.yml @@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user {{ if .http_client_timeout }} request.timeout: {{ .http_client_timeout }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: url.params.startTime diff --git a/x-pack/filebeat/module/google_workspace/groups/manifest.yml b/x-pack/filebeat/module/google_workspace/groups/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/google_workspace/groups/manifest.yml +++ b/x-pack/filebeat/module/google_workspace/groups/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/google_workspace/login/config/config.yml b/x-pack/filebeat/module/google_workspace/login/config/config.yml index 6f1249bd195..cab92bb0ec1 100644 --- a/x-pack/filebeat/module/google_workspace/login/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/login/config/config.yml @@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user {{ if .http_client_timeout }} request.timeout: {{ .http_client_timeout }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: url.params.startTime diff --git a/x-pack/filebeat/module/google_workspace/login/manifest.yml b/x-pack/filebeat/module/google_workspace/login/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/google_workspace/login/manifest.yml +++ b/x-pack/filebeat/module/google_workspace/login/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/google_workspace/saml/config/config.yml b/x-pack/filebeat/module/google_workspace/saml/config/config.yml index d69484c939f..1ce19e2d05e 100644 --- a/x-pack/filebeat/module/google_workspace/saml/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/saml/config/config.yml @@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user {{ if .http_client_timeout }} request.timeout: {{ .http_client_timeout }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: url.params.startTime diff --git a/x-pack/filebeat/module/google_workspace/saml/manifest.yml b/x-pack/filebeat/module/google_workspace/saml/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/google_workspace/saml/manifest.yml +++ b/x-pack/filebeat/module/google_workspace/saml/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml index f67eb420833..5f5142478ae 100644 --- a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml @@ -11,6 +11,9 @@ request.url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user {{ if .http_client_timeout }} request.timeout: {{ .http_client_timeout }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: url.params.startTime diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/manifest.yml b/x-pack/filebeat/module/google_workspace/user_accounts/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/google_workspace/user_accounts/manifest.yml +++ b/x-pack/filebeat/module/google_workspace/user_accounts/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/gsuite/admin/config/config.yml b/x-pack/filebeat/module/gsuite/admin/config/config.yml index 12e3730dc93..b6a7f3b6c9c 100644 --- a/x-pack/filebeat/module/gsuite/admin/config/config.yml +++ b/x-pack/filebeat/module/gsuite/admin/config/config.yml @@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }} pagination.id_field: nextPageToken pagination.url_field: pageToken +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/module/gsuite/admin/manifest.yml b/x-pack/filebeat/module/gsuite/admin/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/gsuite/admin/manifest.yml +++ b/x-pack/filebeat/module/gsuite/admin/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/gsuite/drive/config/config.yml b/x-pack/filebeat/module/gsuite/drive/config/config.yml index 80583ee31b6..45e5385413a 100644 --- a/x-pack/filebeat/module/gsuite/drive/config/config.yml +++ b/x-pack/filebeat/module/gsuite/drive/config/config.yml @@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }} pagination.id_field: nextPageToken pagination.url_field: pageToken +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/module/gsuite/drive/manifest.yml b/x-pack/filebeat/module/gsuite/drive/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/gsuite/drive/manifest.yml +++ b/x-pack/filebeat/module/gsuite/drive/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/gsuite/groups/config/config.yml b/x-pack/filebeat/module/gsuite/groups/config/config.yml index 75482518477..1d5efbbd51b 100644 --- a/x-pack/filebeat/module/gsuite/groups/config/config.yml +++ b/x-pack/filebeat/module/gsuite/groups/config/config.yml @@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }} pagination.id_field: nextPageToken pagination.url_field: pageToken +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/module/gsuite/groups/manifest.yml b/x-pack/filebeat/module/gsuite/groups/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/gsuite/groups/manifest.yml +++ b/x-pack/filebeat/module/gsuite/groups/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/gsuite/login/config/config.yml b/x-pack/filebeat/module/gsuite/login/config/config.yml index ab40715bd4a..565595fc6a4 100644 --- a/x-pack/filebeat/module/gsuite/login/config/config.yml +++ b/x-pack/filebeat/module/gsuite/login/config/config.yml @@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }} pagination.id_field: nextPageToken pagination.url_field: pageToken +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/module/gsuite/login/manifest.yml b/x-pack/filebeat/module/gsuite/login/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/gsuite/login/manifest.yml +++ b/x-pack/filebeat/module/gsuite/login/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/gsuite/saml/config/config.yml b/x-pack/filebeat/module/gsuite/saml/config/config.yml index 62f1e7d9f4e..f83331abbae 100644 --- a/x-pack/filebeat/module/gsuite/saml/config/config.yml +++ b/x-pack/filebeat/module/gsuite/saml/config/config.yml @@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }} pagination.id_field: nextPageToken pagination.url_field: pageToken +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/module/gsuite/saml/manifest.yml b/x-pack/filebeat/module/gsuite/saml/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/gsuite/saml/manifest.yml +++ b/x-pack/filebeat/module/gsuite/saml/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml index c6aa5ded144..32e269007c9 100644 --- a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml @@ -23,6 +23,10 @@ date_cursor.initial_interval: {{ .initial_interval }} pagination.id_field: nextPageToken pagination.url_field: pageToken +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + {{ else if eq .input "file" }} type: log paths: diff --git a/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml b/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml index 48570efe448..c5992776ac0 100644 --- a/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml +++ b/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml @@ -15,6 +15,7 @@ var: default: 2h - name: tags default: [forwarded] + - name: proxy_url input: config/config.yml ingest_pipeline: ../ingest/common.yml diff --git a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml index d1e5c971b80..57a1d61cea3 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml @@ -9,6 +9,10 @@ auth.oauth2: {{ .oauth2 | tojson }} auth.oauth2.provider: azure auth.oauth2.azure.resource: https://api.securitycenter.windows.com/ +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + request.url: "https://api.securitycenter.windows.com/api/alerts" request.method: GET request.transforms: diff --git a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml index 1c93e4d277a..2bf5bf65034 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml @@ -8,6 +8,7 @@ var: - name: tags default: [defender-atp, forwarded] - name: oauth2 + - name: proxy_url ingest_pipeline: ingest/pipeline.yml input: config/atp.yml diff --git a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml index 52ebe56c3b1..e35fc8a88a5 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml @@ -9,6 +9,10 @@ auth.oauth2: {{ .oauth2 | tojson }} auth.oauth2.provider: azure auth.oauth2.azure.resource: https://api.securitycenter.windows.com/ +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + request.url: "https://api.security.microsoft.com/api/incidents" request.method: GET request.transforms: diff --git a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml index 99f00620d9d..e3524259d08 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml @@ -8,6 +8,7 @@ var: - name: tags default: [m365-defender, forwarded] - name: oauth2 + - name: proxy_url ingest_pipeline: ingest/pipeline.yml input: config/defender.yml diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml index 488f0a249c0..c41a10651d4 100644 --- a/x-pack/filebeat/module/misp/threat/config/input.yml +++ b/x-pack/filebeat/module/misp/threat/config/input.yml @@ -6,6 +6,9 @@ interval: {{ .interval }} request.method: POST request.ssl: {{ .ssl | tojson }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.url: {{ .url }} request.timeout: {{ .http_client_timeout }} request.body: {{ .http_request_body | tojson }} diff --git a/x-pack/filebeat/module/misp/threat/manifest.yml b/x-pack/filebeat/module/misp/threat/manifest.yml index 63e3ef6db6e..312c800901f 100644 --- a/x-pack/filebeat/module/misp/threat/manifest.yml +++ b/x-pack/filebeat/module/misp/threat/manifest.yml @@ -13,6 +13,7 @@ var: default: "60s" - name: url - name: ssl + - name: proxy_url input: config/input.yml ingest_pipeline: ingest/pipeline.json diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index 05ff819fad0..8321a36a391 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -12,6 +12,11 @@ request.ssl: {{ .ssl | tojson }} request.timeout: {{ .http_client_timeout }} {{ end }} + +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} + request.method: GET request.url: {{ .url }} request.rate_limit: diff --git a/x-pack/filebeat/module/okta/system/manifest.yml b/x-pack/filebeat/module/okta/system/manifest.yml index 688f1a9ba44..830c8682de5 100644 --- a/x-pack/filebeat/module/okta/system/manifest.yml +++ b/x-pack/filebeat/module/okta/system/manifest.yml @@ -13,6 +13,7 @@ var: - name: tags default: [forwarded] - name: url + - name: proxy_url - name: initial_interval default: 24h diff --git a/x-pack/filebeat/module/snyk/audit/config/config.yml b/x-pack/filebeat/module/snyk/audit/config/config.yml index 73cd5423a02..f83c91327a2 100644 --- a/x-pack/filebeat/module/snyk/audit/config/config.yml +++ b/x-pack/filebeat/module/snyk/audit/config/config.yml @@ -13,6 +13,9 @@ request.url: https://snyk.io/api/v1/org/{{.audit_id}}/audit?page=1&sortOrder=ASC {{ end }} request.method: POST request.ssl: {{ .ssl | tojson }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: header.Authorization diff --git a/x-pack/filebeat/module/snyk/audit/manifest.yml b/x-pack/filebeat/module/snyk/audit/manifest.yml index eed5f4c29c9..b8858144bae 100644 --- a/x-pack/filebeat/module/snyk/audit/manifest.yml +++ b/x-pack/filebeat/module/snyk/audit/manifest.yml @@ -21,6 +21,7 @@ var: - name: event default: "" - name: ssl + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml index ca371361192..b04e945f7e0 100644 --- a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml +++ b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml @@ -7,6 +7,9 @@ interval: {{ .interval }} request.url: {{ .url }} request.method: POST request.ssl: {{ .ssl | tojson }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.transforms: - set: target: header.Authorization diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml b/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml index 391333380dc..771e52b972b 100644 --- a/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml +++ b/x-pack/filebeat/module/snyk/vulnerabilities/manifest.yml @@ -61,6 +61,7 @@ var: default: 0 - name: max_priority_score default: 1000 + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml index 145dfe246dd..ee9e7d30764 100644 --- a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml +++ b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml @@ -8,6 +8,9 @@ request.method: GET {{ if .ssl }} request.ssl: {{ .ssl | tojson }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.url: {{ .url }} request.transforms: - set: diff --git a/x-pack/filebeat/module/threatintel/abusemalware/manifest.yml b/x-pack/filebeat/module/threatintel/abusemalware/manifest.yml index e3159060cd8..5fe3a155180 100644 --- a/x-pack/filebeat/module/threatintel/abusemalware/manifest.yml +++ b/x-pack/filebeat/module/threatintel/abusemalware/manifest.yml @@ -10,6 +10,7 @@ var: - name: ssl - name: tags default: [threatintel-abusemalware, forwarded] + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml b/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml index 96affa7da97..276f5448a1f 100644 --- a/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml +++ b/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml @@ -8,6 +8,9 @@ request.method: GET {{ if .ssl }} request.ssl: {{ .ssl | tojson }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.url: {{ .url }} request.transforms: - set: diff --git a/x-pack/filebeat/module/threatintel/abuseurl/manifest.yml b/x-pack/filebeat/module/threatintel/abuseurl/manifest.yml index 155e9b4ff3f..13b5e663c4a 100644 --- a/x-pack/filebeat/module/threatintel/abuseurl/manifest.yml +++ b/x-pack/filebeat/module/threatintel/abuseurl/manifest.yml @@ -10,6 +10,7 @@ var: - name: ssl - name: tags default: [threatintel-abuseurls, forwarded] + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/threatintel/anomali/config/config.yml b/x-pack/filebeat/module/threatintel/anomali/config/config.yml index fd55b6e07c2..557b71c31bb 100644 --- a/x-pack/filebeat/module/threatintel/anomali/config/config.yml +++ b/x-pack/filebeat/module/threatintel/anomali/config/config.yml @@ -14,6 +14,9 @@ request.method: GET {{ if .ssl }} request.ssl: {{ .ssl | tojson }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.url: {{ .url }} request.redirect.forward_headers: true request.transforms: diff --git a/x-pack/filebeat/module/threatintel/anomali/manifest.yml b/x-pack/filebeat/module/threatintel/anomali/manifest.yml index b7b87d8fe66..1087e00df8a 100644 --- a/x-pack/filebeat/module/threatintel/anomali/manifest.yml +++ b/x-pack/filebeat/module/threatintel/anomali/manifest.yml @@ -16,6 +16,7 @@ var: default: "https://otx.alienvault.com/api/v1/indicators/export" - name: tags default: [threatintel-anomali, forwarded] + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml index e28c6c1d9a7..54e895b5364 100644 --- a/x-pack/filebeat/module/threatintel/misp/config/config.yml +++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml @@ -8,6 +8,9 @@ request.method: POST {{ if .ssl }} request.ssl: {{ .ssl | tojson }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} request.url: {{ .url }} request.body: limit: 100 diff --git a/x-pack/filebeat/module/threatintel/misp/manifest.yml b/x-pack/filebeat/module/threatintel/misp/manifest.yml index a39c1fe4496..41443c01df8 100644 --- a/x-pack/filebeat/module/threatintel/misp/manifest.yml +++ b/x-pack/filebeat/module/threatintel/misp/manifest.yml @@ -14,6 +14,7 @@ var: default: "https://localhost/events/restSearch" - name: tags default: [threatintel-misp, forwarded] + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/threatintel/otx/config/config.yml b/x-pack/filebeat/module/threatintel/otx/config/config.yml index 252c64a21f4..44af7e19acb 100644 --- a/x-pack/filebeat/module/threatintel/otx/config/config.yml +++ b/x-pack/filebeat/module/threatintel/otx/config/config.yml @@ -8,6 +8,9 @@ request.method: GET {{ if .ssl }} request.ssl: {{ .ssl | tojson }} {{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} {{ if .http_client_timeout }} request.timeout: {{ .http_client_timeout }} {{ end }} diff --git a/x-pack/filebeat/module/threatintel/otx/manifest.yml b/x-pack/filebeat/module/threatintel/otx/manifest.yml index c17efa499e9..0fdefa51d76 100644 --- a/x-pack/filebeat/module/threatintel/otx/manifest.yml +++ b/x-pack/filebeat/module/threatintel/otx/manifest.yml @@ -19,6 +19,7 @@ var: default: "https://otx.alienvault.com/api/v1/indicators/export" - name: tags default: [threatintel-otx, forwarded] + - name: proxy_url ingest_pipeline: - ingest/pipeline.yml