diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 39cef21f4ce..ff8730d23c8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -44,6 +44,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Preserve case of http.request.method. ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359] - Adds check on `` config option value for the azure input `resource_manager_endpoint`. {pull}18890[18890] - Okta module now requires objects instead of JSON strings for the `http_headers`, `http_request_body`, `pagination`, `rate_limit`, and `ssl` variables. {pull}18953[18953] +- Adds oauth support for httpjson input. {issue}18415[18415] {pull}18892[18892] *Heartbeat* diff --git a/vendor/golang.org/x/oauth2/endpoints/endpoints.go b/vendor/golang.org/x/oauth2/endpoints/endpoints.go new file mode 100644 index 00000000000..811e101f920 --- /dev/null +++ b/vendor/golang.org/x/oauth2/endpoints/endpoints.go @@ -0,0 +1,238 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package endpoints provides constants for using OAuth2 to access various services. +package endpoints + +import ( + "strings" + + "golang.org/x/oauth2" +) + +// Amazon is the endpoint for Amazon. +var Amazon = oauth2.Endpoint{ + AuthURL: "https://www.amazon.com/ap/oa", + TokenURL: "https://api.amazon.com/auth/o2/token", +} + +// Bitbucket is the endpoint for Bitbucket. +var Bitbucket = oauth2.Endpoint{ + AuthURL: "https://bitbucket.org/site/oauth2/authorize", + TokenURL: "https://bitbucket.org/site/oauth2/access_token", +} + +// Cern is the endpoint for CERN. +var Cern = oauth2.Endpoint{ + AuthURL: "https://oauth.web.cern.ch/OAuth/Authorize", + TokenURL: "https://oauth.web.cern.ch/OAuth/Token", +} + +// Facebook is the endpoint for Facebook. +var Facebook = oauth2.Endpoint{ + AuthURL: "https://www.facebook.com/v3.2/dialog/oauth", + TokenURL: "https://graph.facebook.com/v3.2/oauth/access_token", +} + +// Foursquare is the endpoint for Foursquare. +var Foursquare = oauth2.Endpoint{ + AuthURL: "https://foursquare.com/oauth2/authorize", + TokenURL: "https://foursquare.com/oauth2/access_token", +} + +// Fitbit is the endpoint for Fitbit. +var Fitbit = oauth2.Endpoint{ + AuthURL: "https://www.fitbit.com/oauth2/authorize", + TokenURL: "https://api.fitbit.com/oauth2/token", +} + +// GitHub is the endpoint for Github. +var GitHub = oauth2.Endpoint{ + AuthURL: "https://github.com/login/oauth/authorize", + TokenURL: "https://github.com/login/oauth/access_token", +} + +// GitLab is the endpoint for GitLab. +var GitLab = oauth2.Endpoint{ + AuthURL: "https://gitlab.com/oauth/authorize", + TokenURL: "https://gitlab.com/oauth/token", +} + +// Google is the endpoint for Google. +var Google = oauth2.Endpoint{ + AuthURL: "https://accounts.google.com/o/oauth2/auth", + TokenURL: "https://oauth2.googleapis.com/token", +} + +// Heroku is the endpoint for Heroku. +var Heroku = oauth2.Endpoint{ + AuthURL: "https://id.heroku.com/oauth/authorize", + TokenURL: "https://id.heroku.com/oauth/token", +} + +// HipChat is the endpoint for HipChat. +var HipChat = oauth2.Endpoint{ + AuthURL: "https://www.hipchat.com/users/authorize", + TokenURL: "https://api.hipchat.com/v2/oauth/token", +} + +// Instagram is the endpoint for Instagram. +var Instagram = oauth2.Endpoint{ + AuthURL: "https://api.instagram.com/oauth/authorize", + TokenURL: "https://api.instagram.com/oauth/access_token", +} + +// KaKao is the endpoint for KaKao. +var KaKao = oauth2.Endpoint{ + AuthURL: "https://kauth.kakao.com/oauth/authorize", + TokenURL: "https://kauth.kakao.com/oauth/token", +} + +// LinkedIn is the endpoint for LinkedIn. +var LinkedIn = oauth2.Endpoint{ + AuthURL: "https://www.linkedin.com/oauth/v2/authorization", + TokenURL: "https://www.linkedin.com/oauth/v2/accessToken", +} + +// Mailchimp is the endpoint for Mailchimp. +var Mailchimp = oauth2.Endpoint{ + AuthURL: "https://login.mailchimp.com/oauth2/authorize", + TokenURL: "https://login.mailchimp.com/oauth2/token", +} + +// Mailru is the endpoint for Mail.Ru. +var Mailru = oauth2.Endpoint{ + AuthURL: "https://o2.mail.ru/login", + TokenURL: "https://o2.mail.ru/token", +} + +// MediaMath is the endpoint for MediaMath. +var MediaMath = oauth2.Endpoint{ + AuthURL: "https://api.mediamath.com/oauth2/v1.0/authorize", + TokenURL: "https://api.mediamath.com/oauth2/v1.0/token", +} + +// MediaMathSandbox is the endpoint for MediaMath Sandbox. +var MediaMathSandbox = oauth2.Endpoint{ + AuthURL: "https://t1sandbox.mediamath.com/oauth2/v1.0/authorize", + TokenURL: "https://t1sandbox.mediamath.com/oauth2/v1.0/token", +} + +// Microsoft is the endpoint for Microsoft. +var Microsoft = oauth2.Endpoint{ + AuthURL: "https://login.live.com/oauth20_authorize.srf", + TokenURL: "https://login.live.com/oauth20_token.srf", +} + +// NokiaHealth is the endpoint for Nokia Health. +var NokiaHealth = oauth2.Endpoint{ + AuthURL: "https://account.health.nokia.com/oauth2_user/authorize2", + TokenURL: "https://account.health.nokia.com/oauth2/token", +} + +// Odnoklassniki is the endpoint for Odnoklassniki. +var Odnoklassniki = oauth2.Endpoint{ + AuthURL: "https://www.odnoklassniki.ru/oauth/authorize", + TokenURL: "https://api.odnoklassniki.ru/oauth/token.do", +} + +// PayPal is the endpoint for PayPal. +var PayPal = oauth2.Endpoint{ + AuthURL: "https://www.paypal.com/webapps/auth/protocol/openidconnect/v1/authorize", + TokenURL: "https://api.paypal.com/v1/identity/openidconnect/tokenservice", +} + +// PayPalSandbox is the endpoint for PayPal Sandbox. +var PayPalSandbox = oauth2.Endpoint{ + AuthURL: "https://www.sandbox.paypal.com/webapps/auth/protocol/openidconnect/v1/authorize", + TokenURL: "https://api.sandbox.paypal.com/v1/identity/openidconnect/tokenservice", +} + +// Slack is the endpoint for Slack. +var Slack = oauth2.Endpoint{ + AuthURL: "https://slack.com/oauth/authorize", + TokenURL: "https://slack.com/api/oauth.access", +} + +// Spotify is the endpoint for Spotify. +var Spotify = oauth2.Endpoint{ + AuthURL: "https://accounts.spotify.com/authorize", + TokenURL: "https://accounts.spotify.com/api/token", +} + +// StackOverflow is the endpoint for Stack Overflow. +var StackOverflow = oauth2.Endpoint{ + AuthURL: "https://stackoverflow.com/oauth", + TokenURL: "https://stackoverflow.com/oauth/access_token", +} + +// Twitch is the endpoint for Twitch. +var Twitch = oauth2.Endpoint{ + AuthURL: "https://id.twitch.tv/oauth2/authorize", + TokenURL: "https://id.twitch.tv/oauth2/token", +} + +// Uber is the endpoint for Uber. +var Uber = oauth2.Endpoint{ + AuthURL: "https://login.uber.com/oauth/v2/authorize", + TokenURL: "https://login.uber.com/oauth/v2/token", +} + +// Vk is the endpoint for Vk. +var Vk = oauth2.Endpoint{ + AuthURL: "https://oauth.vk.com/authorize", + TokenURL: "https://oauth.vk.com/access_token", +} + +// Yahoo is the endpoint for Yahoo. +var Yahoo = oauth2.Endpoint{ + AuthURL: "https://api.login.yahoo.com/oauth2/request_auth", + TokenURL: "https://api.login.yahoo.com/oauth2/get_token", +} + +// Yandex is the endpoint for Yandex. +var Yandex = oauth2.Endpoint{ + AuthURL: "https://oauth.yandex.com/authorize", + TokenURL: "https://oauth.yandex.com/token", +} + +// AzureAD returns a new oauth2.Endpoint for the given tenant at Azure Active Directory. +// If tenant is empty, it uses the tenant called `common`. +// +// For more information see: +// https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-v2-protocols#endpoints +func AzureAD(tenant string) oauth2.Endpoint { + if tenant == "" { + tenant = "common" + } + return oauth2.Endpoint{ + AuthURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/authorize", + TokenURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/token", + } +} + +// HipChatServer returns a new oauth2.Endpoint for a HipChat Server instance +// running on the given domain or host. +func HipChatServer(host string) oauth2.Endpoint { + return oauth2.Endpoint{ + AuthURL: "https://" + host + "/users/authorize", + TokenURL: "https://" + host + "/v2/oauth/token", + } +} + +// AWSCognito returns a new oauth2.Endpoint for the supplied AWS Cognito domain which is +// linked to your Cognito User Pool. +// +// Example domain: https://testing.auth.us-east-1.amazoncognito.com +// +// For more information see: +// https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-user-pools-assign-domain.html +// https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-userpools-server-contract-reference.html +func AWSCognito(domain string) oauth2.Endpoint { + domain = strings.TrimRight(domain, "/") + return oauth2.Endpoint{ + AuthURL: domain + "/oauth2/authorize", + TokenURL: domain + "/oauth2/token", + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a0e9358788f..94f523d32fb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -947,6 +947,7 @@ golang.org/x/net/websocket # golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/oauth2 golang.org/x/oauth2/clientcredentials +golang.org/x/oauth2/endpoints golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 61f1d249198..8f47070b967 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -48,6 +48,29 @@ Example configurations: url: http://localhost:9200/_search/scroll ---- +Additionally, it supports authentication via HTTP Headers, API key or oauth2. + +Example configurations with authentication: + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: httpjson + http_headers: + Authorization: 'Basic aGVsbG86d29ybGQ=' + url: http://localhost +---- + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: httpjson + oauth2: + client.id: 12345678901234567890abcdef + client.secret: abcdef12345678901234567890 + token_url: http://localhost/oauth2/token + url: http://localhost +---- ==== Configuration options @@ -249,6 +272,110 @@ information. The URL of the HTTP API. Required. +[float] +==== `oauth2.enabled` + +The `enabled` setting can be used to disable the oauth2 configuration by +setting it to `false`. The default value is `true`. + +NOTE: OAuth2 settings are disabled if either `enabled` is set to `false` or +the `oauth2` section is missing. + +[float] +==== `oauth2.provider` + +The `provider` setting can be used to configure supported oauth2 providers. +Each supported provider will require specific settings. It is not set by default. +Supported providers are: `azure`, `google`. + +[float] +==== `oauth2.client.id` + +The `client.id` setting is used as part of the authentication flow. It is always required +except if using `google` as provider. Required for providers: `default`, `azure`. + +[float] +==== `oauth2.client.secret` + +The `client.secret` setting is used as part of the authentication flow. It is always required +except if using `google` as provider. Required for providers: `default`, `azure`. + +[float] +==== `oauth2.scopes` + +The `scopes` setting defines a list of scopes that will be requested during the oauth2 flow. +It is optional for all providers. + +[float] +==== `oauth2.token_url` + +The `token_url` setting specifies the endpoint that will be used to generate the +tokens during the oauth2 flow. It is required if no provider is specified. + +NOTE: For `azure` provider either `token_url` or `azure.tenant_id` is required. + +[float] +==== `oauth2.endpoint_params` + +The `endpoint_params` setting specifies a set of values that will be sent on each +request to the `token_url`. Each param key can have multiple values. +Can be set for all providers except `google`. + +["source","yaml",subs="attributes"] +---- +- type: httpjson + oauth2: + endpoint_params: + Param1: + - ValueA + - ValueB + Param2: + - Value +---- + +[float] +==== `oauth2.azure.tenant_id` + +The `azure.tenant_id` is used for authentication when using `azure` provider. +Since it is used in the process to generate the `token_url`, it can't be used in +combination with it. It is not required. + +For information about where to find it, you can refer to +https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal. + +[float] +==== `oauth2.azure.resource` + +The `azure.resource` is used to identify the accessed WebAPI resource when using `azure` provider. +It is not required. + +[float] +==== `oauth2.google.credentials_file` + +The `google.credentials_file` setting specifies the credentials file for Google. + +NOTE: Only one of the credentials settings can be set at once. If none is provided, loading +default credentials from the environment will be attempted via ADC. For more information about +how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. + +[float] +==== `oauth2.google.credentials_json` + +The `google.credentials_json` setting allows to write your credentials information as raw JSON. + +NOTE: Only one of the credentials settings can be set at once. If none is provided, loading +default credentials from the environment will be attempted via ADC. For more information about +how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. + +[float] +==== `oauth2.google.jwt_file` + +The `google.jwt_file` setting specifies the JWT Account Key file for Google. + +NOTE: Only one of the credentials settings can be set at once. If none is provided, loading +default credentials from the environment will be attempted via ADC. For more information about +how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. + [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index cb1e12ba417..bd9f584895b 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -17,6 +17,7 @@ import ( // Config contains information about httpjson configuration type config struct { + OAuth2 *OAuth2 `config:"oauth2"` APIKey string `config:"api_key"` AuthenticationScheme string `config:"authentication_scheme"` HTTPClientTimeout time.Duration `config:"http_client_timeout"` @@ -62,9 +63,7 @@ type RateLimit struct { func (c *config) Validate() error { switch strings.ToUpper(c.HTTPMethod) { - case "GET": - break - case "POST": + case "GET", "POST": break default: return errors.Errorf("httpjson input: Invalid http_method, %s", c.HTTPMethod) @@ -84,6 +83,11 @@ func (c *config) Validate() error { } } } + if c.OAuth2.IsEnabled() { + if c.APIKey != "" || c.AuthenticationScheme != "" { + return errors.Errorf("invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously") + } + } return nil } diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go new file mode 100644 index 00000000000..b9bdb45668c --- /dev/null +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -0,0 +1,209 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + "golang.org/x/oauth2/endpoints" + "golang.org/x/oauth2/google" +) + +// An OAuth2Provider represents a supported oauth provider. +type OAuth2Provider string + +const ( + OAuth2ProviderDefault OAuth2Provider = "" // OAuth2ProviderDefault means no specific provider is set. + OAuth2ProviderAzure OAuth2Provider = "azure" // OAuth2ProviderAzure AzureAD. + OAuth2ProviderGoogle OAuth2Provider = "google" // OAuth2ProviderGoogle Google. +) + +func (p *OAuth2Provider) Unpack(in string) error { + *p = OAuth2Provider(in) + return nil +} + +func (p OAuth2Provider) canonical() OAuth2Provider { + return OAuth2Provider(strings.ToLower(string(p))) +} + +// OAuth2 contains information about oauth2 authentication settings. +type OAuth2 struct { + // common oauth fields + ClientID string `config:"client.id"` + ClientSecret string `config:"client.secret"` + Enabled *bool `config:"enabled"` + EndpointParams map[string][]string `config:"endpoint_params"` + Provider OAuth2Provider `config:"provider"` + Scopes []string `config:"scopes"` + TokenURL string `config:"token_url"` + + // google specific + GoogleCredentialsFile string `config:"google.credentials_file"` + GoogleCredentialsJSON []byte `config:"google.credentials_json"` + GoogleJWTFile string `config:"google.jwt_file"` + + // microsoft azure specific + AzureTenantID string `config:"azure.tenant_id"` + AzureResource string `config:"azure.resource"` +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (o *OAuth2) IsEnabled() bool { + return o != nil && (o.Enabled == nil || *o.Enabled) +} + +// Client wraps the given http.Client and returns a new one that will use the oauth authentication. +func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, error) { + ctx = context.WithValue(ctx, oauth2.HTTPClient, client) + + switch o.GetProvider() { + case OAuth2ProviderAzure, OAuth2ProviderDefault: + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.GetTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.GetEndpointParams(), + } + return creds.Client(ctx), nil + case OAuth2ProviderGoogle: + creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) + } + return oauth2.NewClient(ctx, creds.TokenSource), nil + default: + return nil, errors.New("oauth2 client: unknown provider") + } +} + +// GetTokenURL returns the TokenURL. +func (o *OAuth2) GetTokenURL() string { + switch o.GetProvider() { + case OAuth2ProviderAzure: + if o.TokenURL == "" { + return endpoints.AzureAD(o.AzureTenantID).TokenURL + } + } + + return o.TokenURL +} + +// GetProvider returns provider in its canonical form. +func (o OAuth2) GetProvider() OAuth2Provider { + return o.Provider.canonical() +} + +// GetEndpointParams returns endpoint params with any provider ones combined. +func (o OAuth2) GetEndpointParams() map[string][]string { + switch o.GetProvider() { + case OAuth2ProviderAzure: + if o.AzureResource != "" { + if o.EndpointParams == nil { + o.EndpointParams = map[string][]string{} + } + o.EndpointParams["resource"] = []string{o.AzureResource} + } + } + + return o.EndpointParams +} + +// Validate checks if oauth2 config is valid. +func (o *OAuth2) Validate() error { + switch o.GetProvider() { + case OAuth2ProviderAzure: + return o.validateAzureProvider() + case OAuth2ProviderGoogle: + return o.validateGoogleProvider() + case OAuth2ProviderDefault: + if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" { + return errors.New("invalid configuration: both token_url and client credentials must be provided") + } + default: + return fmt.Errorf("invalid configuration: unknown provider %q", o.GetProvider()) + } + return nil +} + +// findDefaultGoogleCredentials will default to google.FindDefaultCredentials and will only be changed for testing purposes +var findDefaultGoogleCredentials = google.FindDefaultCredentials + +func (o *OAuth2) validateGoogleProvider() error { + if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || + o.AzureTenantID != "" || o.AzureResource != "" || len(o.EndpointParams) > 0 { + return errors.New("invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead") + } + + // credentials_json + if len(o.GoogleCredentialsJSON) > 0 { + if !json.Valid(o.GoogleCredentialsJSON) { + return errors.New("invalid configuration: google.credentials_json must be valid JSON") + } + return nil + } + + // credentials_file + if o.GoogleCredentialsFile != "" { + return o.populateCredentialsJSONFromFile(o.GoogleCredentialsFile) + } + + // jwt_file + if o.GoogleJWTFile != "" { + return o.populateCredentialsJSONFromFile(o.GoogleJWTFile) + } + + // Application Default Credentials (ADC) + ctx := context.Background() + if creds, err := findDefaultGoogleCredentials(ctx, o.Scopes...); err == nil { + o.GoogleCredentialsJSON = creds.JSON + return nil + } + + return fmt.Errorf("invalid configuration: no authentication credentials were configured or detected (ADC)") +} + +func (o *OAuth2) populateCredentialsJSONFromFile(file string) error { + if _, err := os.Stat(file); os.IsNotExist(err) { + return fmt.Errorf("invalid configuration: the file %q cannot be found", file) + } + + credBytes, err := ioutil.ReadFile(file) + if err != nil { + return fmt.Errorf("invalid configuration: the file %q cannot be read", file) + } + + if !json.Valid(credBytes) { + return fmt.Errorf("invalid configuration: the file %q does not contain valid JSON", file) + } + + o.GoogleCredentialsJSON = credBytes + + return nil +} + +func (o *OAuth2) validateAzureProvider() error { + if o.TokenURL == "" && o.AzureTenantID == "" { + return errors.New("invalid configuration: at least one of token_url or tenant_id must be provided") + } + if o.TokenURL != "" && o.AzureTenantID != "" { + return errors.New("invalid configuration: only one of token_url and tenant_id can be used") + } + if o.ClientID == "" || o.ClientSecret == "" { + return errors.New("invalid configuration: client credentials must be provided") + } + + return nil +} diff --git a/x-pack/filebeat/input/httpjson/config_oauth_test.go b/x-pack/filebeat/input/httpjson/config_oauth_test.go new file mode 100644 index 00000000000..3fa0eed4284 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/config_oauth_test.go @@ -0,0 +1,94 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "reflect" + "testing" +) + +func TestProviderCanonical(t *testing.T) { + const ( + a OAuth2Provider = "gOoGle" + b OAuth2Provider = "google" + ) + + if a.canonical() != b.canonical() { + t.Fatal("Canonical provider should be equal") + } +} + +func TestGetProviderIsCanonical(t *testing.T) { + const expected OAuth2Provider = "google" + + oauth2 := OAuth2{Provider: "GOogle"} + if oauth2.GetProvider() != expected { + t.Fatal("GetProvider should return canonical provider") + } +} + +func TestIsEnabled(t *testing.T) { + oauth2 := OAuth2{} + if !oauth2.IsEnabled() { + t.Fatal("OAuth2 should be enabled by default") + } + + var enabled = false + oauth2.Enabled = &enabled + + if oauth2.IsEnabled() { + t.Fatal("OAuth2 should be disabled") + } + + enabled = true + if !oauth2.IsEnabled() { + t.Fatal("OAuth2 should be enabled") + } +} + +func TestGetTokenURL(t *testing.T) { + const expected = "http://localhost" + oauth2 := OAuth2{TokenURL: "http://localhost"} + if got := oauth2.GetTokenURL(); got != expected { + t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) + } +} + +func TestGetTokenURLWithAzure(t *testing.T) { + const expectedWithoutTenantID = "http://localhost" + oauth2 := OAuth2{TokenURL: "http://localhost", Provider: "azure"} + if got := oauth2.GetTokenURL(); got != expectedWithoutTenantID { + t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) + } + + oauth2.TokenURL = "" + oauth2.AzureTenantID = "a_tenant_id" + const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" + if got := oauth2.GetTokenURL(); got != expectedWithTenantID { + t.Fatalf("GetTokenURL should return the generated TokenURL but got %q", got) + } +} + +func TestGetEndpointParams(t *testing.T) { + var expected = map[string][]string{"foo": {"bar"}} + oauth2 := OAuth2{EndpointParams: map[string][]string{"foo": {"bar"}}} + if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expected) { + t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) + } +} + +func TestGetEndpointParamsWithAzure(t *testing.T) { + var expectedWithoutResource = map[string][]string{"foo": {"bar"}} + oauth2 := OAuth2{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} + if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expectedWithoutResource) { + t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) + } + + oauth2.AzureResource = "baz" + var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} + if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expectedWithResource) { + t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) + } +} diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go new file mode 100644 index 00000000000..cfec6a2440b --- /dev/null +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -0,0 +1,383 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "context" + "os" + "testing" + + "github.com/pkg/errors" + "golang.org/x/oauth2/google" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestConfigValidationCase1(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "http_request_body": map[string]interface{}{"test": "abc"}, + "no_http_body": true, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. no_http_body and http_request_body cannot coexist.") + } +} + +func TestConfigValidationCase2(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "no_http_body": true, + "pagination": map[string]interface{}{"extra_body_content": map[string]interface{}{"test": "abc"}}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. no_http_body and pagination.extra_body_content cannot coexist.") + } +} + +func TestConfigValidationCase3(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "no_http_body": true, + "pagination": map[string]interface{}{"req_field": "abc"}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. no_http_body and pagination.req_field cannot coexist.") + } +} + +func TestConfigValidationCase4(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "req_field": "abc"}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. pagination.header and pagination.req_field cannot coexist.") + } +} + +func TestConfigValidationCase5(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "id_field": "abc"}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. pagination.header and pagination.id_field cannot coexist.") + } +} + +func TestConfigValidationCase6(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "extra_body_content": map[string]interface{}{"test": "abc"}}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. pagination.header and extra_body_content cannot coexist.") + } +} + +func TestConfigValidationCase7(t *testing.T) { + m := map[string]interface{}{ + "http_method": "DELETE", + "no_http_body": true, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") + } +} + +func TestConfigOauth2Validation(t *testing.T) { + cases := []struct { + name string + expectedErr string + input map[string]interface{} + setup func() + teardown func() + }{ + { + name: "can't set oauth2 and api_key together", + expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", + input: map[string]interface{}{ + "api_key": "an_api_key", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", + }, + }, + { + name: "can set oauth2 and api_key together if oauth2 is disabled", + input: map[string]interface{}{ + "api_key": "an_api_key", + "oauth2": map[string]interface{}{ + "enabled": false, + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", + }, + }, + { + name: "can't set oauth2 and authentication_scheme", + expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", + input: map[string]interface{}{ + "authentication_scheme": "a_scheme", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", + }, + }, + { + name: "token_url and client credentials must be set", + expectedErr: "invalid configuration: both token_url and client credentials must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{}, + "url": "localhost", + }, + }, + { + name: "must fail with an unknown provider", + expectedErr: "invalid configuration: unknown provider \"unknown\" accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "unknown", + }, + "url": "localhost", + }, + }, + { + name: "azure must have either tenant_id or token_url", + expectedErr: "invalid configuration: at least one of token_url or tenant_id must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + }, + "url": "localhost", + }, + }, + { + name: "azure must have only one of token_url and tenant_id", + expectedErr: "invalid configuration: only one of token_url and tenant_id can be used accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + "token_url": "localhost", + }, + "url": "localhost", + }, + }, + { + name: "azure must have client credentials set", + expectedErr: "invalid configuration: client credentials must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + }, + "url": "localhost", + }, + }, + { + name: "azure config is valid", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + }, + "url": "localhost", + }, + }, + { + name: "google can't have token_url or client credentials set", + expectedErr: "invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + "token_url": "localhost", + }, + "url": "localhost", + }, + }, + { + name: "google must fail if no ADC available", + expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { + // we change the default function to force a failure + findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { + return nil, errors.New("failed") + } + }, + teardown: func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }, + }, + { + name: "google must fail if credentials file not found", + expectedErr: "invalid configuration: the file \"./wrong\" cannot be found accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./wrong", + }, + "url": "localhost", + }, + }, + { + name: "google must fail if ADC is wrongly set", + expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") }, + }, + { + name: "google must work if ADC is set up", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") }, + }, + { + name: "google must work if credentials_file is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/credentials.json", + }, + "url": "localhost", + }, + }, + { + name: "google must work if jwt_file is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.jwt_file": "./testdata/credentials.json", + }, + "url": "localhost", + }, + }, + { + name: "google must work if credentials_json is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0" + }`), + }, + "url": "localhost", + }, + }, + { + name: "google must fail if credentials_json is not a valid JSON", + expectedErr: "invalid configuration: google.credentials_json must be valid JSON accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`invalid`), + }, + "url": "localhost", + }, + }, + { + name: "google must fail if the provided credentials file is not a valid JSON", + expectedErr: "invalid configuration: the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/invalid_credentials.json", + }, + "url": "localhost", + }, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + if c.setup != nil { + c.setup() + } + + if c.teardown != nil { + defer c.teardown() + } + + cfg := common.MustNewConfigFrom(c.input) + conf := defaultConfig() + err := cfg.Unpack(&conf) + + switch { + case c.expectedErr == "": + if err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } + + case c.expectedErr != "": + if err == nil || err.Error() != c.expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", c.expectedErr, err) + } + } + }) + } +} diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/httpjson_test.go index 33643ac29ab..3e3bf7f93ba 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/httpjson_test.go @@ -11,6 +11,7 @@ import ( "log" "net/http" "net/http/httptest" + "reflect" "regexp" "strconv" "sync" @@ -185,102 +186,52 @@ func (o *stubOutleter) OnEvent(event beat.Event) bool { return !o.done } -// --- Test Cases +func newOAuth2TestServer(t *testing.T) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() -func TestConfigValidationCase1(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "http_request_body": map[string]interface{}{"test": "abc"}, - "no_http_body": true, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and http_request_body cannot coexist.") - } -} + if r.Method != "POST" { + t.Errorf("expected POST request, got %v", r.Method) + return + } -func TestConfigValidationCase2(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "no_http_body": true, - "pagination": map[string]interface{}{"extra_body_content": map[string]interface{}{"test": "abc"}}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and pagination.extra_body_content cannot coexist.") - } -} + if err := r.ParseForm(); err != nil { + t.Errorf("no error expected, got %q", err) + return + } -func TestConfigValidationCase3(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "no_http_body": true, - "pagination": map[string]interface{}{"req_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and pagination.req_field cannot coexist.") - } -} + if gt := r.FormValue("grant_type"); gt != "client_credentials" { + t.Errorf("expected grant_type was client_credentials, got %q", gt) + return + } -func TestConfigValidationCase4(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "req_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and pagination.req_field cannot coexist.") - } -} + clientID := r.FormValue("client_id") + clientSecret := r.FormValue("client_secret") + if clientID == "" || clientSecret == "" { + clientID, clientSecret, _ = r.BasicAuth() + } + if clientID != "a_client_id" || clientSecret != "a_client_secret" { + t.Errorf("expected client credentials \"a_client_id:a_client_secret\", got \"%s:%s\"", clientID, clientSecret) + } -func TestConfigValidationCase5(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "id_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and pagination.id_field cannot coexist.") - } -} + if s := r.FormValue("scope"); s != "scope1 scope2" { + t.Errorf("expected scope was scope1+scope2, got %q", s) + return + } -func TestConfigValidationCase6(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "extra_body_content": map[string]interface{}{"test": "abc"}}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and extra_body_content cannot coexist.") - } -} + expectedParams := []string{"v1", "v2"} + if p := r.Form["param1"]; !reflect.DeepEqual(expectedParams, p) { + t.Errorf("expected params were %q, but got %q", expectedParams, p) + return + } -func TestConfigValidationCase7(t *testing.T) { - m := map[string]interface{}{ - "http_method": "DELETE", - "no_http_body": true, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") - } + w.Header().Set("content-type", "application/json") + w.Write([]byte(`{"token_type":"Bearer","expires_in":"3599","access_token":"abcdef1234567890"}`)) + })) } +// --- Test Cases + func TestGetNextLinkFromHeader(t *testing.T) { header := make(http.Header) header.Add("Link", "; rel=\"self\"") @@ -506,3 +457,34 @@ func TestRunStop(t *testing.T) { input.Stop() }) } + +func TestOAuth2(t *testing.T) { + ts := newOAuth2TestServer(t) + m := map[string]interface{}{ + "http_method": "GET", + "oauth2.client.id": "a_client_id", + "oauth2.client.secret": "a_client_secret", + "oauth2.token_url": ts.URL, + "oauth2.endpoint_params": map[string][]string{ + "param1": {"v1", "v2"}, + }, + "oauth2.scopes": []string{"scope1", "scope2"}, + "interval": 0, + } + defer ts.Close() + + runTest(t, false, false, m, func(input *HttpjsonInput, out *stubOutleter, t *testing.T) { + group, _ := errgroup.WithContext(context.Background()) + group.Go(input.run) + + events, ok := out.waitForEvents(1) + if !ok { + t.Fatalf("Expected 1 events, but got %d.", len(events)) + } + input.Stop() + + if err := group.Wait(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index bcb2296c039..31829f5e233 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -397,31 +397,11 @@ func (in *HttpjsonInput) run() error { ctx, cancel := context.WithCancel(in.workerCtx) defer cancel() - tlsConfig, err := tlscommon.LoadTLSConfig(in.config.TLS) - if err != nil { - return err - } - - var dialer, tlsDialer transport.Dialer - - dialer = transport.NetDialer(in.config.HTTPClientTimeout) - tlsDialer, err = transport.TLSDialer(dialer, tlsConfig, in.config.HTTPClientTimeout) + client, err := in.newHTTPClient(ctx) if err != nil { return err } - // Make transport client - var client *http.Client - client = &http.Client{ - Transport: &http.Transport{ - Dial: dialer.Dial, - DialTLS: tlsDialer.Dial, - TLSClientConfig: tlsConfig.ToConfig(), - DisableKeepAlives: true, - }, - Timeout: in.config.HTTPClientTimeout, - } - ri := &RequestInfo{ URL: in.URL, ContentMap: common.MapStr{}, @@ -462,6 +442,38 @@ func (in *HttpjsonInput) Wait() { in.Stop() } +func (in *HttpjsonInput) newHTTPClient(ctx context.Context) (*http.Client, error) { + tlsConfig, err := tlscommon.LoadTLSConfig(in.config.TLS) + if err != nil { + return nil, err + } + + var dialer, tlsDialer transport.Dialer + + dialer = transport.NetDialer(in.config.HTTPClientTimeout) + tlsDialer, err = transport.TLSDialer(dialer, tlsConfig, in.config.HTTPClientTimeout) + if err != nil { + return nil, err + } + + // Make transport client + client := &http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + DialTLS: tlsDialer.Dial, + TLSClientConfig: tlsConfig.ToConfig(), + DisableKeepAlives: true, + }, + Timeout: in.config.HTTPClientTimeout, + } + + if in.config.OAuth2.IsEnabled() { + return in.config.OAuth2.Client(ctx, client) + } + + return client, nil +} + func makeEvent(body string) beat.Event { fields := common.MapStr{ "event": common.MapStr{ diff --git a/x-pack/filebeat/input/httpjson/testdata/credentials.json b/x-pack/filebeat/input/httpjson/testdata/credentials.json new file mode 100644 index 00000000000..2b5fdd89e5c --- /dev/null +++ b/x-pack/filebeat/input/httpjson/testdata/credentials.json @@ -0,0 +1,7 @@ +{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0" +} diff --git a/x-pack/filebeat/input/httpjson/testdata/invalid_credentials.json b/x-pack/filebeat/input/httpjson/testdata/invalid_credentials.json new file mode 100644 index 00000000000..9977a2836c1 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/testdata/invalid_credentials.json @@ -0,0 +1 @@ +invalid