From e2aab838bdf1601105efa634df4228b9130019fc Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Tue, 23 Nov 2021 12:30:00 +0530 Subject: [PATCH 01/10] Add grant-type: passoword in httpjson oauth2 --- x-pack/filebeat/input/httpjson/config_auth.go | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index 88eac44edc1..8b27a0a9c78 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -82,6 +82,8 @@ type oAuth2Config struct { // common oauth fields ClientID string `config:"client.id"` ClientSecret string `config:"client.secret"` + User string `config:"user"` + Password string `config:"password"` EndpointParams map[string][]string `config:"endpoint_params"` Provider oAuth2Provider `config:"provider"` Scopes []string `config:"scopes"` @@ -109,14 +111,30 @@ func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.C switch o.getProvider() { case oAuth2ProviderAzure, oAuth2ProviderDefault: - creds := clientcredentials.Config{ - ClientID: o.ClientID, - ClientSecret: o.ClientSecret, - TokenURL: o.getTokenURL(), - Scopes: o.Scopes, - EndpointParams: o.getEndpointParams(), + if o.User != "" || o.Password != "" { + conf := &oauth2.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + Endpoint: oauth2.Endpoint{ + TokenURL: o.TokenURL, + AuthStyle: 1, + }, + } + token, err := conf.PasswordCredentialsToken(ctx, o.User, o.Password) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) + } + return conf.Client(ctx, token), nil + } else { + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.getTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.getEndpointParams(), + } + return creds.Client(ctx), nil } - return creds.Client(ctx), nil case oAuth2ProviderGoogle: if o.GoogleJWTFile != "" { cfg, err := google.JWTConfigFromJSON(o.GoogleCredentialsJSON, o.Scopes...) From d38852d23bb1a651ca2a4d20b0f87067719681fa Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 24 Nov 2021 13:31:19 +0530 Subject: [PATCH 02/10] refactor code and add new properties tests in config_test.go --- x-pack/filebeat/input/httpjson/config_auth.go | 13 ++++-- x-pack/filebeat/input/httpjson/config_test.go | 42 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index 8b27a0a9c78..faf70124843 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -22,6 +22,8 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +const AuthStyleInParams = 1 + type authConfig struct { Basic *basicAuthConfig `config:"basic"` OAuth2 *oAuth2Config `config:"oauth2"` @@ -82,12 +84,12 @@ type oAuth2Config struct { // common oauth fields ClientID string `config:"client.id"` ClientSecret string `config:"client.secret"` - User string `config:"user"` - Password string `config:"password"` EndpointParams map[string][]string `config:"endpoint_params"` + Password string `config:"password"` Provider oAuth2Provider `config:"provider"` Scopes []string `config:"scopes"` TokenURL string `config:"token_url"` + User string `config:"user"` // google specific GoogleCredentialsFile string `config:"google.credentials_file"` @@ -117,12 +119,12 @@ func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.C ClientSecret: o.ClientSecret, Endpoint: oauth2.Endpoint{ TokenURL: o.TokenURL, - AuthStyle: 1, + AuthStyle: AuthStyleInParams, }, } token, err := conf.PasswordCredentialsToken(ctx, o.User, o.Password) if err != nil { - return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) + return nil, fmt.Errorf("oauth2 client: error loading credentials using user and password: %w", err) } return conf.Client(ctx, token), nil } else { @@ -202,6 +204,9 @@ func (o *oAuth2Config) Validate() error { if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" { return errors.New("both token_url and client credentials must be provided") } + if (o.User != "" && o.Password == "") || (o.User == "" && o.Password != "") { + return errors.New("both user and password credentials must be provided") + } default: return fmt.Errorf("unknown provider %q", o.getProvider()) } diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 67662b40752..71ee7edb246 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -152,6 +152,48 @@ func TestConfigOauth2Validation(t *testing.T) { "auth.oauth2": map[string]interface{}{}, }, }, + { + name: "if user and password is set oauth2 must use user-password authentication", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "user": "a_client_user", + "password": "a_client_password", + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + }, + }, + { + name: "if user is set password credentials must be set for user-password authentication", + expectedErr: "both user and password credentials must be provided accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "user": "a_client_user", + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + }, + }, + { + name: "if password is set user credentials must be set for user-password authentication", + expectedErr: "both user and password credentials must be provided accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "password": "a_client_password", + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + }, + }, { name: "must fail with an unknown provider", expectedErr: "unknown provider \"unknown\" accessing 'auth.oauth2'", From 7bbacd87ee6b2b9b3005577534a057e82e039976 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Thu, 25 Nov 2021 16:54:01 +0530 Subject: [PATCH 03/10] Add grant_type: password in oAuth2ProviderDefault --- x-pack/filebeat/input/httpjson/config_auth.go | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index faf70124843..9a859d9350e 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -22,7 +22,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -const AuthStyleInParams = 1 +const authStyleInParams = 1 type authConfig struct { Basic *basicAuthConfig `config:"basic"` @@ -107,19 +107,31 @@ func (o *oAuth2Config) isEnabled() bool { return o != nil && (o.Enabled == nil || *o.Enabled) } +// clientCredentialsGrant creates http client from token_url and client credentials +func (o *oAuth2Config) clientCredentialsGrant(ctx context.Context, client *http.Client) *http.Client { + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.getTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.getEndpointParams(), + } + return creds.Client(ctx) +} + // Client wraps the given http.Client and returns a new one that will use the oauth authentication. func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.Client, error) { ctx = context.WithValue(ctx, oauth2.HTTPClient, client) switch o.getProvider() { - case oAuth2ProviderAzure, oAuth2ProviderDefault: + case oAuth2ProviderDefault: if o.User != "" || o.Password != "" { conf := &oauth2.Config{ ClientID: o.ClientID, ClientSecret: o.ClientSecret, Endpoint: oauth2.Endpoint{ TokenURL: o.TokenURL, - AuthStyle: AuthStyleInParams, + AuthStyle: authStyleInParams, }, } token, err := conf.PasswordCredentialsToken(ctx, o.User, o.Password) @@ -128,15 +140,10 @@ func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.C } return conf.Client(ctx, token), nil } else { - creds := clientcredentials.Config{ - ClientID: o.ClientID, - ClientSecret: o.ClientSecret, - TokenURL: o.getTokenURL(), - Scopes: o.Scopes, - EndpointParams: o.getEndpointParams(), - } - return creds.Client(ctx), nil + return o.clientCredentialsGrant(ctx, client), nil } + case oAuth2ProviderAzure: + return o.clientCredentialsGrant(ctx, client), nil case oAuth2ProviderGoogle: if o.GoogleJWTFile != "" { cfg, err := google.JWTConfigFromJSON(o.GoogleCredentialsJSON, o.Scopes...) From abfae0b8a3bbd60cd0276682e6beeef4a1a0135e Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 29 Nov 2021 10:32:41 +0530 Subject: [PATCH 04/10] Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index fbef98864e6..e6f9ccba8ad 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -347,6 +347,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support in aws-s3 input for custom script parsing of s3 notifications. {pull}28946[28946] - Improve error handling in aws-s3 input for malformed s3 notifications. {issue}28828[28828] {pull}28946[28946] - Add support for parsers on journald input {pull}29070[29070] +- Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087] *Heartbeat* From d30ac17c254762b298a2772b14f43266d616bebd Mon Sep 17 00:00:00 2001 From: Sunny Chaudhari Date: Tue, 30 Nov 2021 19:40:35 +0530 Subject: [PATCH 05/10] update input-httpjson.asciidoc with new extended httpjson authentication method --- .../docs/inputs/input-httpjson.asciidoc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 027ef66e3e7..a6ca2e910a1 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -82,6 +82,8 @@ filebeat.inputs: client.id: 12345678901234567890abcdef client.secret: abcdef12345678901234567890 token_url: http://localhost/oauth2/token + user: abc.xyz@mail.com + password: P@$$W0₹D request.url: http://localhost ---- @@ -265,6 +267,23 @@ except if using `google` as provider. Required for providers: `default`, `azure` The client secret used as part of the authentication flow. It is always required except if using `google` as provider. Required for providers: `default`, `azure`. +[float] +==== `auth.oauth2.user` + +The user used as part of the authentication flow. It is required for authentication +- grant type password. It is always required except if using `google` as provider. +Required for providers: `default`, `azure`. + +[float] +==== `auth.oauth2.password` + +The password used as part of the authentication flow.It is required for authentication +- grant type password. It is always required except if using `google` as provider. +Required for providers: `default`, `azure`. + +NOTE: If user and password is not used then it will automatically use `token_url` and +`client credential` method + [float] ==== `auth.oauth2.scopes` From 0c6f0ae9ca506102e1180fdf39ff617af9cab382 Mon Sep 17 00:00:00 2001 From: Sunny Chaudhari Date: Tue, 30 Nov 2021 20:02:01 +0530 Subject: [PATCH 06/10] add comment for authstyleparam variable --- x-pack/filebeat/input/httpjson/config_auth.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index 9a859d9350e..7e8dae6dc0a 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +// authStyleInParams sends the "client_id" and "client_secret" in the POST body as application/x-www-form-urlencoded parameters. const authStyleInParams = 1 type authConfig struct { From 9bbf09d3e8b47e118ee703bde9ec90fb66bbba07 Mon Sep 17 00:00:00 2001 From: Sunny Chaudhari Date: Tue, 30 Nov 2021 20:48:21 +0530 Subject: [PATCH 07/10] update user dummy value in the doc --- x-pack/filebeat/docs/inputs/input-httpjson.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index a6ca2e910a1..c78ffe6931f 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -82,7 +82,7 @@ filebeat.inputs: client.id: 12345678901234567890abcdef client.secret: abcdef12345678901234567890 token_url: http://localhost/oauth2/token - user: abc.xyz@mail.com + user: user@domain.tld password: P@$$W0₹D request.url: http://localhost ---- From 1df0b0161a8d60842e448213deda7f4ebcf56d2a Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 1 Dec 2021 09:59:38 +0530 Subject: [PATCH 08/10] Update input-httpjson.asciidoc - provider should be default only for user-passowrd method --- .../docs/inputs/input-httpjson.asciidoc | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index c78ffe6931f..cbde7b8b831 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -74,6 +74,17 @@ filebeat.inputs: value: 'Basic aGVsbG86d29ybGQ=' ---- +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + auth.oauth2: + client.id: 12345678901234567890abcdef + client.secret: abcdef12345678901234567890 + token_url: http://localhost/oauth2/token + request.url: http://localhost +---- + ["source","yaml",subs="attributes"] ---- filebeat.inputs: @@ -270,19 +281,18 @@ except if using `google` as provider. Required for providers: `default`, `azure` [float] ==== `auth.oauth2.user` -The user used as part of the authentication flow. It is required for authentication -- grant type password. It is always required except if using `google` as provider. -Required for providers: `default`, `azure`. +The user used as part of the authentication flow. It is required for authentication + - grant type password. It is only available for provider `default`. [float] ==== `auth.oauth2.password` -The password used as part of the authentication flow.It is required for authentication -- grant type password. It is always required except if using `google` as provider. -Required for providers: `default`, `azure`. +The password used as part of the authentication flow. It is required for authentication + - grant type password. It is only available for provider `default`. -NOTE: If user and password is not used then it will automatically use `token_url` and -`client credential` method +NOTE: user and password are required for grant_type password. If user and +password is not used then it will automatically use the `token_url` and +`client credential` method. [float] ==== `auth.oauth2.scopes` From 54735fb959536856c4f6b20e1d90e06de63bcc27 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Wed, 24 Nov 2021 13:31:19 +0530 Subject: [PATCH 09/10] refactor code and add new properties tests in config_test.go --- x-pack/filebeat/input/httpjson/config_auth.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index 7e8dae6dc0a..c0b42e7e2c3 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -22,8 +22,12 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +<<<<<<< HEAD // authStyleInParams sends the "client_id" and "client_secret" in the POST body as application/x-www-form-urlencoded parameters. const authStyleInParams = 1 +======= +const AuthStyleInParams = 1 +>>>>>>> 8e10984a75 (refactor code and add new properties tests in config_test.go) type authConfig struct { Basic *basicAuthConfig `config:"basic"` @@ -132,7 +136,11 @@ func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.C ClientSecret: o.ClientSecret, Endpoint: oauth2.Endpoint{ TokenURL: o.TokenURL, +<<<<<<< HEAD AuthStyle: authStyleInParams, +======= + AuthStyle: AuthStyleInParams, +>>>>>>> 8e10984a75 (refactor code and add new properties tests in config_test.go) }, } token, err := conf.PasswordCredentialsToken(ctx, o.User, o.Password) From d4d21c1c649f0e4d52df0fa8886de67bc61698c3 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Thu, 25 Nov 2021 16:54:01 +0530 Subject: [PATCH 10/10] Add grant_type: password in oAuth2ProviderDefault --- x-pack/filebeat/input/httpjson/config_auth.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index c0b42e7e2c3..7e8dae6dc0a 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -22,12 +22,8 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -<<<<<<< HEAD // authStyleInParams sends the "client_id" and "client_secret" in the POST body as application/x-www-form-urlencoded parameters. const authStyleInParams = 1 -======= -const AuthStyleInParams = 1 ->>>>>>> 8e10984a75 (refactor code and add new properties tests in config_test.go) type authConfig struct { Basic *basicAuthConfig `config:"basic"` @@ -136,11 +132,7 @@ func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.C ClientSecret: o.ClientSecret, Endpoint: oauth2.Endpoint{ TokenURL: o.TokenURL, -<<<<<<< HEAD AuthStyle: authStyleInParams, -======= - AuthStyle: AuthStyleInParams, ->>>>>>> 8e10984a75 (refactor code and add new properties tests in config_test.go) }, } token, err := conf.PasswordCredentialsToken(ctx, o.User, o.Password)