Update filebeat httpjson input to support pagination via Header and O…

…kta module (#16354) * Update filebeat httpjson input to support pagination via Header and Okta module
elastic · Mar 11, 2020 · a6a2d49 · a6a2d49
1 parent df3eb93
commit a6a2d49
Show file tree

Hide file tree

Showing 5 changed files with 464 additions and 74 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -170,6 +170,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow users to override pipeline ID in fileset input config. {issue}9531[9531] {pull}16561[16561]
 - Add `o365audit` input type for consuming events from Office 365 Management Activity API. {issue}16196[16196] {pull}16244[16244]
 - Improve ECS categorization field mappings in logstash module. {issue}16169[16169] {pull}16668[16668]
+- Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc
@@ -91,17 +91,32 @@ Time duration between repeated data retrievals. Default: 0s, meaning no repeated
 If the HTTP API returns data in a JSON array, then this option can be set to decode these records 
 from the array. Default: not used.
 
+[float]
+==== `no_http_body`
+
+If set, do not use HTTP request body. Default: false.
+
 [float]
 ==== `pagination.enabled`
 
-This option specifies whether pagination is enabled. Default: false.
+This option specifies whether pagination is enabled. Default: true.
 
 [float]
 ==== `pagination.extra_body_content`
 
 Any additional data that needs to be set in the HTTP pagination request can be specified in
 this JSON blob. Default: not used.
 
+[float]
+==== `pagination.header.field_name`
+
+The field name in the HTTP Header that is used for pagination control. 
+
+[float]
+==== `pagination.header.regex_pattern`
+
+The regular expression pattern to use for retrieving the pagination information from the HTTP Header field specified above.
+
 [float]
 ==== `pagination.id_field`
 
@@ -120,6 +135,22 @@ Required when pagination is enabled.
 This specifies the URL for sending pagination request. Required if the pagination URL is different 
 than the HTTP API URL.
 
+[float]
+==== `rate_limit.limit`
+
+This specifies the field in the HTTP Header of the response that specifies the total limit.
+
+[float]
+==== `rate_limit.remaining`
+
+This specifies the field in the HTTP Header of the response that specifies the remaining quota of the rate limit.
+
+[float]
+==== `rate_limit.reset`
+
+This specifies the field in the HTTP Header of the response that specifies the epoch time 
+when the rate limit will be reset.
+
 [float]
 ==== `ssl`
 

diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go
@@ -5,6 +5,7 @@
 package httpjson
 
 import (
+	"regexp"
 	"strings"
 	"time"
 
@@ -16,35 +17,72 @@ import (
 
 // Config contains information about httpjson configuration
 type config struct {
-	APIKey            string            `config:"api_key"`
-	HTTPClientTimeout time.Duration     `config:"http_client_timeout"`
-	HTTPHeaders       common.MapStr     `config:"http_headers"`
-	HTTPMethod        string            `config:"http_method" validate:"required"`
-	HTTPRequestBody   common.MapStr     `config:"http_request_body"`
-	Interval          time.Duration     `config:"interval"`
-	JSONObjects       string            `config:"json_objects_array"`
-	Pagination        *Pagination       `config:"pagination"`
-	TLS               *tlscommon.Config `config:"ssl"`
-	URL               string            `config:"url" validate:"required"`
+	APIKey               string            `config:"api_key"`
+	AuthenticationScheme string            `config:"authentication_scheme"`
+	HTTPClientTimeout    time.Duration     `config:"http_client_timeout"`
+	HTTPHeaders          common.MapStr     `config:"http_headers"`
+	HTTPMethod           string            `config:"http_method" validate:"required"`
+	HTTPRequestBody      common.MapStr     `config:"http_request_body"`
+	Interval             time.Duration     `config:"interval"`
+	JSONObjects          string            `config:"json_objects_array"`
+	NoHTTPBody           bool              `config:"no_http_body"`
+	Pagination           *Pagination       `config:"pagination"`
+	RateLimit            *RateLimit        `config:"rate_limit"`
+	TLS                  *tlscommon.Config `config:"ssl"`
+	URL                  string            `config:"url" validate:"required"`
 }
 
 // Pagination contains information about httpjson pagination settings
 type Pagination struct {
-	IsEnabled        bool          `config:"enabled"`
+	Enabled          *bool         `config:"enabled"`
 	ExtraBodyContent common.MapStr `config:"extra_body_content"`
+	Header           *Header       `config:"header"`
 	IDField          string        `config:"id_field"`
 	RequestField     string        `config:"req_field"`
 	URL              string        `config:"url"`
 }
 
+// IsEnabled returns true if the `enable` field is set to true in the yaml.
+func (p *Pagination) IsEnabled() bool {
+	return p != nil && (p.Enabled == nil || *p.Enabled)
+}
+
+// HTTP Header information for pagination
+type Header struct {
+	FieldName    string         `config:"field_name" validate:"required"`
+	RegexPattern *regexp.Regexp `config:"regex_pattern" validate:"required"`
+}
+
+// HTTP Header Rate Limit information
+type RateLimit struct {
+	Limit     string `config:"limit"`
+	Reset     string `config:"reset"`
+	Remaining string `config:"remaining"`
+}
+
 func (c *config) Validate() error {
 	switch strings.ToUpper(c.HTTPMethod) {
 	case "GET":
 		break
 	case "POST":
 		break
 	default:
-		return errors.Errorf("httpjson input: Invalid http_method, %s - ", c.HTTPMethod)
+		return errors.Errorf("httpjson input: Invalid http_method, %s", c.HTTPMethod)
+	}
+	if c.NoHTTPBody {
+		if len(c.HTTPRequestBody) > 0 {
+			return errors.Errorf("invalid configuration: both no_http_body and http_request_body cannot be set simultaneously")
+		}
+		if c.Pagination != nil && (len(c.Pagination.ExtraBodyContent) > 0 || c.Pagination.RequestField != "") {
+			return errors.Errorf("invalid configuration: both no_http_body and pagination.extra_body_content or pagination.req_field cannot be set simultaneously")
+		}
+	}
+	if c.Pagination != nil {
+		if c.Pagination.Header != nil {
+			if c.Pagination.RequestField != "" || c.Pagination.IDField != "" || len(c.Pagination.ExtraBodyContent) > 0 {
+				return errors.Errorf("invalid configuration: both pagination.header and pagination.req_field or pagination.id_field or pagination.extra_body_content cannot be set simultaneously")
+			}
+		}
 	}
 	return nil
 }