Filebeat Okta module

filebeat/docs/modules/okta.asciidoc

@@ -0,0 +1,27 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-okta]]
+[role="xpack"]
+
+:modulename: okta
+:has-dashboards: false
+
+== OKTA module
+
+beta[]
+
+This is a filebeat module for retrieving system logs from OKTA via API.
+
+:has-dashboards!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-okta,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -29,6 +29,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-nats>>
   * <<filebeat-module-netflow>>
   * <<filebeat-module-nginx>>
+  * <<filebeat-module-okta>>
   * <<filebeat-module-osquery>>
   * <<filebeat-module-panw>>
   * <<filebeat-module-postgresql>>
@@ -70,6 +71,7 @@ include::modules/mysql.asciidoc[]
 include::modules/nats.asciidoc[]
 include::modules/netflow.asciidoc[]
 include::modules/nginx.asciidoc[]
+include::modules/okta.asciidoc[]
 include::modules/osquery.asciidoc[]
 include::modules/panw.asciidoc[]
 include::modules/postgresql.asciidoc[]

x-pack/filebeat/filebeat.reference.yml

@@ -642,6 +642,16 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+#--------------------------------- OKTA Module ---------------------------------
+- module: okta
+  system:
+    enabled: true
+    # API key to access OKTA
+    #var.api_key
+
+    # URL of the OKTA REST API
+    #var.url
+
 #------------------------------- Osquery Module -------------------------------
 - module: osquery
   result:

x-pack/filebeat/input/httpjson/config.go

@@ -16,27 +16,44 @@ import (
 
 // Config contains information about httpjson configuration
 type config struct {
-	APIKey            string            `config:"api_key"`
-	HTTPClientTimeout time.Duration     `config:"http_client_timeout"`
-	HTTPHeaders       common.MapStr     `config:"http_headers"`
-	HTTPMethod        string            `config:"http_method" validate:"required"`
-	HTTPRequestBody   common.MapStr     `config:"http_request_body"`
-	Interval          time.Duration     `config:"interval"`
-	JSONObjects       string            `config:"json_objects_array"`
-	Pagination        *Pagination       `config:"pagination"`
-	TLS               *tlscommon.Config `config:"ssl"`
-	URL               string            `config:"url" validate:"required"`
+	APIKey               string            `config:"api_key"`
+	AuthenticationScheme string            `config:"authentication_scheme"`
+	HTTPClientTimeout    time.Duration     `config:"http_client_timeout"`
+	HTTPHeaders          common.MapStr     `config:"http_headers"`
+	HTTPMethod           string            `config:"http_method" validate:"required"`
+	HTTPRequestBody      common.MapStr     `config:"http_request_body"`
+	Interval             time.Duration     `config:"interval"`
+	JSONObjects          string            `config:"json_objects_array"`
+	NoHTTPBody           bool              `config:"no_http_body"`
+	Pagination           *Pagination       `config:"pagination"`
+	RateLimit            *RateLimit        `config:"rate_limit"`
+	TLS                  *tlscommon.Config `config:"ssl"`
+	URL                  string            `config:"url" validate:"required"`
 }
 
 // Pagination contains information about httpjson pagination settings
 type Pagination struct {
 	IsEnabled        bool          `config:"enabled"`
 	ExtraBodyContent common.MapStr `config:"extra_body_content"`
+	Header           *Header       `config:"header"`
 	IDField          string        `config:"id_field"`
 	RequestField     string        `config:"req_field"`
 	URL              string        `config:"url"`
 }
 
+// HTTP Header information for pagination
+type Header struct {
+	FieldName    string `config:"field_name"`
+	RegexPattern string `config:"regex_pattern"`
+}
+
+// HTTP Header Rate Limit information
+type RateLimit struct {
+	Limit     string `config:"limit"`
+	Reset     string `config:"reset"`
+	Remaining string `config:"remaining"`
+}
+
 func (c *config) Validate() error {
 	switch strings.ToUpper(c.HTTPMethod) {
 	case "GET":

x-pack/filebeat/input/httpjson/httpjson_test.go

@@ -35,11 +35,14 @@ func testSetup(t *testing.T) {
 	})
 }
 
-func runTest(t *testing.T, m map[string]interface{}, run func(input *httpjsonInput, out *stubOutleter, t *testing.T)) {
-	// Setup httpbin environment
+func runTest(t *testing.T, isTLS bool, m map[string]interface{}, run func(input *httpjsonInput, out *stubOutleter, t *testing.T)) {
 	testSetup(t)
-	// Create test http server
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	// Create an http test server according to whether TLS is used
+	var newServer = httptest.NewServer
+	if isTLS {
+		newServer = httptest.NewTLSServer
+	}
+	ts := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == http.MethodPost {
 			req, err := ioutil.ReadAll(r.Body)
 			defer r.Body.Close()
@@ -154,7 +157,29 @@ func TestGET(t *testing.T) {
 		"http_method": "GET",
 		"interval":    0,
 	}
-	runTest(t, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
+	runTest(t, false, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
+		group, _ := errgroup.WithContext(context.Background())
+		group.Go(input.run)
+
+		events, ok := out.waitForEvents(1)
+		if !ok {
+			t.Fatalf("Expected 1 events, but got %d.", len(events))
+		}
+		input.Stop()
+
+		if err := group.Wait(); err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func TestGetHTTPS(t *testing.T) {
+	m := map[string]interface{}{
+		"http_method":           "GET",
+		"interval":              0,
+		"ssl.verification_mode": "none",
+	}
+	runTest(t, true, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
 		group, _ := errgroup.WithContext(context.Background())
 		group.Go(input.run)
 
@@ -176,7 +201,7 @@ func TestPOST(t *testing.T) {
 		"http_request_body": map[string]interface{}{"test": "abc", "testNested": map[string]interface{}{"testNested1": 123}},
 		"interval":          0,
 	}
-	runTest(t, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
+	runTest(t, false, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
 		group, _ := errgroup.WithContext(context.Background())
 		group.Go(input.run)
 
@@ -198,7 +223,7 @@ func TestRepeatedPOST(t *testing.T) {
 		"http_request_body": map[string]interface{}{"test": "abc", "testNested": map[string]interface{}{"testNested1": 123}},
 		"interval":          10 ^ 9,
 	}
-	runTest(t, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
+	runTest(t, false, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
 		group, _ := errgroup.WithContext(context.Background())
 		group.Go(input.run)
 
@@ -219,7 +244,7 @@ func TestRunStop(t *testing.T) {
 		"http_method": "GET",
 		"interval":    0,
 	}
-	runTest(t, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
+	runTest(t, false, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) {
 		input.Run()
 		input.Stop()
 		input.Run()