[Heartbeat] Add httpcommon options to ZipURL

heartbeat/heartbeat.yml

@@ -23,7 +23,7 @@ heartbeat.config.monitors:
 heartbeat.monitors:
 - type: http
   # Set enabled to true (or delete the following line) to enable this example monitor
-  enabled: false
+  enabled: true
   # ID used to uniquely identify this monitor in elasticsearch even if the config changes
   id: my-monitor
   # Human readable display name for this service in Uptime UI and elsewhere

monitors.d/plaintodos.yml

@@ -0,0 +1,12 @@
+- name: Todos
+  id: todos
+  type: browser
+  enabled: true
+  schedule: "@every 3m"
+  tags: todos-app
+  params:
+    url: "https://elastic.github.io/synthetics-demo/"
+  source:
+    zip_url: 
+      url: "https://github.com/elastic/synthetics-demo/archive/refs/heads/main.zip"
+      folder: "todos/synthetics-tests"

x-pack/heartbeat/heartbeat.yml

@@ -21,22 +21,18 @@ heartbeat.config.monitors:
 
 # Configure monitors inline
 heartbeat.monitors:
-- type: http
-  # Set enabled to true (or delete the following line) to enable this example monitor
-  enabled: false
-  # ID used to uniquely identify this monitor in elasticsearch even if the config changes
-  id: my-monitor
-  # Human readable display name for this service in Uptime UI and elsewhere
-  name: My Monitor
-  # List or urls to query
-  urls: ["http://localhost:9200"]
-  # Configure task schedule
-  schedule: '@every 10s'
-  # Total test connection and data exchange timeout
-  #timeout: 16s
-  # Name of corresponding APM service, if Elastic APM is in use for the monitored service.
-  #service.name: my-apm-service-name
-
+- name: Todos
+  id: todos
+  type: browser
+  enabled: true
+  schedule: "@every 3m"
+  tags: todos-app
+  params:
+    url: "https://elastic.github.io/synthetics-demo/"
+  source:
+    zip_url: 
+      url: "https://github.com/elastic/synthetics-demo/archive/refs/heads/main.zip"
+      folder: "todos/synthetics-tests"
 # ======================= Elasticsearch template setting =======================
 
 setup.template.settings:
@@ -93,11 +89,11 @@ setup.kibana:
 # ================================== Outputs ===================================
 
 # Configure what output to use when sending the data collected by the beat.
-
+output.console: ~
 # ---------------------------- Elasticsearch Output ----------------------------
-output.elasticsearch:
+#output.elasticsearch:
   # Array of hosts to connect to.
-  hosts: ["localhost:9200"]
+  #hosts: ["localhost:9200"]
 
   # Protocol - either `http` (default) or `https`.
   #protocol: "https"

x-pack/heartbeat/monitors/browser/source/zipurl.go

@@ -14,6 +14,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/elastic/beats/v7/libbeat/common/transport/httpcommon"
 )
 
 type ZipURLSource struct {
@@ -23,13 +25,25 @@ type ZipURLSource struct {
 	Password string `config:"password" json:"password"`
 	Retries  int    `config:"retries" default:"3" json:"retries"`
 	BaseSource
-	// Etag from last successful fetch
-	etag            string
 	TargetDirectory string `config:"target_directory" json:"target_directory"`
+
+	// Etag from last successful fetch
+	etag string
+
+	Transport httpcommon.HTTPTransportSettings `config:",inline" yaml:",inline"`
+
+	httpClient *http.Client
 }
 
 var ErrNoEtag = fmt.Errorf("No ETag header in zip file response. Heartbeat requires an etag to efficiently cache downloaded code")
 
+func (z *ZipURLSource) Validate() (err error) {
+	if z.httpClient == nil {
+		z.httpClient, _ = z.Transport.Client()
+	}
+	return err
+}
+
 func (z *ZipURLSource) Fetch() error {
 	changed, err := checkIfChanged(z)
 	if err != nil {
@@ -181,14 +195,15 @@ func retryingZipRequest(method string, z *ZipURLSource) (resp *http.Response, er
 }
 
 func zipRequest(method string, z *ZipURLSource) (*http.Response, error) {
+
 	req, err := http.NewRequest(method, z.URL, nil)
 	if err != nil {
 		return nil, fmt.Errorf("could not issue request to: %s %w", z.URL, err)
 	}
 	if z.Username != "" && z.Password != "" {
 		req.SetBasicAuth(z.Username, z.Password)
 	}
-	return http.DefaultClient.Do(req)
+	return z.httpClient.Do(req)
 }
 
 func download(z *ZipURLSource, tf *os.File) (etag string, err error) {

x-pack/heartbeat/monitors/browser/source/zipurl_test.go

@@ -5,69 +5,132 @@
 package source
 
 import (
-	"context"
 	"fmt"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"path/filepath"
 	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source/fixtures"
 )
 
-func TestZipUrlFetchNoAuth(t *testing.T) {
-	address, teardown := setupTests()
-	defer teardown()
-
-	zus := &ZipURLSource{
-		URL:     fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:  "/",
-		Retries: 3,
+func TestSimpleCases(t *testing.T) {
+	type testCase struct {
+		name         string
+		cfg          common.MapStr
+		tlsServer    bool
+		wantFetchErr bool
+	}
+	testCases := []testCase{
+		{
+			"basics",
+			common.MapStr{
+				"folder":  "/",
+				"retries": 3,
+			},
+			false,
+			false,
+		},
+		{
+			"targetdir",
+			common.MapStr{
+				"folder":           "/",
+				"retries":          3,
+				"target_directory": "/tmp/synthetics/blah",
+			},
+			false,
+			false,
+		},
+		{
+			"auth success",
+			common.MapStr{
+				"folder":   "/",
+				"retries":  3,
+				"username": "testuser",
+				"password": "testpass",
+			},
+			false,
+			false,
+		},
+		{
+			"auth failure",
+			common.MapStr{
+				"folder":   "/",
+				"retries":  3,
+				"username": "testuser",
+				"password": "badpass",
+			},
+			false,
+			true,
+		},
+		{
+			"ssl ignore cert errors",
+			common.MapStr{
+				"folder":  "/",
+				"retries": 3,
+				"timeout": 123,
+				"ssl": common.MapStr{
+					"enabled":           "true",
+					"verification_mode": "none",
+				},
+			},
+			true,
+			false,
+		},
+		{
+			"bad ssl",
+			common.MapStr{
+				"folder":  "/",
+				"retries": 3,
+				"ssl": common.MapStr{
+					"enabled":                 "true",
+					"certificate_authorities": []string{},
+				},
+			},
+			true,
+			true,
+		},
 	}
-	fetchAndCheckDir(t, zus)
-}
 
-func TestZipUrlFetchWithAuth(t *testing.T) {
-	address, teardown := setupTests()
-	defer teardown()
+	for _, tc := range testCases {
+		url, teardown := setupTests(tc.tlsServer)
+		defer teardown()
+		t.Run(tc.name, func(t *testing.T) {
+			tc.cfg["url"] = fmt.Sprintf("%s/fixtures/todos.zip", url)
+			zus, err := dummyZus(tc.cfg)
+			require.NoError(t, err)
 
-	zus := &ZipURLSource{
-		URL:      fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:   "/",
-		Retries:  3,
-		Username: "testuser",
-		Password: "testpass",
-	}
-	fetchAndCheckDir(t, zus)
-}
+			require.NotNil(t, zus.httpClient)
 
-func TestZipUrlTargetDirectory(t *testing.T) {
-	address, teardown := setupTests()
-	defer teardown()
+			if tc.wantFetchErr == true {
+				err := zus.Fetch()
+				require.Error(t, err)
+				return
+			}
 
-	zus := &ZipURLSource{
-		URL:             fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:          "/",
-		Retries:         3,
-		TargetDirectory: "/tmp/synthetics/blah",
+			fetchAndCheckDir(t, zus)
+		})
 	}
-	fetchAndCheckDir(t, zus)
 }
 
 func TestZipUrlWithSameEtag(t *testing.T) {
-	address, teardown := setupTests()
+	address, teardown := setupTests(false)
 	defer teardown()
 
-	zus := ZipURLSource{
-		URL:     fmt.Sprintf("http://%s/fixtures/todos.zip", address),
-		Folder:  "/",
-		Retries: 3,
-	}
-	err := zus.Fetch()
+	zus, err := dummyZus(common.MapStr{
+		"url":     fmt.Sprintf("http://%s/fixtures/todos.zip", address),
+		"folder":  "/",
+		"retries": 3,
+	})
+	require.NoError(t, err)
+	err = zus.Fetch()
 	defer zus.Close()
 	require.NoError(t, err)
 
@@ -80,32 +143,33 @@ func TestZipUrlWithSameEtag(t *testing.T) {
 }
 
 func TestZipUrlWithBadUrl(t *testing.T) {
-	_, teardown := setupTests()
+	_, teardown := setupTests(false)
 	defer teardown()
 
-	zus := ZipURLSource{
-		URL:     "http://notahost.notadomaintoehutoeuhn",
-		Folder:  "/",
-		Retries: 2,
-	}
-	err := zus.Fetch()
+	zus, err := dummyZus(common.MapStr{
+		"url":     "http://notahost.notadomaintoehutoeuhn",
+		"folder":  "/",
+		"retries": 2,
+	})
+	require.NoError(t, err)
+	err = zus.Fetch()
 	defer zus.Close()
 	require.Error(t, err)
 }
 
-func setupTests() (addr string, teardown func()) {
+func setupTests(tls bool) (addr string, teardown func()) {
 	// go offline, so we dont invoke npm install for unit tests
 	GoOffline()
 
-	srv := createServer()
-	address := srv.Addr
+	srv := createServer(tls)
+	address := srv.URL
 	return address, func() {
 		GoOnline()
-		srv.Shutdown(context.Background())
+		srv.Close()
 	}
 }
 
-func createServer() (addr *http.Server) {
+func createServer(tls bool) (addr *httptest.Server) {
 	_, filename, _, _ := runtime.Caller(0)
 	fixturesPath := path.Join(filepath.Dir(filename), "fixtures")
 	fileServer := http.FileServer(http.Dir(fixturesPath))
@@ -121,10 +185,12 @@ func createServer() (addr *http.Server) {
 		http.StripPrefix("/fixtures", fileServer).ServeHTTP(resp, req)
 	})
 
-	srv := &http.Server{Addr: "localhost:1234", Handler: mux}
-	go func() {
-		srv.ListenAndServe()
-	}()
+	var srv *httptest.Server
+	if tls {
+		srv = httptest.NewTLSServer(mux)
+	} else {
+		srv = httptest.NewServer(mux)
+	}
 
 	return srv
 }
@@ -140,3 +206,14 @@ func fetchAndCheckDir(t *testing.T, zip *ZipURLSource) {
 	_, err = os.Stat(zip.TargetDirectory)
 	require.True(t, os.IsNotExist(err), "TargetDirectory %s should have been deleted", zip.TargetDirectory)
 }
+
+func dummyZus(conf map[string]interface{}) (*ZipURLSource, error) {
+	zus := &ZipURLSource{}
+	y, _ := yaml.Marshal(conf)
+	c, err := common.NewConfigWithYAML(y, string(y))
+	if err != nil {
+		return nil, err
+	}
+	err = c.Unpack(zus)
+	return zus, err
+}