Run kubernetes integration tests inside of a pod and use kind to setu…

…p a kubernetes cluster (#17656)

* Use kind to bring up a local kubernetes cluster and then run the integration tests against the kind cluster.

* Run mage update.

* Add more tests.

* Fix more tests, install kind in prepare-tests.

* Switch to running kubernetes integration tests inside of Kubernetes.

* Use golang 1.13.9.

* Fix for other beats.

* Run mage fmt.

* Don't run kubernetes integration tests if not inside kubernetes environment.

* Fix metricbeat to use mage and the makefile shim for mage.

* Improve the error message when kind or kubectl is not available.

* Refactor the integration tests into a more module system.

* Fix go vet.

* Setup travis to use kind.

* Run kubernetes integration tests in Jenkins.

* Fix filebeat magefile.

* Fix travis and Jenkins.

* Check requirements of the test runner before actually running the tests.

* Add return on parsebool error.

* Don't return err on missing requirements for tester.

* Run make update.

* Move the kubernetes items to its own module, import that module only by metricbeat.

.ci/scripts/install-kind.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+DEFAULT_HOME="/usr/local"
+KIND_VERSION=${KIND_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KIND_CMD="${HOME}/bin/kind"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+chmod +x "${KIND_CMD}"

.ci/scripts/install-kubectl.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+DEFAULT_HOME="/usr/local"
+K8S_VERSION=${K8S_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KUBECTL_CMD="${HOME}/bin/kubectl"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KUBECTL_CMD}"
+

.ci/scripts/kind-setup.sh

@@ -1,18 +1,5 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-MSG="parameter missing."
-K8S_VERSION=${K8S_VERSION:?$MSG}
-HOME=${HOME:?$MSG}
-KBC_CMD="${HOME}/bin/kubectl"
-
-mkdir -p "${HOME}/bin"
-
-curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-chmod +x "${KBC_CMD}"
-
-GO111MODULE="on" go get sigs.k8s.io/kind@v0.5.1
 kind create cluster --image kindest/node:${K8S_VERSION}
-
-export KUBECONFIG="$(kind get kubeconfig-path)"
 kubectl cluster-info

.travis.yml

@@ -116,7 +116,13 @@ jobs:
       stage: test
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="-C metricbeat integration-tests"
+      install: 
+        - .ci/scripts/install-kind.sh
+        - .ci/scripts/install-kubectl.sh
+      env:
+        - TARGETS="-C metricbeat integration-tests"
+        - K8S_VERSION=v1.17.2
+        - KIND_VERSION=v0.7.0
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux

Jenkinsfile

@@ -587,7 +587,7 @@ pipeline {
             }
           }
           steps {
-            k8sTest(["v1.16.2","v1.15.3","v1.14.6","v1.13.10","v1.12.10","v1.11.10"])
+            k8sTest(["v1.18.2","v1.17.2","v1.16.4","v1.15.7","v1.14.10"])
           }
         }
       }
@@ -806,14 +806,14 @@ def dumpFilteredEnvironment(){
 def k8sTest(versions){
   versions.each{ v ->
     stage("k8s ${v}"){
-      withEnv(["K8S_VERSION=${v}"]){
+      withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){
         withGithubNotify(context: "K8s ${v}") {
           withBeatsEnv(false) {
-            sh(label: "Install k8s", script: """
-              eval "\$(gvm use ${GO_VERSION} --format=bash)"
-              .ci/scripts/kind-setup.sh
-            """)
-            sh(label: "Kubernetes Kind",script: "make KUBECONFIG=\"\$(kind get kubeconfig-path)\" -C deploy/kubernetes test")
+            sh(label: "Install kind", script: ".ci/scripts/install-kind.sh")
+            sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh")
+            sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests")
+            sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh")
+            sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test")
             sh(label: 'Delete cluster', script: 'kind delete cluster')
           }
         }

NOTICE.txt

@@ -1371,6 +1371,15 @@ License type (autodetected): Apache-2.0
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: github.com/docker/spdystream
+Revision: 449fdfce4d96
+License type (autodetected): Apache-2.0
+./vendor/github.com/docker/spdystream/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
 --------------------------------------------------------------------
 Dependency: github.com/dop251/goja
 Overwrite: github.com/andrewkroh/goja

dev-tools/mage/common.go

@@ -220,7 +220,19 @@ func dockerInfo() (*DockerInfo, error) {
 // PATH.
 func HaveDockerCompose() error {
 	_, err := exec.LookPath("docker-compose")
-	return errors.Wrap(err, "docker-compose was not found on the PATH")
+	if err != nil {
+		return fmt.Errorf("docker-compose is not available")
+	}
+	return nil
+}
+
+// HaveKubectl returns an error if kind is not found on the PATH.
+func HaveKubectl() error {
+	_, err := exec.LookPath("kubectl")
+	if err != nil {
+		return fmt.Errorf("kubectl is not available")
+	}
+	return nil
 }
 
 // FindReplace reads a file, performs a find/replace operation, then writes the

dev-tools/mage/gotest.go

@@ -26,6 +26,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -137,33 +138,47 @@ func DefaultTestBinaryArgs() TestBinaryArgs {
 // Use RACE_DETECTOR=true to enable the race detector.
 // Use MODULE=module to run only tests for `module`.
 func GoTestIntegrationForModule(ctx context.Context) error {
-	return RunIntegTest("goIntegTest", func() error {
-		module := EnvOr("MODULE", "")
-		if module != "" {
-			err := GoTest(ctx, GoTestIntegrationArgsForModule(module))
-			return errors.Wrapf(err, "integration tests failed for module %s", module)
+	module := EnvOr("MODULE", "")
+	modulesFileInfo, err := ioutil.ReadDir("./module")
+	if err != nil {
+		return err
+	}
+
+	foundModule := false
+	failedModules := []string{}
+	for _, fi := range modulesFileInfo {
+		if !fi.IsDir() {
+			continue
+		}
+		if module != "" && module != fi.Name() {
+			continue
 		}
+		foundModule = true
 
-		modulesFileInfo, err := ioutil.ReadDir("./module")
+		// Set MODULE because only want that modules tests to run inside the testing environment.
+		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), map[string]string{"MODULE": fi.Name()})
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "test setup failed for module %s", fi.Name())
 		}
-
-		var failed bool
-		for _, fi := range modulesFileInfo {
-			if !fi.IsDir() {
-				continue
-			}
+		err = runners.Test("goIntegTest", func() error {
 			err := GoTest(ctx, GoTestIntegrationArgsForModule(fi.Name()))
 			if err != nil {
-				failed = true
+				return err
 			}
+			return nil
+		})
+		if err != nil {
+			// err will already be report to stdout, collect failed module to report at end
+			failedModules = append(failedModules, fi.Name())
 		}
-		if failed {
-			return errors.New("integration tests failed")
-		}
-		return nil
-	})
+	}
+	if module != "" && !foundModule {
+		return fmt.Errorf("no module %s", module)
+	}
+	if len(failedModules) > 0 {
+		return fmt.Errorf("failed modules: %s", strings.Join(failedModules, ", "))
+	}
+	return nil
 }
 
 // GoTest invokes "go test" and reports the results to stdout. It returns an