Add x-pack/filebeat to Travis CI (#9184)

This adds testing of x-pack/filebeat to travis CI. And includes x-pack/filebeat in the
top-level `make check` target.

In x-pack/filebeat I changed the integTest target to pre-build the Docker images to
avoid any timeouts in the EnsureUp() calls.

Bump the version of docker-compose on Travis to 1.22.0 the same as what's being used within
the Docker images that we build.

Add fmt and check to Mage. This adds a means to format code and check for modifications.
These targets behave as follows:

- `mage fmt` - Adds license headers, run goimports, and autopep8.
- `mage check` - Runs fmt and update. Then checks to see if there were any modifications
  using git. And checks if any nosetests files or YAML files are executable.

.editorconfig

@@ -23,7 +23,10 @@ indent_size = 4
 indent_style = space
 indent_size = 2
 
-[Makefile]
+[Makefile*]
+indent_style = tab
+
+[*.mk]
 indent_style = tab
 
 [Vagrantfile]

.travis.yml

@@ -12,7 +12,7 @@ env:
   global:
     # Cross-compile for amd64 only to speed up testing.
     - GOX_FLAGS="-arch amd64"
-    - DOCKER_COMPOSE_VERSION=1.11.1
+    - DOCKER_COMPOSE_VERSION=1.21.0
     - GO_VERSION="$(cat .go-version)"
     # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704
     - TRAVIS_MINIKUBE_VERSION=v0.25.2
@@ -34,6 +34,10 @@ jobs:
       env: TARGETS="TEST_ENVIRONMENT=0 -C filebeat testsuite"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/filebeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Heartbeat
     - os: linux

Makefile

@@ -72,7 +72,7 @@ clean-vendor:
 
 .PHONY: check
 check: python-env
-	@$(foreach var,$(PROJECTS) dev-tools,$(MAKE) -C $(var) check || exit 1;)
+	@$(foreach var,$(PROJECTS) dev-tools x-pack/filebeat,$(MAKE) -C $(var) check || exit 1;)
 	@# Checks also python files which are not part of the beats
 	@$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
 	@# Validate that all updates were committed
@@ -107,7 +107,7 @@ misspell:
 
 .PHONY: fmt
 fmt: add-headers python-env
-	@$(foreach var,$(PROJECTS) dev-tools,$(MAKE) -C $(var) fmt || exit 1;)
+	@$(foreach var,$(PROJECTS) dev-tools x-pack/filebeat,$(MAKE) -C $(var) fmt || exit 1;)
 	@# Cleans also python files which are not part of the beats
 	@$(FIND) -name "*.py" -exec $(PYTHON_ENV)/bin/autopep8 --in-place --max-line-length 120 {} \;
 

dev-tools/mage/check.go

@@ -0,0 +1,180 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/processors/dissect"
+)
+
+// Check looks for created/modified/deleted/renamed files and returns an error
+// if it finds any modifications. If executed in in verbose mode it will write
+// the results of 'git diff' to stdout to indicate what changes have been made.
+//
+// It also checks the file permissions of nosetests test cases and YAML files.
+func Check() error {
+	fmt.Println(">> check: Checking for modified files or incorrect permissions")
+
+	mg.Deps(CheckNosetestsNotExecutable, CheckYAMLNotExecutable)
+
+	changes, err := GitDiffIndex()
+	if err != nil {
+		return errors.Wrap(err, "failed to diff the git index")
+	}
+
+	if len(changes) > 0 {
+		if mg.Verbose() {
+			GitDiff()
+		}
+
+		return errors.Errorf("some files are not up-to-date. "+
+			"Run 'mage fmt update' then review and commit the changes. "+
+			"Modified: %v", changes)
+	}
+	return nil
+}
+
+// GitDiffIndex returns a list of files that differ from what is committed.
+// These could file that were created, deleted, modified, or moved.
+func GitDiffIndex() ([]string, error) {
+	// Ensure the index is updated so that diff-index gives accurate results.
+	if err := sh.Run("git", "update-index", "-q", "--refresh"); err != nil {
+		return nil, err
+	}
+
+	// git diff-index provides a list of modified files.
+	// https://www.git-scm.com/docs/git-diff-index
+	out, err := sh.Output("git", "diff-index", "HEAD", "--", ".")
+	if err != nil {
+		return nil, err
+	}
+
+	// Example formats.
+	// :100644 100644 bcd1234... 0123456... M file0
+	// :100644 100644 abcd123... 1234567... R86 file1 file3
+	d, err := dissect.New(":%{src_mode} %{dst_mode} %{src_sha1} %{dst_sha1} %{status}\t%{paths}")
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse lines.
+	var modified []string
+	s := bufio.NewScanner(bytes.NewBufferString(out))
+	for s.Scan() {
+		m, err := d.Dissect(s.Text())
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to dissect git diff-index output")
+		}
+
+		paths := strings.Split(m["paths"], "\t")
+		if len(paths) > 1 {
+			modified = append(modified, paths[1])
+		} else {
+			modified = append(modified, paths[0])
+		}
+	}
+	if err = s.Err(); err != nil {
+		return nil, err
+	}
+
+	return modified, nil
+}
+
+// GitDiff runs 'git diff' and writes the output to stdout.
+func GitDiff() error {
+	c := exec.Command("git", "--no-pager", "diff", "--minimal")
+	c.Stdin = nil
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	log.Println("exec:", strings.Join(c.Args, " "))
+	err := c.Run()
+	return err
+}
+
+// CheckNosetestsNotExecutable checks that none of the nosetests files are
+// executable. Nosetests silently skips executable .py files and we don't want
+// this to happen.
+func CheckNosetestsNotExecutable() error {
+	if runtime.GOOS == "windows" {
+		// Skip windows because it doesn't have POSIX permissions.
+		return nil
+	}
+
+	tests, err := FindFiles(nosetestsTestFiles...)
+	if err != nil {
+		return err
+	}
+
+	var executableTestFiles []string
+	for _, file := range tests {
+		info, err := os.Stat(file)
+		if err != nil {
+			return err
+		}
+
+		if info.Mode().Perm()&0111 > 0 {
+			executableTestFiles = append(executableTestFiles, file)
+		}
+	}
+
+	if len(executableTestFiles) > 0 {
+		return errors.Errorf("nosetests files cannot be executable because "+
+			"they will be skipped. Fix permissions of %v", executableTestFiles)
+	}
+	return nil
+}
+
+// CheckYAMLNotExecutable checks that no .yml or .yaml files are executable.
+func CheckYAMLNotExecutable() error {
+	if runtime.GOOS == "windows" {
+		// Skip windows because it doesn't have POSIX permissions.
+		return nil
+	}
+
+	executableYAMLFiles, err := FindFilesRecursive(func(path string, info os.FileInfo) bool {
+		switch filepath.Ext(path) {
+		default:
+			return false
+		case ".yml", ".yaml":
+			return info.Mode().Perm()&0111 > 0
+		}
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed search for YAML files")
+	}
+
+	if len(executableYAMLFiles) > 0 {
+		return errors.Errorf("YAML files cannot be executable. Fix "+
+			"permissions of %v", executableYAMLFiles)
+
+	}
+	return nil
+}

dev-tools/mage/common.go

@@ -520,6 +520,29 @@ func FindFiles(globs ...string) ([]string, error) {
 	return configFiles, nil
 }
 
+// FindFilesRecursive recursively traverses from the CWD and invokes the given
+// match function on each regular file to determine if the given path should be
+// returned as a match.
+func FindFilesRecursive(match func(path string, info os.FileInfo) bool) ([]string, error) {
+	var matches []string
+	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.Mode().IsRegular() {
+			// continue
+			return nil
+		}
+
+		if match(filepath.ToSlash(path), info) {
+			matches = append(matches, path)
+		}
+		return nil
+	})
+	return matches, err
+}
+
 // FileConcat concatenates files and writes the output to out.
 func FileConcat(out string, perm os.FileMode, files ...string) error {
 	f, err := os.OpenFile(createDir(out), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)

dev-tools/mage/fmt.go

@@ -0,0 +1,130 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+)
+
+var (
+	// GoImportsImportPath controls the import path used to install goimports.
+	GoImportsImportPath = "github.com/elastic/beats/vendor/golang.org/x/tools/cmd/goimports"
+
+	// GoImportsLocalPrefix is a string prefix matching imports that should be
+	// grouped after third-party packages.
+	GoImportsLocalPrefix = "github.com/elastic"
+
+	// GoLicenserImportPath controls the import path used to install go-licenser.
+	GoLicenserImportPath = "github.com/elastic/go-licenser"
+)
+
+// Format adds license headers, formats .go files with goimports, and formats
+// .py files with autopep8.
+func Format() {
+	// Don't run AddLicenseHeaders and GoImports concurrently because they
+	// both can modify the same files.
+	mg.Deps(AddLicenseHeaders)
+	mg.Deps(GoImports, PythonAutopep8)
+}
+
+// GoImports executes goimports against all .go files in and below the CWD. It
+// ignores vendor/ directories.
+func GoImports() error {
+	goFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool {
+		return filepath.Ext(path) == ".go" && !strings.Contains(path, "vendor/")
+	})
+	if err != nil {
+		return err
+	}
+	if len(goFiles) == 0 {
+		return nil
+	}
+
+	fmt.Println(">> fmt - goimports: Formatting Go code")
+	if err := sh.Run("go", "get", GoImportsImportPath); err != nil {
+		return err
+	}
+
+	args := append(
+		[]string{"-local", GoImportsLocalPrefix, "-l", "-w"},
+		goFiles...,
+	)
+
+	return sh.RunV("goimports", args...)
+}
+
+// PythonAutopep8 executes autopep8 on all .py files in and below the CWD. It
+// ignores build/ directories.
+func PythonAutopep8() error {
+	pyFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool {
+		return filepath.Ext(path) == ".py" && !strings.Contains(path, "build/")
+	})
+	if err != nil {
+		return err
+	}
+	if len(pyFiles) == 0 {
+		return nil
+	}
+
+	fmt.Println(">> fmt - autopep8: Formatting Python code")
+	ve, err := PythonVirtualenv()
+	if err != nil {
+		return err
+	}
+
+	autopep8, err := lookVirtualenvPath(ve, "autopep8")
+	if err != nil {
+		return err
+	}
+
+	args := append(
+		[]string{"--in-place", "--max-line-length", "120"},
+		pyFiles...,
+	)
+
+	return sh.RunV(autopep8, args...)
+}
+
+// AddLicenseHeaders adds license headers to .go files. It applies the
+// appropriate license header based on the value of mage.BeatLicense.
+func AddLicenseHeaders() error {
+	fmt.Println(">> fmt - go-licenser: Adding missing headers")
+
+	if err := sh.Run("go", "get", GoLicenserImportPath); err != nil {
+		return err
+	}
+
+	var license string
+	switch BeatLicense {
+	case "ASL 2.0":
+		license = "ASL2"
+	case "Elastic":
+		license = BeatLicense
+	default:
+		return errors.Errorf("unknown license type %v", BeatLicense)
+	}
+
+	return sh.RunV("go-licenser", "-license", license)
+}