Add module infrastructure to X-Pack Metricbeat (#9242)

So far it was not possible to add modules to X-Pack Metricbeat as all the automation was not in place. With this change new modules can be added, fields.go are generated and config files are updated. There are still things missing like the docs collection.

.travis.yml

@@ -94,6 +94,10 @@ jobs:
       env: TARGETS="-C metricbeat crosscompile"
       go: $GO_VERSION
       stage: test
+    - os: linux
+      env: TARGETS="-C x-pack/metricbeat testsuite"
+      go: $GO_VERSION
+      stage: test
 
     # Packetbeat
     - os: linux

Makefile

@@ -13,12 +13,18 @@ REVIEWDOG_OPTIONS?=-diff "git diff master"
 REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
 XPACK_SUFFIX=x-pack/
 
+# PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
+# Mage. For compatibility with CI testing these projects support a subset of the
+# makefile targets. After all Beats converge to primarily using Mage we can
+# remove this and treat all sub-projects the same.
+PROJECTS_XPACK_MAGE=x-pack/filebeat x-pack/metricbeat
+
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
 # Also it builds the docs and the generators
 
 .PHONY: testsuite
 testsuite:
-	@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;)
+	@$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) testsuite || exit 1;)
 
 .PHONY: setup-commit-hook
 setup-commit-hook:
@@ -54,13 +60,13 @@ coverage-report:
 
 .PHONY: update
 update: notice
-	@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) update || exit 1;)
+	@$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) update || exit 1;)
 	@$(MAKE) -C deploy/kubernetes all
 
 .PHONY: clean
 clean:
 	@rm -rf build
-	@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;)
+	@$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) clean || exit 1;)
 	@$(MAKE) -C generator clean
 	@-mage -clean 2> /dev/null
 
@@ -72,7 +78,7 @@ clean-vendor:
 
 .PHONY: check
 check: python-env
-	@$(foreach var,$(PROJECTS) dev-tools x-pack/filebeat,$(MAKE) -C $(var) check || exit 1;)
+	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
 	@# Checks also python files which are not part of the beats
 	@$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
 	@# Validate that all updates were committed
@@ -107,7 +113,7 @@ misspell:
 
 .PHONY: fmt
 fmt: add-headers python-env
-	@$(foreach var,$(PROJECTS) dev-tools x-pack/filebeat,$(MAKE) -C $(var) fmt || exit 1;)
+	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) fmt || exit 1;)
 	@# Cleans also python files which are not part of the beats
 	@$(FIND) -name "*.py" -exec $(PYTHON_ENV)/bin/autopep8 --in-place --max-line-length 120 {} \;
 

dev-tools/mage/integtest.go

@@ -274,7 +274,7 @@ func dockerComposeBuildImages() error {
 		return err
 	}
 
-	args := []string{"build", "--pull", "--force-rm"}
+	args := []string{"-p", dockerComposeProjectName(), "build", "--pull", "--force-rm"}
 	if _, noCache := os.LookupEnv("DOCKER_NOCACHE"); noCache {
 		args = append(args, "--no-cache")
 	}

dev-tools/make/xpack.mk

@@ -4,17 +4,21 @@
 #
 # Variables
 #
-PWD           := $(CURDIR)
 .DEFAULT_GOAL := help
+PWD           := $(CURDIR)
 
 #
 # Includes
 #
 include $(ES_BEATS)/dev-tools/make/mage.mk
 
 #
-# Targets
+# Targets (alphabetically sorted).
 #
+.PHONY: check
+check: mage
+	mage check
+
 .PHONY: clean
 clean: mage
 	mage clean
@@ -23,16 +27,17 @@ clean: mage
 fmt: mage
 	mage fmt
 
-.PHONY: check
-check: mage
-	mage check
-
-.PHONY: testsuite
-testsuite: mage
-	mage update build unitTest integTest
-
 # Default target.
 .PHONY: help
 help:
 	@echo Use mage rather than make. Here are the available mage targets:
 	@mage -l
+
+.PHONY: testsuite
+testsuite: mage
+	mage update build unitTest integTest
+
+.PHONY: update
+update: mage
+	mage update
+

metricbeat/Dockerfile

@@ -1,21 +1,16 @@
-FROM golang:1.11.2
-MAINTAINER Nicolas Ruflin <ruflin@elastic.co>
+FROM golang:1.10.3
 
-RUN set -x && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv && \
-    apt-get clean
+RUN \
+    apt-get update \
+      && apt-get install -y --no-install-recommends \
+         netcat \
+         python-pip \
+         virtualenv \
+      && rm -rf /var/lib/apt/lists/*
 
 RUN pip install --upgrade pip
 RUN pip install --upgrade setuptools
 RUN pip install --upgrade docker-compose==1.21.0
 
-# Setup work environment
-ENV METRICBEAT_PATH /go/src/github.com/elastic/beats/metricbeat
-
-RUN mkdir -p $METRICBEAT_PATH/build/coverage
-WORKDIR $METRICBEAT_PATH
-
-# Add healthcheck for docker/healthcheck metricset to check during testing
+# Add healthcheck for the docker/healthcheck metricset to check during testing.
 HEALTHCHECK CMD exit 0

x-pack/metricbeat/Makefile

@@ -0,0 +1,3 @@
+ES_BEATS ?= ../..
+
+include $(ES_BEATS)/dev-tools/make/xpack.mk

x-pack/metricbeat/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '2.1'
+services:
+  beat:
+    build: ../../metricbeat
+    environment:
+      - TEST_ENVIRONMENT=false
+    working_dir: /go/src/github.com/elastic/beats/x-pack/metricbeat
+    volumes:
+      - ${PWD}/../..:/go/src/github.com/elastic/beats/
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: make