Generate fields.yml using Mage

CHANGELOG-developer.asciidoc

@@ -25,6 +25,10 @@ The list below covers the major changes between 6.3.0 and master only.
 - Dashboards under _meta/kibana are expected to be decoded. See https://github.com/elastic/beats/pull/7224 for a conversion script. {pull}7265[7265]
 - Constructor `(github.com/elastic/beats/libbeat/output/codec/json).New` expects a new `escapeHTML` parameter. {pull}7445[7445]
 - Packaging has been refactored and updates are required. See the PR for migration details. {pull}7388[7388]
+- `make fields` has been modified to use Mage (https://magefile.org/) in an effort to make
+  the building a Beat more cross-platform friendly (e.g. Windows). This requires that your Beat
+  has a magefile.go with a fields target. The `FIELDS_FILE_PATH` make variable is no longer
+  used because the value is specified in magefile.go. {pull}7670[7670]
 
 ==== Bugfixes
 

auditbeat/Makefile

@@ -6,7 +6,6 @@ GOX_OS?=linux windows ## @Building List of all OS to be supported by "make cross
 DEV_OS?=linux
 TESTING_ENVIRONMENT?=snapshot-noxpack
 ES_BEATS?=..
-FIELDS_FILE_PATH=module
 
 # Path to the libbeat Makefile
 include ${ES_BEATS}/libbeat/scripts/Makefile

auditbeat/magefile.go

@@ -91,6 +91,11 @@ func Update() error {
 	return sh.Run("make", "update")
 }
 
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("module")
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Auditbeat.
 // - Config files are Go templates.

auditbeat/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage
+
+mage %*

dev-tools/jenkins_ci.ps1

@@ -21,6 +21,14 @@ $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH"
 # each run starts from a clean slate.
 $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
 
+exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage }
+exec { go get -u github.com/jstemmer/go-junit-report }
+
+echo "Building libbeat fields.yml"
+cd libbeat
+exec { mage fields }
+cd ..
+
 if (Test-Path "$env:beat") {
     cd "$env:beat"
 } else {
@@ -35,20 +43,11 @@ New-Item -ItemType directory -Path build\coverage | Out-Null
 New-Item -ItemType directory -Path build\system-tests | Out-Null
 New-Item -ItemType directory -Path build\system-tests\run | Out-Null
 
-exec { go get -u github.com/jstemmer/go-junit-report }
+echo "Building fields.yml"
+exec { mage fields }
 
 echo "Building $env:beat"
-exec { go build } "Build FAILURE"
-
-# always build the libbeat fields
-cp ..\libbeat\_meta\fields.common.yml ..\libbeat\_meta\fields.generated.yml
-cat ..\libbeat\processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath ..\libbeat\_meta\fields.generated.yml
-cp ..\libbeat\_meta\fields.generated.yml ..\libbeat\fields.yml
-
-if ($env:beat -eq "metricbeat") {
-    cp .\_meta\fields.common.yml .\_meta\fields.generated.yml
-    python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml
-}
+exec { mage build } "Build FAILURE"
 
 echo "Unit testing $env:beat"
 go test -v $(go list ./... | select-string -Pattern "vendor" -NotMatch) 2>&1 | Out-File -encoding UTF8 build/TEST-go-unit.out

dev-tools/mage/fields.go

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"path/filepath"
+
+	"github.com/magefile/mage/sh"
+)
+
+// GenerateFieldsYAML generates a fields.yml file for a Beat. This will include
+// the common fields specified by libbeat, the common fields for the Beat,
+// and any additional fields.yml files you specify.
+//
+// fieldsFiles specifies additional directories to search recursively for files
+// named fields.yml. The contents of each fields.yml will be included in the
+// generated file.
+func GenerateFieldsYAML(fieldsFiles ...string) error {
+	const globalFieldsCmdPath = "libbeat/scripts/cmd/global_fields/main.go"
+
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+
+	globalFieldsCmd := sh.RunCmd("go", "run",
+		filepath.Join(beatsDir, globalFieldsCmdPath),
+		"-es_beats_path", beatsDir,
+		"-beat_path", CWD(),
+	)
+
+	return globalFieldsCmd(fieldsFiles...)
+}

dev-tools/packaging/templates/common/magefile.go.tmpl

@@ -70,3 +70,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("protos")
+}

filebeat/magefile.go

@@ -91,6 +91,11 @@ func Update() error {
 	return sh.Run("make", "update")
 }
 
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("module")
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Filebeat.
 // - Include modules directory in packages (minus _meta and test files).

filebeat/make.bat

@@ -1,7 +1,11 @@
-REM Batch script to build and test on Windows. You can use this in conjunction
-REM with the Vagrant machine.
-go build
-go test ./...
-go test -c -cover -covermode=count -coverpkg ./...
-cd tests\system
-nosetests
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage
+
+mage %*

generator/beat/Makefile

@@ -1,4 +1 @@
-override FIELDS_FILE_PATH=
-export FIELDS_FILE_PATH
-
 include ../common/Makefile

generator/beat/{beat}/Makefile

@@ -13,7 +13,7 @@ MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 
 # Initial beat setup
 .PHONY: setup
-setup: copy-vendor update git-init
+setup: copy-vendor git-init update git-add
 
 # Copy beats into vendor directory
 .PHONY: copy-vendor
@@ -27,17 +27,11 @@ copy-vendor:
 .PHONY: git-init
 git-init:
 	git init
-	git add README.md CONTRIBUTING.md
-	git commit -m "Initial commit"
-	git add LICENSE.txt
-	git commit -m "Add the LICENSE"
-	git add .gitignore
-	git commit -m "Add git settings"
-	git add .
-	git reset -- .travis.yml
-	git commit -m "Add {beat}"
-	git add .travis.yml
-	git commit -m "Add Travis CI"
+
+.PHONY: git-add
+git-add:
+	git add -A
+	git commit -m "Add generated {beat} files"
 
 # Collects all dependencies and then calls update
 .PHONY: collect

generator/beat/{beat}/magefile.go

@@ -89,3 +89,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML()
+}

generator/beat/{beat}/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install {beat_path}/vendor/github.com/magefile/mage
+
+mage %*

generator/common/Makefile

@@ -11,23 +11,36 @@ PREPARE_COMMAND?=
 test: prepare-test
 	. ${PYTHON_ENV}/bin/activate; \
 	export GOPATH=${PWD}/build ; \
-	export PATH=${PATH}:${PWD}/build/bin; \
+	export PATH=$${GOPATH}/bin:${PATH}; \
 	cd ${BEAT_PATH} ; \
 	$(MAKE) copy-vendor || exit 1  ; \
 	${PREPARE_COMMAND} \
+	$(MAKE) git-init || exit 1 ; \
 	$(MAKE) update || exit 1 ; \
+	$(MAKE) git-add || exit 1 ; \
 	$(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \
 	$(MAKE) || exit 1 ; \
 	$(MAKE) unit
 
+.PHONY: prepare-test
 prepare-test:: python-env
 	# Makes sure to use current version of beats for testing
 	mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/
-	rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
+	rsync -a \
+	  --include=vendor/github.com/magefile/mage/build \
+	  --exclude=build/ \
+	  --exclude=.git/ \
+	  ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
 
 	mkdir -p ${BEAT_PATH}
 	export GOPATH=${PWD}/build ; \
-	. ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/script/generate.py --type=${BEAT_TYPE} --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin"
+	. ${PYTHON_ENV}/bin/activate && \
+	  python ${PWD}/build/src/github.com/elastic/beats/script/generate.py \
+	    --type=${BEAT_TYPE} \
+	    --project_name=Testbeat \
+	    --github_name=ruflin \
+	    --beat_path=beatpath/testbeat \
+	    --full_name="Nicolas Ruflin"
 
 # Runs test build for the created beat
 .PHONY: test-build

generator/metricbeat/Makefile

@@ -1,16 +1,14 @@
-BEAT_TYPE=metricbeat
-PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; FIELDS_FILE_PATH=module
-override FIELDS_FILE_PATH=
-export FIELDS_FILE_PATH
-
+BEAT_TYPE       = metricbeat
+PREPARE_COMMAND = MODULE=elastic METRICSET=test make create-metricset;
 
 include ../common/Makefile
 
+.PHONY: prepare-test
 prepare-test:: python-env
-
 	mkdir -p ${BEAT_PATH}/scripts
 	rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts
 
 # Collects all dependencies and then calls update
 .PHONY: collect
 collect: fields
+

generator/metricbeat/{beat}/Makefile

@@ -13,7 +13,7 @@ MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 
 # Initial beat setup
 .PHONY: setup
-setup: copy-vendor create-metricset collect git-init
+setup: copy-vendor git-init create-metricset collect git-add
 
 # Copy beats into vendor directory
 .PHONY: copy-vendor
@@ -28,5 +28,8 @@ copy-vendor:
 .PHONY: git-init
 git-init:
 	git init
-	git add --all
-	git commit -m 'Initial add by Beat generator'
+
+.PHONY: git-add
+git-add:
+	git add -A
+	git commit -m "Add generated {beat} files"

generator/metricbeat/{beat}/magefile.go

@@ -89,3 +89,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("module")
+}

generator/metricbeat/{beat}/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install {beat_path}/vendor/github.com/magefile/mage
+
+mage %*

heartbeat/Makefile

@@ -2,7 +2,6 @@ BEAT_NAME=heartbeat
 BEAT_TITLE=Heartbeat
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT=false
-FIELDS_FILE_PATH=monitors/active
 
 # Path to the libbeat Makefile
 -include ../libbeat/scripts/Makefile

heartbeat/magefile.go

@@ -88,3 +88,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("monitors/active")
+}

heartbeat/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage
+
+mage %*

libbeat/Makefile

@@ -1,7 +1,6 @@
 BEAT_NAME=libbeat
 TEST_ENVIRONMENT?=true
 SYSTEM_TESTS=true
-FIELDS_FILE_PATH=processors
 
 include scripts/Makefile