Generate fields.yml using Mage (#7670)

Make will now delegate to mage for generating fields.yml. Make will check if the mage command exists and go install it if not. The FIELDS_FILE_PATH make variable is not longer used because the path(s) are specified in magefile.go.

This allows fields.yml to be generated on Windows using Mage. The CI scripts for Windows have been updated so that fields.yml is generated for all Beats during testing.

This also adds a make.bat in each directory where building occurs to give Windows
users a starting point.

Some fixes were made to the generators because:
- rsync was excluding important source files contained in a directory
  named "build"
- the generated project needed to be `git init` before running certain
  magefile targets that detect project's root dir and import path.

CHANGELOG-developer.asciidoc

@@ -25,6 +25,10 @@ The list below covers the major changes between 6.3.0 and master only.
 - Dashboards under _meta/kibana are expected to be decoded. See https://github.com/elastic/beats/pull/7224 for a conversion script. {pull}7265[7265]
 - Constructor `(github.com/elastic/beats/libbeat/output/codec/json).New` expects a new `escapeHTML` parameter. {pull}7445[7445]
 - Packaging has been refactored and updates are required. See the PR for migration details. {pull}7388[7388]
+- `make fields` has been modified to use Mage (https://magefile.org/) in an effort to make
+  the building a Beat more cross-platform friendly (e.g. Windows). This requires that your Beat
+  has a magefile.go with a fields target. The `FIELDS_FILE_PATH` make variable is no longer
+  used because the value is specified in magefile.go. {pull}7670[7670]
 
 ==== Bugfixes
 

auditbeat/Makefile

@@ -6,7 +6,6 @@ GOX_OS?=linux windows ## @Building List of all OS to be supported by "make cross
 DEV_OS?=linux
 TESTING_ENVIRONMENT?=snapshot-noxpack
 ES_BEATS?=..
-FIELDS_FILE_PATH=module
 
 # Path to the libbeat Makefile
 include ${ES_BEATS}/libbeat/scripts/Makefile

auditbeat/magefile.go

@@ -91,6 +91,11 @@ func Update() error {
 	return sh.Run("make", "update")
 }
 
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("module")
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Auditbeat.
 // - Config files are Go templates.

auditbeat/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage
+
+mage %*

dev-tools/jenkins_ci.ps1

@@ -21,6 +21,14 @@ $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH"
 # each run starts from a clean slate.
 $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
 
+exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage }
+exec { go get -u github.com/jstemmer/go-junit-report }
+
+echo "Building libbeat fields.yml"
+cd libbeat
+exec { mage fields }
+cd ..
+
 if (Test-Path "$env:beat") {
     cd "$env:beat"
 } else {
@@ -35,20 +43,11 @@ New-Item -ItemType directory -Path build\coverage | Out-Null
 New-Item -ItemType directory -Path build\system-tests | Out-Null
 New-Item -ItemType directory -Path build\system-tests\run | Out-Null
 
-exec { go get -u github.com/jstemmer/go-junit-report }
+echo "Building fields.yml"
+exec { mage fields }
 
 echo "Building $env:beat"
-exec { go build } "Build FAILURE"
-
-# always build the libbeat fields
-cp ..\libbeat\_meta\fields.common.yml ..\libbeat\_meta\fields.generated.yml
-cat ..\libbeat\processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath ..\libbeat\_meta\fields.generated.yml
-cp ..\libbeat\_meta\fields.generated.yml ..\libbeat\fields.yml
-
-if ($env:beat -eq "metricbeat") {
-    cp .\_meta\fields.common.yml .\_meta\fields.generated.yml
-    python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml
-}
+exec { mage build } "Build FAILURE"
 
 echo "Unit testing $env:beat"
 go test -v $(go list ./... | select-string -Pattern "vendor" -NotMatch) 2>&1 | Out-File -encoding UTF8 build/TEST-go-unit.out

dev-tools/mage/fields.go

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"path/filepath"
+
+	"github.com/magefile/mage/sh"
+)
+
+// GenerateFieldsYAML generates a fields.yml file for a Beat. This will include
+// the common fields specified by libbeat, the common fields for the Beat,
+// and any additional fields.yml files you specify.
+//
+// fieldsFiles specifies additional directories to search recursively for files
+// named fields.yml. The contents of each fields.yml will be included in the
+// generated file.
+func GenerateFieldsYAML(fieldsFiles ...string) error {
+	const globalFieldsCmdPath = "libbeat/scripts/cmd/global_fields/main.go"
+
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+
+	globalFieldsCmd := sh.RunCmd("go", "run",
+		filepath.Join(beatsDir, globalFieldsCmdPath),
+		"-es_beats_path", beatsDir,
+		"-beat_path", CWD(),
+	)
+
+	return globalFieldsCmd(fieldsFiles...)
+}

dev-tools/packaging/templates/common/magefile.go.tmpl

@@ -70,3 +70,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("protos")
+}

filebeat/magefile.go

@@ -91,6 +91,11 @@ func Update() error {
 	return sh.Run("make", "update")
 }
 
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("module")
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Filebeat.
 // - Include modules directory in packages (minus _meta and test files).

filebeat/make.bat

@@ -1,7 +1,11 @@
-REM Batch script to build and test on Windows. You can use this in conjunction
-REM with the Vagrant machine.
-go build
-go test ./...
-go test -c -cover -covermode=count -coverpkg ./...
-cd tests\system
-nosetests
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage
+
+mage %*

generator/beat/Makefile

@@ -1,4 +1 @@
-override FIELDS_FILE_PATH=
-export FIELDS_FILE_PATH
-
 include ../common/Makefile

generator/beat/{beat}/Makefile

@@ -13,7 +13,7 @@ MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 
 # Initial beat setup
 .PHONY: setup
-setup: copy-vendor update git-init
+setup: copy-vendor git-init update git-add
 
 # Copy beats into vendor directory
 .PHONY: copy-vendor
@@ -27,17 +27,11 @@ copy-vendor:
 .PHONY: git-init
 git-init:
 	git init
-	git add README.md CONTRIBUTING.md
-	git commit -m "Initial commit"
-	git add LICENSE.txt
-	git commit -m "Add the LICENSE"
-	git add .gitignore
-	git commit -m "Add git settings"
-	git add .
-	git reset -- .travis.yml
-	git commit -m "Add {beat}"
-	git add .travis.yml
-	git commit -m "Add Travis CI"
+
+.PHONY: git-add
+git-add:
+	git add -A
+	git commit -m "Add generated {beat} files"
 
 # Collects all dependencies and then calls update
 .PHONY: collect

generator/beat/{beat}/magefile.go

@@ -89,3 +89,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML()
+}

generator/beat/{beat}/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install {beat_path}/vendor/github.com/magefile/mage
+
+mage %*

generator/common/Makefile

@@ -11,23 +11,36 @@ PREPARE_COMMAND?=
 test: prepare-test
 	. ${PYTHON_ENV}/bin/activate; \
 	export GOPATH=${PWD}/build ; \
-	export PATH=${PATH}:${PWD}/build/bin; \
+	export PATH=$${GOPATH}/bin:${PATH}; \
 	cd ${BEAT_PATH} ; \
 	$(MAKE) copy-vendor || exit 1  ; \
 	${PREPARE_COMMAND} \
+	$(MAKE) git-init || exit 1 ; \
 	$(MAKE) update || exit 1 ; \
+	$(MAKE) git-add || exit 1 ; \
 	$(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \
 	$(MAKE) || exit 1 ; \
 	$(MAKE) unit
 
+.PHONY: prepare-test
 prepare-test:: python-env
 	# Makes sure to use current version of beats for testing
 	mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/
-	rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
+	rsync -a \
+	  --include=vendor/github.com/magefile/mage/build \
+	  --exclude=build/ \
+	  --exclude=.git/ \
+	  ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
 
 	mkdir -p ${BEAT_PATH}
 	export GOPATH=${PWD}/build ; \
-	. ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/script/generate.py --type=${BEAT_TYPE} --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin"
+	. ${PYTHON_ENV}/bin/activate && \
+	  python ${PWD}/build/src/github.com/elastic/beats/script/generate.py \
+	    --type=${BEAT_TYPE} \
+	    --project_name=Testbeat \
+	    --github_name=ruflin \
+	    --beat_path=beatpath/testbeat \
+	    --full_name="Nicolas Ruflin"
 
 # Runs test build for the created beat
 .PHONY: test-build

generator/metricbeat/Makefile

@@ -1,16 +1,14 @@
-BEAT_TYPE=metricbeat
-PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ; FIELDS_FILE_PATH=module
-override FIELDS_FILE_PATH=
-export FIELDS_FILE_PATH
-
+BEAT_TYPE       = metricbeat
+PREPARE_COMMAND = MODULE=elastic METRICSET=test make create-metricset;
 
 include ../common/Makefile
 
+.PHONY: prepare-test
 prepare-test:: python-env
-
 	mkdir -p ${BEAT_PATH}/scripts
 	rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts
 
 # Collects all dependencies and then calls update
 .PHONY: collect
 collect: fields
+

generator/metricbeat/{beat}/Makefile

@@ -13,7 +13,7 @@ MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 
 # Initial beat setup
 .PHONY: setup
-setup: copy-vendor create-metricset collect git-init
+setup: copy-vendor git-init create-metricset collect git-add
 
 # Copy beats into vendor directory
 .PHONY: copy-vendor
@@ -28,5 +28,8 @@ copy-vendor:
 .PHONY: git-init
 git-init:
 	git init
-	git add --all
-	git commit -m 'Initial add by Beat generator'
+
+.PHONY: git-add
+git-add:
+	git add -A
+	git commit -m "Add generated {beat} files"

generator/metricbeat/{beat}/magefile.go

@@ -89,3 +89,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("module")
+}

generator/metricbeat/{beat}/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install {beat_path}/vendor/github.com/magefile/mage
+
+mage %*

heartbeat/Makefile

@@ -2,7 +2,6 @@ BEAT_NAME=heartbeat
 BEAT_TITLE=Heartbeat
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT=false
-FIELDS_FILE_PATH=monitors/active
 
 # Path to the libbeat Makefile
 -include ../libbeat/scripts/Makefile

heartbeat/magefile.go

@@ -88,3 +88,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("monitors/active")
+}

heartbeat/make.bat

@@ -0,0 +1,11 @@
+@echo off
+
+REM Windows wrapper for Mage (https://magefile.org/) that installs it
+REM to %GOPATH%\bin from the Beats vendor directory.
+REM
+REM After running this once you may invoke mage.exe directly.
+
+WHERE mage
+IF %ERRORLEVEL% NEQ 0 go install github.com/elastic/beats/vendor/github.com/magefile/mage
+
+mage %*

libbeat/Makefile

@@ -1,7 +1,6 @@
 BEAT_NAME=libbeat
 TEST_ENVIRONMENT?=true
 SYSTEM_TESTS=true
-FIELDS_FILE_PATH=processors
 
 include scripts/Makefile