WIP Generate fields, dashboards, config from mage

auditbeat/magefile.go

@@ -98,11 +98,22 @@ func Update() error {
 	return sh.Run("make", "update")
 }
 
-// Fields generates a fields.yml for the Beat.
-func Fields() error {
+// Fields generates a fields.yml and include/fields.go for the Beat.
+func Fields() {
+	mg.SerialDeps(fieldsYML, mage.GenerateAllInOneFieldsGo)
+}
+
+// fieldsYML generates a fields.yml.
+func fieldsYML() error {
 	return mage.GenerateFieldsYAML("module")
 }
 
+// Dashboards collects all the dashboards and generates index patterns.
+func Dashboards() error {
+	mg.Deps(Fields)
+	return mage.KibanaDashboards("module")
+}
+
 // GoTestUnit executes the Go unit tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.

dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -19,51 +19,73 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"os"
+	"log"
+	"path/filepath"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/kibana"
 	"github.com/elastic/beats/libbeat/version"
 )
 
+var usageText = `
+Usage: kibana_index_pattern [flags]
+  kibana_index_pattern generates Kibana index patterns from the Beat's
+  fields.yml file. It will create a index pattern file that is usable with both
+  Kibana 5.x and 6.x.
+Options:
+`[1:]
+
+var (
+	beatName       string
+	beatVersion    string
+	indexPattern   string
+	fieldsYAMLFile string
+	outputDir      string
+)
+
+func init() {
+	flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+	flag.StringVar(&beatVersion, "version", version.GetDefaultVersion(), "Beat version. (Required)")
+	flag.StringVar(&indexPattern, "index", "", "Kibana index pattern. (Required)")
+	flag.StringVar(&fieldsYAMLFile, "fields", "fields.yml", "fields.yml file containing all fields used by the Beat.")
+	flag.StringVar(&outputDir, "out", "build/kibana", "Output dir.")
+}
+
 func main() {
-	index := flag.String("index", "", "The name of the index pattern. (required)")
-	beatName := flag.String("beat-name", "", "The name of the beat. (required)")
-	beatDir := flag.String("beat-dir", "", "The local beat directory. (required)")
-	beatVersion := flag.String("version", version.GetDefaultVersion(), "The beat version.")
+	log.SetFlags(0)
 	flag.Parse()
 
-	if *index == "" {
-		fmt.Fprint(os.Stderr, "The name of the index pattern must be set.")
-		os.Exit(1)
+	if beatName == "" {
+		log.Fatal("Name of the Beat must be set (-beat).")
 	}
 
-	if *beatName == "" {
-		fmt.Fprint(os.Stderr, "The name of the beat must be set.")
-		os.Exit(1)
+	if beatVersion == "" {
+		log.Fatal("Beat version must be set (-version).")
 	}
 
-	if *beatDir == "" {
-		fmt.Fprint(os.Stderr, "The beat directory must be set.")
-		os.Exit(1)
+	if indexPattern == "" {
+		log.Fatal("Index pattern must be set (-index).")
 	}
 
 	version5, _ := common.NewVersion("5.0.0")
 	version6, _ := common.NewVersion("6.0.0")
-	versions := []*common.Version{version5, version6}
+	versions := []common.Version{*version5, *version6}
 	for _, version := range versions {
+		indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, version)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-		indexPatternGenerator, err := kibana.NewGenerator(*index, *beatName, *beatDir, *beatVersion, *version)
+		file, err := indexPattern.Generate()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
-			os.Exit(1)
+			log.Fatal(err)
 		}
-		pattern, err := indexPatternGenerator.Generate()
+
+		// Log output file location.
+		absFile, err := filepath.Abs(file)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
-			os.Exit(1)
+			absFile = file
 		}
-		fmt.Fprintf(os.Stdout, "-- The index pattern was created under %v\n", pattern)
+		log.Printf(">> The index pattern was created under %v", absFile)
 	}
 }

dev-tools/cmd/module_fields/main.go

@@ -0,0 +1,118 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+
+	"github.com/elastic/beats/libbeat/asset"
+	"github.com/elastic/beats/libbeat/generator/fields"
+	"github.com/elastic/beats/licenses"
+)
+
+var usageText = `
+Usage: module_fields [flags] [module-dir]
+  module_fields generates a fields.go file containing a copy of the module's
+  field.yml data in a format that can be embedded in Beat's binary. module-dir
+  should be the directory containing modules (e.g. filebeat/module).
+Options:
+`[1:]
+
+var (
+	beatName string
+	license  string
+)
+
+func init() {
+	flag.StringVar(&beatName, "beat", "", "Name of the beat. (Required)")
+	flag.StringVar(&license, "license", "ASL2", "License header for generated file.")
+	flag.Usage = usageFlag
+}
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+
+	if beatName == "" {
+		log.Fatal("You must use -beat to specify the beat name.")
+	}
+
+	license, err := licenses.Find(license)
+	if err != nil {
+		log.Fatalf("Invalid license specifier: %v", err)
+	}
+
+	args := flag.Args()
+	if len(args) != 1 {
+		log.Fatal("module-dir must be passed as an argument.")
+	}
+	dir := args[0]
+
+	modules, err := fields.GetModules(dir)
+	if err != nil {
+		log.Fatalf("Error fetching modules: %v", err)
+	}
+
+	for _, module := range modules {
+		files, err := fields.CollectFiles(module, dir)
+		if err != nil {
+			log.Fatalf("Error fetching files for module %v: %v", module, err)
+		}
+
+		data, err := fields.GenerateFieldsYml(files)
+		if err != nil {
+			log.Fatalf("Error fetching files for module %v: %v", module, err)
+		}
+
+		encData, err := asset.EncodeData(string(data))
+		if err != nil {
+			log.Fatalf("Error encoding the data: %v", err)
+		}
+
+		var buf bytes.Buffer
+		asset.Template.Execute(&buf, asset.Data{
+			License: license,
+			Beat:    beatName,
+			Name:    module,
+			Data:    encData,
+			Package: module,
+		})
+
+		bs, err := format.Source(buf.Bytes())
+		if err != nil {
+			log.Fatalf("Error creating golang file from template: %v", err)
+		}
+
+		err = ioutil.WriteFile(path.Join(dir, module, "fields.go"), bs, 0644)
+		if err != nil {
+			log.Fatalf("Error writing fields.go: %v", err)
+		}
+	}
+}
+
+func usageFlag() {
+	fmt.Fprintf(os.Stderr, usageText)
+	flag.PrintDefaults()
+}

dev-tools/mage/clean.go

@@ -35,10 +35,6 @@ var DefaultCleanPaths = []string{
 	"_meta/kibana.generated",
 	"_meta/kibana/5/index-pattern/{{.BeatName}}.json",
 	"_meta/kibana/6/index-pattern/{{.BeatName}}.json",
-
-	"../x-pack/{{.BeatName}}/build",
-	"../x-pack/{{.BeatName}}/{{.BeatName}}",
-	"../x-pack/{{.BeatName}}/{{.BeatName}}.exe",
 }
 
 // Clean clean generated build artifacts.