diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 812dba7078f4..dfbf5d17bd99 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -36,6 +36,11 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Extract Elasticsearch client logic from `outputs/elasticsearch` package into new `esclientleg` package. {pull}16150[16150] - Rename `queue.BufferConfig.Events` to `queue.BufferConfig.MaxEvents`. {pull}17622[17622] - Remove `queue.Feature` and replace `queue.RegisterType` with `queue.RegisterQueueType`. {pull}17666[17666] +- The way configuration files are generated has changed to make it easier to customize parts + of the config without requiring changes to libbeat config templates. Generation is now + fully based on Go text/template and no longer uses file concatenation to generate the config. + Your magefile.go will require a change to adapt the devtool API. See the pull request for + more details. {pull}18148[18148] ==== Bugfixes diff --git a/auditbeat/_meta/common.reference.yml b/auditbeat/_meta/common.reference.yml deleted file mode 100644 index 8c9ae27e5db5..000000000000 --- a/auditbeat/_meta/common.reference.yml +++ /dev/null @@ -1,31 +0,0 @@ -########################## Auditbeat Configuration ############################# - -# This is a reference configuration file documenting all non-deprecated options -# in comments. For a shorter configuration example that contains only the most -# common options, please see auditbeat.yml in the same directory. -# -# You can find the full configuration reference here: -# https://www.elastic.co/guide/en/beats/auditbeat/index.html - -#============================ Config Reloading ================================ - -# Config reloading allows to dynamically load modules. Each file which is -# monitored must contain one or multiple modules as a list. -auditbeat.config.modules: - - # Glob pattern for configuration reloading - path: ${path.config}/modules.d/*.yml - - # Period on which files under path should be checked for changes - reload.period: 10s - - # Set to true to enable config reloading - reload.enabled: false - -# Maximum amount of time to randomly delay the start of a dataset. Use 0 to -# disable startup delay. -auditbeat.max_start_delay: 10s - -#========================== Modules configuration ============================= -auditbeat.modules: - diff --git a/auditbeat/_meta/config/auditbeat.config.modules.yml.tmpl b/auditbeat/_meta/config/auditbeat.config.modules.yml.tmpl new file mode 100644 index 000000000000..8108f3edf92e --- /dev/null +++ b/auditbeat/_meta/config/auditbeat.config.modules.yml.tmpl @@ -0,0 +1,18 @@ +{{header "Config Reloading"}} + +# Config reloading allows to dynamically load modules. Each file which is +# monitored must contain one or multiple modules as a list. +auditbeat.config.modules: + + # Glob pattern for configuration reloading + path: ${path.config}/modules.d/*.yml + + # Period on which files under path should be checked for changes + reload.period: 10s + + # Set to true to enable config reloading + reload.enabled: false + +# Maximum amount of time to randomly delay the start of a dataset. Use 0 to +# disable startup delay. +auditbeat.max_start_delay: 10s diff --git a/auditbeat/_meta/config/auditbeat.modules.yml.tmpl b/auditbeat/_meta/config/auditbeat.modules.yml.tmpl new file mode 100644 index 000000000000..6265552a9c18 --- /dev/null +++ b/auditbeat/_meta/config/auditbeat.modules.yml.tmpl @@ -0,0 +1,2 @@ +{{header "Modules configuration"}} +auditbeat.modules: diff --git a/auditbeat/_meta/beat.docker.yml b/auditbeat/_meta/config/beat.docker.yml.tmpl similarity index 100% rename from auditbeat/_meta/beat.docker.yml rename to auditbeat/_meta/config/beat.docker.yml.tmpl diff --git a/auditbeat/_meta/config/beat.reference.yml.tmpl b/auditbeat/_meta/config/beat.reference.yml.tmpl new file mode 100644 index 000000000000..ce2e0f726649 --- /dev/null +++ b/auditbeat/_meta/config/beat.reference.yml.tmpl @@ -0,0 +1,4 @@ +{{template "header.reference.yml.tmpl" .}} +{{template "auditbeat.config.modules.yml.tmpl" .}} +{{template "auditbeat.modules.yml.tmpl" .}} +{{template "config.modules.yml.tmpl" .}} diff --git a/auditbeat/_meta/config/beat.yml.tmpl b/auditbeat/_meta/config/beat.yml.tmpl new file mode 100644 index 000000000000..b90667b2b4ec --- /dev/null +++ b/auditbeat/_meta/config/beat.yml.tmpl @@ -0,0 +1,4 @@ +{{template "header.yml.tmpl" .}} +{{template "auditbeat.modules.yml.tmpl" .}} +{{template "config.modules.yml.tmpl" .}} +{{template "setup.template.yml.tmpl" .}} diff --git a/auditbeat/_meta/config/header.reference.yml.tmpl b/auditbeat/_meta/config/header.reference.yml.tmpl new file mode 100644 index 000000000000..355f8c2b50ed --- /dev/null +++ b/auditbeat/_meta/config/header.reference.yml.tmpl @@ -0,0 +1,8 @@ +########################## Auditbeat Configuration ############################# + +# This is a reference configuration file documenting all non-deprecated options +# in comments. For a shorter configuration example that contains only the most +# common options, please see auditbeat.yml in the same directory. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/auditbeat/index.html diff --git a/auditbeat/_meta/common.p1.yml b/auditbeat/_meta/config/header.yml.tmpl similarity index 80% rename from auditbeat/_meta/common.p1.yml rename to auditbeat/_meta/config/header.yml.tmpl index 9fc4f5ccbc5d..9f703af21408 100644 --- a/auditbeat/_meta/common.p1.yml +++ b/auditbeat/_meta/config/header.yml.tmpl @@ -6,7 +6,3 @@ # # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/auditbeat/index.html - -#========================== Modules configuration ============================= -auditbeat.modules: - diff --git a/auditbeat/_meta/config/setup.template.yml.tmpl b/auditbeat/_meta/config/setup.template.yml.tmpl new file mode 100644 index 000000000000..a241fcbd983c --- /dev/null +++ b/auditbeat/_meta/config/setup.template.yml.tmpl @@ -0,0 +1,5 @@ +{{header "Elasticsearch template setting"}} +setup.template.settings: + index.number_of_shards: 1 + #index.codec: best_compression + #_source.enabled: false diff --git a/auditbeat/auditbeat.docker.yml b/auditbeat/auditbeat.docker.yml index a012bbb6aad9..19c9bd1b4775 100644 --- a/auditbeat/auditbeat.docker.yml +++ b/auditbeat/auditbeat.docker.yml @@ -12,6 +12,7 @@ auditbeat.modules: - /sbin - /usr/sbin - /etc + processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 716d0fbc2c2e..bd819aa8fd44 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/auditbeat/index.html -#============================ Config Reloading ================================ +# ============================== Config Reloading ============================== # Config reloading allows to dynamically load modules. Each file which is # monitored must contain one or multiple modules as a list. @@ -26,7 +26,7 @@ auditbeat.config.modules: # disable startup delay. auditbeat.max_start_delay: 10s -#========================== Modules configuration ============================= +# =========================== Modules configuration ============================ auditbeat.modules: # The auditd module collects events from the audit framework in the Linux @@ -118,7 +118,8 @@ auditbeat.modules: #keep_null: false -#================================ General ====================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -226,7 +227,7 @@ auditbeat.modules: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -389,7 +390,7 @@ auditbeat.modules: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Auditbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -402,11 +403,11 @@ auditbeat.modules: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -547,7 +548,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -661,7 +662,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -840,7 +841,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -958,7 +959,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -992,7 +993,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -1005,7 +1006,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Auditbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -1031,11 +1032,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1079,8 +1082,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1134,7 +1136,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1148,13 +1150,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'auditbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "auditbeat" +#setup.ilm.rollover_alias: 'auditbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'auditbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1169,7 +1171,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1224,9 +1226,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1293,8 +1294,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1436,7 +1436,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1460,12 +1461,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index 0aa50de30cec..79cca537a8a1 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/auditbeat/index.html -#========================== Modules configuration ============================= +# =========================== Modules configuration ============================ auditbeat.modules: - module: auditd @@ -48,13 +48,14 @@ auditbeat.modules: - /etc -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression #_source.enabled: false -#================================ General ===================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -69,8 +70,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -82,7 +82,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -99,7 +99,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Auditbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -112,11 +112,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -129,7 +129,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -144,7 +144,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -153,7 +153,8 @@ processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -164,8 +165,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# auditbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -186,7 +187,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/auditbeat/scripts/mage/config.go b/auditbeat/scripts/mage/config.go index 91f704461232..d6a1f6b14248 100644 --- a/auditbeat/scripts/mage/config.go +++ b/auditbeat/scripts/mage/config.go @@ -61,27 +61,15 @@ func configFileParams(dirs ...string) (devtools.ConfigFileParams, error) { if len(configFiles) == 0 { return devtools.ConfigFileParams{}, errors.Errorf("no config files found in %v", globs) } + devtools.MustFileConcat("build/config.modules.yml.tmpl", 0644, configFiles...) - return devtools.ConfigFileParams{ - ShortParts: join( - devtools.OSSBeatDir("_meta/common.p1.yml"), - configFiles, - devtools.OSSBeatDir("_meta/common.p2.yml"), - devtools.LibbeatDir("_meta/config.yml.tmpl"), - ), - ReferenceParts: join( - devtools.OSSBeatDir("_meta/common.reference.yml"), - configFiles, - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - ), - DockerParts: []string{ - devtools.OSSBeatDir("_meta/beat.docker.yml"), - devtools.LibbeatDir("_meta/config.docker.yml"), - }, - ExtraVars: map[string]interface{}{ - "ArchBits": archBits, - }, - }, nil + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl")) + p.Templates = append(p.Templates, "build/config.modules.yml.tmpl") + p.ExtraVars = map[string]interface{}{ + "ArchBits": archBits, + } + return p, nil } // archBits returns the number of bit width of the GOARCH architecture value. @@ -95,16 +83,3 @@ func archBits(goarch string) int { return 64 } } - -func join(items ...interface{}) []string { - var out []string - for _, item := range items { - switch v := item.(type) { - case string: - out = append(out, v) - case []string: - out = append(out, v...) - } - } - return out -} diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 8ea81b6e9eca..d2e9f8d081ce 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -115,7 +115,7 @@ func joinMaps(args ...map[string]interface{}) map[string]interface{} { return args[0] } - var out map[string]interface{} + out := map[string]interface{}{} for _, m := range args { for k, v := range m { out[k] = v diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go index 6cabac5c9b6c..677b307926f8 100644 --- a/dev-tools/mage/config.go +++ b/dev-tools/mage/config.go @@ -20,12 +20,11 @@ package mage import ( "fmt" "io/ioutil" - "log" "os" "path/filepath" - "regexp" "sort" "strings" + "text/template" "github.com/magefile/mage/mg" @@ -64,44 +63,87 @@ func (t ConfigFileType) IsDocker() bool { return t&DockerConfigType > 0 } // ConfigFileParams defines the files that make up each config file. type ConfigFileParams struct { - ShortParts []string // List of files or globs. - ShortDeps []interface{} - ReferenceParts []string // List of files or globs. - ReferenceDeps []interface{} - DockerParts []string // List of files or globs. - DockerDeps []interface{} - ExtraVars map[string]interface{} + Templates []string // List of files or globs to load. + ExtraVars map[string]interface{} + Short, Reference, Docker ConfigParams } -// Empty checks if configuration files are set. -func (c ConfigFileParams) Empty() bool { - return len(c.ShortParts) == len(c.ReferenceDeps) && len(c.ReferenceParts) == len(c.DockerParts) && len(c.DockerParts) == 0 +type ConfigParams struct { + Template string + Deps []interface{} +} + +func DefaultConfigFileParams() ConfigFileParams { + return ConfigFileParams{ + Templates: []string{LibbeatDir("_meta/config/*.tmpl")}, + ExtraVars: map[string]interface{}{}, + Short: ConfigParams{ + Template: LibbeatDir("_meta/config/default.short.yml.tmpl"), + }, + Reference: ConfigParams{ + Template: LibbeatDir("_meta/config/default.reference.yml.tmpl"), + }, + Docker: ConfigParams{ + Template: LibbeatDir("_meta/config/default.docker.yml.tmpl"), + }, + } } // Config generates config files. Set DEV_OS and DEV_ARCH to change the target // host for the generated configs. Defaults to linux/amd64. func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error { - if args.Empty() { - args = ConfigFileParams{ - ShortParts: []string{ - OSSBeatDir("_meta/beat.yml"), - LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceParts: []string{ - OSSBeatDir("_meta/beat.reference.yml"), - LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - OSSBeatDir("_meta/beat.docker.yml"), - LibbeatDir("_meta/config.docker.yml"), - }, + // Short + if types.IsShort() { + file := filepath.Join(targetDir, BeatName+".yml") + if err := makeConfigTemplate(file, 0600, args, ShortConfigType); err != nil { + return errors.Wrap(err, "failed making short config") } } - if err := makeConfigTemplates(types, args); err != nil { - return errors.Wrap(err, "failed making config templates") + // Reference + if types.IsReference() { + file := filepath.Join(targetDir, BeatName+".reference.yml") + if err := makeConfigTemplate(file, 0644, args, ReferenceConfigType); err != nil { + return errors.Wrap(err, "failed making reference config") + } + } + + // Docker + if types.IsDocker() { + file := filepath.Join(targetDir, BeatName+".docker.yml") + if err := makeConfigTemplate(file, 0600, args, DockerConfigType); err != nil { + return errors.Wrap(err, "failed making docker config") + } } + return nil +} + +func makeConfigTemplate(destination string, mode os.FileMode, confParams ConfigFileParams, typ ConfigFileType) error { + // Determine what type to build and set some parameters. + var confFile ConfigParams + var tmplParams map[string]interface{} + switch typ { + case ShortConfigType: + confFile = confParams.Short + tmplParams = map[string]interface{}{} + case ReferenceConfigType: + confFile = confParams.Reference + tmplParams = map[string]interface{}{"Reference": true} + case DockerConfigType: + confFile = confParams.Docker + tmplParams = map[string]interface{}{"Docker": true} + default: + panic(errors.Errorf("Invalid config file type: %v", typ)) + } + + // Build the dependencies. + mg.SerialDeps(confFile.Deps...) + + // Set variables that are available in templates. + // Rather than adding more "ExcludeX"/"UseX" options consider overwriting + // one of the libbeat templates in your project by adding a file with the + // same name to your _meta/config directory. params := map[string]interface{}{ "GOOS": EnvOr("DEV_OS", "linux"), "GOARCH": EnvOr("DEV_ARCH", "amd64"), @@ -116,90 +158,58 @@ func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error "UseDockerMetadataProcessor": true, "UseKubernetesMetadataProcessor": false, "ExcludeDashboards": false, - "UseProcessorsTemplate": false, - } - for k, v := range args.ExtraVars { - params[k] = v - } - - // Short - if types.IsShort() { - file := filepath.Join(targetDir, BeatName+".yml") - fmt.Printf(">> Building %v for %v/%v\n", file, params["GOOS"], params["GOARCH"]) - if err := ExpandFile(shortTemplate, file, params); err != nil { - return errors.Wrapf(err, "failed building %v", file) - } } + params = joinMaps(params, confParams.ExtraVars, tmplParams) + funcs := joinMaps(FuncMap, template.FuncMap{ + "header": header, + "subheader": subheader, + }) - // Reference - if types.IsReference() { - file := filepath.Join(targetDir, BeatName+".reference.yml") - params["Reference"] = true - fmt.Printf(">> Building %v for %v/%v\n", file, params["GOOS"], params["GOARCH"]) - if err := ExpandFile(referenceTemplate, file, params); err != nil { - return errors.Wrapf(err, "failed building %v", file) + fmt.Printf(">> Building %v for %v/%v\n", destination, params["GOOS"], params["GOARCH"]) + var err error + tmpl := template.New("config").Option("missingkey=error").Funcs(funcs) + for _, templateGlob := range confParams.Templates { + if tmpl, err = tmpl.ParseGlob(templateGlob); err != nil { + return errors.Wrapf(err, "failed to parse config templates in %q", templateGlob) } } - // Docker - if types.IsDocker() { - file := filepath.Join(targetDir, BeatName+".docker.yml") - params["Reference"] = false - params["Docker"] = true - fmt.Printf(">> Building %v for %v/%v\n", file, params["GOOS"], params["GOARCH"]) - if err := ExpandFile(dockerTemplate, file, params); err != nil { - return errors.Wrapf(err, "failed building %v", file) - } + data, err := ioutil.ReadFile(confFile.Template) + if err != nil { + return errors.Wrapf(err, "failed to read config template %q", confFile.Template) } - return nil -} - -func makeConfigTemplates(types ConfigFileType, args ConfigFileParams) error { - var err error - - if types.IsShort() { - mg.SerialDeps(args.ShortDeps...) - if err = makeConfigTemplate(shortTemplate, 0600, args.ShortParts...); err != nil { - return err - } + tmpl, err = tmpl.Parse(string(data)) + if err != nil { + return errors.Wrap(err, "failed to parse template") } - if types.IsReference() { - mg.SerialDeps(args.ReferenceDeps...) - if err = makeConfigTemplate(referenceTemplate, 0644, args.ReferenceParts...); err != nil { - return err - } + out, err := os.OpenFile(CreateDir(destination), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err } + defer out.Close() - if types.IsDocker() { - mg.SerialDeps(args.DockerDeps...) - if err = makeConfigTemplate(dockerTemplate, 0600, args.DockerParts...); err != nil { - return err - } + if err = tmpl.Execute(out, EnvMap(params)); err != nil { + return errors.Wrapf(err, "failed building %v", destination) } return nil } -func makeConfigTemplate(destination string, mode os.FileMode, parts ...string) error { - configFiles, err := FindFiles(parts...) - if err != nil { - return errors.Wrap(err, "failed to find config templates") - } +func header(title string) string { + return makeHeading(title, "=") +} - if IsUpToDate(destination, configFiles...) { - return nil - } +func subheader(title string) string { + return makeHeading(title, "-") +} - log.Println(">> Building", destination) - if err = FileConcat(destination, mode, configFiles...); err != nil { - return err - } - if err = FindReplace(destination, regexp.MustCompile("beatname"), "{{.BeatName}}"); err != nil { - return err - } - return FindReplace(destination, regexp.MustCompile("beat-index-prefix"), "{{.BeatIndexPrefix}}") +func makeHeading(title, separator string) string { + const line = 80 + leftEquals := (line - len("# ") - len(title) - 2*len(" ")) / 2 + rightEquals := leftEquals + len(title)%2 + return "# " + strings.Repeat(separator, leftEquals) + " " + title + " " + strings.Repeat(separator, rightEquals) } const moduleConfigTemplate = ` diff --git a/filebeat/_meta/beat.docker.yml b/filebeat/_meta/config/beat.docker.yml.tmpl similarity index 98% rename from filebeat/_meta/beat.docker.yml rename to filebeat/_meta/config/beat.docker.yml.tmpl index 756c2df52179..3af3b45289c4 100644 --- a/filebeat/_meta/beat.docker.yml +++ b/filebeat/_meta/config/beat.docker.yml.tmpl @@ -2,4 +2,3 @@ filebeat.config: modules: path: ${path.config}/modules.d/*.yml reload.enabled: false - diff --git a/filebeat/_meta/config/beat.reference.yml.tmpl b/filebeat/_meta/config/beat.reference.yml.tmpl new file mode 100644 index 000000000000..542892789b11 --- /dev/null +++ b/filebeat/_meta/config/beat.reference.yml.tmpl @@ -0,0 +1,5 @@ +{{template "header.reference.yml.tmpl" .}} +{{template "config.modules.yml.tmpl" .}} +{{template "filebeat.inputs.reference.yml.tmpl" .}} +{{template "filebeat.autodiscover.reference.yml.tmpl" .}} +{{template "filebeat.global.reference.yml.tmpl" .}} diff --git a/filebeat/_meta/config/beat.yml.tmpl b/filebeat/_meta/config/beat.yml.tmpl new file mode 100644 index 000000000000..95a6019ee706 --- /dev/null +++ b/filebeat/_meta/config/beat.yml.tmpl @@ -0,0 +1,4 @@ +{{template "header.yml.tmpl" .}} +{{template "filebeat.inputs.yml.tmpl" .}} +{{template "filebeat.config.modules.yml.tmpl" .}} +{{template "setup.template.yml.tmpl" .}} diff --git a/filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl b/filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl new file mode 100644 index 000000000000..003559be0dcf --- /dev/null +++ b/filebeat/_meta/config/filebeat.autodiscover.reference.yml.tmpl @@ -0,0 +1,16 @@ +{{header "Filebeat autodiscover"}} + +# Autodiscover allows you to detect changes in the system and spawn new modules +# or inputs as they happen. + +#filebeat.autodiscover: + # List of enabled autodiscover providers +# providers: +# - type: docker +# templates: +# - condition: +# equals.docker.container.image: busybox +# config: +# - type: container +# paths: +# - /var/lib/docker/containers/${data.docker.container.id}/*.log diff --git a/filebeat/_meta/config/filebeat.config.modules.yml.tmpl b/filebeat/_meta/config/filebeat.config.modules.yml.tmpl new file mode 100644 index 000000000000..58214008d2b3 --- /dev/null +++ b/filebeat/_meta/config/filebeat.config.modules.yml.tmpl @@ -0,0 +1,11 @@ +{{header "Filebeat modules"}} + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s diff --git a/filebeat/_meta/common.reference.p2.yml b/filebeat/_meta/config/filebeat.global.reference.yml.tmpl similarity index 74% rename from filebeat/_meta/common.reference.p2.yml rename to filebeat/_meta/config/filebeat.global.reference.yml.tmpl index eb3c1c9cca4c..dccfc790a7c5 100644 --- a/filebeat/_meta/common.reference.p2.yml +++ b/filebeat/_meta/config/filebeat.global.reference.yml.tmpl @@ -1,21 +1,4 @@ -#========================== Filebeat autodiscover ============================== - -# Autodiscover allows you to detect changes in the system and spawn new modules -# or inputs as they happen. - -#filebeat.autodiscover: - # List of enabled autodiscover providers -# providers: -# - type: docker -# templates: -# - condition: -# equals.docker.container.image: busybox -# config: -# - type: container -# paths: -# - /var/lib/docker/containers/${data.docker.container.id}/*.log - -#========================= Filebeat global options ============================ +{{header "Filebeat global options"}} # Registry data path. If a relative path is used, it is considered relative to the # data path. diff --git a/filebeat/_meta/common.reference.inputs.yml b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl similarity index 100% rename from filebeat/_meta/common.reference.inputs.yml rename to filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/config/filebeat.inputs.yml.tmpl similarity index 72% rename from filebeat/_meta/common.p2.yml rename to filebeat/_meta/config/filebeat.inputs.yml.tmpl index b423f0b0494b..a7bd1b5eaa6e 100644 --- a/filebeat/_meta/common.p2.yml +++ b/filebeat/_meta/config/filebeat.inputs.yml.tmpl @@ -1,7 +1,4 @@ -# For more available modules and options, please see the filebeat.reference.yml sample -# configuration file. - -#=========================== Filebeat inputs ============================= +{{header "Filebeat inputs"}} filebeat.inputs: @@ -52,23 +49,3 @@ filebeat.inputs: # that was (not) matched before or after or as long as a pattern is not matched based on negate. # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash #multiline.match: after - - -#============================= Filebeat modules =============================== - -filebeat.config.modules: - # Glob pattern for configuration loading - path: ${path.config}/modules.d/*.yml - - # Set to true to enable config reloading - reload.enabled: false - - # Period on which files under path should be checked for changes - #reload.period: 10s - -#==================== Elasticsearch template setting ========================== - -setup.template.settings: - index.number_of_shards: 1 - #index.codec: best_compression - #_source.enabled: false diff --git a/filebeat/_meta/common.reference.p1.yml b/filebeat/_meta/config/header.reference.yml.tmpl similarity index 99% rename from filebeat/_meta/common.reference.p1.yml rename to filebeat/_meta/config/header.reference.yml.tmpl index c02e11deacb3..9c9e00b66b26 100644 --- a/filebeat/_meta/common.reference.p1.yml +++ b/filebeat/_meta/config/header.reference.yml.tmpl @@ -6,4 +6,3 @@ # # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/filebeat/index.html - diff --git a/filebeat/_meta/common.p1.yml b/filebeat/_meta/config/header.yml.tmpl similarity index 79% rename from filebeat/_meta/common.p1.yml rename to filebeat/_meta/config/header.yml.tmpl index 10b4ef6956df..d0351dc0ff15 100644 --- a/filebeat/_meta/common.p1.yml +++ b/filebeat/_meta/config/header.yml.tmpl @@ -7,3 +7,5 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/filebeat/index.html +# For more available modules and options, please see the filebeat.reference.yml sample +# configuration file. diff --git a/auditbeat/_meta/common.p2.yml b/filebeat/_meta/config/setup.template.yml.tmpl similarity index 58% rename from auditbeat/_meta/common.p2.yml rename to filebeat/_meta/config/setup.template.yml.tmpl index 468cc1d45a91..290fbc27acee 100644 --- a/auditbeat/_meta/common.p2.yml +++ b/filebeat/_meta/config/setup.template.yml.tmpl @@ -1,5 +1,5 @@ +{{header "Elasticsearch template setting"}} -#==================== Elasticsearch template setting ========================== setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 31e4d3eaa92e..06bc14e56af9 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -384,6 +384,7 @@ filebeat.modules: #input: + #=========================== Filebeat inputs ============================= # List of inputs to fetch data. @@ -760,7 +761,8 @@ filebeat.inputs: # Configure stream to filter to a specific stream: stdout, stderr or all (default) #stream: all -#========================== Filebeat autodiscover ============================== + +# =========================== Filebeat autodiscover ============================ # Autodiscover allows you to detect changes in the system and spawn new modules # or inputs as they happen. @@ -777,7 +779,7 @@ filebeat.inputs: # paths: # - /var/lib/docker/containers/${data.docker.container.id}/*.log -#========================= Filebeat global options ============================ +# ========================== Filebeat global options =========================== # Registry data path. If a relative path is used, it is considered relative to the # data path. @@ -824,7 +826,8 @@ filebeat.inputs: #reload.enabled: true #reload.period: 10s -#================================ General ====================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -932,7 +935,7 @@ filebeat.inputs: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -1095,7 +1098,7 @@ filebeat.inputs: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -1108,11 +1111,11 @@ filebeat.inputs: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -1253,7 +1256,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -1367,7 +1370,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -1546,7 +1549,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -1664,7 +1667,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -1698,7 +1701,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -1711,7 +1714,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Filebeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -1737,11 +1740,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1785,8 +1790,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1840,7 +1844,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1854,13 +1858,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'filebeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "filebeat" +#setup.ilm.rollover_alias: 'filebeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'filebeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1875,7 +1879,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1930,9 +1934,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1999,8 +2002,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -2142,7 +2144,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -2166,12 +2169,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 581e1a43f23b..51a0d40224e1 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -10,7 +10,7 @@ # For more available modules and options, please see the filebeat.reference.yml sample # configuration file. -#=========================== Filebeat inputs ============================= +# ============================== Filebeat inputs =============================== filebeat.inputs: @@ -62,8 +62,7 @@ filebeat.inputs: # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash #multiline.match: after - -#============================= Filebeat modules =============================== +# ============================== Filebeat modules ============================== filebeat.config.modules: # Glob pattern for configuration loading @@ -75,14 +74,15 @@ filebeat.config.modules: # Period on which files under path should be checked for changes #reload.period: 10s -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression #_source.enabled: false -#================================ General ===================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -97,8 +97,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -110,7 +109,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -127,7 +126,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -140,11 +139,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -157,7 +156,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -172,7 +171,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -182,7 +181,8 @@ processors: - add_docker_metadata: ~ - add_kubernetes_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -193,8 +193,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# filebeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -215,7 +215,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/filebeat/scripts/mage/config.go b/filebeat/scripts/mage/config.go index 7585c62b634a..0893fbc07439 100644 --- a/filebeat/scripts/mage/config.go +++ b/filebeat/scripts/mage/config.go @@ -18,38 +18,25 @@ package mage import ( + "github.com/magefile/mage/mg" + devtools "github.com/elastic/beats/v7/dev-tools/mage" ) -const modulesConfigYml = "build/config.modules.yml" +const modulesConfigYml = "build/config.modules.yml.tmpl" func configFileParams(moduleDirs ...string) devtools.ConfigFileParams { collectModuleConfig := func() error { return devtools.GenerateModuleReferenceConfig(modulesConfigYml, moduleDirs...) } + mg.Deps(collectModuleConfig) - return devtools.ConfigFileParams{ - ShortParts: []string{ - devtools.OSSBeatDir("_meta/common.p1.yml"), - devtools.OSSBeatDir("_meta/common.p2.yml"), - devtools.LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceDeps: []interface{}{collectModuleConfig}, - ReferenceParts: []string{ - devtools.OSSBeatDir("_meta/common.reference.p1.yml"), - modulesConfigYml, - devtools.OSSBeatDir("_meta/common.reference.inputs.yml"), - devtools.OSSBeatDir("_meta/common.reference.p2.yml"), - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - devtools.OSSBeatDir("_meta/beat.docker.yml"), - devtools.LibbeatDir("_meta/config.docker.yml"), - }, - ExtraVars: map[string]interface{}{ - "UseKubernetesMetadataProcessor": true, - }, + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl"), modulesConfigYml) + p.ExtraVars = map[string]interface{}{ + "UseKubernetesMetadataProcessor": true, } + return p } // OSSConfigFileParams returns the default ConfigFileParams for generating @@ -62,13 +49,6 @@ func OSSConfigFileParams(moduleDirs ...string) devtools.ConfigFileParams { // filebeat*.yml files. func XPackConfigFileParams() devtools.ConfigFileParams { args := configFileParams(devtools.OSSBeatDir("module"), "module") - args.ReferenceParts = []string{ - devtools.OSSBeatDir("_meta/common.reference.p1.yml"), - modulesConfigYml, - devtools.OSSBeatDir("_meta/common.reference.inputs.yml"), - "_meta/common.reference.inputs.yml", // Added only to X-Pack. - devtools.OSSBeatDir("_meta/common.reference.p2.yml"), - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - } + args.Templates = append(args.Templates, "_meta/config/*.tmpl") return args } diff --git a/generator/_templates/beat/{beat}/_meta/beat.docker.yml b/generator/_templates/beat/{beat}/_meta/config/beat.docker.yml.tmpl similarity index 100% rename from generator/_templates/beat/{beat}/_meta/beat.docker.yml rename to generator/_templates/beat/{beat}/_meta/config/beat.docker.yml.tmpl diff --git a/generator/_templates/beat/{beat}/_meta/beat.reference.yml b/generator/_templates/beat/{beat}/_meta/config/beat.reference.yml.tmpl similarity index 100% rename from generator/_templates/beat/{beat}/_meta/beat.reference.yml rename to generator/_templates/beat/{beat}/_meta/config/beat.reference.yml.tmpl diff --git a/generator/_templates/beat/{beat}/_meta/beat.yml b/generator/_templates/beat/{beat}/_meta/config/beat.yml.tmpl similarity index 100% rename from generator/_templates/beat/{beat}/_meta/beat.yml rename to generator/_templates/beat/{beat}/_meta/config/beat.yml.tmpl diff --git a/generator/_templates/beat/{beat}/magefile.go b/generator/_templates/beat/{beat}/magefile.go index d924f3c79461..28638df0f0ad 100644 --- a/generator/_templates/beat/{beat}/magefile.go +++ b/generator/_templates/beat/{beat}/magefile.go @@ -56,7 +56,9 @@ func Fields() error { // Config generates both the short/reference/docker configs. func Config() error { - return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".") + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, "_meta/config/*.tmpl") + return devtools.Config(devtools.AllConfigTypes, p, ".") } // Clean cleans all generated files and build artifacts. diff --git a/generator/_templates/metricbeat/{beat}/_meta/docker.yml b/generator/_templates/metricbeat/{beat}/_meta/config/beat.docker.yml.tmpl similarity index 100% rename from generator/_templates/metricbeat/{beat}/_meta/docker.yml rename to generator/_templates/metricbeat/{beat}/_meta/config/beat.docker.yml.tmpl diff --git a/generator/_templates/metricbeat/{beat}/_meta/reference.yml b/generator/_templates/metricbeat/{beat}/_meta/config/beat.reference.yml.tmpl similarity index 100% rename from generator/_templates/metricbeat/{beat}/_meta/reference.yml rename to generator/_templates/metricbeat/{beat}/_meta/config/beat.reference.yml.tmpl diff --git a/generator/_templates/metricbeat/{beat}/_meta/short.yml b/generator/_templates/metricbeat/{beat}/_meta/config/beat.yml.tmpl similarity index 100% rename from generator/_templates/metricbeat/{beat}/_meta/short.yml rename to generator/_templates/metricbeat/{beat}/_meta/config/beat.yml.tmpl diff --git a/generator/_templates/metricbeat/{beat}/magefile.go b/generator/_templates/metricbeat/{beat}/magefile.go index 934276e633b8..ba9f64fdefbb 100644 --- a/generator/_templates/metricbeat/{beat}/magefile.go +++ b/generator/_templates/metricbeat/{beat}/magefile.go @@ -76,13 +76,9 @@ func Config() { } func configYML() error { - customDeps := devtools.ConfigFileParams{ - ShortParts: []string{"_meta/short.yml", devtools.LibbeatDir("_meta/config.yml.tmpl")}, - ReferenceParts: []string{"_meta/reference.yml", devtools.LibbeatDir("_meta/config.reference.yml.tmpl")}, - DockerParts: []string{"_meta/docker.yml", devtools.LibbeatDir("_meta/config.docker.yml")}, - ExtraVars: map[string]interface{}{"BeatName": devtools.BeatName}, - } - return devtools.Config(devtools.AllConfigTypes, customDeps, ".") + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, "_meta/config/*.tmpl") + return devtools.Config(devtools.AllConfigTypes, p, ".") } // Clean cleans all generated files and build artifacts. diff --git a/heartbeat/_meta/beat.docker.yml b/heartbeat/_meta/config/beat.docker.yml.tmpl similarity index 99% rename from heartbeat/_meta/beat.docker.yml rename to heartbeat/_meta/config/beat.docker.yml.tmpl index f845c4a47a12..2d5c7b43afdc 100644 --- a/heartbeat/_meta/beat.docker.yml +++ b/heartbeat/_meta/config/beat.docker.yml.tmpl @@ -21,4 +21,3 @@ heartbeat.monitors: hosts: - elasticsearch - kibana - diff --git a/heartbeat/_meta/beat.reference.yml b/heartbeat/_meta/config/beat.reference.yml.tmpl similarity index 100% rename from heartbeat/_meta/beat.reference.yml rename to heartbeat/_meta/config/beat.reference.yml.tmpl diff --git a/heartbeat/_meta/beat.yml b/heartbeat/_meta/config/beat.yml.tmpl similarity index 94% rename from heartbeat/_meta/beat.yml rename to heartbeat/_meta/config/beat.yml.tmpl index 5459f28f9890..04c9b71f7a18 100644 --- a/heartbeat/_meta/beat.yml +++ b/heartbeat/_meta/config/beat.yml.tmpl @@ -33,7 +33,7 @@ heartbeat.monitors: # Total test connection and data exchange timeout #timeout: 16s -#==================== Elasticsearch template setting ========================== +{{header "Elasticsearch template setting"}} setup.template.settings: index.number_of_shards: 1 diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index baae1a24d27c..b37deb1d74e1 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -269,7 +269,7 @@ heartbeat.scheduler: # Set the scheduler it's timezone #location: '' -#================================ General ====================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -377,7 +377,7 @@ heartbeat.scheduler: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -540,7 +540,7 @@ heartbeat.scheduler: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Heartbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -553,11 +553,11 @@ heartbeat.scheduler: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -698,7 +698,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -812,7 +812,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -991,7 +991,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -1109,7 +1109,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -1143,7 +1143,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -1156,7 +1156,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Heartbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -1182,11 +1182,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1230,8 +1232,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1285,7 +1286,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1299,13 +1300,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'heartbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "heartbeat" +#setup.ilm.rollover_alias: 'heartbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'heartbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1320,7 +1321,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1375,9 +1376,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1444,8 +1444,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1587,7 +1586,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1611,12 +1611,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index aa3e1283f70f..425329e7c9aa 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -33,14 +33,14 @@ heartbeat.monitors: # Total test connection and data exchange timeout #timeout: 16s -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 index.codec: best_compression #_source.enabled: false -#================================ General ===================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -56,7 +56,7 @@ setup.template.settings: # env: staging -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -73,7 +73,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Heartbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -86,11 +86,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -103,7 +103,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -118,7 +118,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= processors: - add_observer_metadata: @@ -129,7 +129,8 @@ processors: # Lat, Lon " #location: "37.926868, -78.024902" -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -140,8 +141,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# heartbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -162,7 +163,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/heartbeat/scripts/mage/config.go b/heartbeat/scripts/mage/config.go index 5128c7d6672b..c1cc2e3f7aca 100644 --- a/heartbeat/scripts/mage/config.go +++ b/heartbeat/scripts/mage/config.go @@ -22,24 +22,13 @@ import ( ) // ConfigFileParams returns the default ConfigFileParams for generating -// packetbeat*.yml files. +// heartbeat*.yml files. func ConfigFileParams() devtools.ConfigFileParams { - return devtools.ConfigFileParams{ - ShortParts: []string{ - devtools.OSSBeatDir("_meta/beat.yml"), - devtools.LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceParts: []string{ - devtools.OSSBeatDir("_meta/beat.reference.yml"), - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - devtools.OSSBeatDir("_meta/beat.docker.yml"), - devtools.LibbeatDir("_meta/config.docker.yml"), - }, - ExtraVars: map[string]interface{}{ - "UseObserverProcessor": true, - "ExcludeDashboards": true, - }, + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl")) + p.ExtraVars = map[string]interface{}{ + "UseObserverProcessor": true, + "ExcludeDashboards": true, } + return p } diff --git a/journalbeat/_meta/beat.docker.yml b/journalbeat/_meta/config/beat.docker.yml.tmpl similarity index 97% rename from journalbeat/_meta/beat.docker.yml rename to journalbeat/_meta/config/beat.docker.yml.tmpl index b7d83bd0111c..20ae6ee130d2 100644 --- a/journalbeat/_meta/beat.docker.yml +++ b/journalbeat/_meta/config/beat.docker.yml.tmpl @@ -1,4 +1,3 @@ journalbeat.inputs: - paths: [] seek: cursor - diff --git a/journalbeat/_meta/beat.reference.yml b/journalbeat/_meta/config/beat.reference.yml.tmpl similarity index 89% rename from journalbeat/_meta/beat.reference.yml rename to journalbeat/_meta/config/beat.reference.yml.tmpl index c50755936d79..3d4bc90b6f14 100644 --- a/journalbeat/_meta/beat.reference.yml +++ b/journalbeat/_meta/config/beat.reference.yml.tmpl @@ -10,7 +10,7 @@ # For more available modules and options, please see the journalbeat.reference.yml sample # configuration file. -#=========================== Journalbeat inputs ============================= +{{header "Journalbeat inputs"}} journalbeat.inputs: # Paths that should be crawled and fetched. Possible values files and directories. @@ -44,13 +44,13 @@ journalbeat.inputs: # env: staging -#========================= Journalbeat global options ============================ +{{header "Journalbeat global options"}} #journalbeat: # Name of the registry file. If a relative path is used, it is considered relative to the # data path. #registry_file: registry -#==================== Elasticsearch template setting ========================== +{{header "Elasticsearch template setting"}} setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression diff --git a/journalbeat/_meta/beat.yml b/journalbeat/_meta/config/beat.yml.tmpl similarity index 87% rename from journalbeat/_meta/beat.yml rename to journalbeat/_meta/config/beat.yml.tmpl index 2bd2c91ce914..9410e82a9254 100644 --- a/journalbeat/_meta/beat.yml +++ b/journalbeat/_meta/config/beat.yml.tmpl @@ -10,7 +10,7 @@ # For more available modules and options, please see the journalbeat.reference.yml sample # configuration file. -#=========================== Journalbeat inputs ============================= +{{header "Journalbeat inputs"}} journalbeat.inputs: # Paths that should be crawled and fetched. Possible values files and directories. @@ -39,13 +39,13 @@ journalbeat.inputs: # env: staging -#========================= Journalbeat global options ============================ +{{header "Journalbeat global options"}} #journalbeat: # Name of the registry file. If a relative path is used, it is considered relative to the # data path. #registry_file: registry -#==================== Elasticsearch template setting ========================== +{{header "Elasticsearch template setting"}} setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index bd9cad66a871..40bae12d2a1a 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -10,7 +10,7 @@ # For more available modules and options, please see the journalbeat.reference.yml sample # configuration file. -#=========================== Journalbeat inputs ============================= +# ============================= Journalbeat inputs ============================= journalbeat.inputs: # Paths that should be crawled and fetched. Possible values files and directories. @@ -44,19 +44,19 @@ journalbeat.inputs: # env: staging -#========================= Journalbeat global options ============================ +# ========================= Journalbeat global options ========================= #journalbeat: # Name of the registry file. If a relative path is used, it is considered relative to the # data path. #registry_file: registry -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression #_source.enabled: false -#================================ General ====================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -164,7 +164,7 @@ setup.template.settings: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -327,7 +327,7 @@ setup.template.settings: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Journalbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -340,11 +340,11 @@ setup.template.settings: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -485,7 +485,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -599,7 +599,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -778,7 +778,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -896,7 +896,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -930,7 +930,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -943,7 +943,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Journalbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -969,11 +969,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1017,8 +1019,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1072,7 +1073,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1086,13 +1087,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'journalbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "journalbeat" +#setup.ilm.rollover_alias: 'journalbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'journalbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1107,7 +1108,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1162,9 +1163,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1231,8 +1231,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Journalbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1374,7 +1373,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1398,12 +1398,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml index 9767cd72a0ff..2dfa8a07f5f2 100644 --- a/journalbeat/journalbeat.yml +++ b/journalbeat/journalbeat.yml @@ -10,7 +10,7 @@ # For more available modules and options, please see the journalbeat.reference.yml sample # configuration file. -#=========================== Journalbeat inputs ============================= +# ============================= Journalbeat inputs ============================= journalbeat.inputs: # Paths that should be crawled and fetched. Possible values files and directories. @@ -39,19 +39,19 @@ journalbeat.inputs: # env: staging -#========================= Journalbeat global options ============================ +# ========================= Journalbeat global options ========================= #journalbeat: # Name of the registry file. If a relative path is used, it is considered relative to the # data path. #registry_file: registry -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression #_source.enabled: false -#================================ General ===================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -66,8 +66,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -79,7 +78,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -96,7 +95,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Journalbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -109,11 +108,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -126,7 +125,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -141,7 +140,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -150,7 +149,8 @@ processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -161,8 +161,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# journalbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Journalbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -183,7 +183,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/journalbeat/magefile.go b/journalbeat/magefile.go index 58fdf91dfbe5..f4ef3c694aad 100644 --- a/journalbeat/magefile.go +++ b/journalbeat/magefile.go @@ -231,5 +231,7 @@ func selectImage(platform string) (string, error) { // Config generates both the short/reference/docker configs. func Config() error { - return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".") + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl")) + return devtools.Config(devtools.AllConfigTypes, p, ".") } diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl deleted file mode 100644 index 64a3a81566bb..000000000000 --- a/libbeat/_meta/config.reference.yml.tmpl +++ /dev/null @@ -1,1352 +0,0 @@ - -#================================ General ====================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -# If this options is not defined, the hostname is used. -#name: - -# The tags of the shipper are included in their own field with each -# transaction published. Tags make it easy to group servers by different -# logical properties. -#tags: ["service-X", "web-tier"] - -# Optional fields that you can specify to add additional information to the -# output. Fields can be scalar values, arrays, dictionaries, or any nested -# combination of these. -#fields: -# env: staging - -# If this option is set to true, the custom fields are stored as top-level -# fields in the output document instead of being grouped under a fields -# sub-dictionary. Default is false. -#fields_under_root: false - -# Internal queue configuration for buffering events to be published. -#queue: - # Queue type by name (default 'mem') - # The memory queue will present all available events (up to the outputs - # bulk_max_size) to the output, the moment the output is ready to server - # another batch of events. - #mem: - # Max number of events the queue can buffer. - #events: 4096 - - # Hints the minimum number of events stored in the queue, - # before providing a batch of events to the outputs. - # The default value is set to 2048. - # A value of 0 ensures events are immediately available - # to be sent to the outputs. - #flush.min_events: 2048 - - # Maximum duration after which events are available to the outputs, - # if the number of events stored in the queue is < `flush.min_events`. - #flush.timeout: 1s - - # The spool queue will store events in a local spool file, before - # forwarding the events to the outputs. - # - # Beta: spooling to disk is currently a beta feature. Use with care. - # - # The spool file is a circular buffer, which blocks once the file/buffer is full. - # Events are put into a write buffer and flushed once the write buffer - # is full or the flush_timeout is triggered. - # Once ACKed by the output, events are removed immediately from the queue, - # making space for new events to be persisted. - #spool: - # The file namespace configures the file path and the file creation settings. - # Once the file exists, the `size`, `page_size` and `prealloc` settings - # will have no more effect. - #file: - # Location of spool file. The default value is ${path.data}/spool.dat. - #path: "${path.data}/spool.dat" - - # Configure file permissions if file is created. The default value is 0600. - #permissions: 0600 - - # File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB. - #size: 100MiB - - # The files page size. A file is split into multiple pages of the same size. The default value is 4KiB. - #page_size: 4KiB - - # If prealloc is set, the required space for the file is reserved using - # truncate. The default value is true. - #prealloc: true - - # Spool writer settings - # Events are serialized into a write buffer. The write buffer is flushed if: - # - The buffer limit has been reached. - # - The configured limit of buffered events is reached. - # - The flush timeout is triggered. - #write: - # Sets the write buffer size. - #buffer_size: 1MiB - - # Maximum duration after which events are flushed if the write buffer - # is not full yet. The default value is 1s. - #flush.timeout: 1s - - # Number of maximum buffered events. The write buffer is flushed once the - # limit is reached. - #flush.events: 16384 - - # Configure the on-disk event encoding. The encoding can be changed - # between restarts. - # Valid encodings are: json, ubjson, and cbor. - #codec: cbor - #read: - # Reader flush timeout, waiting for more events to become available, so - # to fill a complete batch as required by the outputs. - # If flush_timeout is 0, all available events are forwarded to the - # outputs immediately. - # The default value is 0s. - #flush.timeout: 0s - -# Sets the maximum number of CPUs that can be executing simultaneously. The -# default is the number of logical CPUs available in the system. -#max_procs: - -#================================ Processors =================================== - -# Processors are used to reduce the number of fields in the exported event or to -# enhance the event with external metadata. This section defines a list of -# processors that are applied one by one and the first one receives the initial -# event: -# -# event -> filter1 -> event1 -> filter2 ->event2 ... -# -# The supported processors are drop_fields, drop_event, include_fields, -# decode_json_fields, and add_cloud_metadata. -# -# For example, you can use the following processors to keep the fields that -# contain CPU load percentages, but remove the fields that contain CPU ticks -# values: -# -#processors: -# - include_fields: -# fields: ["cpu"] -# - drop_fields: -# fields: ["cpu.user", "cpu.system"] -# -# The following example drops the events that have the HTTP response code 200: -# -#processors: -# - drop_event: -# when: -# equals: -# http.code: 200 -# -# The following example renames the field a to b: -# -#processors: -# - rename: -# fields: -# - from: "a" -# to: "b" -# -# The following example tokenizes the string into fields: -# -#processors: -# - dissect: -# tokenizer: "%{key1} - %{key2}" -# field: "message" -# target_prefix: "dissect" -# -# The following example enriches each event with metadata from the cloud -# provider about the host machine. It works on EC2, GCE, DigitalOcean, -# Tencent Cloud, and Alibaba Cloud. -# -#processors: -# - add_cloud_metadata: ~ -# -# The following example enriches each event with the machine's local time zone -# offset from UTC. -# -#processors: -# - add_locale: -# format: offset -# -# The following example enriches each event with docker metadata, it matches -# given fields to an existing container id and adds info from that container: -# -#processors: -# - add_docker_metadata: -# host: "unix:///var/run/docker.sock" -# match_fields: ["system.process.cgroup.id"] -# match_pids: ["process.pid", "process.ppid"] -# match_source: true -# match_source_index: 4 -# match_short_id: false -# cleanup_timeout: 60 -# labels.dedot: false -# # To connect to Docker over TLS you must specify a client and CA certificate. -# #ssl: -# # certificate_authority: "/etc/pki/root/ca.pem" -# # certificate: "/etc/pki/client/cert.pem" -# # key: "/etc/pki/client/cert.key" -# -# The following example enriches each event with docker metadata, it matches -# container id from log path available in `source` field (by default it expects -# it to be /var/lib/docker/containers/*/*.log). -# -#processors: -# - add_docker_metadata: ~ -# -# The following example enriches each event with host metadata. -# -#processors: -# - add_host_metadata: ~ -# -# The following example enriches each event with process metadata using -# process IDs included in the event. -# -#processors: -# - add_process_metadata: -# match_pids: ["system.process.ppid"] -# target: system.process.parent -# -# The following example decodes fields containing JSON strings -# and replaces the strings with valid JSON objects. -# -#processors: -# - decode_json_fields: -# fields: ["field1", "field2", ...] -# process_array: false -# max_depth: 1 -# target: "" -# overwrite_keys: false -# -#processors: -# - decompress_gzip_field: -# from: "field1" -# to: "field2" -# ignore_missing: false -# fail_on_error: true -# -# The following example copies the value of message to message_copied -# -#processors: -# - copy_fields: -# fields: -# - from: message -# to: message_copied -# fail_on_error: true -# ignore_missing: false -# -# The following example truncates the value of message to 1024 bytes -# -#processors: -# - truncate_fields: -# fields: -# - message -# max_bytes: 1024 -# fail_on_error: false -# ignore_missing: true -# -# The following example preserves the raw message under event.original -# -#processors: -# - copy_fields: -# fields: -# - from: message -# to: event.original -# fail_on_error: false -# ignore_missing: true -# - truncate_fields: -# fields: -# - event.original -# max_bytes: 1024 -# fail_on_error: false -# ignore_missing: true -# -# The following example URL-decodes the value of field1 to field2 -# -#processors: -# - urldecode: -# fields: -# - from: "field1" -# to: "field2" -# ignore_missing: false -# fail_on_error: true - -#============================= Elastic Cloud ================================== - -# These settings simplify using {{.BeatName | title}} with the Elastic Cloud (https://cloud.elastic.co/). - -# The cloud.id setting overwrites the `output.elasticsearch.hosts` and -# `setup.kibana.host` options. -# You can find the `cloud.id` in the Elastic Cloud web UI. -#cloud.id: - -# The cloud.auth setting overwrites the `output.elasticsearch.username` and -# `output.elasticsearch.password` settings. The format is `:`. -#cloud.auth: - -#================================ Outputs ====================================== - -# Configure what output to use when sending the data collected by the beat. - -#-------------------------- Elasticsearch output ------------------------------- -output.elasticsearch: - # Boolean flag to enable or disable the output module. - #enabled: true - - # Array of hosts to connect to. - # Scheme and port can be left out and will be set to the default (http and 9200) - # In case you specify and additional path, the scheme is required: http://localhost:9200/path - # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 - hosts: ["localhost:9200"] - - # Set gzip compression level. - #compression_level: 0 - - # Configure escaping HTML symbols in strings. - #escape_html: false - - # Protocol - either `http` (default) or `https`. - #protocol: "https" - - # Authentication credentials - either API key or username/password. - #api_key: "id:api_key" - #username: "elastic" - #password: "changeme" - - # Dictionary of HTTP parameters to pass within the URL with index operations. - #parameters: - #param1: value1 - #param2: value2 - - # Number of workers per Elasticsearch host. - #worker: 1 - - # Optional index name. The default is "beat-index-prefix" plus date - # and generates [beat-index-prefix-]YYYY.MM.DD keys. - # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. - #index: "beat-index-prefix-%{[agent.version]}-%{+yyyy.MM.dd}" - - # Optional ingest node pipeline. By default no pipeline will be used. - #pipeline: "" - - # Optional HTTP path - #path: "/elasticsearch" - - # Custom HTTP headers to add to each request - #headers: - # X-My-Header: Contents of the header - - # Proxy server URL - #proxy_url: http://proxy:3128 - - # Whether to disable proxy settings for outgoing connections. If true, this - # takes precedence over both the proxy_url field and any environment settings - # (HTTP_PROXY, HTTPS_PROXY). The default is false. - #proxy_disable: false - - # The number of times a particular Elasticsearch index operation is attempted. If - # the indexing operation doesn't succeed after this many retries, the events are - # dropped. The default is 3. - #max_retries: 3 - - # The maximum number of events to bulk in a single Elasticsearch bulk API index request. - # The default is 50. - #bulk_max_size: 50 - - # The number of seconds to wait before trying to reconnect to Elasticsearch - # after a network error. After waiting backoff.init seconds, the Beat - # tries to reconnect. If the attempt fails, the backoff timer is increased - # exponentially up to backoff.max. After a successful connection, the backoff - # timer is reset. The default is 1s. - #backoff.init: 1s - - # The maximum number of seconds to wait before attempting to connect to - # Elasticsearch after a network error. The default is 60s. - #backoff.max: 60s - - # Configure HTTP request timeout before failing a request to Elasticsearch. - #timeout: 90 - - # Use SSL settings for HTTPS. - #ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL-based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions from 1.1 - # up to 1.3 are enabled. - #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] - - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client certificate key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the certificate key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE-based cipher suites - #ssl.curve_types: [] - - # Configure what types of renegotiation are supported. Valid options are - # never, once, and freely. Default is never. - #ssl.renegotiation: never - - # Configure a pin that can be used to do extra validation of the verified certificate chain, - # this allow you to ensure that a specific certificate is used to validate the chain of trust. - # - # The pin is a base64 encoded string of the SHA-256 fingerprint. - #ssl.ca_sha256 - - # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. - #kerberos.enabled: true - - # Authentication type to use with Kerberos. Available options: keytab, password. - #kerberos.auth_type: password - - # Path to the keytab file. It is used when auth_type is set to keytab. - #kerberos.keytab: /etc/elastic.keytab - - # Path to the Kerberos configuration. - #kerberos.config_path: /etc/krb5.conf - - # Name of the Kerberos user. - #kerberos.username: elastic - - # Password of the Kerberos user. It is used when auth_type is set to password. - #kerberos.password: changeme - - # Kerberos realm. - #kerberos.realm: ELASTIC -{{if not .ExcludeLogstash}} -#----------------------------- Logstash output --------------------------------- -#output.logstash: - # Boolean flag to enable or disable the output module. - #enabled: true - - # The Logstash hosts - #hosts: ["localhost:5044"] - - # Number of workers per Logstash host. - #worker: 1 - - # Set gzip compression level. - #compression_level: 3 - - # Configure escaping HTML symbols in strings. - #escape_html: false - - # Optional maximum time to live for a connection to Logstash, after which the - # connection will be re-established. A value of `0s` (the default) will - # disable this feature. - # - # Not yet supported for async connections (i.e. with the "pipelining" option set) - #ttl: 30s - - # Optionally load-balance events between Logstash hosts. Default is false. - #loadbalance: false - - # Number of batches to be sent asynchronously to Logstash while processing - # new batches. - #pipelining: 2 - - # If enabled only a subset of events in a batch of events is transferred per - # transaction. The number of events to be sent increases up to `bulk_max_size` - # if no error is encountered. - #slow_start: false - - # The number of seconds to wait before trying to reconnect to Logstash - # after a network error. After waiting backoff.init seconds, the Beat - # tries to reconnect. If the attempt fails, the backoff timer is increased - # exponentially up to backoff.max. After a successful connection, the backoff - # timer is reset. The default is 1s. - #backoff.init: 1s - - # The maximum number of seconds to wait before attempting to connect to - # Logstash after a network error. The default is 60s. - #backoff.max: 60s - - # Optional index name. The default index name is set to beat-index-prefix - # in all lowercase. - #index: 'beat-index-prefix' - - # SOCKS5 proxy server URL - #proxy_url: socks5://user:password@socks5-server:2233 - - # Resolve names locally when using a proxy server. Defaults to false. - #proxy_use_local_resolver: false - - # Enable SSL support. SSL is automatically enabled if any SSL setting is set. - #ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions from 1.1 - # up to 1.3 are enabled. - #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] - - # Optional SSL configuration options. SSL is off by default. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client certificate key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the Certificate Key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE-based cipher suites - #ssl.curve_types: [] - - # Configure what types of renegotiation are supported. Valid options are - # never, once, and freely. Default is never. - #ssl.renegotiation: never - - # Configure a pin that can be used to do extra validation of the verified certificate chain, - # this allow you to ensure that a specific certificate is used to validate the chain of trust. - # - # The pin is a base64 encoded string of the SHA-256 fingerprint. - #ssl.ca_sha256 - - # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, the events are typically dropped. - # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting - # and retry until all events are published. Set max_retries to a value less - # than 0 to retry until all events are published. The default is 3. - #max_retries: 3 - - # The maximum number of events to bulk in a single Logstash request. The - # default is 2048. - #bulk_max_size: 2048 - - # The number of seconds to wait for responses from the Logstash server before - # timing out. The default is 30s. - #timeout: 30s -{{end}}{{if not .ExcludeKafka}} -#------------------------------- Kafka output ---------------------------------- -#output.kafka: - # Boolean flag to enable or disable the output module. - #enabled: true - - # The list of Kafka broker addresses from which to fetch the cluster metadata. - # The cluster metadata contain the actual Kafka brokers events are published - # to. - #hosts: ["localhost:9092"] - - # The Kafka topic used for produced events. The setting can be a format string - # using any event field. To set the topic from document type use `%{[type]}`. - #topic: beats - - # The Kafka event key setting. Use format string to create a unique event key. - # By default no event key will be generated. - #key: '' - - # The Kafka event partitioning strategy. Default hashing strategy is `hash` - # using the `output.kafka.key` setting or randomly distributes events if - # `output.kafka.key` is not configured. - #partition.hash: - # If enabled, events will only be published to partitions with reachable - # leaders. Default is false. - #reachable_only: false - - # Configure alternative event field names used to compute the hash value. - # If empty `output.kafka.key` setting will be used. - # Default value is empty list. - #hash: [] - - # Authentication details. Password is required if username is set. - #username: '' - #password: '' - - # Kafka version {{.BeatName | title}} is assumed to run against. Defaults to the "1.0.0". - #version: '1.0.0' - - # Configure JSON encoding - #codec.json: - # Pretty-print JSON event - #pretty: false - - # Configure escaping HTML symbols in strings. - #escape_html: false - - # Metadata update configuration. Metadata contains leader information - # used to decide which broker to use when publishing. - #metadata: - # Max metadata request retry attempts when cluster is in middle of leader - # election. Defaults to 3 retries. - #retry.max: 3 - - # Wait time between retries during leader elections. Default is 250ms. - #retry.backoff: 250ms - - # Refresh metadata interval. Defaults to every 10 minutes. - #refresh_frequency: 10m - - # Strategy for fetching the topics metadata from the broker. Default is false. - #full: false - - # The number of concurrent load-balanced Kafka output workers. - #worker: 1 - - # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, events are typically dropped. - # Some Beats, such as Filebeat, ignore the max_retries setting and retry until - # all events are published. Set max_retries to a value less than 0 to retry - # until all events are published. The default is 3. - #max_retries: 3 - - # The maximum number of events to bulk in a single Kafka request. The default - # is 2048. - #bulk_max_size: 2048 - - # Duration to wait before sending bulk Kafka request. 0 is no delay. The default - # is 0. - #bulk_flush_frequency: 0s - - # The number of seconds to wait for responses from the Kafka brokers before - # timing out. The default is 30s. - #timeout: 30s - - # The maximum duration a broker will wait for number of required ACKs. The - # default is 10s. - #broker_timeout: 10s - - # The number of messages buffered for each Kafka broker. The default is 256. - #channel_buffer_size: 256 - - # The keep-alive period for an active network connection. If 0s, keep-alives - # are disabled. The default is 0 seconds. - #keep_alive: 0 - - # Sets the output compression codec. Must be one of none, snappy and gzip. The - # default is gzip. - #compression: gzip - - # Set the compression level. Currently only gzip provides a compression level - # between 0 and 9. The default value is chosen by the compression algorithm. - #compression_level: 4 - - # The maximum permitted size of JSON-encoded messages. Bigger messages will be - # dropped. The default value is 1000000 (bytes). This value should be equal to - # or less than the broker's message.max.bytes. - #max_message_bytes: 1000000 - - # The ACK reliability level required from broker. 0=no response, 1=wait for - # local commit, -1=wait for all replicas to commit. The default is 1. Note: - # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently - # on error. - #required_acks: 1 - - # The configurable ClientID used for logging, debugging, and auditing - # purposes. The default is "beats". - #client_id: beats - - # Enable SSL support. SSL is automatically enabled if any SSL setting is set. - #ssl.enabled: true - - # Optional SSL configuration options. SSL is off by default. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions from 1.1 - # up to 1.3 are enabled. - #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the Certificate Key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE-based cipher suites - #ssl.curve_types: [] - - # Configure what types of renegotiation are supported. Valid options are - # never, once, and freely. Default is never. - #ssl.renegotiation: never - - # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. - #kerberos.enabled: true - - # Authentication type to use with Kerberos. Available options: keytab, password. - #kerberos.auth_type: password - - # Path to the keytab file. It is used when auth_type is set to keytab. - #kerberos.keytab: /etc/security/keytabs/kafka.keytab - - # Path to the Kerberos configuration. - #kerberos.config_path: /etc/krb5.conf - - # The service name. Service principal name is contructed from - # service_name/hostname@realm. - #kerberos.service_name: kafka - - # Name of the Kerberos user. - #kerberos.username: elastic - - # Password of the Kerberos user. It is used when auth_type is set to password. - #kerberos.password: changeme - - # Kerberos realm. - #kerberos.realm: ELASTIC -{{end}}{{if not .ExcludeRedis}} -#------------------------------- Redis output ---------------------------------- -#output.redis: - # Boolean flag to enable or disable the output module. - #enabled: true - - # Configure JSON encoding - #codec.json: - # Pretty print json event - #pretty: false - - # Configure escaping HTML symbols in strings. - #escape_html: false - - # The list of Redis servers to connect to. If load-balancing is enabled, the - # events are distributed to the servers in the list. If one server becomes - # unreachable, the events are distributed to the reachable servers only. - # The hosts setting supports redis and rediss urls with custom password like - # redis://:password@localhost:6379. - #hosts: ["localhost:6379"] - - # The name of the Redis list or channel the events are published to. The - # default is {{.BeatName}}. - #key: {{.BeatName}} - - # The password to authenticate to Redis with. The default is no authentication. - #password: - - # The Redis database number where the events are published. The default is 0. - #db: 0 - - # The Redis data type to use for publishing events. If the data type is list, - # the Redis RPUSH command is used. If the data type is channel, the Redis - # PUBLISH command is used. The default value is list. - #datatype: list - - # The number of workers to use for each host configured to publish events to - # Redis. Use this setting along with the loadbalance option. For example, if - # you have 2 hosts and 3 workers, in total 6 workers are started (3 for each - # host). - #worker: 1 - - # If set to true and multiple hosts or workers are configured, the output - # plugin load balances published events onto all Redis hosts. If set to false, - # the output plugin sends all events to only one host (determined at random) - # and will switch to another host if the currently selected one becomes - # unreachable. The default value is true. - #loadbalance: true - - # The Redis connection timeout in seconds. The default is 5 seconds. - #timeout: 5s - - # The number of times to retry publishing an event after a publishing failure. - # After the specified number of retries, the events are typically dropped. - # Some Beats, such as Filebeat, ignore the max_retries setting and retry until - # all events are published. Set max_retries to a value less than 0 to retry - # until all events are published. The default is 3. - #max_retries: 3 - - # The number of seconds to wait before trying to reconnect to Redis - # after a network error. After waiting backoff.init seconds, the Beat - # tries to reconnect. If the attempt fails, the backoff timer is increased - # exponentially up to backoff.max. After a successful connection, the backoff - # timer is reset. The default is 1s. - #backoff.init: 1s - - # The maximum number of seconds to wait before attempting to connect to - # Redis after a network error. The default is 60s. - #backoff.max: 60s - - # The maximum number of events to bulk in a single Redis request or pipeline. - # The default is 2048. - #bulk_max_size: 2048 - - # The URL of the SOCKS5 proxy to use when connecting to the Redis servers. The - # value must be a URL with a scheme of socks5://. - #proxy_url: - - # This option determines whether Redis hostnames are resolved locally when - # using a proxy. The default value is false, which means that name resolution - # occurs on the proxy server. - #proxy_use_local_resolver: false - - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. - #ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions from 1.1 - # up to 1.3 are enabled. - #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] - - # Optional SSL configuration options. SSL is off by default. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the Certificate Key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE based cipher suites - #ssl.curve_types: [] - - # Configure what types of renegotiation are supported. Valid options are - # never, once, and freely. Default is never. - #ssl.renegotiation: never -{{end}}{{if not .ExcludeFileOutput}} -#------------------------------- File output ----------------------------------- -#output.file: - # Boolean flag to enable or disable the output module. - #enabled: true - - # Configure JSON encoding - #codec.json: - # Pretty-print JSON event - #pretty: false - - # Configure escaping HTML symbols in strings. - #escape_html: false - - # Path to the directory where to save the generated files. The option is - # mandatory. - #path: "/tmp/{{.BeatName}}" - - # Name of the generated files. The default is `{{.BeatName}}` and it generates - # files: `{{.BeatName}}`, `{{.BeatName}}.1`, `{{.BeatName}}.2`, etc. - #filename: {{.BeatName}} - - # Maximum size in kilobytes of each file. When this size is reached, and on - # every {{.BeatName | title}} restart, the files are rotated. The default value is 10240 - # kB. - #rotate_every_kb: 10000 - - # Maximum number of files under path. When this number of files is reached, - # the oldest file is deleted and the rest are shifted from last to first. The - # default is 7 files. - #number_of_files: 7 - - # Permissions to use for file creation. The default is 0600. - #permissions: 0600 -{{end}}{{if not .ExcludeConsole}} -#----------------------------- Console output --------------------------------- -#output.console: - # Boolean flag to enable or disable the output module. - #enabled: true - - # Configure JSON encoding - #codec.json: - # Pretty-print JSON event - #pretty: false - - # Configure escaping HTML symbols in strings. - #escape_html: false -{{end}} -#================================= Paths ====================================== - -# The home path for the {{.BeatName | title}} installation. This is the default base path -# for all other path settings and for miscellaneous files that come with the -# distribution (for example, the sample dashboards). -# If not set by a CLI flag or in the configuration file, the default for the -# home path is the location of the binary. -#path.home: - -# The configuration path for the {{.BeatName | title}} installation. This is the default -# base path for configuration files, including the main YAML configuration file -# and the Elasticsearch template file. If not set by a CLI flag or in the -# configuration file, the default for the configuration path is the home path. -#path.config: ${path.home} - -# The data path for the {{.BeatName | title}} installation. This is the default base path -# for all the files in which {{.BeatName | title}} needs to store its data. If not set by a -# CLI flag or in the configuration file, the default for the data path is a data -# subdirectory inside the home path. -#path.data: ${path.home}/data - -# The logs path for a {{.BeatName | title}} installation. This is the default location for -# the Beat's log files. If not set by a CLI flag or in the configuration file, -# the default for the logs path is a logs subdirectory inside the home path. -#path.logs: ${path.home}/logs - -#================================ Keystore ========================================== -# Location of the Keystore containing the keys and their sensitive values. -#keystore.path: "${path.config}/beats.keystore" - -#============================== Dashboards ===================================== -# These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards are disabled by default and can be enabled either by setting the -# options here, or by using the `-setup` CLI flag or the `setup` command. -#setup.dashboards.enabled: false - -# The directory from where to read the dashboards. The default is the `kibana` -# folder in the home path. -#setup.dashboards.directory: ${path.home}/kibana - -# The URL from where to download the dashboards archive. It is used instead of -# the directory if it has a value. -#setup.dashboards.url: - -# The file archive (zip file) from where to read the dashboards. It is used instead -# of the directory when it has a value. -#setup.dashboards.file: - -# In case the archive contains the dashboards from multiple Beats, this lets you -# select which one to load. You can load all the dashboards in the archive by -# setting this to the empty string. -#setup.dashboards.beat: {{.BeatName}} - -# The name of the Kibana index to use for setting the configuration. Default is ".kibana" -#setup.dashboards.kibana_index: .kibana - -# The Elasticsearch index name. This overwrites the index name defined in the -# dashboards and index pattern. Example: testbeat-* -#setup.dashboards.index: - -# Always use the Kibana API for loading the dashboards instead of autodetecting -# how to install the dashboards by first querying Elasticsearch. -#setup.dashboards.always_kibana: false - -# If true and Kibana is not reachable at the time when dashboards are loaded, -# it will retry to reconnect to Kibana instead of exiting with an error. -#setup.dashboards.retry.enabled: false - -# Duration interval between Kibana connection retries. -#setup.dashboards.retry.interval: 1s - -# Maximum number of retries before exiting with an error, 0 for unlimited retrying. -#setup.dashboards.retry.maximum: 0 - - -#============================== Template ===================================== - -# A template is used to set the mapping in Elasticsearch -# By default template loading is enabled and the template is loaded. -# These settings can be adjusted to load your own template or overwrite existing ones. - -# Set to false to disable template loading. -#setup.template.enabled: true - -# Template name. By default the template name is "beat-index-prefix-%{[agent.version]}" -# The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.name: "beat-index-prefix-%{[agent.version]}" - -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. -# The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "beat-index-prefix-%{[agent.version]}-*" - -# Path to fields.yml file to generate the template -#setup.template.fields: "${path.config}/fields.yml" - -# A list of fields to be added to the template and Kibana index pattern. Also -# specify setup.template.overwrite: true to overwrite the existing template. -#setup.template.append_fields: -#- name: field_name -# type: field_type - -# Enable JSON template loading. If this is enabled, the fields.yml is ignored. -#setup.template.json.enabled: false - -# Path to the JSON template file -#setup.template.json.path: "${path.config}/template.json" - -# Name under which the template is stored in Elasticsearch -#setup.template.json.name: "" - -# Overwrite existing template -#setup.template.overwrite: false - -# Elasticsearch template settings -setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - #index: - #number_of_shards: 1 - #codec: best_compression - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false - -#============================== Setup ILM ===================================== - -# Configure index lifecycle management (ILM). These settings create a write -# alias and add additional settings to the index template. When ILM is enabled, -# output.elasticsearch.index is ignored, and the write alias is used to set the -# index name. - -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'beatname-%{[agent.version]}'. -#setup.ilm.rollover_alias: "beat-index-prefix" - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" - -# Set the lifecycle policy name. The default policy name is -# 'beatname'. -#setup.ilm.policy_name: "mypolicy" - -# The path to a JSON file that contains a lifecycle policy configuration. Used -# to load your own lifecycle policy. -#setup.ilm.policy_file: - -# Disable the check for an existing lifecycle policy. The default is true. If -# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy -# can be installed. -#setup.ilm.check_exists: true - -# Overwrite the lifecycle policy at startup. The default is false. -#setup.ilm.overwrite: false - -#============================== Kibana ===================================== - -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. -# This requires a Kibana endpoint configuration. -setup.kibana: - - # Kibana Host - # Scheme and port can be left out and will be set to the default (http and 5601) - # In case you specify and additional path, the scheme is required: http://localhost:5601/path - # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 - #host: "localhost:5601" - - # Optional protocol and basic auth credentials. - #protocol: "https" - #username: "elastic" - #password: "changeme" - - # Optional HTTP path - #path: "" - - # Optional Kibana space ID. - #space.id: "" - - # Use SSL settings for HTTPS. Default is true. - #ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions from 1.1 - # up to 1.3 are enabled. - #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] - - # SSL configuration. The default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client certificate key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the certificate key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE-based cipher suites - #ssl.curve_types: [] - - - -#================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to stderr. The default is false. -#logging.to_stderr: false - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - #path: /var/log/{{.BeatName}} - - # The name of the files where the logs are written to. - #name: {{.BeatName}} - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - #keepfiles: 7 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - - # Enable log file rotation on time intervals in addition to size-based rotation. - # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h - # are boundary-aligned with minutes, hours, days, weeks, months, and years as - # reported by the local system clock. All other intervals are calculated from the - # Unix epoch. Defaults to disabled. - #interval: 0 - - # Rotate existing logs on startup rather than appending to the existing - # file. Defaults to true. - # rotateonstartup: true - -# Set to true to log messages in JSON format. -#logging.json: false - - -#============================== X-Pack Monitoring =============================== -# {{.BeatName | title}} can export internal metrics to a central Elasticsearch monitoring -# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The -# reporting is disabled by default. - -# Set to true to enable the monitoring reporter. -#monitoring.enabled: false - -# Sets the UUID of the Elasticsearch cluster under which monitoring data for this -# {{.BeatName | title}} instance will appear in the Stack Monitoring UI. If output.elasticsearch -# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. -#monitoring.cluster_uuid: - -# Uncomment to send the metrics to Elasticsearch. Most settings from the -# Elasticsearch output are accepted here as well. -# Note that the settings should point to your Elasticsearch *monitoring* cluster. -# Any setting that is not set is automatically inherited from the Elasticsearch -# output configuration, so if you have the Elasticsearch output configured such -# that it is pointing to your Elasticsearch monitoring cluster, you can simply -# uncomment the following line. -#monitoring.elasticsearch: - - # Array of hosts to connect to. - # Scheme and port can be left out and will be set to the default (http and 9200) - # In case you specify and additional path, the scheme is required: http://localhost:9200/path - # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 - #hosts: ["localhost:9200"] - - # Set gzip compression level. - #compression_level: 0 - - # Protocol - either `http` (default) or `https`. - #protocol: "https" - - # Authentication credentials - either API key or username/password. - #api_key: "id:api_key" - #username: "beats_system" - #password: "changeme" - - # Dictionary of HTTP parameters to pass within the URL with index operations. - #parameters: - #param1: value1 - #param2: value2 - - # Custom HTTP headers to add to each request - #headers: - # X-My-Header: Contents of the header - - # Proxy server url - #proxy_url: http://proxy:3128 - - # The number of times a particular Elasticsearch index operation is attempted. If - # the indexing operation doesn't succeed after this many retries, the events are - # dropped. The default is 3. - #max_retries: 3 - - # The maximum number of events to bulk in a single Elasticsearch bulk API index request. - # The default is 50. - #bulk_max_size: 50 - - # The number of seconds to wait before trying to reconnect to Elasticsearch - # after a network error. After waiting backoff.init seconds, the Beat - # tries to reconnect. If the attempt fails, the backoff timer is increased - # exponentially up to backoff.max. After a successful connection, the backoff - # timer is reset. The default is 1s. - #backoff.init: 1s - - # The maximum number of seconds to wait before attempting to connect to - # Elasticsearch after a network error. The default is 60s. - #backoff.max: 60s - - # Configure HTTP request timeout before failing an request to Elasticsearch. - #timeout: 90 - - # Use SSL settings for HTTPS. - #ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions from 1.1 - # up to 1.3 are enabled. - #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] - - # SSL configuration. The default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client certificate key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the certificate key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE-based cipher suites - #ssl.curve_types: [] - - # Configure what types of renegotiation are supported. Valid options are - # never, once, and freely. Default is never. - #ssl.renegotiation: never - - # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. - #kerberos.enabled: true - - # Authentication type to use with Kerberos. Available options: keytab, password. - #kerberos.auth_type: password - - # Path to the keytab file. It is used when auth_type is set to keytab. - #kerberos.keytab: /etc/elastic.keytab - - # Path to the Kerberos configuration. - #kerberos.config_path: /etc/krb5.conf - - # Name of the Kerberos user. - #kerberos.username: elastic - - # Password of the Kerberos user. It is used when auth_type is set to password. - #kerberos.password: changeme - - # Kerberos realm. - #kerberos.realm: ELASTIC - - #metrics.period: 10s - #state.period: 1m - -# The `monitoring.cloud.id` setting overwrites the `monitoring.elasticsearch.hosts` -# setting. You can find the value for this setting in the Elastic Cloud web UI. -#monitoring.cloud.id: - -# The `monitoring.cloud.auth` setting overwrites the `monitoring.elasticsearch.username` -# and `monitoring.elasticsearch.password` settings. The format is `:`. -#monitoring.cloud.auth: - -#================================ HTTP Endpoint ====================================== -# Each beat can expose internal metrics through a HTTP endpoint. For security -# reasons the endpoint is disabled by default. This feature is currently experimental. -# Stats can be access through http://localhost:5066/stats . For pretty JSON output -# append ?pretty to the URL. - -# Defines if the HTTP endpoint is enabled. -#http.enabled: false - -# The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. -# When using IP addresses, it is recommended to only use localhost. -#http.host: localhost - -# Port on which the HTTP endpoint will bind. Default is 5066. -#http.port: 5066 - -# Define which user should be owning the named pipe. -#http.named_pipe.user: - -# Define which the permissions that should be applied to the named pipe, use the Security -# Descriptor Definition Language (SDDL) to define the permission. This option cannot be used with -# `http.user`. -#http.named_pipe.security_descriptor: - -#============================= Process Security ================================ - -# Enable or disable seccomp system call filtering on Linux. Default is enabled. -#seccomp.enabled: true - -#================================= Migration ================================== - -# This allows to enable 6.7 migration aliases -#migration.6_to_7.enabled: false diff --git a/libbeat/_meta/config.docker.yml b/libbeat/_meta/config/default.docker.yml.tmpl similarity index 83% rename from libbeat/_meta/config.docker.yml rename to libbeat/_meta/config/default.docker.yml.tmpl index 6ce79dc1c427..720aafc117df 100644 --- a/libbeat/_meta/config.docker.yml +++ b/libbeat/_meta/config/default.docker.yml.tmpl @@ -1,3 +1,4 @@ +{{block "beat.docker.yml.tmpl" .}}{{end}} processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ diff --git a/libbeat/_meta/config/default.reference.yml.tmpl b/libbeat/_meta/config/default.reference.yml.tmpl new file mode 100644 index 000000000000..c7c75d51cf47 --- /dev/null +++ b/libbeat/_meta/config/default.reference.yml.tmpl @@ -0,0 +1,22 @@ +{{block "beat.reference.yml.tmpl" . }}{{end}} +{{template "general.reference.yml.tmpl" .}} +{{template "processors.reference.yml.tmpl" .}} +{{template "elastic-cloud.yml.tmpl" .}} +{{template "outputs.yml.tmpl" .}} +{{template "output-elasticsearch.reference.yml.tmpl" .}} +{{template "output-logstash.reference.yml.tmpl" .}} +{{if not .ExcludeKafka}}{{template "output-kafka.reference.yml.tmpl" .}}{{end}} +{{if not .ExcludeRedis}}{{template "output-redis.reference.yml.tmpl" .}}{{end}} +{{if not .ExcludeFileOutput}}{{template "output-file.reference.yml.tmpl" .}}{{end}} +{{if not .ExcludeConsole}}{{template "output-console.reference.yml.tmpl" .}}{{end}} +{{template "paths.reference.yml.tmpl" .}} +{{template "keystore.reference.yml.tmpl" .}} +{{template "setup.dashboards.reference.yml.tmpl" .}} +{{template "setup.template.reference.yml.tmpl" .}} +{{template "setup.ilm.reference.yml.tmpl" .}} +{{template "setup.kibana.reference.yml.tmpl" .}} +{{template "logging.reference.yml.tmpl" .}} +{{template "monitoring.reference.yml.tmpl" .}} +{{template "http.reference.yml.tmpl" .}} +{{template "seccomp.reference.yml.tmpl" .}} +{{template "migration.yml.tmpl" .}} diff --git a/libbeat/_meta/config/default.short.yml.tmpl b/libbeat/_meta/config/default.short.yml.tmpl new file mode 100644 index 000000000000..c18d2abd8a57 --- /dev/null +++ b/libbeat/_meta/config/default.short.yml.tmpl @@ -0,0 +1,12 @@ +{{block "beat.yml.tmpl" .}}{{end}} +{{template "general.yml.tmpl" .}} +{{if not .ExcludeDashboards}}{{template "setup.dashboards.yml.tmpl" .}}{{end}} +{{template "setup.kibana.yml.tmpl" .}} +{{template "elastic-cloud.yml.tmpl" .}} +{{template "outputs.yml.tmpl" .}} +{{template "output-elasticsearch.yml.tmpl" .}} +{{template "output-logstash.yml.tmpl" .}} +{{template "processors.yml.tmpl" .}} +{{template "logging.yml.tmpl" .}} +{{template "monitoring.yml.tmpl" .}} +{{template "migration.yml.tmpl" .}} diff --git a/libbeat/_meta/config/elastic-cloud.yml.tmpl b/libbeat/_meta/config/elastic-cloud.yml.tmpl new file mode 100644 index 000000000000..f736f6ff659e --- /dev/null +++ b/libbeat/_meta/config/elastic-cloud.yml.tmpl @@ -0,0 +1,12 @@ +{{header "Elastic Cloud"}} + +# These settings simplify using {{ .BeatName | title }} with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: diff --git a/libbeat/_meta/config/general.reference.yml.tmpl b/libbeat/_meta/config/general.reference.yml.tmpl new file mode 100644 index 000000000000..8500d01c39e1 --- /dev/null +++ b/libbeat/_meta/config/general.reference.yml.tmpl @@ -0,0 +1,107 @@ +{{header "General"}} + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Internal queue configuration for buffering events to be published. +#queue: + # Queue type by name (default 'mem') + # The memory queue will present all available events (up to the outputs + # bulk_max_size) to the output, the moment the output is ready to server + # another batch of events. + #mem: + # Max number of events the queue can buffer. + #events: 4096 + + # Hints the minimum number of events stored in the queue, + # before providing a batch of events to the outputs. + # The default value is set to 2048. + # A value of 0 ensures events are immediately available + # to be sent to the outputs. + #flush.min_events: 2048 + + # Maximum duration after which events are available to the outputs, + # if the number of events stored in the queue is < `flush.min_events`. + #flush.timeout: 1s + + # The spool queue will store events in a local spool file, before + # forwarding the events to the outputs. + # + # Beta: spooling to disk is currently a beta feature. Use with care. + # + # The spool file is a circular buffer, which blocks once the file/buffer is full. + # Events are put into a write buffer and flushed once the write buffer + # is full or the flush_timeout is triggered. + # Once ACKed by the output, events are removed immediately from the queue, + # making space for new events to be persisted. + #spool: + # The file namespace configures the file path and the file creation settings. + # Once the file exists, the `size`, `page_size` and `prealloc` settings + # will have no more effect. + #file: + # Location of spool file. The default value is ${path.data}/spool.dat. + #path: "${path.data}/spool.dat" + + # Configure file permissions if file is created. The default value is 0600. + #permissions: 0600 + + # File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB. + #size: 100MiB + + # The files page size. A file is split into multiple pages of the same size. The default value is 4KiB. + #page_size: 4KiB + + # If prealloc is set, the required space for the file is reserved using + # truncate. The default value is true. + #prealloc: true + + # Spool writer settings + # Events are serialized into a write buffer. The write buffer is flushed if: + # - The buffer limit has been reached. + # - The configured limit of buffered events is reached. + # - The flush timeout is triggered. + #write: + # Sets the write buffer size. + #buffer_size: 1MiB + + # Maximum duration after which events are flushed if the write buffer + # is not full yet. The default value is 1s. + #flush.timeout: 1s + + # Number of maximum buffered events. The write buffer is flushed once the + # limit is reached. + #flush.events: 16384 + + # Configure the on-disk event encoding. The encoding can be changed + # between restarts. + # Valid encodings are: json, ubjson, and cbor. + #codec: cbor + #read: + # Reader flush timeout, waiting for more events to become available, so + # to fill a complete batch as required by the outputs. + # If flush_timeout is 0, all available events are forwarded to the + # outputs immediately. + # The default value is 0s. + #flush.timeout: 0s + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: diff --git a/libbeat/_meta/config/general.yml.tmpl b/libbeat/_meta/config/general.yml.tmpl new file mode 100644 index 000000000000..1c85044c1b93 --- /dev/null +++ b/libbeat/_meta/config/general.yml.tmpl @@ -0,0 +1,14 @@ +{{header "General"}} + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. +#fields: +# env: staging diff --git a/libbeat/_meta/config/http.reference.yml.tmpl b/libbeat/_meta/config/http.reference.yml.tmpl new file mode 100644 index 000000000000..19a9f5fcd501 --- /dev/null +++ b/libbeat/_meta/config/http.reference.yml.tmpl @@ -0,0 +1,24 @@ +{{header "HTTP Endpoint"}} + +# Each beat can expose internal metrics through a HTTP endpoint. For security +# reasons the endpoint is disabled by default. This feature is currently experimental. +# Stats can be access through http://localhost:5066/stats . For pretty JSON output +# append ?pretty to the URL. + +# Defines if the HTTP endpoint is enabled. +#http.enabled: false + +# The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# When using IP addresses, it is recommended to only use localhost. +#http.host: localhost + +# Port on which the HTTP endpoint will bind. Default is 5066. +#http.port: 5066 + +# Define which user should be owning the named pipe. +#http.named_pipe.user: + +# Define which the permissions that should be applied to the named pipe, use the Security +# Descriptor Definition Language (SDDL) to define the permission. This option cannot be used with +# `http.user`. +#http.named_pipe.security_descriptor: diff --git a/libbeat/_meta/config/keystore.reference.yml.tmpl b/libbeat/_meta/config/keystore.reference.yml.tmpl new file mode 100644 index 000000000000..2c4de9757a19 --- /dev/null +++ b/libbeat/_meta/config/keystore.reference.yml.tmpl @@ -0,0 +1,4 @@ +{{header "Keystore"}} + +# Location of the Keystore containing the keys and their sensitive values. +#keystore.path: "${path.config}/beats.keystore" diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl new file mode 100644 index 000000000000..0c3000dc0602 --- /dev/null +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -0,0 +1,67 @@ +{{header "Logging"}} + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +#logging.to_stderr: false + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false diff --git a/libbeat/_meta/config/logging.yml.tmpl b/libbeat/_meta/config/logging.yml.tmpl new file mode 100644 index 000000000000..a639acc76ab3 --- /dev/null +++ b/libbeat/_meta/config/logging.yml.tmpl @@ -0,0 +1,10 @@ +{{header "Logging"}} + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] diff --git a/libbeat/_meta/config/migration.yml.tmpl b/libbeat/_meta/config/migration.yml.tmpl new file mode 100644 index 000000000000..8abeab86be4e --- /dev/null +++ b/libbeat/_meta/config/migration.yml.tmpl @@ -0,0 +1,4 @@ +{{header "Migration"}} + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: {{not .Reference}} diff --git a/libbeat/_meta/config/monitoring.reference.yml.tmpl b/libbeat/_meta/config/monitoring.reference.yml.tmpl new file mode 100644 index 000000000000..187b92678eb4 --- /dev/null +++ b/libbeat/_meta/config/monitoring.reference.yml.tmpl @@ -0,0 +1,141 @@ +{{header "X-Pack Monitoring"}} +# {{.BeatName | title}} can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# {{.BeatName | title}} instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + #hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "beats_system" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the URL with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # SSL configuration. The default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. + #kerberos.enabled: true + + # Authentication type to use with Kerberos. Available options: keytab, password. + #kerberos.auth_type: password + + # Path to the keytab file. It is used when auth_type is set to keytab. + #kerberos.keytab: /etc/elastic.keytab + + # Path to the Kerberos configuration. + #kerberos.config_path: /etc/krb5.conf + + # Name of the Kerberos user. + #kerberos.username: elastic + + # Password of the Kerberos user. It is used when auth_type is set to password. + #kerberos.password: changeme + + # Kerberos realm. + #kerberos.realm: ELASTIC + + #metrics.period: 10s + #state.period: 1m + +# The `monitoring.cloud.id` setting overwrites the `monitoring.elasticsearch.hosts` +# setting. You can find the value for this setting in the Elastic Cloud web UI. +#monitoring.cloud.id: + +# The `monitoring.cloud.auth` setting overwrites the `monitoring.elasticsearch.username` +# and `monitoring.elasticsearch.password` settings. The format is `:`. +#monitoring.cloud.auth: diff --git a/libbeat/_meta/config/monitoring.yml.tmpl b/libbeat/_meta/config/monitoring.yml.tmpl new file mode 100644 index 000000000000..6253cb167d59 --- /dev/null +++ b/libbeat/_meta/config/monitoring.yml.tmpl @@ -0,0 +1,21 @@ +{{header "X-Pack Monitoring"}} +# {{.BeatName | title }} can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# {{ .BeatName | title }} instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: diff --git a/libbeat/_meta/config/output-console.reference.yml.tmpl b/libbeat/_meta/config/output-console.reference.yml.tmpl new file mode 100644 index 000000000000..163e93828076 --- /dev/null +++ b/libbeat/_meta/config/output-console.reference.yml.tmpl @@ -0,0 +1,12 @@ +{{subheader "Console Output"}} +#output.console: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl new file mode 100644 index 000000000000..5de92febf3c2 --- /dev/null +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -0,0 +1,140 @@ +{{subheader "Elasticsearch Output"}} +output.elasticsearch: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the URL with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Number of workers per Elasticsearch host. + #worker: 1 + + # Optional index name. The default is "{{.BeatIndexPrefix}}" plus date + # and generates [{{.BeatIndexPrefix}}-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. + #index: "{{.BeatIndexPrefix}}-%{[agent.version]}-%{+yyyy.MM.dd}" + + # Optional ingest node pipeline. By default no pipeline will be used. + #pipeline: "" + + # Optional HTTP path + #path: "/elasticsearch" + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server URL + #proxy_url: http://proxy:3128 + + # Whether to disable proxy settings for outgoing connections. If true, this + # takes precedence over both the proxy_url field and any environment settings + # (HTTP_PROXY, HTTPS_PROXY). The default is false. + #proxy_disable: false + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing a request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL-based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. + #kerberos.enabled: true + + # Authentication type to use with Kerberos. Available options: keytab, password. + #kerberos.auth_type: password + + # Path to the keytab file. It is used when auth_type is set to keytab. + #kerberos.keytab: /etc/elastic.keytab + + # Path to the Kerberos configuration. + #kerberos.config_path: /etc/krb5.conf + + # Name of the Kerberos user. + #kerberos.username: elastic + + # Password of the Kerberos user. It is used when auth_type is set to password. + #kerberos.password: changeme + + # Kerberos realm. + #kerberos.realm: ELASTIC diff --git a/libbeat/_meta/config/output-elasticsearch.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.yml.tmpl new file mode 100644 index 000000000000..7c1287e01a11 --- /dev/null +++ b/libbeat/_meta/config/output-elasticsearch.yml.tmpl @@ -0,0 +1,12 @@ +{{subheader "Elasticsearch Output"}} +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" diff --git a/libbeat/_meta/config/output-file.reference.yml.tmpl b/libbeat/_meta/config/output-file.reference.yml.tmpl new file mode 100644 index 000000000000..2c3834441078 --- /dev/null +++ b/libbeat/_meta/config/output-file.reference.yml.tmpl @@ -0,0 +1,33 @@ +{{subheader "File Output"}} +#output.file: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Path to the directory where to save the generated files. The option is + # mandatory. + #path: "/tmp/{{.BeatName}}" + + # Name of the generated files. The default is `{{.BeatName}}` and it generates + # files: `{{.BeatName}}`, `{{.BeatName}}.1`, `{{.BeatName}}.2`, etc. + #filename: {{.BeatName}} + + # Maximum size in kilobytes of each file. When this size is reached, and on + # every {{.BeatName | title}} restart, the files are rotated. The default value is 10240 + # kB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, + # the oldest file is deleted and the rest are shifted from last to first. The + # default is 7 files. + #number_of_files: 7 + + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 diff --git a/libbeat/_meta/config/output-kafka.reference.yml.tmpl b/libbeat/_meta/config/output-kafka.reference.yml.tmpl new file mode 100644 index 000000000000..50efb87fbb15 --- /dev/null +++ b/libbeat/_meta/config/output-kafka.reference.yml.tmpl @@ -0,0 +1,178 @@ +{{subheader "Kafka Output"}} +#output.kafka: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The list of Kafka broker addresses from which to fetch the cluster metadata. + # The cluster metadata contain the actual Kafka brokers events are published + # to. + #hosts: ["localhost:9092"] + + # The Kafka topic used for produced events. The setting can be a format string + # using any event field. To set the topic from document type use `%{[type]}`. + #topic: beats + + # The Kafka event key setting. Use format string to create a unique event key. + # By default no event key will be generated. + #key: '' + + # The Kafka event partitioning strategy. Default hashing strategy is `hash` + # using the `output.kafka.key` setting or randomly distributes events if + # `output.kafka.key` is not configured. + #partition.hash: + # If enabled, events will only be published to partitions with reachable + # leaders. Default is false. + #reachable_only: false + + # Configure alternative event field names used to compute the hash value. + # If empty `output.kafka.key` setting will be used. + # Default value is empty list. + #hash: [] + + # Authentication details. Password is required if username is set. + #username: '' + #password: '' + + # Kafka version {{.BeatName | title}} is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Metadata update configuration. Metadata contains leader information + # used to decide which broker to use when publishing. + #metadata: + # Max metadata request retry attempts when cluster is in middle of leader + # election. Defaults to 3 retries. + #retry.max: 3 + + # Wait time between retries during leader elections. Default is 250ms. + #retry.backoff: 250ms + + # Refresh metadata interval. Defaults to every 10 minutes. + #refresh_frequency: 10m + + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false + + # The number of concurrent load-balanced Kafka output workers. + #worker: 1 + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Kafka request. The default + # is 2048. + #bulk_max_size: 2048 + + # Duration to wait before sending bulk Kafka request. 0 is no delay. The default + # is 0. + #bulk_flush_frequency: 0s + + # The number of seconds to wait for responses from the Kafka brokers before + # timing out. The default is 30s. + #timeout: 30s + + # The maximum duration a broker will wait for number of required ACKs. The + # default is 10s. + #broker_timeout: 10s + + # The number of messages buffered for each Kafka broker. The default is 256. + #channel_buffer_size: 256 + + # The keep-alive period for an active network connection. If 0s, keep-alives + # are disabled. The default is 0 seconds. + #keep_alive: 0 + + # Sets the output compression codec. Must be one of none, snappy and gzip. The + # default is gzip. + #compression: gzip + + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + + # The maximum permitted size of JSON-encoded messages. Bigger messages will be + # dropped. The default value is 1000000 (bytes). This value should be equal to + # or less than the broker's message.max.bytes. + #max_message_bytes: 1000000 + + # The ACK reliability level required from broker. 0=no response, 1=wait for + # local commit, -1=wait for all replicas to commit. The default is 1. Note: + # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently + # on error. + #required_acks: 1 + + # The configurable ClientID used for logging, debugging, and auditing + # purposes. The default is "beats". + #client_id: beats + + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. + #ssl.enabled: true + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. + #kerberos.enabled: true + + # Authentication type to use with Kerberos. Available options: keytab, password. + #kerberos.auth_type: password + + # Path to the keytab file. It is used when auth_type is set to keytab. + #kerberos.keytab: /etc/security/keytabs/kafka.keytab + + # Path to the Kerberos configuration. + #kerberos.config_path: /etc/krb5.conf + + # The service name. Service principal name is contructed from + # service_name/hostname@realm. + #kerberos.service_name: kafka + + # Name of the Kerberos user. + #kerberos.username: elastic + + # Password of the Kerberos user. It is used when auth_type is set to password. + #kerberos.password: changeme + + # Kerberos realm. + #kerberos.realm: ELASTIC diff --git a/libbeat/_meta/config/output-logstash.reference.yml.tmpl b/libbeat/_meta/config/output-logstash.reference.yml.tmpl new file mode 100644 index 000000000000..da182d0496eb --- /dev/null +++ b/libbeat/_meta/config/output-logstash.reference.yml.tmpl @@ -0,0 +1,113 @@ +{{subheader "Logstash Output"}} +#output.logstash: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Number of workers per Logstash host. + #worker: 1 + + # Set gzip compression level. + #compression_level: 3 + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Optional maximum time to live for a connection to Logstash, after which the + # connection will be re-established. A value of `0s` (the default) will + # disable this feature. + # + # Not yet supported for async connections (i.e. with the "pipelining" option set) + #ttl: 30s + + # Optionally load-balance events between Logstash hosts. Default is false. + #loadbalance: false + + # Number of batches to be sent asynchronously to Logstash while processing + # new batches. + #pipelining: 2 + + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` + # if no error is encountered. + #slow_start: false + + # The number of seconds to wait before trying to reconnect to Logstash + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Logstash after a network error. The default is 60s. + #backoff.max: 60s + + # Optional index name. The default index name is set to {{.BeatIndexPrefix}} + # in all lowercase. + #index: '{{.BeatIndexPrefix}}' + + # SOCKS5 proxy server URL + #proxy_url: socks5://user:password@socks5-server:2233 + + # Resolve names locally when using a proxy server. Defaults to false. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled if any SSL setting is set. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s diff --git a/libbeat/_meta/config/output-logstash.yml.tmpl b/libbeat/_meta/config/output-logstash.yml.tmpl new file mode 100644 index 000000000000..a937e0de99e6 --- /dev/null +++ b/libbeat/_meta/config/output-logstash.yml.tmpl @@ -0,0 +1,14 @@ +{{subheader "Logstash Output"}} +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" diff --git a/libbeat/_meta/config/output-redis.reference.yml.tmpl b/libbeat/_meta/config/output-redis.reference.yml.tmpl new file mode 100644 index 000000000000..3b8fa47f2924 --- /dev/null +++ b/libbeat/_meta/config/output-redis.reference.yml.tmpl @@ -0,0 +1,117 @@ +{{subheader "Redis Output"}} +#output.redis: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # The list of Redis servers to connect to. If load-balancing is enabled, the + # events are distributed to the servers in the list. If one server becomes + # unreachable, the events are distributed to the reachable servers only. + # The hosts setting supports redis and rediss urls with custom password like + # redis://:password@localhost:6379. + #hosts: ["localhost:6379"] + + # The name of the Redis list or channel the events are published to. The + # default is {{.BeatName}}. + #key: {{.BeatName}} + + # The password to authenticate to Redis with. The default is no authentication. + #password: + + # The Redis database number where the events are published. The default is 0. + #db: 0 + + # The Redis data type to use for publishing events. If the data type is list, + # the Redis RPUSH command is used. If the data type is channel, the Redis + # PUBLISH command is used. The default value is list. + #datatype: list + + # The number of workers to use for each host configured to publish events to + # Redis. Use this setting along with the loadbalance option. For example, if + # you have 2 hosts and 3 workers, in total 6 workers are started (3 for each + # host). + #worker: 1 + + # If set to true and multiple hosts or workers are configured, the output + # plugin load balances published events onto all Redis hosts. If set to false, + # the output plugin sends all events to only one host (determined at random) + # and will switch to another host if the currently selected one becomes + # unreachable. The default value is true. + #loadbalance: true + + # The Redis connection timeout in seconds. The default is 5 seconds. + #timeout: 5s + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The number of seconds to wait before trying to reconnect to Redis + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Redis after a network error. The default is 60s. + #backoff.max: 60s + + # The maximum number of events to bulk in a single Redis request or pipeline. + # The default is 2048. + #bulk_max_size: 2048 + + # The URL of the SOCKS5 proxy to use when connecting to the Redis servers. The + # value must be a URL with a scheme of socks5://. + #proxy_url: + + # This option determines whether Redis hostnames are resolved locally when + # using a proxy. The default value is false, which means that name resolution + # occurs on the proxy server. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never diff --git a/libbeat/_meta/config/outputs.yml.tmpl b/libbeat/_meta/config/outputs.yml.tmpl new file mode 100644 index 000000000000..662c88cfbcf8 --- /dev/null +++ b/libbeat/_meta/config/outputs.yml.tmpl @@ -0,0 +1,3 @@ +{{header "Outputs"}} + +# Configure what output to use when sending the data collected by the beat. diff --git a/libbeat/_meta/config/paths.reference.yml.tmpl b/libbeat/_meta/config/paths.reference.yml.tmpl new file mode 100644 index 000000000000..7907777f5e12 --- /dev/null +++ b/libbeat/_meta/config/paths.reference.yml.tmpl @@ -0,0 +1,25 @@ +{{header "Paths"}} + +# The home path for the {{.BeatName | title}} installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: + +# The configuration path for the {{.BeatName | title}} installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: ${path.home} + +# The data path for the {{.BeatName | title}} installation. This is the default base path +# for all the files in which {{.BeatName | title}} needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: ${path.home}/data + +# The logs path for a {{.BeatName | title}} installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: ${path.home}/logs diff --git a/libbeat/_meta/config/processors.reference.yml.tmpl b/libbeat/_meta/config/processors.reference.yml.tmpl new file mode 100644 index 000000000000..c7c081b49bbc --- /dev/null +++ b/libbeat/_meta/config/processors.reference.yml.tmpl @@ -0,0 +1,162 @@ +{{header "Processors"}} + +# Processors are used to reduce the number of fields in the exported event or to +# enhance the event with external metadata. This section defines a list of +# processors that are applied one by one and the first one receives the initial +# event: +# +# event -> filter1 -> event1 -> filter2 ->event2 ... +# +# The supported processors are drop_fields, drop_event, include_fields, +# decode_json_fields, and add_cloud_metadata. +# +# For example, you can use the following processors to keep the fields that +# contain CPU load percentages, but remove the fields that contain CPU ticks +# values: +# +#processors: +# - include_fields: +# fields: ["cpu"] +# - drop_fields: +# fields: ["cpu.user", "cpu.system"] +# +# The following example drops the events that have the HTTP response code 200: +# +#processors: +# - drop_event: +# when: +# equals: +# http.code: 200 +# +# The following example renames the field a to b: +# +#processors: +# - rename: +# fields: +# - from: "a" +# to: "b" +# +# The following example tokenizes the string into fields: +# +#processors: +# - dissect: +# tokenizer: "%{key1} - %{key2}" +# field: "message" +# target_prefix: "dissect" +# +# The following example enriches each event with metadata from the cloud +# provider about the host machine. It works on EC2, GCE, DigitalOcean, +# Tencent Cloud, and Alibaba Cloud. +# +#processors: +# - add_cloud_metadata: ~ +# +# The following example enriches each event with the machine's local time zone +# offset from UTC. +# +#processors: +# - add_locale: +# format: offset +# +# The following example enriches each event with docker metadata, it matches +# given fields to an existing container id and adds info from that container: +# +#processors: +# - add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_fields: ["system.process.cgroup.id"] +# match_pids: ["process.pid", "process.ppid"] +# match_source: true +# match_source_index: 4 +# match_short_id: false +# cleanup_timeout: 60 +# labels.dedot: false +# # To connect to Docker over TLS you must specify a client and CA certificate. +# #ssl: +# # certificate_authority: "/etc/pki/root/ca.pem" +# # certificate: "/etc/pki/client/cert.pem" +# # key: "/etc/pki/client/cert.key" +# +# The following example enriches each event with docker metadata, it matches +# container id from log path available in `source` field (by default it expects +# it to be /var/lib/docker/containers/*/*.log). +# +#processors: +# - add_docker_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +# - add_host_metadata: ~ +# +# The following example enriches each event with process metadata using +# process IDs included in the event. +# +#processors: +# - add_process_metadata: +# match_pids: ["system.process.ppid"] +# target: system.process.parent +# +# The following example decodes fields containing JSON strings +# and replaces the strings with valid JSON objects. +# +#processors: +# - decode_json_fields: +# fields: ["field1", "field2", ...] +# process_array: false +# max_depth: 1 +# target: "" +# overwrite_keys: false +# +#processors: +# - decompress_gzip_field: +# from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true +# +# The following example copies the value of message to message_copied +# +#processors: +# - copy_fields: +# fields: +# - from: message +# to: message_copied +# fail_on_error: true +# ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +# - truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +# - copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +# - truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +# - urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true diff --git a/libbeat/_meta/config/processors.yml.tmpl b/libbeat/_meta/config/processors.yml.tmpl new file mode 100644 index 000000000000..b935f85dbac0 --- /dev/null +++ b/libbeat/_meta/config/processors.yml.tmpl @@ -0,0 +1,24 @@ +{{header "Processors"}} +{{if not .UseObserverProcessor}} +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ +{{- if .UseDockerMetadataProcessor }} + - add_docker_metadata: ~{{ end }} + +{{- if .UseKubernetesMetadataProcessor }} + - add_kubernetes_metadata: ~ +{{- else -}} +{{ end }} +{{else}} +processors: + - add_observer_metadata: + # Optional, but recommended geo settings for the location {{ .BeatName | title }} is running in + #geo: + # Token describing this location + #name: us-east-1a + # Lat, Lon " + #location: "37.926868, -78.024902" +{{end}} diff --git a/libbeat/_meta/config/seccomp.reference.yml.tmpl b/libbeat/_meta/config/seccomp.reference.yml.tmpl new file mode 100644 index 000000000000..54edc3326e38 --- /dev/null +++ b/libbeat/_meta/config/seccomp.reference.yml.tmpl @@ -0,0 +1,4 @@ +{{header "Process Security"}} + +# Enable or disable seccomp system call filtering on Linux. Default is enabled. +#seccomp.enabled: true diff --git a/libbeat/_meta/config/setup.dashboards.reference.yml.tmpl b/libbeat/_meta/config/setup.dashboards.reference.yml.tmpl new file mode 100644 index 000000000000..1d3b07980070 --- /dev/null +++ b/libbeat/_meta/config/setup.dashboards.reference.yml.tmpl @@ -0,0 +1,44 @@ +{{header "Dashboards"}} + +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards are disabled by default and can be enabled either by setting the +# options here, or by using the `-setup` CLI flag or the `setup` command. +#setup.dashboards.enabled: false + +# The directory from where to read the dashboards. The default is the `kibana` +# folder in the home path. +#setup.dashboards.directory: ${path.home}/kibana + +# The URL from where to download the dashboards archive. It is used instead of +# the directory if it has a value. +#setup.dashboards.url: + +# The file archive (zip file) from where to read the dashboards. It is used instead +# of the directory when it has a value. +#setup.dashboards.file: + +# In case the archive contains the dashboards from multiple Beats, this lets you +# select which one to load. You can load all the dashboards in the archive by +# setting this to the empty string. +#setup.dashboards.beat: {{.BeatName}} + +# The name of the Kibana index to use for setting the configuration. Default is ".kibana" +#setup.dashboards.kibana_index: .kibana + +# The Elasticsearch index name. This overwrites the index name defined in the +# dashboards and index pattern. Example: testbeat-* +#setup.dashboards.index: + +# Always use the Kibana API for loading the dashboards instead of autodetecting +# how to install the dashboards by first querying Elasticsearch. +#setup.dashboards.always_kibana: false + +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 diff --git a/libbeat/_meta/config/setup.dashboards.yml.tmpl b/libbeat/_meta/config/setup.dashboards.yml.tmpl new file mode 100644 index 000000000000..227b742a86e4 --- /dev/null +++ b/libbeat/_meta/config/setup.dashboards.yml.tmpl @@ -0,0 +1,11 @@ +{{header "Dashboards"}} +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here or by using the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: diff --git a/libbeat/_meta/config/setup.ilm.reference.yml.tmpl b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl new file mode 100644 index 000000000000..1416da0cc8dc --- /dev/null +++ b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl @@ -0,0 +1,34 @@ +{{header "Index Lifecycle Management (ILM)"}} + +# Configure index lifecycle management (ILM). These settings create a write +# alias and add additional settings to the index template. When ILM is enabled, +# output.elasticsearch.index is ignored, and the write alias is used to set the +# index name. + +# Enable ILM support. Valid values are true, false, and auto. When set to auto +# (the default), the Beat uses index lifecycle management when it connects to a +# cluster that supports ILM; otherwise, it creates daily indices. +#setup.ilm.enabled: auto + +# Set the prefix used in the index lifecycle write alias name. The default alias +# name is '{{.BeatName}}-%{[agent.version]}'. +#setup.ilm.rollover_alias: '{{.BeatIndexPrefix}}' + +# Set the rollover index pattern. The default is "%{now/d}-000001". +#setup.ilm.pattern: "{now/d}-000001" + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.ilm.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.ilm.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy +# can be installed. +#setup.ilm.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.ilm.overwrite: false diff --git a/libbeat/_meta/config/setup.kibana.reference.yml.tmpl b/libbeat/_meta/config/setup.kibana.reference.yml.tmpl new file mode 100644 index 000000000000..603b3da4196c --- /dev/null +++ b/libbeat/_meta/config/setup.kibana.reference.yml.tmpl @@ -0,0 +1,54 @@ +{{header "Kibana"}} + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + # Optional HTTP path + #path: "" + + # Optional Kibana space ID. + #space.id: "" + + # Use SSL settings for HTTPS. Default is true. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # SSL configuration. The default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] diff --git a/libbeat/_meta/config/setup.kibana.yml.tmpl b/libbeat/_meta/config/setup.kibana.yml.tmpl new file mode 100644 index 000000000000..6954fb814c76 --- /dev/null +++ b/libbeat/_meta/config/setup.kibana.yml.tmpl @@ -0,0 +1,16 @@ +{{header "Kibana"}} + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: diff --git a/libbeat/_meta/config/setup.template.reference.yml.tmpl b/libbeat/_meta/config/setup.template.reference.yml.tmpl new file mode 100644 index 000000000000..48d23d9d0c90 --- /dev/null +++ b/libbeat/_meta/config/setup.template.reference.yml.tmpl @@ -0,0 +1,53 @@ +{{header "Template"}} + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is "{{.BeatIndexPrefix}}-%{[agent.version]}" +# The template name and pattern has to be set in case the Elasticsearch index pattern is modified. +#setup.template.name: "{{.BeatIndexPrefix}}-%{[agent.version]}" + +# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. +# The template name and pattern has to be set in case the Elasticsearch index pattern is modified. +#setup.template.pattern: "{{.BeatIndexPrefix}}-%{[agent.version]}-*" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# A list of fields to be added to the template and Kibana index pattern. Also +# specify setup.template.overwrite: true to overwrite the existing template. +#setup.template.append_fields: +#- name: field_name +# type: field_type + +# Enable JSON template loading. If this is enabled, the fields.yml is ignored. +#setup.template.json.enabled: false + +# Path to the JSON template file +#setup.template.json.path: "${path.config}/template.json" + +# Name under which the template is stored in Elasticsearch +#setup.template.json.name: "" + +# Overwrite existing template +#setup.template.overwrite: false + +# Elasticsearch template settings +setup.template.settings: + + # A dictionary of settings to place into the settings.index dictionary + # of the Elasticsearch template. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html + #index: + #number_of_shards: 1 + #codec: best_compression + + # A dictionary of settings for the _source field. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html + #_source: + #enabled: false diff --git a/libbeat/magefile.go b/libbeat/magefile.go index e5a22799b6dc..67e84e63be32 100644 --- a/libbeat/magefile.go +++ b/libbeat/magefile.go @@ -49,5 +49,5 @@ func Fields() error { // Config generates example and reference configuration for libbeat. func Config() error { - return devtools.Config(devtools.ShortConfigType|devtools.ReferenceConfigType, devtools.ConfigFileParams{}, ".") + return devtools.Config(devtools.ShortConfigType|devtools.ReferenceConfigType, devtools.DefaultConfigFileParams(), ".") } diff --git a/metricbeat/_meta/beat.docker.yml b/metricbeat/_meta/config/beat.docker.yml.tmpl similarity index 98% rename from metricbeat/_meta/beat.docker.yml rename to metricbeat/_meta/config/beat.docker.yml.tmpl index 16b19a866dd7..10c25e6a5cc7 100644 --- a/metricbeat/_meta/beat.docker.yml +++ b/metricbeat/_meta/config/beat.docker.yml.tmpl @@ -1,4 +1,3 @@ metricbeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false - diff --git a/metricbeat/_meta/config/beat.reference.yml.tmpl b/metricbeat/_meta/config/beat.reference.yml.tmpl new file mode 100644 index 000000000000..7d00457d5c65 --- /dev/null +++ b/metricbeat/_meta/config/beat.reference.yml.tmpl @@ -0,0 +1,2 @@ +{{template "header.reference.yml.tmpl" .}} +{{template "config.modules.yml.tmpl" .}} diff --git a/metricbeat/_meta/config/beat.yml.tmpl b/metricbeat/_meta/config/beat.yml.tmpl new file mode 100644 index 000000000000..f34d7c555156 --- /dev/null +++ b/metricbeat/_meta/config/beat.yml.tmpl @@ -0,0 +1,3 @@ +{{template "header.yml.tmpl" .}} +{{template "metricbeat.config.modules.yml.tmpl" .}} +{{template "setup.template.yml.tmpl" .}} diff --git a/metricbeat/_meta/common.reference.yml b/metricbeat/_meta/config/header.reference.yml.tmpl similarity index 100% rename from metricbeat/_meta/common.reference.yml rename to metricbeat/_meta/config/header.reference.yml.tmpl diff --git a/metricbeat/_meta/common.yml b/metricbeat/_meta/config/header.yml.tmpl similarity index 100% rename from metricbeat/_meta/common.yml rename to metricbeat/_meta/config/header.yml.tmpl diff --git a/metricbeat/_meta/config/metricbeat.config.modules.yml.tmpl b/metricbeat/_meta/config/metricbeat.config.modules.yml.tmpl new file mode 100644 index 000000000000..e616bc21b63a --- /dev/null +++ b/metricbeat/_meta/config/metricbeat.config.modules.yml.tmpl @@ -0,0 +1,11 @@ +{{header "Modules configuration"}} + +metricbeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s diff --git a/metricbeat/_meta/config/setup.template.yml.tmpl b/metricbeat/_meta/config/setup.template.yml.tmpl new file mode 100644 index 000000000000..2b8d28f2a303 --- /dev/null +++ b/metricbeat/_meta/config/setup.template.yml.tmpl @@ -0,0 +1,6 @@ +{{header "Elasticsearch template setting"}} + +setup.template.settings: + index.number_of_shards: 1 + index.codec: best_compression + #_source.enabled: false diff --git a/metricbeat/_meta/setup.yml b/metricbeat/_meta/setup.yml deleted file mode 100644 index 337067373f1a..000000000000 --- a/metricbeat/_meta/setup.yml +++ /dev/null @@ -1,19 +0,0 @@ - -#========================== Modules configuration ============================ - -metricbeat.config.modules: - # Glob pattern for configuration loading - path: ${path.config}/modules.d/*.yml - - # Set to true to enable config reloading - reload.enabled: false - - # Period on which files under path should be checked for changes - #reload.period: 10s - -#==================== Elasticsearch template setting ========================== - -setup.template.settings: - index.number_of_shards: 1 - index.codec: best_compression - #_source.enabled: false diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index aa0aac52e97d..ee91fcd3355a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -53,6 +53,7 @@ metricbeat.max_start_delay: 10s #timeseries.enabled: false + #========================== Modules configuration ============================= metricbeat.modules: @@ -871,7 +872,8 @@ metricbeat.modules: -#================================ General ====================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -979,7 +981,7 @@ metricbeat.modules: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -1142,7 +1144,7 @@ metricbeat.modules: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -1155,11 +1157,11 @@ metricbeat.modules: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -1300,7 +1302,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -1414,7 +1416,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -1593,7 +1595,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -1711,7 +1713,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -1745,7 +1747,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -1758,7 +1760,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Metricbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -1784,11 +1786,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1832,8 +1836,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1887,7 +1890,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1901,13 +1904,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'metricbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "metricbeat" +#setup.ilm.rollover_alias: 'metricbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'metricbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1922,7 +1925,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1977,9 +1980,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -2046,8 +2048,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -2189,7 +2190,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -2213,12 +2215,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 5bd19f3030c2..96975ee6027f 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/metricbeat/index.html -#========================== Modules configuration ============================ +# =========================== Modules configuration ============================ metricbeat.config.modules: # Glob pattern for configuration loading @@ -19,14 +19,15 @@ metricbeat.config.modules: # Period on which files under path should be checked for changes #reload.period: 10s -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 index.codec: best_compression #_source.enabled: false -#================================ General ===================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -41,8 +42,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -54,7 +54,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -71,7 +71,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -84,11 +84,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -101,7 +101,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -116,7 +116,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -126,7 +126,8 @@ processors: - add_docker_metadata: ~ - add_kubernetes_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -137,8 +138,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# metricbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -159,7 +160,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/metricbeat/scripts/mage/config.go b/metricbeat/scripts/mage/config.go index ffa30d27bad3..e49e223753fe 100644 --- a/metricbeat/scripts/mage/config.go +++ b/metricbeat/scripts/mage/config.go @@ -18,36 +18,25 @@ package mage import ( + "github.com/magefile/mage/mg" + devtools "github.com/elastic/beats/v7/dev-tools/mage" ) -const modulesConfigYml = "build/config.modules.yml" +const modulesConfigYml = "build/config.modules.yml.tmpl" func configFileParams(moduleDirs ...string) devtools.ConfigFileParams { collectModuleConfig := func() error { return devtools.GenerateModuleReferenceConfig(modulesConfigYml, moduleDirs...) } + mg.Deps(collectModuleConfig) - return devtools.ConfigFileParams{ - ShortParts: []string{ - devtools.OSSBeatDir("_meta/common.yml"), - devtools.OSSBeatDir("_meta/setup.yml"), - devtools.LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceDeps: []interface{}{collectModuleConfig}, - ReferenceParts: []string{ - devtools.OSSBeatDir("_meta/common.reference.yml"), - modulesConfigYml, - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - devtools.OSSBeatDir("_meta/beat.docker.yml"), - devtools.LibbeatDir("_meta/config.docker.yml"), - }, - ExtraVars: map[string]interface{}{ - "UseKubernetesMetadataProcessor": true, - }, + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl"), modulesConfigYml) + p.ExtraVars = map[string]interface{}{ + "UseKubernetesMetadataProcessor": true, } + return p } // OSSConfigFileParams returns the default ConfigFileParams for generating @@ -59,11 +48,5 @@ func OSSConfigFileParams(moduleDirs ...string) devtools.ConfigFileParams { // XPackConfigFileParams returns the default ConfigFileParams for generating // metricbeat*.yml files. func XPackConfigFileParams() devtools.ConfigFileParams { - args := configFileParams(devtools.OSSBeatDir("module"), "module") - args.ReferenceParts = []string{ - devtools.OSSBeatDir("_meta/common.reference.yml"), - modulesConfigYml, - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - } - return args + return configFileParams(devtools.OSSBeatDir("module"), "module") } diff --git a/packetbeat/_meta/beat.docker.yml b/packetbeat/_meta/config/beat.docker.yml.tmpl similarity index 99% rename from packetbeat/_meta/beat.docker.yml rename to packetbeat/_meta/config/beat.docker.yml.tmpl index 90c4c24e6657..f4f0db1f7e6e 100644 --- a/packetbeat/_meta/beat.docker.yml +++ b/packetbeat/_meta/config/beat.docker.yml.tmpl @@ -36,4 +36,3 @@ packetbeat.protocols.cassandra: packetbeat.protocols.tls: ports: [443, 993, 995, 5223, 8443, 8883, 9243] - diff --git a/packetbeat/_meta/beat.reference.yml b/packetbeat/_meta/config/beat.reference.yml.tmpl similarity index 98% rename from packetbeat/_meta/beat.reference.yml rename to packetbeat/_meta/config/beat.reference.yml.tmpl index 6ddd057b4c9c..1a3aab315d76 100644 --- a/packetbeat/_meta/beat.reference.yml +++ b/packetbeat/_meta/config/beat.reference.yml.tmpl @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/packetbeat/index.html -#============================== Network device ================================ +{{header "Network device"}} # Select the network interface to sniff the data. You can use the "any" # keyword to sniff on all connected interfaces. @@ -47,7 +47,7 @@ packetbeat.interfaces.device: {{ call .device .GOOS }} # can stay enabled even after beat is shut down. #packetbeat.interfaces.auto_promisc_mode: true -#================================== Flows ===================================== +{{header "Flows"}} packetbeat.flows: # Enable Network flows. Default: true @@ -63,7 +63,7 @@ packetbeat.flows: # Set to true to publish fields with null values in events. #keep_null: false -#========================== Transaction protocols ============================= +{{header "Transaction protocols"}} packetbeat.protocols: - type: icmp @@ -531,7 +531,7 @@ packetbeat.protocols: # Set to true to publish fields with null values in events. #keep_null: false -#=========================== Monitored processes ============================== +{{header "Monitored processes"}} # Packetbeat can enrich events with information about the process associated # the socket that sent or received the packet if Packetbeat is monitoring diff --git a/packetbeat/_meta/beat.yml b/packetbeat/_meta/config/beat.yml.tmpl similarity index 91% rename from packetbeat/_meta/beat.yml rename to packetbeat/_meta/config/beat.yml.tmpl index 8c8037a5e7c0..fb221cba3c9a 100644 --- a/packetbeat/_meta/beat.yml +++ b/packetbeat/_meta/config/beat.yml.tmpl @@ -7,13 +7,13 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/packetbeat/index.html -#============================== Network device ================================ +{{header "Network device"}} # Select the network interface to sniff the data. On Linux, you can use the # "any" keyword to sniff on all connected interfaces. packetbeat.interfaces.device: {{ call .device .GOOS }} -#================================== Flows ===================================== +{{header "Flows"}} # Set `enabled: false` or comment out all options to disable flows reporting. packetbeat.flows: @@ -24,7 +24,7 @@ packetbeat.flows: # Configure reporting period. If set to -1, only killed flows will be reported period: 10s -#========================== Transaction protocols ============================= +{{header "Transaction protocols"}} packetbeat.protocols: - type: icmp @@ -101,7 +101,7 @@ packetbeat.protocols: - 8883 # Secure MQTT - 9243 # Elasticsearch -#==================== Elasticsearch template setting ========================== +{{header "Elasticsearch template setting"}} setup.template.settings: index.number_of_shards: 1 diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 759687a288bf..cf91e7c70276 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/packetbeat/index.html -#============================== Network device ================================ +# =============================== Network device =============================== # Select the network interface to sniff the data. You can use the "any" # keyword to sniff on all connected interfaces. @@ -47,7 +47,7 @@ packetbeat.interfaces.device: any # can stay enabled even after beat is shut down. #packetbeat.interfaces.auto_promisc_mode: true -#================================== Flows ===================================== +# =================================== Flows ==================================== packetbeat.flows: # Enable Network flows. Default: true @@ -63,7 +63,7 @@ packetbeat.flows: # Set to true to publish fields with null values in events. #keep_null: false -#========================== Transaction protocols ============================= +# =========================== Transaction protocols ============================ packetbeat.protocols: - type: icmp @@ -531,7 +531,7 @@ packetbeat.protocols: # Set to true to publish fields with null values in events. #keep_null: false -#=========================== Monitored processes ============================== +# ============================ Monitored processes ============================= # Packetbeat can enrich events with information about the process associated # the socket that sent or received the packet if Packetbeat is monitoring @@ -545,7 +545,7 @@ packetbeat.procs.enabled: false # false. packetbeat.ignore_outgoing: false -#================================ General ====================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -653,7 +653,7 @@ packetbeat.ignore_outgoing: false # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -816,7 +816,7 @@ packetbeat.ignore_outgoing: false # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Packetbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -829,11 +829,11 @@ packetbeat.ignore_outgoing: false # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -974,7 +974,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -1088,7 +1088,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -1267,7 +1267,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -1385,7 +1385,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -1419,7 +1419,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -1432,7 +1432,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Packetbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -1458,11 +1458,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1506,8 +1508,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1561,7 +1562,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1575,13 +1576,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'packetbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "packetbeat" +#setup.ilm.rollover_alias: 'packetbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'packetbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1596,7 +1597,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1651,9 +1652,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1720,8 +1720,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1863,7 +1862,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1887,12 +1887,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 9d06f982f611..66e1bc85991a 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -7,13 +7,13 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/packetbeat/index.html -#============================== Network device ================================ +# =============================== Network device =============================== # Select the network interface to sniff the data. On Linux, you can use the # "any" keyword to sniff on all connected interfaces. packetbeat.interfaces.device: any -#================================== Flows ===================================== +# =================================== Flows ==================================== # Set `enabled: false` or comment out all options to disable flows reporting. packetbeat.flows: @@ -24,7 +24,7 @@ packetbeat.flows: # Configure reporting period. If set to -1, only killed flows will be reported period: 10s -#========================== Transaction protocols ============================= +# =========================== Transaction protocols ============================ packetbeat.protocols: - type: icmp @@ -101,14 +101,14 @@ packetbeat.protocols: - 8883 # Secure MQTT - 9243 # Elasticsearch -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression #_source.enabled: false -#================================ General ===================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -123,8 +123,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -136,7 +135,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -153,7 +152,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Packetbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -166,11 +165,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -183,7 +182,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -198,7 +197,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -207,7 +206,8 @@ processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -218,8 +218,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# packetbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -240,7 +240,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/packetbeat/scripts/mage/config.go b/packetbeat/scripts/mage/config.go index fac457cd7efd..a143cda22e71 100644 --- a/packetbeat/scripts/mage/config.go +++ b/packetbeat/scripts/mage/config.go @@ -43,24 +43,10 @@ func device(goos string) string { // ConfigFileParams returns the default ConfigFileParams for generating // packetbeat*.yml files. func ConfigFileParams() devtools.ConfigFileParams { - return devtools.ConfigFileParams{ - ShortParts: []string{ - devtools.OSSBeatDir("_meta/beat.yml"), - configTemplateGlob, - devtools.LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceParts: []string{ - devtools.OSSBeatDir("_meta/beat.reference.yml"), - configTemplateGlob, - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - devtools.OSSBeatDir("_meta/beat.docker.yml"), - configTemplateGlob, - devtools.LibbeatDir("_meta/config.docker.yml"), - }, - ExtraVars: map[string]interface{}{ - "device": device, - }, + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl")) + p.ExtraVars = map[string]interface{}{ + "device": device, } + return p } diff --git a/winlogbeat/_meta/beat.reference.yml b/winlogbeat/_meta/beat.reference.yml deleted file mode 100644 index c3f14bd412c5..000000000000 --- a/winlogbeat/_meta/beat.reference.yml +++ /dev/null @@ -1,34 +0,0 @@ -########################## Winlogbeat Configuration ########################### - -# This file is a full configuration example documenting all non-deprecated -# options in comments. For a shorter configuration example, that contains only -# the most common options, please see winlogbeat.yml in the same directory. -# -# You can find the full configuration reference here: -# https://www.elastic.co/guide/en/beats/winlogbeat/index.html - -#======================= Winlogbeat specific options ========================== - -# The registry file is where Winlogbeat persists its state so that the beat -# can resume after shutdown or an outage. The default is .winlogbeat.yml -# in the directory in which it was started. -#winlogbeat.registry_file: .winlogbeat.yml - -# The maximum amount of time Winlogbeat should wait for events to finish -# publishing when shutting down. -#winlogbeat.shutdown_timeout: 0s - -# event_logs specifies a list of event logs to monitor as well as any -# accompanying options. The YAML data type of event_logs is a list of -# dictionaries. -# -# The supported keys are name (required), tags, fields, fields_under_root, -# forwarded, ignore_older, level, no_more_events, event_id, provider, and -# include_xml. Please visit the documentation for the complete details of each -# option. -# https://go.es.io/WinlogbeatConfig -winlogbeat.event_logs: - - name: Application - ignore_older: 72h - - name: Security - - name: System diff --git a/winlogbeat/_meta/config/beat.reference.yml.tmpl b/winlogbeat/_meta/config/beat.reference.yml.tmpl new file mode 100644 index 000000000000..e84dd482b55d --- /dev/null +++ b/winlogbeat/_meta/config/beat.reference.yml.tmpl @@ -0,0 +1,2 @@ +{{template "header.yml.tmpl" .}} +{{template "winlogbeat.event_logs.yml.tmpl" .}} diff --git a/winlogbeat/_meta/config/beat.yml.tmpl b/winlogbeat/_meta/config/beat.yml.tmpl new file mode 100644 index 000000000000..39cc766b2b30 --- /dev/null +++ b/winlogbeat/_meta/config/beat.yml.tmpl @@ -0,0 +1,3 @@ +{{template "header.yml.tmpl" .}} +{{template "winlogbeat.event_logs.yml.tmpl" .}} +{{template "setup.template.yml.tmpl" .}} diff --git a/winlogbeat/_meta/common.yml.tmpl b/winlogbeat/_meta/config/header.yml.tmpl similarity index 66% rename from winlogbeat/_meta/common.yml.tmpl rename to winlogbeat/_meta/config/header.yml.tmpl index 63aa30fa0b5e..7ace0063a189 100644 --- a/winlogbeat/_meta/common.yml.tmpl +++ b/winlogbeat/_meta/config/header.yml.tmpl @@ -1,4 +1,3 @@ -{{define "header" -}} ###################### Winlogbeat Configuration Example ######################## # This file is an example configuration file highlighting only the most common @@ -8,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/winlogbeat/index.html -#======================= Winlogbeat specific options =========================== +{{header "Winlogbeat specific options"}} {{if .Reference -}} # The registry file is where Winlogbeat persists its state so that the beat can @@ -24,19 +23,4 @@ # The supported keys are name (required), tags, fields, fields_under_root, # forwarded, ignore_older, level, event_id, provider, and include_xml. Please # visit the documentation for the complete details of each option. -# https://go.es.io/WinlogbeatConfig{{end -}} - -{{define "elasticsearch_settings" -}} -#==================== Elasticsearch template settings ========================== - -setup.template.settings: - index.number_of_shards: 1 - #index.codec: best_compression - #_source.enabled: false -{{end -}} -{{define "processors.yml.tmpl"}} -processors: - - add_host_metadata: - when.not.contains.tags: forwarded - - add_cloud_metadata: ~ -{{end -}} +# https://go.es.io/WinlogbeatConfig diff --git a/winlogbeat/_meta/config/processors.yml.tmpl b/winlogbeat/_meta/config/processors.yml.tmpl new file mode 100644 index 000000000000..4642130fe8b5 --- /dev/null +++ b/winlogbeat/_meta/config/processors.yml.tmpl @@ -0,0 +1,5 @@ +{{header "Processors"}} +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + - add_cloud_metadata: ~ diff --git a/winlogbeat/_meta/config/setup.template.yml.tmpl b/winlogbeat/_meta/config/setup.template.yml.tmpl new file mode 100644 index 000000000000..4eb23e9da927 --- /dev/null +++ b/winlogbeat/_meta/config/setup.template.yml.tmpl @@ -0,0 +1,6 @@ +{{header "Elasticsearch template settings"}} + +setup.template.settings: + index.number_of_shards: 1 + #index.codec: best_compression + #_source.enabled: false diff --git a/winlogbeat/_meta/beat.yml.tmpl b/winlogbeat/_meta/config/winlogbeat.event_logs.yml.tmpl similarity index 61% rename from winlogbeat/_meta/beat.yml.tmpl rename to winlogbeat/_meta/config/winlogbeat.event_logs.yml.tmpl index 093c6f69c042..64a01bb83529 100644 --- a/winlogbeat/_meta/beat.yml.tmpl +++ b/winlogbeat/_meta/config/winlogbeat.event_logs.yml.tmpl @@ -1,4 +1,3 @@ -{{ template "header" . }} winlogbeat.event_logs: - name: Application ignore_older: 72h @@ -9,4 +8,3 @@ winlogbeat.event_logs: - name: ForwardedEvents tags: [forwarded] -{{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/winlogbeat/scripts/mage/config.go b/winlogbeat/scripts/mage/config.go index 70cc8cb43a6c..15240127193d 100644 --- a/winlogbeat/scripts/mage/config.go +++ b/winlogbeat/scripts/mage/config.go @@ -28,34 +28,11 @@ func config() error { } func configFileParams() devtools.ConfigFileParams { - beatDir := devtools.OSSBeatDir - switch SelectLogic { - case devtools.OSSProject: - beatDir = devtools.OSSBeatDir - case devtools.XPackProject: - beatDir = devtools.XPackBeatDir - default: - panic(devtools.ErrUnknownProjectType) - } - - return devtools.ConfigFileParams{ - ShortParts: []string{ - devtools.OSSBeatDir("_meta/common.yml.tmpl"), - beatDir("_meta/beat.yml.tmpl"), - devtools.LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceParts: []string{ - devtools.OSSBeatDir("_meta/common.yml.tmpl"), - beatDir("_meta/beat.yml.tmpl"), - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - DockerParts: []string{ - devtools.OSSBeatDir("_meta/beat.docker.yml"), - devtools.LibbeatDir("_meta/config.docker.yml"), - }, - ExtraVars: map[string]interface{}{ - "GOOS": "windows", - "UseProcessorsTemplate": true, - }, + conf := devtools.DefaultConfigFileParams() + conf.ExtraVars = map[string]interface{}{"GOOS": "windows"} + conf.Templates = append(conf.Templates, devtools.OSSBeatDir("_meta/config/*.tmpl")) + if devtools.XPackProject == SelectLogic { + conf.Templates = append(conf.Templates, devtools.XPackBeatDir("_meta/config/*.tmpl")) } + return conf } diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index d6c0cb898750..a45d5e8f9f01 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/winlogbeat/index.html -#======================= Winlogbeat specific options =========================== +# ======================== Winlogbeat specific options ========================= # The registry file is where Winlogbeat persists its state so that the beat can # resume after shutdown or an outage. The default is .winlogbeat.yml in the @@ -22,6 +22,7 @@ # forwarded, ignore_older, level, event_id, provider, and include_xml. Please # visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig + winlogbeat.event_logs: - name: Application ignore_older: 72h @@ -34,7 +35,7 @@ winlogbeat.event_logs: tags: [forwarded] -#================================ General ====================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -142,7 +143,7 @@ winlogbeat.event_logs: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -305,7 +306,7 @@ winlogbeat.event_logs: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Winlogbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -318,11 +319,11 @@ winlogbeat.event_logs: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -463,7 +464,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -577,7 +578,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -756,7 +757,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -874,7 +875,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -908,7 +909,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -921,7 +922,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Winlogbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -947,11 +948,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -995,8 +998,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1050,7 +1052,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1064,13 +1066,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'winlogbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "winlogbeat" +#setup.ilm.rollover_alias: 'winlogbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'winlogbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1085,7 +1087,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1140,9 +1142,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1209,8 +1210,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1352,7 +1352,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1376,12 +1377,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index d816327de992..d1308e2a2ed5 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/winlogbeat/index.html -#======================= Winlogbeat specific options =========================== +# ======================== Winlogbeat specific options ========================= # event_logs specifies a list of event logs to monitor as well as any # accompanying options. The YAML data type of event_logs is a list of @@ -17,6 +17,7 @@ # forwarded, ignore_older, level, event_id, provider, and include_xml. Please # visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig + winlogbeat.event_logs: - name: Application ignore_older: 72h @@ -27,7 +28,8 @@ winlogbeat.event_logs: - name: ForwardedEvents tags: [forwarded] -#==================== Elasticsearch template settings ========================== + +# ====================== Elasticsearch template settings ======================= setup.template.settings: index.number_of_shards: 1 @@ -35,7 +37,7 @@ setup.template.settings: #_source.enabled: false -#================================ General ===================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -50,8 +52,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -63,7 +64,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -80,7 +81,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Winlogbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -93,11 +94,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -110,7 +111,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -125,14 +126,13 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== - +# ================================= Processors ================================= processors: - add_host_metadata: when.not.contains.tags: forwarded - add_cloud_metadata: ~ -#================================ Logging ===================================== +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -143,8 +143,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# winlogbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -165,7 +165,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/x-pack/auditbeat/auditbeat.docker.yml b/x-pack/auditbeat/auditbeat.docker.yml index a012bbb6aad9..19c9bd1b4775 100644 --- a/x-pack/auditbeat/auditbeat.docker.yml +++ b/x-pack/auditbeat/auditbeat.docker.yml @@ -12,6 +12,7 @@ auditbeat.modules: - /sbin - /usr/sbin - /etc + processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 1abf0a808362..c8af3fa92f7c 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/auditbeat/index.html -#============================ Config Reloading ================================ +# ============================== Config Reloading ============================== # Config reloading allows to dynamically load modules. Each file which is # monitored must contain one or multiple modules as a list. @@ -26,7 +26,7 @@ auditbeat.config.modules: # disable startup delay. auditbeat.max_start_delay: 10s -#========================== Modules configuration ============================= +# =========================== Modules configuration ============================ auditbeat.modules: # The auditd module collects events from the audit framework in the Linux @@ -174,7 +174,8 @@ auditbeat.modules: login.wtmp_file_pattern: /var/log/wtmp* login.btmp_file_pattern: /var/log/btmp* -#================================ General ====================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -282,7 +283,7 @@ auditbeat.modules: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -445,7 +446,7 @@ auditbeat.modules: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Auditbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -458,11 +459,11 @@ auditbeat.modules: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -603,7 +604,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -717,7 +718,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -896,7 +897,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -1014,7 +1015,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -1048,7 +1049,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -1061,7 +1062,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Auditbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -1087,11 +1088,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1135,8 +1138,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1190,7 +1192,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1204,13 +1206,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'auditbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "auditbeat" +#setup.ilm.rollover_alias: 'auditbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'auditbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1225,7 +1227,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1280,9 +1282,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1349,8 +1350,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1492,7 +1492,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1516,12 +1517,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/x-pack/auditbeat/auditbeat.yml b/x-pack/auditbeat/auditbeat.yml index f174d7a793e3..408e52004e6d 100644 --- a/x-pack/auditbeat/auditbeat.yml +++ b/x-pack/auditbeat/auditbeat.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/auditbeat/index.html -#========================== Modules configuration ============================= +# =========================== Modules configuration ============================ auditbeat.modules: - module: auditd @@ -75,13 +75,14 @@ auditbeat.modules: login.wtmp_file_pattern: /var/log/wtmp* login.btmp_file_pattern: /var/log/btmp* -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression #_source.enabled: false -#================================ General ===================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -96,8 +97,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -109,7 +109,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -126,7 +126,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Auditbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -139,11 +139,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -156,7 +156,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -171,7 +171,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -180,7 +180,8 @@ processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -191,8 +192,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# auditbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -213,7 +214,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/x-pack/elastic-agent/_meta/common.p1.yml b/x-pack/elastic-agent/_meta/config/common.p1.yml.tmpl similarity index 100% rename from x-pack/elastic-agent/_meta/common.p1.yml rename to x-pack/elastic-agent/_meta/config/common.p1.yml.tmpl diff --git a/x-pack/elastic-agent/_meta/common.p2.yml b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl similarity index 100% rename from x-pack/elastic-agent/_meta/common.p2.yml rename to x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl diff --git a/x-pack/elastic-agent/_meta/common.reference.p1.yml b/x-pack/elastic-agent/_meta/config/common.reference.p1.yml.tmpl similarity index 100% rename from x-pack/elastic-agent/_meta/common.reference.p1.yml rename to x-pack/elastic-agent/_meta/config/common.reference.p1.yml.tmpl diff --git a/x-pack/elastic-agent/_meta/common.reference.p2.yml b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl similarity index 100% rename from x-pack/elastic-agent/_meta/common.reference.p2.yml rename to x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl diff --git a/x-pack/elastic-agent/_meta/elastic-agent.docker.yml b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl similarity index 100% rename from x-pack/elastic-agent/_meta/elastic-agent.docker.yml rename to x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.reference.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.reference.yml.tmpl new file mode 100644 index 000000000000..d3d5225f0ec2 --- /dev/null +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.reference.yml.tmpl @@ -0,0 +1,2 @@ +{{template "common.reference.p1.yml.tmpl" .}} +{{template "common.reference.p2.yml.tmpl" .}} diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.yml.tmpl new file mode 100644 index 000000000000..016345047289 --- /dev/null +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.yml.tmpl @@ -0,0 +1,2 @@ +{{template "common.p1.yml.tmpl" .}} +{{template "common.p2.yml.tmpl" .}} diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index 547053af6b26..ae06f02c8168 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -3,6 +3,7 @@ # This file is an example configuration file highlighting only the most common # options. The elastic-agent.reference.yml file from the same directory contains all the # supported options with more comments. You can use it as a reference. + ###################################### # Fleet configuration ###################################### @@ -128,3 +129,4 @@ datasources: # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug #logging.level: trace + diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index 547053af6b26..ae06f02c8168 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -3,6 +3,7 @@ # This file is an example configuration file highlighting only the most common # options. The elastic-agent.reference.yml file from the same directory contains all the # supported options with more comments. You can use it as a reference. + ###################################### # Fleet configuration ###################################### @@ -128,3 +129,4 @@ datasources: # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug #logging.level: trace + diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 8168890998ee..1816dd3b205c 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -383,19 +383,12 @@ func configYML() error { // ConfigFileParams returns the parameters for generating OSS config. func ConfigFileParams() devtools.ConfigFileParams { - return devtools.ConfigFileParams{ - ShortParts: []string{ - devtools.XPackBeatDir("_meta/common.p1.yml"), - devtools.XPackBeatDir("_meta/common.p2.yml"), - }, - ReferenceParts: []string{ - devtools.XPackBeatDir("_meta/common.reference.p1.yml"), - devtools.XPackBeatDir("_meta/common.reference.p2.yml"), - }, - DockerParts: []string{ - devtools.XPackBeatDir("_meta/elastic-agent.docker.yml"), - }, - } + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, "_meta/config/*.tmpl") + p.Short.Template = "_meta/config/elastic-agent.yml.tmpl" + p.Reference.Template = "_meta/config/elastic-agent.reference.yml.tmpl" + p.Docker.Template = "_meta/config/elastic-agent.docker.yml.tmpl" + return p } // fieldDocs generates docs/fields.asciidoc containing all fields diff --git a/x-pack/filebeat/_meta/config/beat.reference.yml.tmpl b/x-pack/filebeat/_meta/config/beat.reference.yml.tmpl new file mode 100644 index 000000000000..d90bd36c7b91 --- /dev/null +++ b/x-pack/filebeat/_meta/config/beat.reference.yml.tmpl @@ -0,0 +1,6 @@ +{{template "header.reference.yml.tmpl" .}} +{{template "config.modules.yml.tmpl" .}} +{{template "filebeat.inputs.reference.yml.tmpl" .}} +{{template "filebeat.inputs.reference.xpack.yml.tmpl" .}} +{{template "filebeat.autodiscover.reference.yml.tmpl" .}} +{{template "filebeat.global.reference.yml.tmpl" .}} diff --git a/x-pack/filebeat/_meta/common.reference.inputs.yml b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl similarity index 100% rename from x-pack/filebeat/_meta/common.reference.inputs.yml rename to x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 0ec62c8e01f2..43984533d62e 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1047,6 +1047,7 @@ filebeat.modules: #var.paths: + #=========================== Filebeat inputs ============================= # List of inputs to fetch data. @@ -1423,6 +1424,7 @@ filebeat.inputs: # Configure stream to filter to a specific stream: stdout, stderr or all (default) #stream: all + #------------------------------ NetFlow input -------------------------------- # Experimental: Config options for the Netflow/IPFIX collector over UDP input #- type: netflow @@ -1498,7 +1500,8 @@ filebeat.inputs: # The duration (in seconds) that the received messages are hidden from subsequent # retrieve requests after being retrieved by a ReceiveMessage request. #visibility_timeout: 300 -#========================== Filebeat autodiscover ============================== + +# =========================== Filebeat autodiscover ============================ # Autodiscover allows you to detect changes in the system and spawn new modules # or inputs as they happen. @@ -1515,7 +1518,7 @@ filebeat.inputs: # paths: # - /var/lib/docker/containers/${data.docker.container.id}/*.log -#========================= Filebeat global options ============================ +# ========================== Filebeat global options =========================== # Registry data path. If a relative path is used, it is considered relative to the # data path. @@ -1562,7 +1565,8 @@ filebeat.inputs: #reload.enabled: true #reload.period: 10s -#================================ General ====================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -1670,7 +1674,7 @@ filebeat.inputs: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -1833,7 +1837,7 @@ filebeat.inputs: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -1846,11 +1850,11 @@ filebeat.inputs: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -1991,7 +1995,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -2105,7 +2109,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -2284,7 +2288,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -2402,7 +2406,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -2436,7 +2440,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -2449,7 +2453,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Filebeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -2475,11 +2479,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -2523,8 +2529,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -2578,7 +2583,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -2592,13 +2597,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'filebeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "filebeat" +#setup.ilm.rollover_alias: 'filebeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'filebeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -2613,7 +2618,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -2668,9 +2673,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -2737,8 +2741,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -2880,7 +2883,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -2904,12 +2908,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 581e1a43f23b..51a0d40224e1 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -10,7 +10,7 @@ # For more available modules and options, please see the filebeat.reference.yml sample # configuration file. -#=========================== Filebeat inputs ============================= +# ============================== Filebeat inputs =============================== filebeat.inputs: @@ -62,8 +62,7 @@ filebeat.inputs: # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash #multiline.match: after - -#============================= Filebeat modules =============================== +# ============================== Filebeat modules ============================== filebeat.config.modules: # Glob pattern for configuration loading @@ -75,14 +74,15 @@ filebeat.config.modules: # Period on which files under path should be checked for changes #reload.period: 10s -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 #index.codec: best_compression #_source.enabled: false -#================================ General ===================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -97,8 +97,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -110,7 +109,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -127,7 +126,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -140,11 +139,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -157,7 +156,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -172,7 +171,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -182,7 +181,8 @@ processors: - add_docker_metadata: ~ - add_kubernetes_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -193,8 +193,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# filebeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -215,7 +215,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/x-pack/functionbeat/_meta/beat.reference.yml b/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl similarity index 99% rename from x-pack/functionbeat/_meta/beat.reference.yml rename to x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl index dc0b0832bd91..c306fb0ac2af 100644 --- a/x-pack/functionbeat/_meta/beat.reference.yml +++ b/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl @@ -6,8 +6,8 @@ # # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/functionbeat/index.html -# -#============================ Provider =============================== + +{{header "Provider"}} # Configure functions to run on AWS Lambda, currently we assume that the credentials # are present in the environment to correctly create the function when using the CLI. # diff --git a/x-pack/functionbeat/_meta/beat.yml b/x-pack/functionbeat/_meta/config/beat.yml.tmpl similarity index 99% rename from x-pack/functionbeat/_meta/beat.yml rename to x-pack/functionbeat/_meta/config/beat.yml.tmpl index ae1c7ee97dc9..533d33dc5992 100644 --- a/x-pack/functionbeat/_meta/beat.yml +++ b/x-pack/functionbeat/_meta/config/beat.yml.tmpl @@ -8,7 +8,7 @@ # https://www.elastic.co/guide/en/beats/functionbeat/index.html # -#============================ Provider =============================== +{{header "Provider"}} # Configure functions to run on AWS Lambda, currently we assume that the credentials # are present in the environment to correctly create the function when using the CLI. # @@ -337,7 +337,7 @@ functionbeat.provider.gcp.functions: # Define custom processors for this function. #processors: # - dissect: - # tokenizer: "%{key1} %{key2}" + # tokenizer: "%{key1} %{key2}" #==================== Elasticsearch template setting ========================== setup.template.settings: diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index c2db1e47b256..2d1c33ed0455 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -6,8 +6,8 @@ # # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/functionbeat/index.html -# -#============================ Provider =============================== + +# ================================== Provider ================================== # Configure functions to run on AWS Lambda, currently we assume that the credentials # are present in the environment to correctly create the function when using the CLI. # @@ -384,7 +384,7 @@ functionbeat.provider.gcp.functions: # - dissect: # tokenizer: "%{key1} %{key2}" -#================================ General ====================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -492,7 +492,7 @@ functionbeat.provider.gcp.functions: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -655,7 +655,7 @@ functionbeat.provider.gcp.functions: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Functionbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -668,11 +668,11 @@ functionbeat.provider.gcp.functions: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -813,7 +813,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -927,7 +927,11 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#================================= Paths ====================================== + + + + +# =================================== Paths ==================================== # The home path for the Functionbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -953,11 +957,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1001,8 +1007,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1056,7 +1061,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1070,13 +1075,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'functionbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "functionbeat" +#setup.ilm.rollover_alias: 'functionbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'functionbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1091,7 +1096,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1146,9 +1151,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1215,8 +1219,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Functionbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1358,7 +1361,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1382,12 +1386,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/x-pack/functionbeat/functionbeat.yml b/x-pack/functionbeat/functionbeat.yml index f26658b15e6f..bb8322840c80 100644 --- a/x-pack/functionbeat/functionbeat.yml +++ b/x-pack/functionbeat/functionbeat.yml @@ -8,7 +8,7 @@ # https://www.elastic.co/guide/en/beats/functionbeat/index.html # -#============================ Provider =============================== +# ================================== Provider ================================== # Configure functions to run on AWS Lambda, currently we assume that the credentials # are present in the environment to correctly create the function when using the CLI. # @@ -337,7 +337,7 @@ functionbeat.provider.gcp.functions: # Define custom processors for this function. #processors: # - dissect: - # tokenizer: "%{key1} %{key2}" + # tokenizer: "%{key1} %{key2}" #==================== Elasticsearch template setting ========================== setup.template.settings: @@ -345,7 +345,7 @@ setup.template.settings: #index.codec: best_compression #_source.enabled: false -#================================ General ===================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -360,8 +360,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -373,7 +372,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -390,7 +389,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Functionbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -403,11 +402,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -420,7 +419,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -435,7 +434,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -443,7 +442,8 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -454,8 +454,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# functionbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Functionbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -476,7 +476,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/x-pack/functionbeat/scripts/mage/config.go b/x-pack/functionbeat/scripts/mage/config.go index 576fc765b121..7fb4395e2bcd 100644 --- a/x-pack/functionbeat/scripts/mage/config.go +++ b/x-pack/functionbeat/scripts/mage/config.go @@ -10,21 +10,14 @@ import ( // XPackConfigFileParams returns the configuration of sample and reference configuration data. func XPackConfigFileParams() devtools.ConfigFileParams { - return devtools.ConfigFileParams{ - ShortParts: []string{ - devtools.OSSBeatDir("_meta/beat.yml"), - devtools.LibbeatDir("_meta/config.yml.tmpl"), - }, - ReferenceParts: []string{ - devtools.OSSBeatDir("_meta/beat.reference.yml"), - devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), - }, - ExtraVars: map[string]interface{}{ - "ExcludeConsole": true, - "ExcludeFileOutput": true, - "ExcludeKafka": true, - "ExcludeRedis": true, - "UseDockerMetadataProcessor": false, - }, + p := devtools.DefaultConfigFileParams() + p.Templates = append(p.Templates, "_meta/config/*.tmpl") + p.ExtraVars = map[string]interface{}{ + "ExcludeConsole": true, + "ExcludeFileOutput": true, + "ExcludeKafka": true, + "ExcludeRedis": true, + "UseDockerMetadataProcessor": false, } + return p } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 4cfc5e8b7505..1518ef5ba229 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -53,6 +53,7 @@ metricbeat.max_start_delay: 10s #timeseries.enabled: false + #========================== Modules configuration ============================= metricbeat.modules: @@ -1297,7 +1298,8 @@ metricbeat.modules: -#================================ General ====================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -1405,7 +1407,7 @@ metricbeat.modules: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -1568,7 +1570,7 @@ metricbeat.modules: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -1581,11 +1583,11 @@ metricbeat.modules: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -1726,7 +1728,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -1840,7 +1842,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -2019,7 +2021,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -2137,7 +2139,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -2171,7 +2173,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -2184,7 +2186,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Metricbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -2210,11 +2212,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -2258,8 +2262,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -2313,7 +2316,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -2327,13 +2330,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'metricbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "metricbeat" +#setup.ilm.rollover_alias: 'metricbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'metricbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -2348,7 +2351,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -2403,9 +2406,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -2472,8 +2474,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -2615,7 +2616,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -2639,12 +2641,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/x-pack/metricbeat/metricbeat.yml b/x-pack/metricbeat/metricbeat.yml index 5bd19f3030c2..96975ee6027f 100644 --- a/x-pack/metricbeat/metricbeat.yml +++ b/x-pack/metricbeat/metricbeat.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/metricbeat/index.html -#========================== Modules configuration ============================ +# =========================== Modules configuration ============================ metricbeat.config.modules: # Glob pattern for configuration loading @@ -19,14 +19,15 @@ metricbeat.config.modules: # Period on which files under path should be checked for changes #reload.period: 10s -#==================== Elasticsearch template setting ========================== +# ======================= Elasticsearch template setting ======================= setup.template.settings: index.number_of_shards: 1 index.codec: best_compression #_source.enabled: false -#================================ General ===================================== + +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -41,8 +42,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -54,7 +54,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -71,7 +71,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -84,11 +84,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -101,7 +101,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -116,7 +116,7 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== +# ================================= Processors ================================= # Configure processors to enhance or manipulate events generated by the beat. @@ -126,7 +126,8 @@ processors: - add_docker_metadata: ~ - add_kubernetes_metadata: ~ -#================================ Logging ===================================== + +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -137,8 +138,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# metricbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -159,7 +160,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true + diff --git a/x-pack/winlogbeat/_meta/beat.yml.tmpl b/x-pack/winlogbeat/_meta/config/winlogbeat.event_logs.yml.tmpl similarity index 90% rename from x-pack/winlogbeat/_meta/beat.yml.tmpl rename to x-pack/winlogbeat/_meta/config/winlogbeat.event_logs.yml.tmpl index 1ea8cdcc879c..6c29d94f6db1 100644 --- a/x-pack/winlogbeat/_meta/beat.yml.tmpl +++ b/x-pack/winlogbeat/_meta/config/winlogbeat.event_logs.yml.tmpl @@ -1,4 +1,3 @@ -{{ template "header" . }} winlogbeat.event_logs: - name: Application ignore_older: 72h @@ -32,5 +31,3 @@ winlogbeat.event_logs: lang: javascript id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js - -{{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 5fa8a93532fa..9cf4eba1c171 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/winlogbeat/index.html -#======================= Winlogbeat specific options =========================== +# ======================== Winlogbeat specific options ========================= # The registry file is where Winlogbeat persists its state so that the beat can # resume after shutdown or an outage. The default is .winlogbeat.yml in the @@ -22,6 +22,7 @@ # forwarded, ignore_older, level, event_id, provider, and include_xml. Please # visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig + winlogbeat.event_logs: - name: Application ignore_older: 72h @@ -57,8 +58,7 @@ winlogbeat.event_logs: file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js - -#================================ General ====================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -166,7 +166,7 @@ winlogbeat.event_logs: # default is the number of logical CPUs available in the system. #max_procs: -#================================ Processors =================================== +# ================================= Processors ================================= # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of @@ -329,7 +329,7 @@ winlogbeat.event_logs: # ignore_missing: false # fail_on_error: true -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Winlogbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -342,11 +342,11 @@ winlogbeat.event_logs: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ====================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------- +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Boolean flag to enable or disable the output module. #enabled: true @@ -487,7 +487,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#----------------------------- Logstash output --------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. #enabled: true @@ -601,7 +601,7 @@ output.elasticsearch: # timing out. The default is 30s. #timeout: 30s -#------------------------------- Kafka output ---------------------------------- +# -------------------------------- Kafka Output -------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true @@ -780,7 +780,7 @@ output.elasticsearch: # Kerberos realm. #kerberos.realm: ELASTIC -#------------------------------- Redis output ---------------------------------- +# -------------------------------- Redis Output -------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true @@ -898,7 +898,7 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never -#------------------------------- File output ----------------------------------- +# -------------------------------- File Output --------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true @@ -932,7 +932,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 -#----------------------------- Console output --------------------------------- +# ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true @@ -945,7 +945,7 @@ output.elasticsearch: # Configure escaping HTML symbols in strings. #escape_html: false -#================================= Paths ====================================== +# =================================== Paths ==================================== # The home path for the Winlogbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the @@ -971,11 +971,13 @@ output.elasticsearch: # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs -#================================ Keystore ========================================== +# ================================== Keystore ================================== + # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= + # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards are disabled by default and can be enabled either by setting the # options here, or by using the `-setup` CLI flag or the `setup` command. @@ -1019,8 +1021,7 @@ output.elasticsearch: # Maximum number of retries before exiting with an error, 0 for unlimited retrying. #setup.dashboards.retry.maximum: 0 - -#============================== Template ===================================== +# ================================== Template ================================== # A template is used to set the mapping in Elasticsearch # By default template loading is enabled and the template is loaded. @@ -1074,7 +1075,7 @@ setup.template.settings: #_source: #enabled: false -#============================== Setup ILM ===================================== +# ====================== Index Lifecycle Management (ILM) ====================== # Configure index lifecycle management (ILM). These settings create a write # alias and add additional settings to the index template. When ILM is enabled, @@ -1088,13 +1089,13 @@ setup.template.settings: # Set the prefix used in the index lifecycle write alias name. The default alias # name is 'winlogbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: "winlogbeat" +#setup.ilm.rollover_alias: 'winlogbeat' # Set the rollover index pattern. The default is "%{now/d}-000001". #setup.ilm.pattern: "{now/d}-000001" # Set the lifecycle policy name. The default policy name is -# 'winlogbeat'. +# 'beatname'. #setup.ilm.policy_name: "mypolicy" # The path to a JSON file that contains a lifecycle policy configuration. Used @@ -1109,7 +1110,7 @@ setup.template.settings: # Overwrite the lifecycle policy at startup. The default is false. #setup.ilm.overwrite: false -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -1164,9 +1165,8 @@ setup.kibana: # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] +# ================================== Logging =================================== - -#================================ Logging ====================================== # There are four options for the log output: file, stderr, syslog, eventlog # The file output is the default. @@ -1233,8 +1233,7 @@ logging.files: # Set to true to log messages in JSON format. #logging.json: false - -#============================== X-Pack Monitoring =============================== +# ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -1376,7 +1375,8 @@ logging.files: # and `monitoring.elasticsearch.password` settings. The format is `:`. #monitoring.cloud.auth: -#================================ HTTP Endpoint ====================================== +# =============================== HTTP Endpoint ================================ + # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output @@ -1400,12 +1400,13 @@ logging.files: # `http.user`. #http.named_pipe.security_descriptor: -#============================= Process Security ================================ +# ============================== Process Security ============================== # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false + diff --git a/x-pack/winlogbeat/winlogbeat.yml b/x-pack/winlogbeat/winlogbeat.yml index e718fb91d413..bb852a289db5 100644 --- a/x-pack/winlogbeat/winlogbeat.yml +++ b/x-pack/winlogbeat/winlogbeat.yml @@ -7,7 +7,7 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/winlogbeat/index.html -#======================= Winlogbeat specific options =========================== +# ======================== Winlogbeat specific options ========================= # event_logs specifies a list of event logs to monitor as well as any # accompanying options. The YAML data type of event_logs is a list of @@ -17,6 +17,7 @@ # forwarded, ignore_older, level, event_id, provider, and include_xml. Please # visit the documentation for the complete details of each option. # https://go.es.io/WinlogbeatConfig + winlogbeat.event_logs: - name: Application ignore_older: 72h @@ -51,7 +52,7 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js -#==================== Elasticsearch template settings ========================== +# ====================== Elasticsearch template settings ======================= setup.template.settings: index.number_of_shards: 1 @@ -59,7 +60,7 @@ setup.template.settings: #_source.enabled: false -#================================ General ===================================== +# ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. @@ -74,8 +75,7 @@ setup.template.settings: #fields: # env: staging - -#============================== Dashboards ===================================== +# ================================= Dashboards ================================= # These settings control loading the sample dashboards to the Kibana index. Loading # the dashboards is disabled by default and can be enabled either by setting the # options here or by using the `setup` command. @@ -87,7 +87,7 @@ setup.template.settings: # website. #setup.dashboards.url: -#============================== Kibana ===================================== +# =================================== Kibana =================================== # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. # This requires a Kibana endpoint configuration. @@ -104,7 +104,7 @@ setup.kibana: # the Default Space will be used. #space.id: -#============================= Elastic Cloud ================================== +# =============================== Elastic Cloud ================================ # These settings simplify using Winlogbeat with the Elastic Cloud (https://cloud.elastic.co/). @@ -117,11 +117,11 @@ setup.kibana: # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: -#================================ Outputs ===================================== +# ================================== Outputs =================================== # Configure what output to use when sending the data collected by the beat. -#-------------------------- Elasticsearch output ------------------------------ +# ---------------------------- Elasticsearch Output ---------------------------- output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] @@ -134,7 +134,7 @@ output.elasticsearch: #username: "elastic" #password: "changeme" -#----------------------------- Logstash output -------------------------------- +# ------------------------------ Logstash Output ------------------------------- #output.logstash: # The Logstash hosts #hosts: ["localhost:5044"] @@ -149,14 +149,13 @@ output.elasticsearch: # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" -#================================ Processors ===================================== - +# ================================= Processors ================================= processors: - add_host_metadata: when.not.contains.tags: forwarded - add_cloud_metadata: ~ -#================================ Logging ===================================== +# ================================== Logging =================================== # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug @@ -167,8 +166,8 @@ processors: # "publish", "service". #logging.selectors: ["*"] -#============================== X-Pack Monitoring =============================== -# winlogbeat can export internal metrics to a central Elasticsearch monitoring +# ============================= X-Pack Monitoring ============================== +# Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The # reporting is disabled by default. @@ -189,7 +188,8 @@ processors: # uncomment the following line. #monitoring.elasticsearch: -#================================= Migration ================================== +# ================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: true +