From 00ecbb0ac2d403cfc9fc30a1ab010c1899be0a82 Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Mon, 14 Aug 2023 12:03:21 -0700 Subject: [PATCH] Add --force-enable-module-filesets option to filebeat setup (#36286) * Add --force-enable-module-filesets option to filebeat setup Closes #30916 --- CHANGELOG.next.asciidoc | 2 + NOTICE.txt | 48 +++++ filebeat/autodiscover/builder/hints/logs.go | 6 +- filebeat/beater/filebeat.go | 16 +- .../docs/howto/load-ingest-pipelines.asciidoc | 37 +++- filebeat/fileset/factory.go | 2 +- filebeat/fileset/modules.go | 33 ++- filebeat/fileset/modules_integration_test.go | 24 +-- filebeat/fileset/modules_test.go | 8 +- filebeat/fileset/setup.go | 8 +- .../tests/integration/setup_pipelines_test.go | 202 ++++++++++++++++++ go.mod | 1 + go.sum | 3 + libbeat/cmd/instance/beat.go | 20 +- libbeat/cmd/setup.go | 36 ++-- libbeat/docs/command-reference.asciidoc | 6 + 16 files changed, 393 insertions(+), 59 deletions(-) create mode 100644 filebeat/tests/integration/setup_pipelines_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f28ef6e2f728..1b6e902fdb35 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -236,6 +236,8 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Added support for Okta OAuth2 provider in the httpjson input. {pull}36273[36273] - Add support of the interval parameter in Salesforce setupaudittrail-rest fileset. {issue}35917[35917] {pull}35938[35938] - Add device handling to Okta input package for entity analytics. {pull}36049[36049] +- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999] +- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}36286[36286] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 6575b082a584..f3af08adf814 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21183,6 +21183,37 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/otiai10/copy +Version: v1.12.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/otiai10/copy@v1.12.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2018 otiai10 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/pierrec/lz4/v4 Version: v4.1.15 @@ -46273,6 +46304,23 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-sp limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/otiai10/mint +Version: v1.5.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.5.1/LICENSE: + +Copyright 2017 otiai10 (Hiromu OCHIAI) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/oxtoacart/bpool Version: v0.0.0-20150712133111-4e1c5567d7c2 diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 59d2eea70ae1..72d8e4a63dd2 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -69,7 +69,7 @@ func NewLogHints(cfg *conf.C) (autodiscover.Builder, error) { return nil, fmt.Errorf("unable to unpack hints config due to error: %w", err) } - moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, false) + moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, fileset.FilesetOverrides{}) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf return template.ApplyConfigTemplate(event, configs) } - var configs []*conf.C + var configs []*conf.C //nolint:prealloc //breaks tests inputs := l.getInputs(hints) for _, h := range inputs { // Clone original config, enable it if disabled @@ -262,7 +262,7 @@ func (l *logHints) getFilesets(hints mapstr.M, module string) map[string]*filese func (l *logHints) getInputs(hints mapstr.M) []mapstr.M { modules := utils.GetHintsAsList(hints, l.config.Key) - var output []mapstr.M + var output []mapstr.M //nolint:prealloc //breaks tests for _, mod := range modules { output = append(output, mapstr.M{ diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 532d8132110f..2fddfd3bb4fb 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -108,7 +108,13 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea return nil, err } - moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, false) + enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1) + forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1) + filesetOverrides := fileset.FilesetOverrides{ + EnableAllFilesets: enableAllFilesets, + ForceEnableModuleFilesets: forceEnableModuleFilesets, + } + moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, filesetOverrides) if err != nil { return nil, err } @@ -199,7 +205,13 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error { // have to be loaded using cfg.Reloader. Otherwise those configurations are skipped. pipelineLoaderFactory := newPipelineLoaderFactory(b.Config.Output.Config()) enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1) - modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, enableAllFilesets) + forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1) + filesetOverrides := fileset.FilesetOverrides{ + EnableAllFilesets: enableAllFilesets, + ForceEnableModuleFilesets: forceEnableModuleFilesets, + } + + modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, filesetOverrides) if fb.config.ConfigModules.Enabled() { if enableAllFilesets { // All module configs need to be loaded to enable all the filesets diff --git a/filebeat/docs/howto/load-ingest-pipelines.asciidoc b/filebeat/docs/howto/load-ingest-pipelines.asciidoc index a6ec44e7ea28..23dc1a2ac64c 100644 --- a/filebeat/docs/howto/load-ingest-pipelines.asciidoc +++ b/filebeat/docs/howto/load-ingest-pipelines.asciidoc @@ -5,7 +5,7 @@ The ingest pipelines used to parse log lines are set up automatically the first time you run {beatname_uc}, assuming the {es} output is enabled. If you're sending events to {ls} you need to load the ingest pipelines manually. To do this, run the `setup` command with the `--pipelines` option specified. You also need to enable -the modules and filesets, this can be accomplished one of two ways. +the modules and filesets, this can be accomplished several ways. First you can use the `--modules` option to enable the module, and the `-M` option to enable the fileset. For example, the following command @@ -39,7 +39,40 @@ loads the access pipeline from the nginx module. PS > .{backslash}{beatname_lc}.exe setup --pipelines --modules nginx -M "nginx.access.enabled=true" ---- -The second option is to use the `--enable-all-filesets` option to +The second option is to use the `--modules` option to enable the +module, and the `--force-enable-module-filesets` option to enable all +the filesets in the module. For example, the following command loads +the access pipeline from the nginx module. + +*deb and rpm:* + +["source","sh",subs="attributes"] +---- +{beatname_lc} setup --pipelines --modules nginx --force-enable-module-filesets +---- + +*mac:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} setup --pipelines --modules nginx --force-enable-module-filesets +---- + +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} setup --pipelines --modules nginx --force-enable-module-filesets +---- + +*win:* + +["source","sh",subs="attributes"] +---- +PS > .{backslash}{beatname_lc}.exe setup --pipelines --modules nginx --force-enable-module-filesets +---- + +The third option is to use the `--enable-all-filesets` option to enable all the modules and all the filesets so all of the ingest pipelines are loaded. For example, the following command loads all the ingest pipelines. diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index bdc6350618e2..0e5308946fa2 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -129,7 +129,7 @@ func (f *Factory) CheckConfig(c *conf.C) error { // createRegistry starts a registry for a set of filesets, it returns the registry and // its input configurations func (f *Factory) createRegistry(c *conf.C) (*ModuleRegistry, []*conf.C, error) { - m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, false) + m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, FilesetOverrides{}) if err != nil { return nil, nil, err } diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go index 158b45f56d39..b1482033c911 100644 --- a/filebeat/fileset/modules.go +++ b/filebeat/fileset/modules.go @@ -46,12 +46,17 @@ type Module struct { config ModuleConfig } +type FilesetOverrides struct { + EnableAllFilesets bool + ForceEnableModuleFilesets bool +} + // newModuleRegistry reads and loads the configured module into the registry. func newModuleRegistry(modulesPath string, moduleConfigs []*ModuleConfig, overrides *ModuleOverrides, beatInfo beat.Info, - enableAllFilesets bool, + filesetOverrides FilesetOverrides, ) (*ModuleRegistry, error) { reg := ModuleRegistry{ registry: []Module{}, @@ -61,7 +66,7 @@ func newModuleRegistry(modulesPath string, for _, mcfg := range moduleConfigs { // an empty ModuleConfig can reach this so we only force enable a // config if the Module name is set and Enabled pointer is valid. - if enableAllFilesets && mcfg.Module != "" && mcfg.Enabled != nil { + if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && mcfg.Module != "" && mcfg.Enabled != nil { *mcfg.Enabled = true } if mcfg.Module == "" || (mcfg.Enabled != nil && !(*mcfg.Enabled)) { @@ -80,8 +85,18 @@ func newModuleRegistry(modulesPath string, config: *mcfg, filesets: []Fileset{}, } - for filesetName, fcfg := range mcfg.Filesets { + if filesetOverrides.ForceEnableModuleFilesets { + if mcfg.Filesets == nil { + mcfg.Filesets = make(map[string]*FilesetConfig) + } + for _, fName := range moduleFilesets { + if _, ok := mcfg.Filesets[fName]; !ok { + mcfg.Filesets[fName] = &FilesetConfig{Enabled: func() *bool { b := true; return &b }()} + } + } + } + for filesetName, fcfg := range mcfg.Filesets { fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides) if err != nil { return nil, fmt.Errorf("error applying overrides on fileset %s/%s: %w", mcfg.Module, filesetName, err) @@ -89,7 +104,7 @@ func newModuleRegistry(modulesPath string, // ModuleConfig can have empty Filesets so we only force // enable if the Enabled pointer is valid - if enableAllFilesets && fcfg.Enabled != nil { + if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && fcfg.Enabled != nil { *fcfg.Enabled = true } if fcfg.Enabled != nil && !(*fcfg.Enabled) { @@ -128,14 +143,14 @@ func newModuleRegistry(modulesPath string, } // NewModuleRegistry reads and loads the configured module into the registry. -func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, enableAllFilesets bool) (*ModuleRegistry, error) { +func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, filesetOverrides FilesetOverrides) (*ModuleRegistry, error) { modulesPath := paths.Resolve(paths.Home, "module") stat, err := os.Stat(modulesPath) if err != nil || !stat.IsDir() { log := logp.NewLogger(logName) log.Errorf("Not loading modules. Module directory not found: %s", modulesPath) - return &ModuleRegistry{log: log}, nil // empty registry, no error + return &ModuleRegistry{log: log}, nil //nolint:nilerr // empty registry, no error } var modulesCLIList []string @@ -146,7 +161,7 @@ func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, e return nil, err } } - var mcfgs []*ModuleConfig + var mcfgs []*ModuleConfig //nolint:prealloc //breaks tests for _, cfg := range moduleConfigs { cfg, err = mergePathDefaults(cfg) if err != nil { @@ -166,7 +181,7 @@ func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, e } enableFilesetsFromOverrides(mcfgs, modulesOverrides) - return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, enableAllFilesets) + return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, filesetOverrides) } // enableFilesetsFromOverrides enables in mcfgs the filesets mentioned in overrides, @@ -433,7 +448,7 @@ func (reg *ModuleRegistry) Empty() bool { // ModuleNames returns the names of modules in the ModuleRegistry. func (reg *ModuleRegistry) ModuleNames() []string { - var modules []string + var modules []string //nolint:prealloc //breaks tests for _, m := range reg.registry { modules = append(modules, m.config.Module) } diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 2d06db83c093..0d5ad2172c06 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -46,7 +46,7 @@ func TestLoadPipeline(t *testing.T) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } - client.Request("DELETE", "/_ingest/pipeline/my-pipeline-id", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/my-pipeline-id", "", nil, nil) content := map[string]interface{}{ "description": "describe pipeline", @@ -98,8 +98,8 @@ func TestSetupNginx(t *testing.T) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } - client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-default", "", nil, nil) - client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-default", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil) modulesPath, err := filepath.Abs("../module") require.NoError(t, err) @@ -114,7 +114,7 @@ func TestSetupNginx(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), false) + reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), FilesetOverrides{}) if err != nil { t.Fatal(err) } @@ -176,9 +176,9 @@ func TestLoadMultiplePipelines(t *testing.T) { t.Skip("Skip tests because ingest is missing the pipeline processor: ", client.GetVersion()) } - client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-pipeline", "", nil, nil) - client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-json_logs", "", nil, nil) - client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-plain_logs", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-pipeline", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-json_logs", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multi-plain_logs", "", nil, nil) modulesPath, err := filepath.Abs("../_meta/test/module") require.NoError(t, err) @@ -193,7 +193,7 @@ func TestLoadMultiplePipelines(t *testing.T) { {"foo", &enabled, filesetConfigs}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false) + reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{}) if err != nil { t.Fatal(err) } @@ -221,9 +221,9 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { t.Skip("Skip tests because ingest is missing the pipeline processor: ", client.GetVersion()) } - client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-pipeline", "", nil, nil) - client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-json_logs", "", nil, nil) - client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-plain_logs_bad", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-pipeline", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-json_logs", "", nil, nil) + _, _, _ = client.Request("DELETE", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-plain_logs_bad", "", nil, nil) modulesPath, err := filepath.Abs("../_meta/test/module") require.NoError(t, err) @@ -238,7 +238,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { {"foo", &enabled, filesetConfigs}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false) + reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{}) if err != nil { t.Fatal(err) } diff --git a/filebeat/fileset/modules_test.go b/filebeat/fileset/modules_test.go index f1baed9e5c50..2328e327c004 100644 --- a/filebeat/fileset/modules_test.go +++ b/filebeat/fileset/modules_test.go @@ -81,7 +81,7 @@ func TestNewModuleRegistry(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false) + reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) @@ -148,7 +148,7 @@ func TestNewModuleRegistryConfig(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false) + reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) @@ -174,7 +174,7 @@ func TestMovedModule(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false) + reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) } @@ -445,7 +445,7 @@ func TestMissingModuleFolder(t *testing.T) { load(t, map[string]interface{}{"module": "nginx"}), } - reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, false) + reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) diff --git a/filebeat/fileset/setup.go b/filebeat/fileset/setup.go index 3ef5a6f2997e..a17a504ab7f3 100644 --- a/filebeat/fileset/setup.go +++ b/filebeat/fileset/setup.go @@ -29,22 +29,22 @@ type SetupFactory struct { beatInfo beat.Info pipelineLoaderFactory PipelineLoaderFactory overwritePipelines bool - enableAllFilesets bool + filesetOverrides FilesetOverrides } // NewSetupFactory creates a SetupFactory -func NewSetupFactory(beatInfo beat.Info, pipelineLoaderFactory PipelineLoaderFactory, enableAllFilesets bool) *SetupFactory { +func NewSetupFactory(beatInfo beat.Info, pipelineLoaderFactory PipelineLoaderFactory, filesetOverrides FilesetOverrides) *SetupFactory { return &SetupFactory{ beatInfo: beatInfo, pipelineLoaderFactory: pipelineLoaderFactory, overwritePipelines: true, - enableAllFilesets: enableAllFilesets, + filesetOverrides: filesetOverrides, } } // Create creates a new SetupCfgRunner to setup module configuration. func (sf *SetupFactory) Create(_ beat.PipelineConnector, c *conf.C) (cfgfile.Runner, error) { - m, err := NewModuleRegistry([]*conf.C{c}, sf.beatInfo, false, sf.enableAllFilesets) + m, err := NewModuleRegistry([]*conf.C{c}, sf.beatInfo, false, sf.filesetOverrides) if err != nil { return nil, err } diff --git a/filebeat/tests/integration/setup_pipelines_test.go b/filebeat/tests/integration/setup_pipelines_test.go new file mode 100644 index 000000000000..bb74ecbe24f2 --- /dev/null +++ b/filebeat/tests/integration/setup_pipelines_test.go @@ -0,0 +1,202 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package integration + +import ( + "fmt" + "net/http" + "net/url" + "path/filepath" + "testing" + "time" + + cp "github.com/otiai10/copy" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/tests/integration" +) + +func TestSetupNoModules(t *testing.T) { + cfg := ` +filebeat: +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module directory") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + filebeat.Start("setup") + filebeat.WaitForLogs("Setup called, but no modules enabled.", 10*time.Second) +} + +func TestSetupModulesNoFileset(t *testing.T) { + cfg := ` +filebeat.config: + modules: + enabled: true + path: modules.d/*.yml +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +logging.level: debug +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module directory") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + filebeat.Start("setup", "--pipelines") + filebeat.WaitForLogs("Number of module configs found: 0", 10*time.Second) +} + +func TestSetupModulesOneEnabled(t *testing.T) { + cfg := ` +filebeat.config: + modules: + enabled: true + path: modules.d/*.yml +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +logging.level: debug +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module directory") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + filebeat.Start("setup", "--pipelines", "--modules", "apache") + filebeat.WaitForLogs("Exiting: module apache is configured but has no enabled filesets", 10*time.Second) +} + +func TestSetupModulesOneEnabledOverride(t *testing.T) { + cfg := ` +filebeat.config: + modules: + enabled: true + path: modules.d/*.yml +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +logging.level: debug +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module directory") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + + t.Cleanup(func() { + delURL, err := url.Parse(esURL.String()) + if err != nil { + t.Fatalf("could not parse ES url: %s", err) + } + delURL.Path = "/_ingest/pipeline/filebeat-*" + ret, _, err := integration.HttpDo(t, http.MethodDelete, *delURL) + if err != nil { + t.Logf("error while deleting filebeat-* pipelines: %s", err) + } + if ret != http.StatusOK { + t.Logf("status was %d while deleting filebeat-* pipelines", ret) + } + }) + filebeat.Start("setup", "--pipelines", "--modules", "apache", "--force-enable-module-filesets") + filebeat.WaitForLogs("Elasticsearch pipeline loaded.", 10*time.Second) + filebeat.WaitForLogs("Elasticsearch pipeline loaded.", 10*time.Second) +} diff --git a/go.mod b/go.mod index 67c9c980f5b9..256c343b109f 100644 --- a/go.mod +++ b/go.mod @@ -214,6 +214,7 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/lestrrat-go/jwx/v2 v2.0.11 + github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.15 github.com/shirou/gopsutil/v3 v3.22.10 go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 diff --git a/go.sum b/go.sum index 4cca126049fa..cb386cea46c5 100644 --- a/go.sum +++ b/go.sum @@ -1325,6 +1325,9 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/osquery/osquery-go v0.0.0-20220706183148-4e1f83012b42 h1:Epwxipb+y/e8ss/SJ7947F8J6dwjv3RHRCz2g0OkCII= github.com/osquery/osquery-go v0.0.0-20220706183148-4e1f83012b42/go.mod h1:0KzmMhe0PL19cdYq6nd1cT9/5bMMJBTssAfuEgM2i34= +github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= +github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 0f1d44a777c6..b881fc4bdeec 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -201,7 +201,6 @@ func initRand() { // instance. // XXX Move this as a *Beat method? func Run(settings Settings, bt beat.Creator) error { - return handleError(func() error { defer func() { if r := recover(); r != nil { @@ -502,7 +501,7 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { } ctx, cancel := context.WithCancel(context.Background()) - var stopBeat = func() { + stopBeat := func() { b.Instrumentation.Tracer().Close() beater.Stop() } @@ -615,8 +614,9 @@ type SetupSettings struct { // Deprecated: use IndexManagementKey instead Template bool // Deprecated: use IndexManagementKey instead - ILMPolicy bool - EnableAllFilesets bool + ILMPolicy bool + EnableAllFilesets bool + ForceEnableModuleFilesets bool } // Setup registers ES index template, kibana dashboards, ml jobs and pipelines. @@ -628,16 +628,19 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if err != nil { return err } - // Tell the beat that we're in the setup command b.InSetupCmd = true + if setup.ForceEnableModuleFilesets { + if err := b.Beat.BeatConfig.SetBool("config.modules.force_enable_module_filesets", -1, true); err != nil { + return fmt.Errorf("error setting force_enable_module_filesets config option %w", err) + } + } // Create beater to give it the opportunity to set loading callbacks _, err = b.createBeater(bt) if err != nil { return err } - if setup.IndexManagement || setup.Template || setup.ILMPolicy { outCfg := b.Config.Output if !isElasticsearchOutput(outCfg.Name()) { @@ -648,7 +651,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er return err } - var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset + loadTemplate, loadILM := idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset if setup.IndexManagement || setup.Template { loadTemplate = idxmgmt.LoadModeOverwrite } @@ -687,6 +690,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er return fmt.Errorf("error setting enable_all_filesets config option %w", err) } } + esConfig := b.Config.Output.Config() err = b.OverwritePipelinesCallback(esConfig) if err != nil { @@ -900,7 +904,7 @@ func (b *Beat) loadMeta(metaPath string) error { // write temporary file first tempFile := metaPath + ".new" - f, err = os.OpenFile(tempFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + f, err = os.OpenFile(tempFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return fmt.Errorf("failed to create Beat meta file: %w", err) } diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index f1d2432ead94..49a2cb5cfbea 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -28,14 +28,18 @@ import ( ) const ( - //DashboardKey used for registering dashboards in setup cmd + // DashboardKey used for registering dashboards in setup cmd DashboardKey = "dashboards" - //PipelineKey used for registering pipelines in setup cmd + // PipelineKey used for registering pipelines in setup cmd PipelineKey = "pipelines" - //IndexManagementKey used for loading all components related to ES index management in setup cmd + // IndexManagementKey used for loading all components related to ES index management in setup cmd IndexManagementKey = "index-management" - //EnableAllFilesetsKey enables all modules and filesets regardless of config + // EnableAllFilesetsKey enables all modules and filesets regardless of config EnableAllFilesetsKey = "enable-all-filesets" + // ForceEnableModuleFilesets enables all the filesets contained + // in the modules that have been explicitly enabled. The + // requires "modules" to be used. + ForceEnableModuleFilesets = "force-enable-module-filesets" ) func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Command { @@ -56,32 +60,33 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co os.Exit(1) } - var registeredFlags = map[string]bool{ - DashboardKey: false, - PipelineKey: false, - IndexManagementKey: false, - EnableAllFilesetsKey: false, + registeredFlags := map[string]bool{ + DashboardKey: false, + PipelineKey: false, + IndexManagementKey: false, + EnableAllFilesetsKey: false, + ForceEnableModuleFilesets: false, } - var setupAll = true + setupAll := true // create collection with registered flags and their values for k := range registeredFlags { val, err := cmd.Flags().GetBool(k) - //if flag is not registered, an error is thrown + // if flag is not registered, an error is thrown if err != nil { delete(registeredFlags, k) continue } registeredFlags[k] = val - //if any flag is set via cmd line then only this flag should be run + // if any flag is set via cmd line then only this flag should be run if val { setupAll = false } } - //create the struct to pass on - var s = instance.SetupSettings{} + // create the struct to pass on + s := instance.SetupSettings{} for k, v := range registeredFlags { if setupAll || v { switch k { @@ -93,6 +98,8 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co s.IndexManagement = true case EnableAllFilesetsKey: s.EnableAllFilesets = true + case ForceEnableModuleFilesets: + s.ForceEnableModuleFilesets = true } } } @@ -108,6 +115,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co setup.Flags().Bool(IndexManagementKey, false, "Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias") setup.Flags().Bool("enable-all-filesets", false, "Behave as if all modules and filesets had been enabled") + setup.Flags().Bool("force-enable-module-filesets", false, "Behave as if all filesets, within enabled modules, are enabled") return &setup } diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index d3d88a57ab8c..a4246e8c5e37 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -842,6 +842,12 @@ would have to list every module with the <> command and enable every fileset within the module with a `-M` option, to load all of the ingest pipelines. +*`--force-enable-module-filesets`*:: +Enables all filesets in the enabled modules. This is useful with +`--pipelines` if you want to load ingest pipelines. Without this +option you enable every fileset within the module with a `-M` option, +to load the ingest pipelines. + endif::[] *`--index-management`*::