Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Elastic Agent] Fix issues with dynamic inputs and conditions #23886

Merged
merged 5 commits into from
Feb 11, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions x-pack/elastic-agent/CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
- Fix issue of missing log messages from filebeat monitor {pull}23514[23514]
- Increase checkin grace period to 30 seconds {pull}23568[23568]
- Fix libbeat from reporting back degraded on config update {pull}23537[23537]
- Fix issues with dynamic inputs and conditions {pull}23886[23886]

==== New features

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upg
// Load configuration from disk to understand in which mode of operation
// we must start the elastic-agent, the mode of operation cannot be changed without restarting the
// elastic-agent.
rawConfig, err := LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return nil, err
}
Expand Down
58 changes: 0 additions & 58 deletions x-pack/elastic-agent/pkg/agent/application/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,8 @@
package application

import (
"io/ioutil"

"github.com/elastic/go-ucfg"

"gopkg.in/yaml.v2"

"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
)

Expand All @@ -34,53 +26,3 @@ func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*conf
}
return cfg, nil
}

// LoadConfigFromFile loads the Agent configuration from a file.
//
// This must be used to load the Agent configuration, so that variables defined in the inputs are not
// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
func LoadConfigFromFile(path string) (*config.Config, error) {
in, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
var m map[string]interface{}
if err := yaml.Unmarshal(in, &m); err != nil {
return nil, err
}
return LoadConfig(m)
}

// LoadConfig loads the Agent configuration from a map.
//
// This must be used to load the Agent configuration, so that variables defined in the inputs are not
// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
func LoadConfig(in map[string]interface{}) (*config.Config, error) {
// make copy of a map so we dont affect a caller
m := common.MapStr(in).Clone()

inputs, ok := m["inputs"]
if ok {
// remove the inputs
delete(m, "inputs")
}
cfg, err := config.NewConfigFrom(m)
if err != nil {
return nil, err
}
if ok {
inputsOnly := map[string]interface{}{
"inputs": inputs,
}
// convert to config without variable substitution
inputsCfg, err := config.NewConfigFrom(inputsOnly, ucfg.PathSep("."), ucfg.ResolveNOOP)
if err != nil {
return nil, err
}
err = cfg.Merge(inputsCfg, ucfg.PathSep("."), ucfg.ResolveNOOP)
if err != nil {
return nil, err
}
}
return cfg, err
}
40 changes: 0 additions & 40 deletions x-pack/elastic-agent/pkg/agent/application/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ package application

import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"

Expand All @@ -20,44 +18,6 @@ import (
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
)

func TestLoadConfig(t *testing.T) {
contents := map[string]interface{}{
"outputs": map[string]interface{}{
"default": map[string]interface{}{
"type": "elasticsearch",
"hosts": []interface{}{"127.0.0.1:9200"},
"username": "elastic",
"password": "changeme",
},
},
"inputs": []interface{}{
map[string]interface{}{
"type": "logfile",
"streams": []interface{}{
map[string]interface{}{
"paths": []interface{}{"/var/log/${host.name}"},
},
},
},
},
}

tmp, err := ioutil.TempDir("", "config")
require.NoError(t, err)
defer os.RemoveAll(tmp)

cfgPath := filepath.Join(tmp, "config.yml")
dumpToYAML(t, cfgPath, contents)

cfg, err := LoadConfigFromFile(cfgPath)
require.NoError(t, err)

cfgData, err := cfg.ToMapStr()
require.NoError(t, err)

assert.Equal(t, contents, cfgData)
}

func TestConfig(t *testing.T) {
testMgmtMode(t)
testLocalConfig(t)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetA
return fmt.Errorf("invalid type, expected ActionPolicyChange and received %T", a)
}

c, err := LoadConfig(action.Policy)
c, err := config.NewConfigFrom(action.Policy)
if err != nil {
return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (c *InspectConfigCmd) inspectConfig() error {
}

func loadConfig(configPath string) (*config.Config, error) {
rawConfig, err := LoadConfigFromFile(configPath)
rawConfig, err := config.LoadFile(configPath)
if err != nil {
return nil, err
}
Expand Down
3 changes: 2 additions & 1 deletion x-pack/elastic-agent/pkg/agent/cmd/enroll.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"time"

"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"

"github.com/spf13/cobra"

Expand Down Expand Up @@ -92,7 +93,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args
}

pathConfigFile := flags.Config()
rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return errors.New(err,
fmt.Sprintf("could not read configuration file %s", pathConfigFile),
Expand Down
2 changes: 1 addition & 1 deletion x-pack/elastic-agent/pkg/agent/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se
service.HandleSignals(stopBeat, cancel)

pathConfigFile := flags.Config()
rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return errors.New(err,
fmt.Sprintf("could not read configuration file %s", pathConfigFile),
Expand Down
3 changes: 2 additions & 1 deletion x-pack/elastic-agent/pkg/agent/cmd/watch.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
)
Expand Down Expand Up @@ -182,7 +183,7 @@ func gracePeriod(marker *upgrade.UpdateMarker) (bool, time.Duration) {

func configuredLogger(flags *globalFlags) (*logger.Logger, error) {
pathConfigFile := flags.Config()
rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
rawConfig, err := config.LoadFile(pathConfigFile)
if err != nil {
return nil, errors.New(err,
fmt.Sprintf("could not read configuration file %s", pathConfigFile),
Expand Down
Loading