From 5d75a6534af77753480683a26440f932bcc953cd Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Fri, 26 Oct 2018 13:18:28 -0400 Subject: [PATCH] The 'export config' subcommand should display field reference instead of values Change the behavior of the export config to not display the values from the keystore or the environment. --- CHANGELOG-developer.asciidoc | 1 + CHANGELOG.asciidoc | 2 ++ NOTICE.txt | 4 +-- libbeat/cmd/export/config.go | 2 ++ libbeat/cmd/instance/beat.go | 35 ++++++++++++++++--- libbeat/cmd/instance/settings.go | 13 +++---- libbeat/keystore/keystore.go | 11 ------ libbeat/tests/system/beat/beat.py | 22 ++++++++---- libbeat/tests/system/test_cmd.py | 18 ++++++++++ libbeat/tests/system/test_keystore.py | 21 +++++++++++ .../github.com/elastic/go-ucfg/CHANGELOG.md | 8 ++++- vendor/github.com/elastic/go-ucfg/opts.go | 13 +++++++ vendor/vendor.json | 10 +++--- 13 files changed, 124 insertions(+), 36 deletions(-) diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index ac0b60a293e..0e99f83776e 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -61,3 +61,4 @@ The list below covers the major changes between 6.3.0 and master only. - Add `mage.GenerateModuleReferenceConfig` for generating reference config files that include configuration sections from the module directory. {pull}8615[8615] - Add `mage.GenerateFieldsGo` for generating fields.go files. {pull}8615[8615] - Add `mage.KibanaDashboards` for collecting Kibana dashboards and generating index patterns. {pull}8615[8615] +- Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769] diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 15bff0c2915..15a7b726a1d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -53,6 +53,8 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Fix race condition when publishing monitoring data. {pull}8646[8646] - Fix in-cluster kubernetes configuration on IPv6. {pull}8754[8754] - Fix bug in loading dashboards from zip file. {issue}8051[8051] +- The export config subcommand should not display real value for field reference. {pull}xxx[xxx] +- The export config subcommand should not display real value for field reference. {pull}8769[8769] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 4c77d72651a..2047473f854 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -567,8 +567,8 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/elastic/go-ucfg -Version: v0.6.4 -Revision: e81c02ad8f1ab46b9e8b07f0832245c0c2e1d13c +Version: v0.6.5 +Revision: 92d43887f91851c9936621665af7f796f4d03412 License type (autodetected): Apache-2.0 ./vendor/github.com/elastic/go-ucfg/LICENSE: -------------------------------------------------------------------- diff --git a/libbeat/cmd/export/config.go b/libbeat/cmd/export/config.go index 60d296a70a8..a678f63c560 100644 --- a/libbeat/cmd/export/config.go +++ b/libbeat/cmd/export/config.go @@ -45,6 +45,8 @@ func exportConfig(settings instance.Settings, name, idxPrefix, beatVersion strin return fmt.Errorf("error initializing beat: %s", err) } + settings.DisableConfigResolver = true + err = b.InitWithSettings(settings) if err != nil { return fmt.Errorf("error initializing beat: %s", err) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index ffbb8df90ab..c36ca8da02a 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -38,6 +38,7 @@ import ( "github.com/elastic/go-sysinfo" "github.com/elastic/go-sysinfo/types" + ucfg "github.com/elastic/go-ucfg" "github.com/elastic/beats/libbeat/api" "github.com/elastic/beats/libbeat/asset" @@ -149,12 +150,13 @@ func init() { // CryptGenRandom is used. func initRand() { n, err := cryptRand.Int(cryptRand.Reader, big.NewInt(math.MaxInt64)) - seed := n.Int64() + var seed int64 if err != nil { // fallback to current timestamp seed = time.Now().UnixNano() + } else { + seed = n.Int64() } - rand.Seed(seed) } @@ -563,8 +565,13 @@ func (b *Beat) configure(settings Settings) error { return fmt.Errorf("could not initialize the keystore: %v", err) } - // TODO: Allow the options to be more flexible for dynamic changes - common.OverwriteConfigOpts(keystore.ConfigOpts(store)) + if settings.DisableConfigResolver { + common.OverwriteConfigOpts(obfuscateConfigOpts()) + } else { + // TODO: Allow the options to be more flexible for dynamic changes + common.OverwriteConfigOpts(configOpts(store)) + } + b.keystore = store err = cloudid.OverwriteSettings(cfg) if err != nil { @@ -880,3 +887,23 @@ func logSystemInfo(info beat.Info) { } } } + +// configOpts returns ucfg config options with a resolver linked to the current keystore. +// TODO: Refactor to allow insert into the config option array without having to redefine everything +func configOpts(store keystore.Keystore) []ucfg.Option { + return []ucfg.Option{ + ucfg.PathSep("."), + ucfg.Resolve(keystore.ResolverWrap(store)), + ucfg.ResolveEnv, + ucfg.VarExp, + } +} + +// obfuscateConfigOpts disables any resolvers in the configuration, instead we return the field +// reference string directly. +func obfuscateConfigOpts() []ucfg.Option { + return []ucfg.Option{ + ucfg.PathSep("."), + ucfg.ResolveNOOP, + } +} diff --git a/libbeat/cmd/instance/settings.go b/libbeat/cmd/instance/settings.go index 09c1180a56f..8275282daad 100644 --- a/libbeat/cmd/instance/settings.go +++ b/libbeat/cmd/instance/settings.go @@ -26,10 +26,11 @@ import ( // Settings contains basic settings for any beat to pass into GenRootCmd type Settings struct { - Name string - IndexPrefix string - Version string - Monitoring report.Settings - RunFlags *pflag.FlagSet - ConfigOverrides *common.Config + Name string + IndexPrefix string + Version string + Monitoring report.Settings + RunFlags *pflag.FlagSet + ConfigOverrides *common.Config + DisableConfigResolver bool } diff --git a/libbeat/keystore/keystore.go b/libbeat/keystore/keystore.go index b7ec9820834..eb95c3e041d 100644 --- a/libbeat/keystore/keystore.go +++ b/libbeat/keystore/keystore.go @@ -120,14 +120,3 @@ func ResolverWrap(keystore Keystore) func(string) (string, error) { return string(v), nil } } - -// ConfigOpts returns ucfg config options with a resolver linked to the current keystore. -// TODO: Refactor to allow insert into the config option array without having to redefine everything -func ConfigOpts(keystore Keystore) []ucfg.Option { - return []ucfg.Option{ - ucfg.PathSep("."), - ucfg.Resolve(ResolverWrap(keystore)), - ucfg.ResolveEnv, - ucfg.VarExp, - } -} diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index daf10fbe3fb..ecceb017542 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -38,21 +38,27 @@ class Proc(object): the object gets collected. """ - def __init__(self, args, outputfile): + def __init__(self, args, outputfile, env={}): self.args = args self.output = open(outputfile, "ab") self.stdin_read, self.stdin_write = os.pipe() + self.env = env def start(self): if sys.platform.startswith("win"): + # ensure that the environment is inherited to the subprocess. + variables = os.environ.copy() + variables = variables.update(self.env) + self.proc = subprocess.Popen( self.args, stdin=self.stdin_read, stdout=self.output, stderr=subprocess.STDOUT, bufsize=0, - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + env=variables) else: self.proc = subprocess.Popen( self.args, @@ -60,7 +66,7 @@ def start(self): stdout=self.output, stderr=subprocess.STDOUT, bufsize=0, - ) + env=self.env) # If a "No such file or directory" error points you here, run # "make metricbeat.test" on metricbeat folder return self.proc @@ -145,7 +151,8 @@ def run_beat(self, output=None, logging_args=["-e", "-v", "-d", "*"], extra_args=[], - exit_code=None): + exit_code=None, + env={}): """ Executes beat. Waits for the process to finish before returning to @@ -153,7 +160,7 @@ def run_beat(self, """ proc = self.start_beat(cmd=cmd, config=config, output=output, logging_args=logging_args, - extra_args=extra_args) + extra_args=extra_args, env=env) if exit_code != None: return proc.check_wait(exit_code) @@ -164,7 +171,8 @@ def start_beat(self, config=None, output=None, logging_args=["-e", "-v", "-d", "*"], - extra_args=[]): + extra_args=[], + env={}): """ Starts beat and returns the process handle. The caller is responsible for stopping / waiting for the @@ -195,7 +203,7 @@ def start_beat(self, if extra_args: args.extend(extra_args) - proc = Proc(args, os.path.join(self.working_dir, output)) + proc = Proc(args, os.path.join(self.working_dir, output), env) proc.start() return proc diff --git a/libbeat/tests/system/test_cmd.py b/libbeat/tests/system/test_cmd.py index ee9471b4f72..c657a3cad3c 100644 --- a/libbeat/tests/system/test_cmd.py +++ b/libbeat/tests/system/test_cmd.py @@ -149,6 +149,24 @@ def test_export_config(self): assert self.log_contains("filename: mockbeat") assert self.log_contains("period: 1234") + def test_export_config_environment_variable(self): + """ + Test export config works but doesn"t expose environment variable. + """ + self.render_config_template("mockbeat", + os.path.join(self.working_dir, + "libbeat.yml"), + metrics_period="${METRIC_PERIOD}") + + exit_code = self.run_beat( + logging_args=[], + extra_args=["export", "config"], + config="libbeat.yml", env={'METRIC_PERIOD': '1234'}) + + assert exit_code == 0 + assert self.log_contains("filename: mockbeat") + assert self.log_contains("period: ${METRIC_PERIOD}") + def test_export_template(self): """ Test export template works diff --git a/libbeat/tests/system/test_keystore.py b/libbeat/tests/system/test_keystore.py index 996a11c9ed1..1f3b86d2586 100644 --- a/libbeat/tests/system/test_keystore.py +++ b/libbeat/tests/system/test_keystore.py @@ -70,3 +70,24 @@ def test_keystore_with_nested_key(self): self.wait_until(lambda: self.log_contains("Elasticsearch url: http://myeleasticsearchsecrethost:9200")) assert self.log_contains(secret) proc.check_kill_and_wait() + + def test_export_config_with_keystore(self): + """ + Test export config works and doesn't expose keystore value + """ + key = "asecret" + secret = "asecretvalue" + + self.render_config_template(keystore_path=self.keystore_path, elasticsearch={ + 'hosts': "${%s}" % key + }) + + exit_code = self.run_beat(extra_args=["keystore", "create"]) + assert exit_code == 0 + + self.add_secret(key, value=secret) + exit_code = self.run_beat(extra_args=["export", "config"]) + + assert exit_code == 0 + assert self.log_contains(secret) == False + assert self.log_contains("${%s}" % key) diff --git a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md index 2dce3c26f5f..3ec656a8887 100644 --- a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md +++ b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md @@ -14,6 +14,11 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +## [0.6.5] + +### Added +- Added a NOOP Resolver that will return the key wrapped in the field reference syntax. #122 + ## [0.6.4] ### Fixed @@ -220,7 +225,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Introduced CHANGELOG.md for documenting changes to ucfg. -[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.6.4...HEAD +[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.6.5...HEAD +[0.6.4]: https://github.com/elastic/go-ucfg/compare/v0.6.4...v0.6.5 [0.6.4]: https://github.com/elastic/go-ucfg/compare/v0.6.3...v0.6.4 [0.6.3]: https://github.com/elastic/go-ucfg/compare/v0.6.2...v0.6.3 [0.6.2]: https://github.com/elastic/go-ucfg/compare/v0.6.1...v0.6.2 diff --git a/vendor/github.com/elastic/go-ucfg/opts.go b/vendor/github.com/elastic/go-ucfg/opts.go index 80075d946d9..69937340c77 100644 --- a/vendor/github.com/elastic/go-ucfg/opts.go +++ b/vendor/github.com/elastic/go-ucfg/opts.go @@ -123,6 +123,19 @@ func doResolveEnv(o *options) { }) } +// ResolveNOOP option add a resolver that will not search the value but instead will return the +// provided key wrap with the field reference syntax. This is useful if you don't to expose values +// from envionment variable or other resolvers. +// +// Example: "mysecret" => ${mysecret}" +var ResolveNOOP Option = doResolveNOOP + +func doResolveNOOP(o *options) { + o.resolvers = append(o.resolvers, func(name string) (string, error) { + return "${" + name + "}", nil + }) +} + var ( // ReplacesValues option configures all merging and unpacking operations to // replace old dictionaries and arrays while merging. Value merging can be diff --git a/vendor/vendor.json b/vendor/vendor.json index bb52089c9ae..65243faf845 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1002,12 +1002,12 @@ "versionExact": "v0.0.3" }, { - "checksumSHA1": "+FZjRtpNIIvf+ZteS0y6IGw/JA8=", + "checksumSHA1": "Yb61Nqnh+3igFci61hv9WYgk/hc=", "path": "github.com/elastic/go-ucfg", - "revision": "e81c02ad8f1ab46b9e8b07f0832245c0c2e1d13c", - "revisionTime": "2018-10-05T15:55:04Z", - "version": "v0.6.4", - "versionExact": "v0.6.4" + "revision": "92d43887f91851c9936621665af7f796f4d03412", + "revisionTime": "2018-10-26T17:42:06Z", + "version": "v0.6.5", + "versionExact": "v0.6.5" }, { "checksumSHA1": "X+R/CD8SokJrmlxFTx2nSevRDhQ=",