fix(config): Restore old environment var behavior with option (influx…

…data#13457)

cmd/telegraf/main.go

@@ -219,17 +219,18 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
 		filters := processFilterFlags(cCtx)
 
 		g := GlobalFlags{
-			config:      cCtx.StringSlice("config"),
-			configDir:   cCtx.StringSlice("config-directory"),
-			testWait:    cCtx.Int("test-wait"),
-			watchConfig: cCtx.String("watch-config"),
-			pidFile:     cCtx.String("pidfile"),
-			plugindDir:  cCtx.String("plugin-directory"),
-			password:    cCtx.String("password"),
-			test:        cCtx.Bool("test"),
-			debug:       cCtx.Bool("debug"),
-			once:        cCtx.Bool("once"),
-			quiet:       cCtx.Bool("quiet"),
+			config:         cCtx.StringSlice("config"),
+			configDir:      cCtx.StringSlice("config-directory"),
+			testWait:       cCtx.Int("test-wait"),
+			watchConfig:    cCtx.String("watch-config"),
+			pidFile:        cCtx.String("pidfile"),
+			plugindDir:     cCtx.String("plugin-directory"),
+			password:       cCtx.String("password"),
+			oldEnvBehavior: cCtx.Bool("old-env-behavior"),
+			test:           cCtx.Bool("test"),
+			debug:          cCtx.Bool("debug"),
+			once:           cCtx.Bool("once"),
+			quiet:          cCtx.Bool("quiet"),
 		}
 
 		w := WindowFlags{
@@ -294,6 +295,10 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
 				},
 				//
 				// Bool flags
+				&cli.BoolFlag{
+					Name:  "old-env-behavior",
+					Usage: "switch back to pre v1.27 environment replacement behavior",
+				},
 				&cli.BoolFlag{
 					Name:  "once",
 					Usage: "run one gather and exit",

cmd/telegraf/telegraf.go

@@ -32,17 +32,18 @@ import (
 var stop chan struct{}
 
 type GlobalFlags struct {
-	config      []string
-	configDir   []string
-	testWait    int
-	watchConfig string
-	pidFile     string
-	plugindDir  string
-	password    string
-	test        bool
-	debug       bool
-	once        bool
-	quiet       bool
+	config         []string
+	configDir      []string
+	testWait       int
+	watchConfig    string
+	pidFile        string
+	plugindDir     string
+	password       string
+	oldEnvBehavior bool
+	test           bool
+	debug          bool
+	once           bool
+	quiet          bool
 }
 
 type WindowFlags struct {
@@ -87,6 +88,9 @@ func (t *Telegraf) Init(pprofErr <-chan error, f Filters, g GlobalFlags, w Windo
 	if g.password != "" {
 		config.Password = config.NewSecret([]byte(g.password))
 	}
+
+	// Set environment replacement behavior
+	config.OldEnvVarReplacement = g.oldEnvBehavior
 }
 
 func (t *Telegraf) ListSecretStores() ([]string, error) {

config/config.go

@@ -40,21 +40,19 @@ import (
 	"github.com/influxdata/telegraf/plugins/serializers"
 )
 
-// envVarPattern is a regex to determine environment variables in the
-// config file for substitution. Those should start with a dollar signs.
-// Expression modified from
-// https://github.com/compose-spec/compose-go/blob/v1.14.0/template/template.go
-const envVarPattern = `\\(?P<escaped>\$)|\$(?i:(?P<named>[_a-z][_a-z0-9]*)|\${(?:(?P<braced>[_a-z][_a-z0-9]*(?::?[-+?](.*))?)}|(?P<invalid>)))`
-
 var (
 	httpLoadConfigRetryInterval = 10 * time.Second
 
 	// fetchURLRe is a regex to determine whether the requested file should
 	// be fetched from a remote or read from the filesystem.
 	fetchURLRe = regexp.MustCompile(`^\w+://`)
 
-	// envVarRe is the compiled regex of envVarPattern
-	envVarRe = regexp.MustCompile(envVarPattern)
+	// oldVarRe is a regex to reproduce pre v1.27.0 environment variable
+	// replacement behavior
+	oldVarRe = regexp.MustCompile(`\$(?i:(?P<named>[_a-z][_a-z0-9]*)|{(?:(?P<braced>[_a-z][_a-z0-9]*(?::?[-+?](.*))?)}|(?P<invalid>)))`)
+	// OldEnvVarReplacement is a switch to allow going back to pre v1.27.0
+	// environment variable replacement behavior
+	OldEnvVarReplacement = false
 
 	// Password specified via command-line
 	Password Secret
@@ -794,7 +792,7 @@ func parseConfig(contents []byte) (*ast.Table, error) {
 	if err != nil {
 		return nil, err
 	}
-	outputBytes, err := substituteEnvironment(contents)
+	outputBytes, err := substituteEnvironment(contents, OldEnvVarReplacement)
 	if err != nil {
 		return nil, err
 	}
@@ -857,19 +855,36 @@ func removeComments(contents []byte) ([]byte, error) {
 	return output.Bytes(), nil
 }
 
-func substituteEnvironment(contents []byte) ([]byte, error) {
+func substituteEnvironment(contents []byte, oldReplacementBehavior bool) ([]byte, error) {
+	options := []template.Option{
+		template.WithReplacementFunction(func(s string, m template.Mapping, cfg *template.Config) (string, error) {
+			result, err := template.DefaultReplacementFunc(s, m, cfg)
+			if err == nil && result == "" {
+				// Keep undeclared environment-variable patterns to reproduce
+				// pre-v1.27 behavior
+				return s, nil
+			}
+			if err != nil && strings.HasPrefix(err.Error(), "Invalid template:") {
+				// Keep invalid template patterns to ignore regexp substitutions
+				// like ${1}
+				return s, nil
+			}
+			return result, err
+		}),
+		template.WithoutLogging,
+	}
+	if oldReplacementBehavior {
+		options = append(options, template.WithPattern(oldVarRe))
+	}
+
 	envMap := utils.GetAsEqualsMap(os.Environ())
-	retVal, err := template.SubstituteWith(string(contents), func(k string) (string, bool) {
+	retVal, err := template.SubstituteWithOptions(string(contents), func(k string) (string, bool) {
 		if v, ok := envMap[k]; ok {
 			return v, ok
 		}
 		return "", false
-	}, envVarRe)
-	var invalidTmplError *template.InvalidTemplateError
-	if err != nil && !errors.As(err, &invalidTmplError) {
-		return nil, err
-	}
-	return []byte(retVal), nil
+	}, options...)
+	return []byte(retVal), err
 }
 
 func (c *Config) addAggregator(name string, table *ast.Table) error {

config/config_test.go

@@ -71,7 +71,7 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
 
 	input := inputs.Inputs["memcached"]().(*MockupInputPlugin)
 	input.Servers = []string{"192.168.1.1"}
-	input.Command = `Raw command which may or may not contain # or ${var} in it
+	input.Command = `Raw command which may or may not contain # in it
 # is unique`
 
 	filter := models.Filter{