Support dynamic TOML

core/cmd/app.go

@@ -71,7 +71,7 @@ func NewApp(client *Client) *cli.App {
 		},
 		cli.StringSliceFlag{
 			Name:   "config, c",
-			Usage:  "TOML configuration file(s) via flag, or raw TOML via env var. If used, legacy env vars must not be set. Multiple files can be used (-c configA.toml -c configB.toml), and they are applied in order with duplicated fields overriding any earlier values.",
+			Usage:  "TOML configuration file(s) via flag, or raw TOML via env var. If used, legacy env vars must not be set. Multiple files can be used (-c configA.toml -c configB.toml), and they are applied in order with duplicated fields overriding any earlier values. If the env var is specified, it is always processed last with the effect of being the final override.",
 			EnvVar: "CL_CONFIG",
 		},
 		cli.StringFlag{
@@ -83,22 +83,9 @@ func NewApp(client *Client) *cli.App {
 		if c.IsSet("config") {
 			// TOML
 			var opts chainlink.GeneralConfigOpts
-			if configTOML := v2.EnvConfig.Get(); configTOML != "" {
-				if err := opts.ParseConfig(configTOML); err != nil {
-					return errors.Wrapf(err, "failed to parse env var %q", v2.EnvConfig)
-				}
-			} else {
-				fileNames := c.StringSlice("config")
-				for _, fileName := range fileNames {
-					b, err := os.ReadFile(fileName)
-					if err != nil {
-						return errors.Wrapf(err, "failed to read config file: %s", fileName)
-					}
-					if err := opts.ParseConfig(string(b)); err != nil {
-						return errors.Wrapf(err, "failed to parse file: %s", fileName)
-					}
-				}
-			}
+
+			fileNames := c.StringSlice("config")
+			loadOpts(&opts, fileNames...)
 
 			secretsTOML := ""
 			if c.IsSet("secrets") {
@@ -1231,3 +1218,22 @@ func logDeprecatedClientEnvWarnings(lggr logger.Logger) {
 		lggr.Errorf("ADMIN_CREDENTIALS_FILE env var has been deprecated and will be removed in a future release. Use flag instead: --admin-credentials-file=%s", s)
 	}
 }
+
+// loadOpts applies file configs and then overlays env config
+func loadOpts(opts *chainlink.GeneralConfigOpts, fileNames ...string) error {
+	for _, fileName := range fileNames {
+		b, err := os.ReadFile(fileName)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read config file: %s", fileName)
+		}
+		if err := opts.ParseConfig(string(b)); err != nil {
+			return errors.Wrapf(err, "failed to parse file: %s", fileName)
+		}
+	}
+	if configTOML := v2.EnvConfig.Get(); configTOML != "" {
+		if err := opts.ParseConfig(configTOML); err != nil {
+			return errors.Wrapf(err, "failed to parse env var %q", v2.EnvConfig)
+		}
+	}
+	return nil
+}

core/cmd/app_test.go

@@ -0,0 +1,128 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pelletier/go-toml/v2"
+	v2 "github.com/smartcontractkit/chainlink/core/config/v2"
+	"github.com/smartcontractkit/chainlink/core/services/chainlink"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	setInFile = "set in config file"
+	setInEnv  = "set in env"
+
+	testEnvContents = fmt.Sprintf("P2P.V2.AnnounceAddresses = ['%s']", setInEnv)
+
+	testConfigFileContents = chainlink.Config{
+		Core: v2.Core{
+			RootDir: &setInFile,
+			P2P: v2.P2P{
+				V2: v2.P2PV2{
+					AnnounceAddresses: &[]string{setInFile},
+					ListenAddresses:   &[]string{setInFile},
+				},
+			},
+		},
+	}
+)
+
+func makeTestConfigFile(t *testing.T) string {
+	d := t.TempDir()
+	p := filepath.Join(d, "test.toml")
+
+	b, err := toml.Marshal(testConfigFileContents)
+	require.NoError(t, err)
+
+	require.NoError(t, os.WriteFile(p, b, 0777))
+	return p
+}
+
+func Test_loadOpts(t *testing.T) {
+	type args struct {
+		opts      *chainlink.GeneralConfigOpts
+		fileNames []string
+		envVar    string
+	}
+	tests := []struct {
+		name     string
+		args     args
+		wantErr  bool
+		wantOpts *chainlink.GeneralConfigOpts
+	}{
+		{
+			name: "env only",
+			args: args{
+				opts:   new(chainlink.GeneralConfigOpts),
+				envVar: testEnvContents,
+			},
+			wantOpts: &chainlink.GeneralConfigOpts{
+				Config: chainlink.Config{
+					Core: v2.Core{
+						P2P: v2.P2P{
+							V2: v2.P2PV2{
+								AnnounceAddresses: &[]string{setInEnv},
+							},
+						},
+					},
+				},
+			},
+		},
+
+		{
+			name: "files only",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{makeTestConfigFile(t)},
+			},
+			wantOpts: &chainlink.GeneralConfigOpts{
+				Config: testConfigFileContents,
+			},
+		},
+		{
+			name: "file error",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{"notexist"},
+			},
+			wantErr: true,
+		},
+
+		{
+			name: "env overlay of file",
+			args: args{
+				opts:      new(chainlink.GeneralConfigOpts),
+				fileNames: []string{makeTestConfigFile(t)},
+				envVar:    testEnvContents,
+			},
+			wantOpts: &chainlink.GeneralConfigOpts{
+				Config: chainlink.Config{
+					Core: v2.Core{
+						RootDir: &setInFile,
+						P2P: v2.P2P{
+							V2: v2.P2PV2{
+								// env should override this specific field
+								AnnounceAddresses: &[]string{setInEnv},
+								ListenAddresses:   &[]string{setInFile},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.args.envVar != "" {
+				t.Setenv(string(v2.EnvConfig), tt.args.envVar)
+			}
+			if err := loadOpts(tt.args.opts, tt.args.fileNames...); (err != nil) != tt.wantErr {
+				t.Errorf("loadOpts() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

docs/CHANGELOG.md

@@ -29,6 +29,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Removed `KEEPER_TURN_FLAG_ENABLED` as all networks/nodes have switched this to `true` now. The variable should be completely removed my NOPs.
 - Removed `Keeper.UpkeepCheckGasPriceEnabled` config (`KEEPER_CHECK_UPKEEP_GAS_PRICE_FEATURE_ENABLED` in old env var configuration) as this feature is deprecated now. The variable should be completely removed by NOPs.
+- TOML env var `CL_CONFIG` always processed as the last configuration, with the effect of being the final override 
+of any values provided via configuration files.
 
 ### Fixed