Warn on shorthand options overriding scenarios

This is currently implemented only for scenarios as that is the case
that is most confusing. Implementing this for duration overriding stages
(for example) is possible, but it seems to not be nearly hit enough.

Unfortunately due to too many of things of #883 and the way shorthand
options are handled in particular, the code can't be in any Apply
function with enough information to have a nice message.

Fixes #2529

cmd/config.go

@@ -198,9 +198,15 @@ func getConsolidatedConfig(gs *state.GlobalState, cliConf Config, runnerOpts lib
 
 	conf = cliConf.Apply(fileConf)
 
+	warnOnShortHandOverride(conf.Options, runnerOpts, "script", gs.Logger)
 	conf = conf.Apply(Config{Options: runnerOpts})
 
-	conf = conf.Apply(envConf).Apply(cliConf)
+	warnOnShortHandOverride(conf.Options, envConf.Options, "env", gs.Logger)
+	conf = conf.Apply(envConf)
+
+	warnOnShortHandOverride(conf.Options, cliConf.Options, "cli", gs.Logger)
+	conf = conf.Apply(cliConf)
+
 	conf = applyDefault(conf)
 
 	// TODO(imiric): Move this validation where it makes sense in the configuration
@@ -215,6 +221,15 @@ func getConsolidatedConfig(gs *state.GlobalState, cliConf Config, runnerOpts lib
 	return conf, nil
 }
 
+func warnOnShortHandOverride(a, b lib.Options, bName string, logger logrus.FieldLogger) {
+	if a.Scenarios != nil &&
+		(b.Duration.Valid || b.Iterations.Valid || b.Stages != nil || b.Scenarios != nil) {
+		logger.Warnf(
+			"%q level configuration overrode scenarios configuration entirely",
+			bName)
+	}
+}
+
 // applyDefault applies the default options value if it is not specified.
 // This happens with types which are not supported by "gopkg.in/guregu/null.v3".
 //

cmd/config_consolidation_test.go

@@ -293,7 +293,9 @@ func getConfigConsolidationTestCases() []configConsolidationTestCase {
 				env: []string{"K6_ITERATIONS=25"},
 				cli: []string{"--vus", "12"},
 			},
-			exp{},
+			exp{
+				logWarning: true,
+			},
 			verifySharedIters(I(12), I(25)),
 		},
 		{