Config merge

_tests/integration/modular_config_main.yml

@@ -0,0 +1,22 @@
+format_version: "15"
+default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+
+include:
+- path: modular_config_module.yml
+- path: print-hello.yml
+  repository: https://github.com/bitrise-io/test-bitrise-config-modules.git
+  branch: main
+
+workflows:
+  print_hello_bitrise:
+    envs:
+    - NAME: Bitrise
+    steps:
+    - script:
+        inputs:
+        - content: echo "Hello $NAME!"
+
+  # This overwrites the NAME env from the included print-hello.yml
+  print_hello:
+    envs:
+    - NAME: John Doe

_tests/integration/modular_config_module.yml

@@ -0,0 +1,8 @@
+workflows:
+  print_hello_world:
+    envs:
+    - NAME: World
+    steps:
+    - script:
+        inputs:
+        - content: echo "Hello $NAME!"

_tests/integration/modular_config_test.go

@@ -0,0 +1,43 @@
+package integration
+
+import (
+	"os"
+	"testing"
+
+	"github.com/bitrise-io/go-utils/command"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_ModularConfig_Run(t *testing.T) {
+	configPth := "modular_config_main.yml"
+	deployDir := os.Getenv("BITRISE_DEPLOY_DIR")
+
+	cmd := command.New(binPath(), "merge", configPth, "-o", deployDir)
+	out, err := cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+
+	cmd = command.New(binPath(), "validate", "--config", configPth)
+	out, err = cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+	require.Equal(t, "Config is valid: \u001B[32;1mtrue\u001B[0m", out)
+
+	cmd = command.New(binPath(), "workflows", "--id-only", "--config", configPth)
+	out, err = cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+	require.Equal(t, "print_hello print_hello_bitrise print_hello_world", out)
+
+	cmd = command.New(binPath(), "run", "print_hello", "--config", configPth)
+	out, err = cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+	require.Contains(t, out, "Hello John Doe!")
+
+	cmd = command.New(binPath(), "run", "print_hello_bitrise", "--config", configPth)
+	out, err = cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+	require.Contains(t, out, "Hello Bitrise!")
+
+	cmd = command.New(binPath(), "run", "print_hello_world", "--config", configPth)
+	out, err = cmd.RunAndReturnTrimmedCombinedOutput()
+	require.NoError(t, err, out)
+	require.Contains(t, out, "Hello World!")
+}

_tests/integration/validate_test.go

@@ -112,7 +112,7 @@ func Test_ValidateTest(t *testing.T) {
 			out, err = cmd.RunAndReturnTrimmedCombinedOutput()
 		}, runtimeLimit)
 		require.Error(t, err, out)
-		expected := fmt.Sprintf("Config is valid: \x1b[31;1mfalse\x1b[0m\nError: \x1b[31;1mConfig (path:%s) is not valid: empty config\x1b[0m", configPth)
+		expected := fmt.Sprintf("Config is valid: \x1b[31;1mfalse\x1b[0m\nError: \x1b[31;1mconfig (%s) is not valid: empty config\x1b[0m", configPth)
 		require.Equal(t, expected, out)
 		require.Equal(t, true, elapsed < runtimeLimit, runningTimeMsg, elapsed, elapsed-runtimeLimit)
 	}
@@ -184,7 +184,7 @@ func Test_ValidateTestJSON(t *testing.T) {
 			out, err = cmd.RunAndReturnTrimmedCombinedOutput()
 		}, runtimeLimit)
 		require.Error(t, err, out)
-		expected := fmt.Sprintf("{\"data\":{\"config\":{\"is_valid\":false,\"error\":\"Config (path:%s) is not valid: empty config\"}}}", configPth)
+		expected := fmt.Sprintf("{\"data\":{\"config\":{\"is_valid\":false,\"error\":\"config (%s) is not valid: empty config\"}}}", configPth)
 		require.Equal(t, expected, out)
 		require.Equal(t, true, elapsed < runtimeLimit, runningTimeMsg, elapsed, elapsed-runtimeLimit)
 	}
@@ -201,7 +201,7 @@ func Test_ValidateTestJSON(t *testing.T) {
 			out, err = cmd.RunAndReturnTrimmedCombinedOutput()
 		}, runtimeLimit)
 		require.Error(t, err, out)
-		expected := fmt.Sprintf("{\"data\":{\"config\":{\"is_valid\":false,\"error\":\"Config (path:%s) is not valid: missing format_version\"}}}", configPth)
+		expected := fmt.Sprintf("{\"data\":{\"config\":{\"is_valid\":false,\"error\":\"config (%s) is not valid: missing format_version\"}}}", configPth)
 		require.Equal(t, expected, out)
 		require.Equal(t, true, elapsed < runtimeLimit, runningTimeMsg, elapsed, elapsed-runtimeLimit)
 	}

bitrise/util.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"github.com/bitrise-io/bitrise/configs"
-	"github.com/bitrise-io/bitrise/log"
 	"github.com/bitrise-io/bitrise/models"
 	envmanModels "github.com/bitrise-io/envman/models"
 	"github.com/bitrise-io/go-utils/command"
@@ -20,7 +19,6 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-// InventoryModelFromYAMLBytes ...
 func InventoryModelFromYAMLBytes(inventoryBytes []byte) (inventory envmanModels.EnvsSerializeModel, err error) {
 	if err = yaml.Unmarshal(inventoryBytes, &inventory); err != nil {
 		return
@@ -55,7 +53,6 @@ func searchEnvInSlice(searchForEnvKey string, searchIn []envmanModels.Environmen
 	return envmanModels.EnvironmentItemModel{}, -1, nil
 }
 
-// ApplyOutputAliases ...
 func ApplyOutputAliases(onEnvs, basedOnEnvs []envmanModels.EnvironmentItemModel) ([]envmanModels.EnvironmentItemModel, error) {
 	for _, basedOnEnv := range basedOnEnvs {
 		envKey, envKeyAlias, err := basedOnEnv.GetKeyValuePair()
@@ -88,7 +85,6 @@ func ApplyOutputAliases(onEnvs, basedOnEnvs []envmanModels.EnvironmentItemModel)
 	return onEnvs, nil
 }
 
-// ApplySensitiveOutputs ...
 func ApplySensitiveOutputs(onEnvs, basedOnEnvs []envmanModels.EnvironmentItemModel) ([]envmanModels.EnvironmentItemModel, error) {
 	for _, basedOnEnv := range basedOnEnvs {
 		envKey, _, err := basedOnEnv.GetKeyValuePair()
@@ -134,7 +130,6 @@ func ApplySensitiveOutputs(onEnvs, basedOnEnvs []envmanModels.EnvironmentItemMod
 	return onEnvs, nil
 }
 
-// CollectEnvironmentsFromFile ...
 func CollectEnvironmentsFromFile(pth string) ([]envmanModels.EnvironmentItemModel, error) {
 	bytes, err := fileutil.ReadBytesFromFile(pth)
 	if err != nil {
@@ -144,7 +139,6 @@ func CollectEnvironmentsFromFile(pth string) ([]envmanModels.EnvironmentItemMode
 	return CollectEnvironmentsFromFileContent(bytes)
 }
 
-// CollectEnvironmentsFromFileContent ...
 func CollectEnvironmentsFromFileContent(bytes []byte) ([]envmanModels.EnvironmentItemModel, error) {
 	var envstore envmanModels.EnvsSerializeModel
 	if err := yaml.Unmarshal(bytes, &envstore); err != nil {
@@ -166,7 +160,6 @@ func CollectEnvironmentsFromFileContent(bytes []byte) ([]envmanModels.Environmen
 	return envstore.Envs, nil
 }
 
-// CleanupStepWorkDir ...
 func CleanupStepWorkDir() error {
 	stepYMLPth := filepath.Join(configs.BitriseWorkDirPath, "current_step.yml")
 	if err := command.RemoveFile(stepYMLPth); err != nil {
@@ -219,7 +212,13 @@ func normalizeValidateFillMissingDefaults(bitriseData *models.BitriseDataModel)
 	return warnings, nil
 }
 
-// ConfigModelFromYAMLBytes ...
+func ConfigModelFromFileContent(configBytes []byte, isJSON bool) (models.BitriseDataModel, []string, error) {
+	if isJSON {
+		return ConfigModelFromJSONBytes(configBytes)
+	}
+	return ConfigModelFromYAMLBytes(configBytes)
+}
+
 func ConfigModelFromYAMLBytes(configBytes []byte) (bitriseData models.BitriseDataModel, warnings []string, err error) {
 	if err = yaml.Unmarshal(configBytes, &bitriseData); err != nil {
 		return
@@ -233,7 +232,6 @@ func ConfigModelFromYAMLBytes(configBytes []byte) (bitriseData models.BitriseDat
 	return
 }
 
-// ConfigModelFromJSONBytes ...
 func ConfigModelFromJSONBytes(configBytes []byte) (bitriseData models.BitriseDataModel, warnings []string, err error) {
 	if err = json.Unmarshal(configBytes, &bitriseData); err != nil {
 		return
@@ -246,7 +244,6 @@ func ConfigModelFromJSONBytes(configBytes []byte) (bitriseData models.BitriseDat
 	return
 }
 
-// ReadBitriseConfig ...
 func ReadBitriseConfig(pth string) (models.BitriseDataModel, []string, error) {
 	if isExists, err := pathutil.IsPathExists(pth); err != nil {
 		return models.BitriseDataModel{}, []string{}, err
@@ -263,16 +260,9 @@ func ReadBitriseConfig(pth string) (models.BitriseDataModel, []string, error) {
 		return models.BitriseDataModel{}, []string{}, errors.New("empty config")
 	}
 
-	if strings.HasSuffix(pth, ".json") {
-		log.Debug("=> Using JSON parser for: ", pth)
-		return ConfigModelFromJSONBytes(bytes)
-	}
-
-	log.Debug("=> Using YAML parser for: ", pth)
-	return ConfigModelFromYAMLBytes(bytes)
+	return ConfigModelFromFileContent(bytes, strings.HasSuffix(pth, ".json"))
 }
 
-// ReadSpecStep ...
 func ReadSpecStep(pth string) (stepmanModels.StepModel, error) {
 	if isExists, err := pathutil.IsPathExists(pth); err != nil {
 		return stepmanModels.StepModel{}, err

cli/commands.go

@@ -114,5 +114,6 @@ var (
 		},
 		pluginCommand,
 		envmanCommand,
+		mergeConfigCommand,
 	}
 )

cli/merge.go

@@ -0,0 +1,101 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+
+	"github.com/bitrise-io/bitrise/configmerge"
+	"github.com/bitrise-io/bitrise/log"
+	"github.com/bitrise-io/bitrise/models"
+	"github.com/bitrise-io/go-utils/fileutil"
+	"github.com/bitrise-io/go-utils/pathutil"
+	"github.com/urfave/cli"
+)
+
+var mergeConfigCommand = cli.Command{
+	Name:      "merge",
+	Usage:     "Resolves includes in a modular bitrise.yml and merges included config modules into a single bitrise.yml file.",
+	ArgsUsage: "args[0]: By default, the command looks for a bitrise.yml in the current directory, custom path can be specified as an argument.",
+	Action:    mergeConfig,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "output, o",
+			Usage: "Output directory for the merged config file (bitrise.yml) and related config file tree (config_tree.json).",
+		},
+	},
+}
+
+func mergeConfig(c *cli.Context) error {
+	var configPth string
+	if c.Args().Present() {
+		configPth = c.Args().First()
+	} else {
+		configPth = "bitrise.yml"
+	}
+	outputDir := c.String("output")
+
+	opts := log.GetGlobalLoggerOpts()
+	logger := log.NewLogger(opts)
+
+	repoCache := configmerge.NewRepoCache()
+	configReader, err := configmerge.NewConfigReader(repoCache, logger)
+	if err != nil {
+		return fmt.Errorf("failed to create config module reader: %w", err)
+	}
+	merger := configmerge.NewMerger(configReader, logger)
+	mergedConfigContent, configFileTree, err := merger.MergeConfig(configPth)
+	if err != nil {
+		return fmt.Errorf("failed to merge config: %w", err)
+	}
+
+	if outputDir == "" {
+		if err := printOutputFiles(mergedConfigContent, *configFileTree, logger); err != nil {
+			return fmt.Errorf("failed to print output files: %w", err)
+		}
+	} else {
+		if err := writeOutputFiles(mergedConfigContent, *configFileTree, outputDir); err != nil {
+			return fmt.Errorf("failed to write output files: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func printOutputFiles(mergedConfigContent string, configFileTree models.ConfigFileTreeModel, logger log.Logger) error {
+	logger.Printf("config tree:")
+	configTreeBytes, err := json.MarshalIndent(configFileTree, "", "\t")
+	if err != nil {
+		return fmt.Errorf("failed to parse config tree: %s", err)
+	}
+	logger.Printf(string(configTreeBytes))
+
+	logger.Print()
+	logger.Printf("merged config:")
+	logger.Printf(mergedConfigContent)
+
+	return nil
+}
+
+func writeOutputFiles(mergedConfigContent string, configFileTree models.ConfigFileTreeModel, outputDir string) error {
+	if err := pathutil.EnsureDirExist(outputDir); err != nil {
+		return err
+	}
+
+	configTreeBytes, err := json.MarshalIndent(configFileTree, "", "\t")
+	if err != nil {
+		return fmt.Errorf("failed to parse config tree: %s", err)
+	}
+
+	configTreePth := filepath.Join(outputDir, "config_tree.json")
+	if err := fileutil.WriteBytesToFile(configTreePth, configTreeBytes); err != nil {
+		return fmt.Errorf("failed to write config tree to file: %s", err)
+	}
+
+	mergedConfigPth := filepath.Join(outputDir, "bitrise.yml")
+	if err := fileutil.WriteStringToFile(mergedConfigPth, mergedConfigContent); err != nil {
+		return fmt.Errorf("failed to write merged config to file: %s", err)
+	}
+
+	return nil
+}