Merge pull request #90 from uselagoon/feature/config-ordering

Feature/config ordering

.gitignore

@@ -28,5 +28,6 @@ dist/
 builds/
 
 # Config files
+/lagoon
 *.lagoon-sync
 *.lagoon-sync-defaults

README.md

@@ -141,54 +141,18 @@ This command would attempt to sync mariadb databases from `prod` to `dev` enviro
 
 ## Configuring lagoon-sync
 
-It is possible to configure the data consumed by lagoon-sync via adding a `lagoon-sync:` key to an existing `.lagoon.yml` file or via a configuration file such as (`.lagoon-sync`). See the `.lagoon.yml` and `.example-lagoon-sync` in the root of this repo for examples.
+Lagoon-sync configuration can be managed via yaml-formatted configuration files. The paths to these config files can be defined either by the `--config` argument, or by environment variables (`LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH`).
 
-If a `.lagoon.yml` is available within the project, then this file will be used as the active configuration file to attempt to gather configuration data from by default.
+The order of configuration precedence is as follows:
 
-Next, if a `.lagoon-sync` or `.lagoon-sync-defaults` file is added to the `/lagoon` directory then these will be used as the active configuration file. Running the sync with `--show-debug` you are able to see the configuration that will be run prior to running the process:
+1. `--config` argument (e.g. `lagoon-sync [command] --config ./.custom-lagoon-sync-config.yaml`).
+2.  `.lagoon.yaml` files (i.e. in project root, or `lagoon` directory). If an `.lagoon.yml` is available within the project, then this file will be used as the active configuration file by default.
+3. `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH` environment variables.
+4. Finally, if no config file can be found the default configuration will be used a safely written to a new '.lagoon.yml`
 
-```
-$ lagoon-sync sync mariadb -p mysite-com -e dev --show-debug
-
-2021/01/22 11:34:10 (DEBUG) Using config file: /lagoon/.lagoon-sync
-2021/01/22 11:34:10 (DEBUG) Config that will be used for sync:
- {
-  "Config": {
-    "DbHostname": "$MARIADB_HOST",
-    "DbUsername": "$MARIADB_USERNAME",
-    "DbPassword": "$MARIADB_PASSWORD",
-    "DbPort": "$MARIADB_PORT",
-    "DbDatabase": "$MARIADB_DATABASE",
-    ...
-```
-
-To recap, the configuration files that can be used by default, in order of priority when available are:
-* /lagoon/.lagoon-sync-defaults
-* /lagoon/.lagoon-sync
-* .lagoon.yml
-
-### Custom configuration files
-If you don't want your configuration file inside `/lagoon` and want to give it another name then you can define a custom file and tell sync to use that by providing the file path. This can be done with `--config` flag such as:Config files that can be used in order of priority:
-- .lagoon-sync-defaults _(no yaml ext neeeded)_
-- .lagoon-sync _(no yaml ext neeeded)_
-- .lagoon.yml _Main config file - path can be given as an argument with `--config`, default is `.lagoon.yml`_
-å
-```
-$ lagoon-sync sync mariadb -p mysite-com -e dev --config=/app/.lagoon-sync --show-debug
-
-2021/01/22 11:43:50 (DEBUG) Using config file: /app/.lagoon-sync
-```
-
-You can also use an environment variable to set the config sync path with either `LAGOON_SYNC_PATH` or `LAGOON_SYNC_DEFAULTS_PATH`.
-
-```
-$ LAGOON_SYNC_PATH=/app/.lagoon-sync lagoon-sync sync mariadb -p mysite-com -e dev --show-debug
-
-2021/01/22 11:46:42 (DEBUG) LAGOON_SYNC_PATH env var found: /app/.lagoon-sync
-2021/01/22 11:46:42 (DEBUG) Using config file: /app/.lagoon-sync
-```
+There are some configuration examples in the `examples` directory of this repo.
 
-To double check which config file is active you can also run the `$ lagoon-sync config` command.
+To double check which config file is loaded you can also run the `lagoon-sync config` command.
 
 ### Example sync config overrides
 ```

assets/lagoon.yml

@@ -0,0 +1,44 @@
+lagoon-sync:
+  ssh:
+    host: "ssh.lagoon.amazeeio.cloud"
+    port: "32222"
+  mariadb:
+    config:
+      hostname: "${MARIADB_HOST:-mariadb}"
+      username: "${MARIADB_USERNAME:-drupal}"
+      password: "${MARIADB_PASSWORD:-drupal}"
+      port: "${MARIADB_PORT:-3306}"
+      database: "${MARIADB_DATABASE:-drupal}"
+  postgres:
+    config:
+      hostname: "${POSTGRES_HOST:-postgres}"
+      username: "${POSTGRES_USERNAME:-drupal}"
+      password: "${POSTGRES_PASSWORD:-drupal}"
+      port: "5432"
+      database: "${POSTGRES_DATABASE:-drupal}"
+    local:
+      config:
+        port: "3306"
+  mongodb:
+    config:
+      hostname: "$MONGODB_HOST"
+      port: "$MONGODB_SERVICE_PORT"
+      database: "MONGODB_DATABASE"
+    local:
+      config:
+        hostname: "$MONGODB_HOST"
+        port: "27017"
+        database: "local"
+  files:
+    config:
+      sync-directory: "/app/web/sites/default/files"
+    local:
+      config:
+        sync-directory: "/app/web/sites/default/files"
+  drupalconfig:
+    config:
+      syncpath: "./config/sync"
+    local:
+      overrides:
+        config:
+          syncpath: "./config/sync"

assets/main.go

@@ -8,13 +8,20 @@ import (
 //go:embed .version
 var Version []byte
 
+//go:embed lagoon.yml
+var DefaultConfigData []byte
+
 //go:embed rsync
 var RsyncLinuxBinBytes []byte
 
 func GetVersion() string {
 	return strings.TrimSuffix(string(Version), "\n")
 }
 
+func GetDefaultConfig() ([]byte, error) {
+	return DefaultConfigData, nil
+}
+
 func RsyncBin() []byte {
 	return RsyncLinuxBinBytes
-}
+}

cmd/config.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 
@@ -39,6 +40,14 @@ var configCmd = &cobra.Command{
 	},
 }
 
+func LoadLagoonConfig(lagoonYamlPath string) ([]byte, error) {
+	var data, err = ioutil.ReadFile(lagoonYamlPath)
+	if err != nil {
+		return []byte{}, err
+	}
+	return data, nil
+}
+
 func PrintConfigOut() []byte {
 	lagoonSyncDefaultsFile, exists := os.LookupEnv("LAGOON_SYNC_DEFAULTS_PATH")
 	if !exists {
@@ -53,7 +62,7 @@ func PrintConfigOut() []byte {
 	activeLagoonYmlFile := viper.ConfigFileUsed()
 
 	// Gather lagoon.yml configuration
-	lagoonConfigBytestream, err := LoadLagoonConfig(cfgFile)
+	lagoonConfigBytestream, err := LoadLagoonConfig(activeLagoonYmlFile)
 	if err != nil {
 		utils.LogFatalError("Couldn't load lagoon config file - ", err.Error())
 	}

cmd/root.go

@@ -1,9 +1,13 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
-	"github.com/uselagoon/lagoon-sync/assets"
 	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/uselagoon/lagoon-sync/assets"
 
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
@@ -47,7 +51,7 @@ func init() {
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
 
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "./.lagoon.yml", "config file (default is .lagoon.yaml)")
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "Path to the file used to set lagoon-sync configuration")
 	rootCmd.PersistentFlags().BoolVar(&ShowDebug, "show-debug", false, "Shows debug information")
 	viper.BindPFlag("show-debug", rootCmd.PersistentFlags().Lookup("show-debug"))
 
@@ -56,66 +60,132 @@ func init() {
 	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 }
 
-// initConfig reads in config file and ENV variables if set.
 func initConfig() {
-	// Find home directory.
-	home, err := homedir.Dir()
+	err := processConfig(cfgFile)
 	if err != nil {
-		fmt.Println(err)
+		utils.LogFatalError("Unable to read in config file", err)
 		os.Exit(1)
 	}
-	// Search config in home directory with name ".lagoon-sync" (without extension).
-	viper.AddConfigPath(home)
-	viper.AddConfigPath("/lagoon")
-	viper.AddConfigPath("/tmp")
+}
+
+func processConfig(cfgFile string) error {
+	// If cfgFile arg given, return early
+	if cfgFile != "" {
+		viper.SetConfigFile(cfgFile)
+		viper.AutomaticEnv()
+
+		if err := viper.ReadInConfig(); err == nil {
+			utils.LogDebugInfo("Using config file", viper.ConfigFileUsed())
+		} else {
+			return fmt.Errorf("failed to read config file: %v", err)
+		}
+
+		return nil
+	}
+
+	home, err := homedir.Dir()
+	if err != nil {
+		return fmt.Errorf("unable to find home directory: %v", err)
+	}
+
+	paths := []string{".", "./lagoon", "/tmp", home}
+	for _, path := range paths {
+		viper.AddConfigPath(path)
+	}
 	viper.SetConfigName(cfgFile)
 	viper.SetConfigType("yaml")
 
-	// Find default config file for env vars (e.g. 'lagoon-sync-defaults')
-	lagoonSyncDefaultsFile, exists := os.LookupEnv("LAGOON_SYNC_DEFAULTS_PATH")
-	if exists {
-		utils.LogDebugInfo("LAGOON_SYNC_DEFAULTS_PATH env var found", lagoonSyncDefaultsFile)
-	} else {
-		lagoonSyncDefaultsFile = "/lagoon/.lagoon-sync-defaults"
+	// Find config file from env vars (e.g., 'LAGOON_SYNC_DEFAULTS_PATH' and 'LAGOON_SYNC_PATH')
+	defaultFiles := map[string]string{
+		"LAGOON_SYNC_DEFAULTS_PATH": "/lagoon/.lagoon-sync-defaults",
+		"LAGOON_SYNC_PATH":          "/lagoon/.lagoon-sync",
 	}
 
-	// Find lagoon-sync config file (e.g. 'lagoon-sync')
-	lagoonSyncCfgFile, exists := os.LookupEnv("LAGOON_SYNC_PATH")
-	if exists {
-		utils.LogDebugInfo("LAGOON_SYNC_PATH env var found", lagoonSyncCfgFile)
-	} else {
-		lagoonSyncCfgFile = "/lagoon/.lagoon-sync"
+	for envVar, defaultFile := range defaultFiles {
+		filePath, exists := os.LookupEnv(envVar)
+		if exists {
+			utils.LogDebugInfo(envVar+" env var found", filePath)
+			if utils.FileExists(filePath) {
+				viper.SetConfigFile(filePath)
+				cfgFile = filePath
+				break
+			}
+		} else {
+			if utils.FileExists(defaultFile) {
+				viper.SetConfigFile(defaultFile)
+				cfgFile = defaultFile
+				break
+			}
+		}
 	}
 
-	if cfgFile != "" {
-		// Use config file from the flag, default for this is '.lagoon.yml'
-		if utils.FileExists(cfgFile) {
-			viper.SetConfigName(cfgFile)
-			viper.SetConfigFile(cfgFile)
+	// Next, check for common lagoon config files and override defaults.
+	for _, path := range paths {
+		cfgFiles := []string{
+			// filepath.Join(path, ".lagoon.yml"),
+			filepath.Join(path, ".lagoon-sync.yml"),
+			filepath.Join(path, ".lagoon-sync"),
 		}
-
-		// Set '.lagoon-sync-defaults' as config file is it exists.
-		if utils.FileExists(lagoonSyncDefaultsFile) {
-			viper.SetConfigName(lagoonSyncDefaultsFile)
-			viper.SetConfigFile(lagoonSyncDefaultsFile)
-			cfgFile = lagoonSyncDefaultsFile
+		for _, filePath := range cfgFiles {
+			if utils.FileExists(filePath) {
+				cfgFile = filePath
+				break
+			}
 		}
-
-		// Set '.lagoon-sync' as config file is it exists.
-		if utils.FileExists(lagoonSyncCfgFile) {
-			viper.SetConfigName(lagoonSyncCfgFile)
-			viper.SetConfigFile(lagoonSyncCfgFile)
-			cfgFile = lagoonSyncCfgFile
+		if cfgFile != "" {
+			break
 		}
 	}
 
-	viper.AutomaticEnv() // read in environment variables that match
-
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		utils.LogDebugInfo("Using config file", viper.ConfigFileUsed())
-	}
-	if err := viper.ReadInConfig(); err != nil {
-		utils.LogFatalError("Unable to read in config file", err)
+	// Set the config file if found
+	if cfgFile != "" {
+		viper.SetConfigFile(cfgFile)
+		viper.AutomaticEnv()
+
+		// If a config file is found, read it in.
+		if err := viper.ReadInConfig(); err == nil {
+			utils.LogDebugInfo("Using config file", viper.ConfigFileUsed())
+		} else {
+			return fmt.Errorf("failed to read config file: %v", err)
+		}
+	} else {
+		// If no config file is found, load default config stored as an asset
+		if !utils.FileExists(".lagoon-sync.yml") && !noCliInteraction {
+			var response string
+			// Ask user to safe-write config to '.lagoon-sync.yml'
+			fmt.Print("No configuration file found, do you want to add the file '.lagoon-sync.yml'? (y/n): ")
+			_, err = fmt.Scanln(&response)
+			if err != nil {
+				return fmt.Errorf("failed to read user input: %v", err)
+			}
+
+			defaultConfigData, err := assets.GetDefaultConfig()
+			if err != nil {
+				return fmt.Errorf("failed to load default config: %v", err)
+			}
+
+			viper.SetConfigType("yaml")
+			viper.SetConfigName("default")
+
+			err = viper.ReadConfig(bytes.NewBuffer(defaultConfigData))
+			if err != nil {
+				return fmt.Errorf("failed to read default config: %v", err)
+			}
+
+			if strings.ToLower(strings.TrimSpace(response)) == "y" {
+				err = viper.SafeWriteConfigAs(".lagoon-sync.yml")
+				if err != nil {
+					return fmt.Errorf("failed to write '.lagoon-sync.yml': %v", err)
+				}
+
+				fmt.Println("The file '.lagoon-sync.yml' has been added.")
+			} else {
+				fmt.Println("Skipping the creation of '.lagoon-sync.yml'.")
+			}
+
+			viper.SetConfigFile(".lagoon-sync.yml")
+		}
 	}
+
+	return nil
 }