🌈 use viper to read config (#80)

* ✨feature: add viper * 🐞 fix: rename private_key to privateKey * ✨feature: use environment variable in build * 🔵 other: refactor: add debug logging and use map to check for duplication * ✨feature: chi v5 * 🔵 other: disable azure in e2e test for now
maxisam · Oct 1, 2023 · 3273175 · 3273175
1 parent d80f0f2
commit 3273175
Show file tree

Hide file tree

Showing 19 changed files with 351 additions and 144 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -89,19 +89,21 @@ jobs:
       - name: Setup key
         env:
           GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }}
-          AZURE_CONNECTIONSTRING: ${{ secrets.AZURE_CONNECTIONSTRING }}
         run: |
           echo "$GCLOUD_KEY" | base64 --decode > ${{ github.workspace }}/test/gh-actions/gcloud.json
-          sed -i "s|BlobEndpoint|$AZURE_CONNECTIONSTRING|g" ${{ github.workspace }}/test/gh-actions/mongo-test.yml
 
       - name: Run Mgob image
-        run: >
-          docker run -d 
-          --name mgob 
-          --network "host" 
-          -v ${{ github.workspace }}/test/gh-actions:/config 
-          -v ${{ github.workspace }}/test/backups:/storage
+        env:
+          AZURE_CONNECTIONSTRING: ${{ secrets.AZURE_CONNECTIONSTRING }}
+        run: |
+          docker run -d \
+          --name mgob \
+          --network "host" \
+          -e MONGO-TEST_AZURE_CONNECTIONSTRING="$AZURE_CONNECTIONSTRING" \
+          -v ${{ github.workspace }}/test/gh-actions:/config \
+          -v ${{ github.workspace }}/test/backups:/storage \
           ${{ github.repository }}:${{ env.APP_VERSION }}.${{ github.run_number }}
+
       - name: Verify mgob backup
         run: |
           sleep 90
@@ -137,7 +139,7 @@ jobs:
             echo "GCloud integration test failed"
             exit 1
           fi
-          # verify for azure
+          # # verify for azure
           # if ! grep -q "Azure upload finished" logs.txt; then
           #   echo "Azure integration test failed, check if the connection string is expired"
           #   exit 1

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -6,7 +6,7 @@ on:
       version:
         type: string
         description: Version
-        default: 1.12
+        default: 2.0
         required: false
 
 jobs:

diff --git a/README.md b/README.md
@@ -25,6 +25,7 @@
 - Multiple Docker image releases catering to different backup solutions
 - Option to skip local backup when retention is set to 0 ([#42](https://github.com/maxisam/mgob/pull/42), Credit: @aneagoe)
 - On-demand restore API
+- Load config from environment variables to override config file. syntax: `PLAN-ID_KEY_PROPERTY` (e.g. `mongo-test_SMTP_SERVER=smtp.company.com`)
 
 ### Helm Chart
 
@@ -34,6 +35,10 @@ helm repo update
 helm upgrade --install mgob maxisam/mgob --namespace mgob --create-namespace
 ```
 
+### Breaking Changes
+
+- v2: in config, sftp.private_key -> sftp.privateKey
+
 ## Original Features
 
 - schedule backups

diff --git a/charts/mgob/values.yaml b/charts/mgob/values.yaml
@@ -121,7 +121,7 @@ config: {}
 #        username: user
 #        password: secret
 #        # you can also specify path to a private key and a passphrase
-#        private_key: /etc/ssh/ssh_host_rsa_key
+#        privateKey: /etc/ssh/ssh_host_rsa_key
 #        passphrase: secretpassphrase
 #        # dir must exist on the SFTP server
 #        dir: backup

diff --git a/cmd/mgob/mgob.go b/cmd/mgob/mgob.go
@@ -4,9 +4,11 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"strings"
 	"syscall"
 
 	"github.com/kelseyhightower/envconfig"
+	"github.com/spf13/viper"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -22,7 +24,7 @@ var (
 	appConfig = &config.AppConfig{}
 	modules   = &config.ModuleConfig{}
 	name      = "mgob"
-	version   = "v1.12.0-dev"
+	version   = "v2.0.0-dev"
 )
 
 func beforeApp(c *cli.Context) error {
@@ -39,6 +41,9 @@ func beforeApp(c *cli.Context) error {
 	}
 
 	log.Debug("log level set to ", c.GlobalString("LogLevel"))
+
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	viper.AutomaticEnv()
 	return nil
 }
 
@@ -93,9 +98,13 @@ func main() {
 	app.Run(os.Args)
 }
 
-func start(c *cli.Context) error {
-	log.Infof("mgob %v", version)
+func handleErr(err error, message string) {
+	if err != nil {
+		log.Fatalf("%s: %+v", message, err)
+	}
+}
 
+func loadConfiguration(c *cli.Context) {
 	appConfig.LogLevel = c.String("LogLevel")
 	appConfig.JSONLog = c.Bool("JSONLog")
 	appConfig.Port = c.Int("Port")
@@ -109,118 +118,96 @@ func start(c *cli.Context) error {
 	log.Infof("starting with config: %+v", appConfig)
 
 	err := envconfig.Process(name, modules)
-	if err != nil {
-		log.Fatal(err.Error())
-	}
+	handleErr(err, "Error processing environment configuration")
 
 	appConfig.UseAwsCli = true
 	appConfig.HasGpg = true
+}
 
+func start(c *cli.Context) error {
+	log.Infof("mgob %v", version)
+
+	// Load the configuration from the command-line flags and environment variables.
+	loadConfiguration(c)
+
+	// Check if mongodump is installed and print the version information.
 	info, err := backup.CheckMongodump()
-	if err != nil {
-		log.Fatal(err)
-	}
+	handleErr(err, "Failed to check mongodump")
 	log.Info(info)
 
+	// Check if all required clients are installed.
 	checkClients()
 
+	// Load the backup plans from the configuration directory.
 	plans, err := config.LoadPlans(appConfig.ConfigPath)
-	if err != nil {
-		log.Fatal(err)
-	}
+	handleErr(err, "Failed to load backup plans")
 
+	// Open the database store for status information.
 	store, err := db.Open(path.Join(appConfig.DataPath, "mgob.db"))
-	if err != nil {
-		log.Fatal(err)
-	}
+	handleErr(err, "Failed to open database store")
+	defer store.Close()
+
+	// Create a new status store for the scheduler.
 	statusStore, err := db.NewStatusStore(store)
-	if err != nil {
-		log.Fatal(err)
-	}
+	handleErr(err, "Failed to create status store")
 
+	// Create a new scheduler and start it.
 	sch := scheduler.New(plans, appConfig, modules, statusStore)
 	sch.Start()
 
+	// Create a new HTTP server and start it in a separate goroutine.
 	server := &api.HttpServer{
 		Config:  appConfig,
 		Modules: modules,
 		Stats:   statusStore,
 	}
-	log.Infof("starting http server on port %v", appConfig.Port)
+	log.Infof("Starting HTTP server on port %v", appConfig.Port)
 	go server.Start(appConfig.Version)
 
-	// wait for SIGINT (Ctrl+C) or SIGTERM (docker stop)
+	// Wait for a SIGINT (Ctrl+C) or SIGTERM (docker stop) signal.
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
 	sig := <-sigChan
 
-	log.Infof("shutting down %v signal received", sig)
+	log.Infof("Shutting down (%v signal received)", sig)
 
 	return nil
 }
 
 func checkClients() {
-	if modules.MinioClient {
-		info, err := backup.CheckMinioClient()
-		if err != nil {
-			log.Fatal(err)
-		}
-		log.Info(info)
-	} else {
-		log.Info("Minio Client is disabled.")
-	}
+	checkClient("Minio Client", modules.MinioClient, backup.CheckMinioClient)
+	checkClient("AWS CLI", modules.AWSClient, backup.CheckAWSClient)
+	checkClient("GPG", modules.GnuPG, backup.CheckGpg)
+	checkClient("Google Storage", modules.GCloudClient, backup.CheckGCloudClient)
+	checkClient("Azure Storage", modules.AzureClient, backup.CheckAzureClient)
+	checkClient("RClone", modules.RCloneClient, backup.CheckRCloneClient)
+}
 
-	if modules.AWSClient {
-		info, err := backup.CheckAWSClient()
-		if err != nil {
-			log.Warn(err)
-			appConfig.UseAwsCli = false
-		}
-		log.Info(info)
-	} else {
-		appConfig.UseAwsCli = false
-		log.Info("AWS CLI is disabled.")
+func checkClient(name string, enabled bool, checkFunc func() (string, error)) {
+	if !enabled {
+		log.Infof("%s is disabled.", name)
+		disableConfig(name)
+		return
 	}
 
-	if modules.GnuPG {
-		info, err := backup.CheckGpg()
-		if err != nil {
+	info, err := checkFunc()
+	if err != nil {
+		if name == "AWS CLI" || name == "GPG" {
 			log.Warn(err)
-			appConfig.HasGpg = false
-		}
-		log.Info(info)
-	} else {
-		appConfig.HasGpg = false
-		log.Info("GPG is disabled.")
-	}
-
-	if modules.GCloudClient {
-		info, err := backup.CheckGCloudClient()
-		if err != nil {
+			disableConfig(name)
+		} else {
 			log.Fatal(err)
 		}
-		log.Info(info)
 	} else {
-		log.Info("Google Storage is disabled.")
-	}
-
-	if modules.AzureClient {
-		info, err := backup.CheckAzureClient()
-		if err != nil {
-			log.Fatal(err)
-		}
 		log.Info(info)
-	} else {
-		log.Info("Azure Storage is disabled.")
 	}
+}
 
-	if modules.RCloneClient {
-		info, err := backup.CheckRCloneClient()
-		if err != nil {
-			log.Fatal(err)
-		}
-		log.Info(info)
-	} else {
-		log.Info("RClone is disabled.")
+func disableConfig(name string) {
+	switch name {
+	case "AWS CLI":
+		appConfig.UseAwsCli = false
+	case "GPG":
+		appConfig.HasGpg = false
 	}
 }
diff --git a/go.mod b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/boltdb/bolt v1.3.1
 	github.com/codeskyblue/go-sh v0.0.0-20200712050446-30169cf553fe
 	github.com/dustin/go-humanize v1.0.1
-	github.com/go-chi/chi v1.5.5
+	github.com/go-chi/chi/v5 v5.0.10
 	github.com/go-chi/render v1.0.3
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/pkg/errors v0.9.1
@@ -23,17 +23,29 @@ require (
 require (
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.16.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )