Skip to content

Commit

Permalink
Changed the way I send webhook alarms
Browse files Browse the repository at this point in the history
  • Loading branch information
@LobbyLobster
LobbyLobster committed Jul 17, 2024
1 parent b2d4ee1 commit c77ab3a
Show file tree
Hide file tree
Showing 10 changed files with 126 additions and 94 deletions.
69 changes: 38 additions & 31 deletions backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@ import (
"time"
)

var didItWork bool = true // this variable is for determining if the app should send a notification after a failed backup to inform that it works now
func itWorksNow(message string, worked bool) {
oldOnlyOnError := params.Notify.Webhook.OnlyOnError
if !didItWork && worked && oldOnlyOnError {
params.Notify.Webhook.OnlyOnError = false
notify.SendAlarm(message, false)
params.Notify.Webhook.OnlyOnError = oldOnlyOnError
}
didItWork = worked
}
// var didItWork bool = true // this variable is for determining if the app should send a notification after a failed backup to inform that it works now
// func tWorksNow(message string, worked bool) {
// oldOnlyOnError := params.Notify.Webhook.OnlyOnError
// if !didItWork && worked && oldOnlyOnError {
// params.Notify.Webhook.OnlyOnError = false
// // notify.SendAlarm(message, false)
// params.Notify.Webhook.OnlyOnError = oldOnlyOnError
// }
// didItWork = worked
// }

func getDBList() (dbList []string) {
switch params.Database {
Expand Down Expand Up @@ -65,7 +65,7 @@ func dumpDB(db string, dst string) (string, string, error) {

func Backup() {
logger.Info("monodb-backup job started.")
notify.SendAlarm("Database backup started.", false)
// notify.SendAlarm("Database backup started.", false)
streamable := (params.Database == "" || params.Database == "postgresql" || (params.Database == "mysql" && !params.BackupAsTables)) && params.ArchivePass == ""

dateNow = rightNow{
Expand Down Expand Up @@ -107,11 +107,11 @@ func Backup() {
if params.BackupAsTables && db != "mysql" {
dumpPaths, names, err := dumpDBWithTables(db, dst)
if err != nil {
notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
itWorksNow("", false)
// notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
// itWorksNow("", false)
} else {
logger.Info("Successfully backed up database:" + db + " with its tables separately, at " + params.BackupDestination + "/" + db)
notify.SendAlarm("Successfully backed up "+db+" at "+params.BackupDestination+"/"+db, false)
// notify.SendAlarm("Successfully backed up "+db+" at "+params.BackupDestination+"/"+db, false)
for i, filePath := range dumpPaths {
name := names[i]
upload(name, db, filePath)
Expand All @@ -121,11 +121,13 @@ func Backup() {
} else {
filePath, name, err := dumpDB(db, dst)
if err != nil {
notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
itWorksNow("", false)
// notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
notify.FailedDBList = append(notify.FailedDBList, db+" - Error: "+err.Error())
// itWorksNow("", false)
} else {
logger.Info("Successfully backed up database:" + db + " at " + filePath)
notify.SendAlarm("Successfully backed up "+db+" at "+filePath, false)
// notify.SendAlarm("Successfully backed up "+db+" at "+filePath, false)
notify.SuccessfulDBList = append(notify.SuccessfulDBList, db)

upload(name, db, filePath)
}
Expand Down Expand Up @@ -175,8 +177,8 @@ func uploadWhileDumping(db string) {
err := dumpAndUpload(db, pipeWriters)
if err != nil {
cancel()
notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
itWorksNow("", false)
// notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
// itWorksNow("", false)
}
for _, writer := range pipeWriters {
writer.Close()
Expand All @@ -185,13 +187,15 @@ func uploadWhileDumping(db string) {
uploadErr := <-channel
if uploadErr != nil {
logger.Error(strconv.Itoa(i+1) + ") " + db + " - " + "Couldn't upload to S3: " + uploaders[i].instance.Endpoint + " - Error: " + uploadErr.Error())
notify.SendAlarm(strconv.Itoa(i+1)+") "+db+" - "+"Couldn't upload to S3: "+uploaders[i].instance.Endpoint+" - Error: "+uploadErr.Error(), true)
itWorksNow("", false)
// notify.SendAlarm(strconv.Itoa(i+1)+") "+db+" - "+"Couldn't upload to S3: "+uploaders[i].instance.Endpoint+" - Error: "+uploadErr.Error(), true)
notify.FailedDBList = append(notify.FailedDBList, db+" to "+uploaders[i].instance.Endpoint+" - Error: "+uploadErr.Error())
// itWorksNow("", false)
} else {
logger.Info(strconv.Itoa(i+1) + ") " + db + " - " + "Successfully uploaded to S3: " + uploaders[i].instance.Endpoint)
message := strconv.Itoa(i+1) + ") " + db + " - " + "Successfully uploaded to S3: " + uploaders[i].instance.Endpoint
notify.SendAlarm(message, false)
itWorksNow(message, true)
logger.Info(message)
// notify.SendAlarm(message, false)
notify.SuccessfulDBList = append(notify.SuccessfulDBList, db+" to "+uploaders[i].instance.Endpoint)
// itWorksNow(message, true)
}
}
}
Expand All @@ -200,22 +204,25 @@ func upload(name, db, filePath string) {
var err error
switch params.BackupType.Type {
case "s3", "minio":
err = uploadToS3(filePath, name, db)
if err != nil {
itWorksNow("", false)
}
uploadToS3(filePath, name, db)
case "sftp":
for _, target := range params.BackupType.Info[0].Targets {
err = SendSFTP(filePath, name, db, target)
if err != nil {
itWorksNow("", false)
// itWorksNow("", false)
notify.FailedDBList = append(notify.FailedDBList, db+" - "+name+" - Error: "+err.Error())
} else {
notify.SuccessfulDBList = append(notify.SuccessfulDBList, db+" - "+name)
}
}
case "rsync":
for _, target := range params.BackupType.Info[0].Targets {
err = SendRsync(filePath, name, db, target)
message, err := SendRsync(filePath, name, db, target)
if err != nil {
itWorksNow("", false)
// itWorksNow("", false)
notify.FailedDBList = append(notify.FailedDBList, db+" - "+message)
} else {
notify.SuccessfulDBList = append(notify.SuccessfulDBList, db+" - "+name)
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions backup/mssql_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ import (
"context"
"database/sql"
"fmt"
_ "github.com/denisenkom/go-mssqldb"
"log"
"monodb-backup/notify"
"os"
"path/filepath"
"strconv"

_ "github.com/denisenkom/go-mssqldb"
)

var mssqlDB *sql.DB
Expand All @@ -20,13 +20,13 @@ func InitializeMSSQL() {
var host, port, user, password string
var err error

if params.Remote.IsRemote == true {
if params.Remote.IsRemote {
host = params.Remote.Host
port = params.Remote.Port
user = params.Remote.User
password = params.Remote.Password
} else {
notify.SendAlarm("Remote should be enabled when backing up MSSQL databases.", true)
// notify.SendAlarm("Remote should be enabled when backing up MSSQL databases.", true)
logger.Fatal("Remote should be enabled when backing up MSSQL databases.")
return
}
Expand Down
3 changes: 1 addition & 2 deletions backup/mssql_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"database/sql"
"fmt"
"log"
"monodb-backup/notify"
"os"
"path/filepath"
"strconv"
Expand All @@ -30,7 +29,7 @@ func InitializeMSSQL() {
user = params.Remote.User
password = params.Remote.Password
} else {
notify.SendAlarm("Remote should be enabled when backing up MSSQL databases.", true)
// notify.SendAlarm("Remote should be enabled when backing up MSSQL databases.", true)
logger.Fatal("Remote should be enabled when backing up MSSQL databases.")
return
}
Expand Down
16 changes: 11 additions & 5 deletions backup/mysql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func getMySQLList() []string {
cmd := exec.Command("/usr/bin/mysql", mysqlArgs...)
out, err := cmd.Output()
if err != nil {
notify.SendAlarm("Couldn't get the list of databases - Error: "+string(out), true)
// notify.SendAlarm("Couldn't get the list of databases - Error: "+string(out), true)
logger.Fatal("Couldn't get the list of databases - Error: " + string(out))
return nil
}
Expand Down Expand Up @@ -59,7 +59,7 @@ func getMySQLList() []string {
// cmd.Stdout = &stdout
// err := cmd.Run()
// if err != nil {
// notify.SendAlarm("Couldn't get the list of databases - Error: "+stdout.String()+"\n"+stderr.String()+"\n"+err.Error(), true)
// // notify.SendAlarm("Couldn't get the list of databases - Error: "+stdout.String()+"\n"+stderr.String()+"\n"+err.Error(), true)
// logger.Fatal("Couldn't get the list of databases - Error: " + stdout.String() + "\n" + stderr.String() + "\n" + err.Error())
// return make([]string, 0)
// }
Expand Down Expand Up @@ -215,12 +215,16 @@ func dumpDBWithTables(db, dst string) ([]string, []string, error) {
var dumpPaths, names []string
oldDst := dst
if err := os.MkdirAll(dst, 0770); err != nil {
logger.Error("Couldn't create parent direectories at backup destination. dst: " + dst + " - Error: " + err.Error())
message := "Couldn't create parent direectories at backup destination. dst: " + dst + " - Error: " + err.Error()
logger.Error(message)
notify.FailedDBList = append(notify.FailedDBList, db+" - "+message)
return make([]string, 0), make([]string, 0), err
}
tableList, metaFile, err := getTableList(db, dst)
if err != nil {
logger.Error("Couldn't get the list of tables. Error: " + err.Error())
message := "Couldn't get the list of tables. Error: " + err.Error()
logger.Error(message)
notify.FailedDBList = append(notify.FailedDBList, db+" - "+message)
return nil, nil, err
}
dumpPaths = append(dumpPaths, metaFile)
Expand All @@ -231,7 +235,9 @@ func dumpDBWithTables(db, dst string) ([]string, []string, error) {
}
filePath, name, err := dumpTable(db, table, oldDst) //TODO +
if err != nil {
logger.Error("Couldn't dump databases. Error: " + err.Error())
message := "Couldn't dump databases. Error: " + err.Error()
logger.Error(message)
notify.FailedDBList = append(notify.FailedDBList, db+" - table: "+table+" - "+message)
return nil, nil, err
}
dumpPaths = append(dumpPaths, filePath)
Expand Down
3 changes: 1 addition & 2 deletions backup/postgresql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"io"
"monodb-backup/config"
"monodb-backup/notify"
"os"
"os/exec"
"path/filepath"
Expand All @@ -30,7 +29,7 @@ func getPSQLList() []string {
cmd.Stderr = &stderr
out, err := cmd.Output()
if err != nil {
notify.SendAlarm("Couldn't get the list of databases - Error: "+string(out), true)
// notify.SendAlarm("Couldn't get the list of databases - Error: "+string(out), true)
logger.Fatal("Couldn't get the list of databases - Error: " + string(out) + "\nError: " + stderr.String())
return nil
}
Expand Down
45 changes: 22 additions & 23 deletions backup/rsync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@ package backup
import (
"bytes"
"monodb-backup/config"
"monodb-backup/notify"
"os/exec"
"strings"
)

var lastDB string
var folderCreated bool

func SendRsync(srcPath, dstPath, db string, target config.Target) error {
func SendRsync(srcPath, dstPath, db string, target config.Target) (string, error) {
var dst string

if target.Path != "" {
Expand All @@ -20,9 +19,9 @@ func SendRsync(srcPath, dstPath, db string, target config.Target) error {
dst = nameWithPath(dstPath)
}

err := sendRsync(srcPath, dst, db, target)
message, err := sendRsync(srcPath, dst, db, target)
if err != nil {
return err
return message, err
}

if params.Rotation.Enabled {
Expand All @@ -35,16 +34,13 @@ func SendRsync(srcPath, dstPath, db string, target config.Target) error {
dstPath = target.Path + "/" + dstPath
}
if shouldRotate {
err = sendRsync(srcPath, dstPath, db, target)
if err != nil {
return err
}
return sendRsync(srcPath, dstPath, db, target)
}
}
return nil
return "", nil
}

func sendRsync(srcPath, dstPath, db string, target config.Target) error {
func sendRsync(srcPath, dstPath, db string, target config.Target) (string, error) {
var stderr1, stderr2, stdout bytes.Buffer

logger.Info("rsync transfer started.\n Source: " + srcPath + " - Destination: " + target.Host + ":" + dstPath)
Expand All @@ -66,21 +62,23 @@ func sendRsync(srcPath, dstPath, db string, target config.Target) error {
err := cmdMkdir.Run()
if err != nil {
cmdMkdir.Stderr = &stderr1
notify.SendAlarm("Couldn't create folder "+newPath+" to upload backups at"+target.Host+":"+dstPath+"\nError: "+err.Error()+" "+stderr1.String(), true)
logger.Error("Couldn't create folder " + newPath + " to upload backups at" + target.Host + ":" + dstPath + "\nError: " + err.Error() + " " + stderr1.String())
message := "Couldn't create folder " + newPath + " to upload backups at" + target.Host + ":" + dstPath + "\nError: " + err.Error() + " " + stderr1.String()
// notify.SendAlarm(message, true)
logger.Error(message)
lastDB = db
return err
return message, err
}
} else {
if lastDB != db && !folderCreated {
cmdMkdir := exec.Command("ssh", "-o", "HostKeyAlgorithms=+ssh-rsa", "-o", "PubKeyAcceptedKeyTypes=+ssh-rsa", target.Host, "mkdir -p "+newPath)
err := cmdMkdir.Run()
if err != nil {
cmdMkdir.Stderr = &stderr1
notify.SendAlarm("Couldn't create folder "+newPath+" to upload backups at"+target.Host+":"+dstPath+"\nError: "+err.Error()+" "+stderr1.String(), true)
logger.Error("Couldn't create folder " + newPath + " to upload backups at" + target.Host + ":" + dstPath + "\nError: " + err.Error() + " " + stderr1.String())
message := "Couldn't create folder " + newPath + " to upload backups at" + target.Host + ":" + dstPath + "\nError: " + err.Error() + " " + stderr1.String()
// notify.SendAlarm(message, true)
logger.Error(message)
lastDB = db
return err
return message, err
}
folderCreated = true
}
Expand All @@ -92,17 +90,18 @@ func sendRsync(srcPath, dstPath, db string, target config.Target) error {

err := cmdRsync.Run()
if err != nil {
notify.SendAlarm("Couldn't send "+srcPath+" to "+target.Host+":"+dstPath+"\nError: "+err.Error()+" "+stderr2.String()+" Stdout: "+stdout.String(), true)
logger.Error("Couldn't send " + srcPath + " to " + target.Host + ":" + dstPath + "\nError: " + err.Error() + " " + stderr2.String() + " Stdout: " + stdout.String())
message := "Couldn't send " + srcPath + " to " + target.Host + ":" + dstPath + "\nError: " + err.Error() + " " + stderr2.String() + " Stdout: " + stdout.String()
// notify.SendAlarm(message, true)
logger.Error(message)
lastDB = db
return err
return message, err
}

logger.Info("Successfully uploaded " + srcPath + " to " + target.Host + ":" + dstPath)
message := "Successfully uploaded " + srcPath + " to " + target.Host + ":" + dstPath
notify.SendAlarm(message, false)
itWorksNow(message, true)
logger.Info(message)
// notify.SendAlarm(message, false)
// itWorksNow(message, true)

lastDB = db
return nil
return "", nil
}
Loading

0 comments on commit c77ab3a

Please sign in to comment.