fix errorhandling for retry functions

go.mod

@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/ahmetb/go-linq/v3 v3.2.0
-	github.com/avast/retry-go/v4 v4.5.0
+	github.com/avast/retry-go/v4 v4.5.1
 	github.com/briandowns/spinner v1.23.0
 	github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
 	github.com/google/uuid v1.3.1

go.sum

@@ -42,8 +42,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/ahmetb/go-linq/v3 v3.2.0 h1:BEuMfp+b59io8g5wYzNoFe9pWPalRklhlhbiU3hYZDE=
 github.com/ahmetb/go-linq/v3 v3.2.0/go.mod h1:haQ3JfOeWK8HpVxMtHHEMPVgBKiYyQ+f1/kLZh/cj9U=
-github.com/avast/retry-go/v4 v4.5.0 h1:QoRAZZ90cj5oni2Lsgl2GW8mNTnUCnmpx/iKpwVisHg=
-github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5Aq1fboC3+I=
+github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
+github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
 github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A=
 github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

internal/cmdutil/cmdutil.go

@@ -2,18 +2,29 @@ package cmdutil
 
 import (
 	"github.com/avast/retry-go/v4"
+	"github.com/conplementag/cops-hq/v2/pkg/error_handling"
 	"github.com/sirupsen/logrus"
 	"os/exec"
 	"time"
 )
 
 // ExecuteFunctionWithRetry - reruns a function in case of error and logs error
 func ExecuteFunctionWithRetry(function func() error, maxAttempts uint) error {
+	if error_handling.PanicOnAnyError {
+		defer func() {
+			logrus.Infof("Reenable errorhandling panic on error")
+			error_handling.PanicOnAnyError = true
+		}()
+
+		logrus.Info("Disable error handling panic on error for retryable function")
+		error_handling.PanicOnAnyError = false
+	}
+
 	return retry.Do(function,
 		retry.Delay(time.Second),
 		retry.DelayType(retry.BackOffDelay),
 		retry.OnRetry(func(n uint, err error) {
-			logrus.Debugf("Retry %d - happend because of %s", n+1, err)
+			logrus.Infof("Retry %d - happend because of %s", n+1, err)
 		}),
 		retry.Attempts(maxAttempts),
 	)

internal/cmdutil/cmdutil_test.go

@@ -0,0 +1,45 @@
+package cmdutil
+
+import (
+	"errors"
+	"strconv"
+	"testing"
+)
+
+// MockFunction simulates a function that can fail up to a certain number of times before succeeding.
+func MockFunction(maxFailures string) func(string) (string, error) {
+	attempts := uint64(0)
+	maxFailuresInternal, _ := strconv.ParseUint(maxFailures, 10, 64)
+	return func(cmd string) (string, error) {
+		if attempts < maxFailuresInternal {
+			attempts++
+			return "", errors.New("error")
+		}
+		return "success", nil
+	}
+}
+
+func TestExecuteWithRetryCommand(t *testing.T) {
+	tests := []struct {
+		name        string
+		maxFailures string
+		maxAttempts uint
+		expectError bool
+	}{
+		{"SuccessOnFirstTry", "0", 3, false},
+		{"SuccessOnRetry", "2", 3, false},
+		{"FailAfterMaxAttempts", "3", 3, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			function := MockFunction(tt.maxFailures)
+			err := ExecuteWithRetry(function, "cmd", tt.maxAttempts)
+			if tt.expectError && err == nil {
+				t.Errorf("Expected an error but got none")
+			} else if !tt.expectError && err != nil {
+				t.Errorf("Did not expect an error but got one: %v", err)
+			}
+		})
+	}
+}

pkg/recipes/terraform/terraform.go

@@ -188,7 +188,7 @@ func (tf *terraformWrapper) Init() error {
 
 	logrus.Info("Creating the remote state blob container named " + tf.storageSettings.BlobContainerName + "...")
 	err = cmdutil.ExecuteWithRetry(
-		tf.executor.Execute,
+		tf.executor.ExecuteSilent,
 		"az storage container create"+
 			" --account-name "+tf.stateStorageAccountName+
 			" --account-key "+storageAccountKey+