Merge pull request #271 from Juniper/270-retry-IBA-calls

code changes and tests to retry when there is a mount point error

apstra/api_iba_dashboards.go

@@ -2,7 +2,9 @@ package apstra
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"time"
 )
@@ -151,17 +153,54 @@ func (o *Client) getIbaDashboardByLabel(ctx context.Context, blueprintId ObjectI
 }
 
 func (o *Client) createIbaDashboard(ctx context.Context, blueprintId ObjectId, in *rawIbaDashboard) (ObjectId, error) {
-	response := &objectIdResponse{}
-
+	var response objectIdResponse
 	err := o.talkToApstra(ctx, &talkToApstraIn{
-		method: http.MethodPost, urlStr: fmt.Sprintf(apiUrlIbaDashboards, blueprintId),
-		apiInput: in, apiResponse: response,
+		method:      http.MethodPost,
+		urlStr:      fmt.Sprintf(apiUrlIbaDashboards, blueprintId),
+		apiInput:    in,
+		apiResponse: &response,
 	})
-	if err != nil {
-		return "", convertTtaeToAceWherePossible(err)
+	if err == nil {
+		return response.Id, nil
+	}
+
+	err = convertTtaeToAceWherePossible(err)
+
+	var ace ClientErr
+	if !(errors.As(err, &ace) && ace.IsRetryable()) {
+		return "", err // fatal error
+	}
+
+	retryMax := o.GetTuningParam("ibaDashboardMaxRetries")
+	retryInterval := time.Duration(o.GetTuningParam("ibaDashboardRetryIntervalMs")) * time.Millisecond
+
+	for i := 0; i < retryMax; i++ {
+		// Make a random wait, in case multiple threads are running
+		if rand.Int()%2 == 0 {
+			time.Sleep(retryInterval)
+		}
+
+		time.Sleep(retryInterval * time.Duration(i))
+
+		e := o.talkToApstra(ctx, &talkToApstraIn{
+			method:      http.MethodPost,
+			urlStr:      fmt.Sprintf(apiUrlIbaDashboards, blueprintId),
+			apiInput:    in,
+			apiResponse: &response,
+		})
+		if e == nil {
+			return response.Id, nil // success!
+		}
+
+		e = convertTtaeToAceWherePossible(e)
+		if !(errors.As(e, &ace) && ace.IsRetryable()) {
+			return "", e // return the fatal error
+		}
+
+		err = errors.Join(err, e) // the error is retryable; stack it with the rest
 	}
 
-	return response.Id, nil
+	return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax))
 }
 
 func (o *Client) updateIbaDashboard(ctx context.Context, blueprintId ObjectId, id ObjectId, in *rawIbaDashboard) error {

apstra/api_iba_predefined_probes.go

@@ -3,8 +3,11 @@ package apstra
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"math/rand"
 	"net/http"
+	"time"
 )
 
 const (
@@ -60,17 +63,52 @@ func (o *Client) getIbaPredefinedProbeByName(ctx context.Context, bpId ObjectId,
 }
 
 func (o *Client) instantiatePredefinedIbaProbe(ctx context.Context, bpId ObjectId, in *IbaPredefinedProbeRequest) (ObjectId, error) {
-	response := &objectIdResponse{}
-
+	var response objectIdResponse
 	err := o.talkToApstra(ctx, &talkToApstraIn{
 		method:      http.MethodPost,
 		urlStr:      fmt.Sprintf(apiUrlIbaPredefinedProbesByName, bpId, in.Name),
 		apiInput:    in.Data,
-		apiResponse: response,
+		apiResponse: &response,
 	})
-	if err != nil {
-		return "", convertTtaeToAceWherePossible(err)
+	if err == nil {
+		return response.Id, nil
+	}
+
+	err = convertTtaeToAceWherePossible(err)
+
+	var ace ClientErr
+	if !(errors.As(err, &ace) && ace.IsRetryable()) {
+		return "", err // fatal error
+	}
+
+	retryMax := o.GetTuningParam("ibaPredefinedProbeMaxRetries")
+	retryInterval := time.Duration(o.GetTuningParam("ibaPredefinedProbeRetryIntervalMs")) * time.Millisecond
+
+	for i := 0; i < retryMax; i++ {
+		// Make a random wait, in case multiple threads are running
+		if rand.Int()%2 == 0 {
+			time.Sleep(retryInterval)
+		}
+
+		time.Sleep(retryInterval * time.Duration(i))
+
+		e := o.talkToApstra(ctx, &talkToApstraIn{
+			method:      http.MethodPost,
+			urlStr:      fmt.Sprintf(apiUrlIbaPredefinedProbesByName, bpId, in.Name),
+			apiInput:    in.Data,
+			apiResponse: &response,
+		})
+		if e == nil {
+			return response.Id, nil // success!
+		}
+
+		e = convertTtaeToAceWherePossible(e)
+		if !(errors.As(e, &ace) && ace.IsRetryable()) {
+			return "", e // return the fatal error
+		}
+
+		err = errors.Join(err, e) // the error is retryable; stack it with the rest
 	}
 
-	return response.Id, nil
+	return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax))
 }

apstra/api_iba_probes.go

@@ -3,8 +3,11 @@ package apstra
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"math/rand"
 	"net/http"
+	"time"
 )
 
 const (
@@ -131,17 +134,53 @@ func (o *Client) deleteIbaProbe(ctx context.Context, bpId ObjectId, id ObjectId)
 	}))
 }
 
-func (o *Client) createIbaProbeFromJson(ctx context.Context, bpId ObjectId, probeJson json.RawMessage) (ObjectId,
-	error) {
-	response := objectIdResponse{}
+func (o *Client) createIbaProbeFromJson(ctx context.Context, bpId ObjectId, probeJson json.RawMessage) (ObjectId, error) {
+	var response objectIdResponse
 	err := o.talkToApstra(ctx, &talkToApstraIn{
 		method:      http.MethodPost,
 		urlStr:      fmt.Sprintf(apiUrlIbaProbes, bpId),
 		apiInput:    probeJson,
 		apiResponse: &response,
 	})
-	if err != nil {
-		return "", convertTtaeToAceWherePossible(err)
+	if err == nil {
+		return response.Id, nil
+	}
+
+	err = convertTtaeToAceWherePossible(err)
+
+	var ace ClientErr
+	if !(errors.As(err, &ace) && ace.IsRetryable()) {
+		return "", err // fatal error
 	}
-	return response.Id, nil
+
+	retryMax := o.GetTuningParam("createProbeMaxRetries")
+	retryInterval := time.Duration(o.GetTuningParam("createProbeRetryIntervalMs")) * time.Millisecond
+
+	for i := 0; i < retryMax; i++ {
+		// Make a random wait, in case multiple threads are running
+		if rand.Int()%2 == 0 {
+			time.Sleep(retryInterval)
+		}
+
+		time.Sleep(retryInterval * time.Duration(i))
+
+		e := o.talkToApstra(ctx, &talkToApstraIn{
+			method:      http.MethodPost,
+			urlStr:      fmt.Sprintf(apiUrlIbaProbes, bpId),
+			apiInput:    probeJson,
+			apiResponse: &response,
+		})
+		if e == nil {
+			return response.Id, nil // success!
+		}
+
+		e = convertTtaeToAceWherePossible(e)
+		if !(errors.As(e, &ace) && ace.IsRetryable()) {
+			return "", e // return the fatal error
+		}
+
+		err = errors.Join(err, e) // the error is retryable; stack it with the rest
+	}
+
+	return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax))
 }

apstra/api_iba_widgets.go

@@ -2,7 +2,9 @@ package apstra
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"time"
 )
@@ -233,18 +235,54 @@ func (o *Client) getIbaWidgetByLabel(ctx context.Context, bpId ObjectId, label s
 }
 
 func (o *Client) createIbaWidget(ctx context.Context, bpId ObjectId, widget *rawIbaWidget) (ObjectId, error) {
-	response := &objectIdResponse{}
-
+	var response objectIdResponse
 	err := o.talkToApstra(ctx, &talkToApstraIn{
 		method:      http.MethodPost,
 		urlStr:      fmt.Sprintf(apiUrlIbaWidgets, bpId),
 		apiInput:    &widget,
 		apiResponse: &response,
 	})
-	if err != nil {
-		return "", err
+	if err == nil {
+		return response.Id, nil
 	}
-	return response.Id, nil
+
+	err = convertTtaeToAceWherePossible(err)
+
+	var ace ClientErr
+	if !(errors.As(err, &ace) && ace.IsRetryable()) {
+		return "", err // fatal error
+	}
+
+	retryMax := o.GetTuningParam("createIbaWidgetMaxRetries")
+	retryInterval := time.Duration(o.GetTuningParam("createIbaWidgetRetryIntervalMs")) * time.Millisecond
+
+	for i := 0; i < retryMax; i++ {
+		// Make a random wait, in case multiple threads are running
+		if rand.Int()%2 == 0 {
+			time.Sleep(retryInterval)
+		}
+
+		time.Sleep(retryInterval * time.Duration(i))
+
+		e := o.talkToApstra(ctx, &talkToApstraIn{
+			method:      http.MethodPost,
+			urlStr:      fmt.Sprintf(apiUrlIbaWidgets, bpId),
+			apiInput:    &widget,
+			apiResponse: &response,
+		})
+		if e == nil {
+			return response.Id, nil // success!
+		}
+
+		e = convertTtaeToAceWherePossible(e)
+		if !(errors.As(e, &ace) && ace.IsRetryable()) {
+			return "", e // return the fatal error
+		}
+
+		err = errors.Join(err, e) // the error is retryable; stack it with the rest
+	}
+
+	return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax))
 }
 
 func (o *Client) updateIbaWidget(ctx context.Context, bpId ObjectId, id ObjectId, widget *rawIbaWidget) error {

apstra/client.go

@@ -37,10 +37,12 @@ const (
 	ErrLagHasAssignedStructrues
 	ErrTimeout
 	ErrAgentProfilePlatformRequired
+	ErrIbaCurrentMountConflictsWithExistingMount
 
 	clientPollingIntervalMs = 1000
 
 	defaultTimerPollingIntervalMs = 1000
+	defaultTimerRetryIntervalMs   = 100
 	defaultTimerTimeoutSec        = 10
 	defaultMaxRetries             = 5
 
@@ -158,6 +160,8 @@ func (o *Client) GetTuningParam(name string) int {
 	switch {
 	case strings.Contains(name, "TimeoutSec"):
 		return defaultTimerTimeoutSec
+	case strings.Contains(name, "RetryIntervalMs"):
+		return defaultTimerRetryIntervalMs
 	case strings.Contains(name, "PollingIntervalMs"):
 		return defaultTimerPollingIntervalMs
 	case strings.Contains(name, "MaxRetries"):

apstra/talk_to_apstra.go

@@ -105,6 +105,8 @@ func convertTtaeToAceWherePossible(err error) error {
 			case strings.Contains(ttae.Msg, "Error executing facade API GET /obj-policy-export") &&
 				strings.Contains(ttae.Msg, "'NoneType' object has no attribute 'id'"):
 				return ClientErr{errType: ErrNotfound, err: errors.New(ttae.Msg)}
+			case strings.Contains(ttae.Msg, "The current mount is conflicting with an existing mount"):
+				return ClientErr{errType: ErrIbaCurrentMountConflictsWithExistingMount, retryable: true, err: errors.New(ttae.Msg)}
 			}
 		}
 	}