diff --git a/apstra/api_iba_dashboards.go b/apstra/api_iba_dashboards.go index ea68c287..99ae8a62 100644 --- a/apstra/api_iba_dashboards.go +++ b/apstra/api_iba_dashboards.go @@ -2,7 +2,9 @@ package apstra import ( "context" + "errors" "fmt" + "math/rand" "net/http" "time" ) @@ -151,17 +153,54 @@ func (o *Client) getIbaDashboardByLabel(ctx context.Context, blueprintId ObjectI } func (o *Client) createIbaDashboard(ctx context.Context, blueprintId ObjectId, in *rawIbaDashboard) (ObjectId, error) { - response := &objectIdResponse{} - + var response objectIdResponse err := o.talkToApstra(ctx, &talkToApstraIn{ - method: http.MethodPost, urlStr: fmt.Sprintf(apiUrlIbaDashboards, blueprintId), - apiInput: in, apiResponse: response, + method: http.MethodPost, + urlStr: fmt.Sprintf(apiUrlIbaDashboards, blueprintId), + apiInput: in, + apiResponse: &response, }) - if err != nil { - return "", convertTtaeToAceWherePossible(err) + if err == nil { + return response.Id, nil + } + + err = convertTtaeToAceWherePossible(err) + + var ace ClientErr + if !(errors.As(err, &ace) && ace.IsRetryable()) { + return "", err // fatal error + } + + retryMax := o.GetTuningParam("ibaDashboardMaxRetries") + retryInterval := time.Duration(o.GetTuningParam("ibaDashboardRetryIntervalMs")) * time.Millisecond + + for i := 0; i < retryMax; i++ { + // Make a random wait, in case multiple threads are running + if rand.Int()%2 == 0 { + time.Sleep(retryInterval) + } + + time.Sleep(retryInterval * time.Duration(i)) + + e := o.talkToApstra(ctx, &talkToApstraIn{ + method: http.MethodPost, + urlStr: fmt.Sprintf(apiUrlIbaDashboards, blueprintId), + apiInput: in, + apiResponse: &response, + }) + if e == nil { + return response.Id, nil // success! + } + + e = convertTtaeToAceWherePossible(e) + if !(errors.As(e, &ace) && ace.IsRetryable()) { + return "", e // return the fatal error + } + + err = errors.Join(err, e) // the error is retryable; stack it with the rest } - return response.Id, nil + return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax)) } func (o *Client) updateIbaDashboard(ctx context.Context, blueprintId ObjectId, id ObjectId, in *rawIbaDashboard) error { diff --git a/apstra/api_iba_predefined_probes.go b/apstra/api_iba_predefined_probes.go index 27a08554..83b1b8d0 100644 --- a/apstra/api_iba_predefined_probes.go +++ b/apstra/api_iba_predefined_probes.go @@ -3,8 +3,11 @@ package apstra import ( "context" "encoding/json" + "errors" "fmt" + "math/rand" "net/http" + "time" ) const ( @@ -60,17 +63,52 @@ func (o *Client) getIbaPredefinedProbeByName(ctx context.Context, bpId ObjectId, } func (o *Client) instantiatePredefinedIbaProbe(ctx context.Context, bpId ObjectId, in *IbaPredefinedProbeRequest) (ObjectId, error) { - response := &objectIdResponse{} - + var response objectIdResponse err := o.talkToApstra(ctx, &talkToApstraIn{ method: http.MethodPost, urlStr: fmt.Sprintf(apiUrlIbaPredefinedProbesByName, bpId, in.Name), apiInput: in.Data, - apiResponse: response, + apiResponse: &response, }) - if err != nil { - return "", convertTtaeToAceWherePossible(err) + if err == nil { + return response.Id, nil + } + + err = convertTtaeToAceWherePossible(err) + + var ace ClientErr + if !(errors.As(err, &ace) && ace.IsRetryable()) { + return "", err // fatal error + } + + retryMax := o.GetTuningParam("ibaPredefinedProbeMaxRetries") + retryInterval := time.Duration(o.GetTuningParam("ibaPredefinedProbeRetryIntervalMs")) * time.Millisecond + + for i := 0; i < retryMax; i++ { + // Make a random wait, in case multiple threads are running + if rand.Int()%2 == 0 { + time.Sleep(retryInterval) + } + + time.Sleep(retryInterval * time.Duration(i)) + + e := o.talkToApstra(ctx, &talkToApstraIn{ + method: http.MethodPost, + urlStr: fmt.Sprintf(apiUrlIbaPredefinedProbesByName, bpId, in.Name), + apiInput: in.Data, + apiResponse: &response, + }) + if e == nil { + return response.Id, nil // success! + } + + e = convertTtaeToAceWherePossible(e) + if !(errors.As(e, &ace) && ace.IsRetryable()) { + return "", e // return the fatal error + } + + err = errors.Join(err, e) // the error is retryable; stack it with the rest } - return response.Id, nil + return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax)) } diff --git a/apstra/api_iba_probes.go b/apstra/api_iba_probes.go index 1b474fb6..8aa4806e 100644 --- a/apstra/api_iba_probes.go +++ b/apstra/api_iba_probes.go @@ -3,8 +3,11 @@ package apstra import ( "context" "encoding/json" + "errors" "fmt" + "math/rand" "net/http" + "time" ) const ( @@ -131,17 +134,53 @@ func (o *Client) deleteIbaProbe(ctx context.Context, bpId ObjectId, id ObjectId) })) } -func (o *Client) createIbaProbeFromJson(ctx context.Context, bpId ObjectId, probeJson json.RawMessage) (ObjectId, - error) { - response := objectIdResponse{} +func (o *Client) createIbaProbeFromJson(ctx context.Context, bpId ObjectId, probeJson json.RawMessage) (ObjectId, error) { + var response objectIdResponse err := o.talkToApstra(ctx, &talkToApstraIn{ method: http.MethodPost, urlStr: fmt.Sprintf(apiUrlIbaProbes, bpId), apiInput: probeJson, apiResponse: &response, }) - if err != nil { - return "", convertTtaeToAceWherePossible(err) + if err == nil { + return response.Id, nil + } + + err = convertTtaeToAceWherePossible(err) + + var ace ClientErr + if !(errors.As(err, &ace) && ace.IsRetryable()) { + return "", err // fatal error } - return response.Id, nil + + retryMax := o.GetTuningParam("createProbeMaxRetries") + retryInterval := time.Duration(o.GetTuningParam("createProbeRetryIntervalMs")) * time.Millisecond + + for i := 0; i < retryMax; i++ { + // Make a random wait, in case multiple threads are running + if rand.Int()%2 == 0 { + time.Sleep(retryInterval) + } + + time.Sleep(retryInterval * time.Duration(i)) + + e := o.talkToApstra(ctx, &talkToApstraIn{ + method: http.MethodPost, + urlStr: fmt.Sprintf(apiUrlIbaProbes, bpId), + apiInput: probeJson, + apiResponse: &response, + }) + if e == nil { + return response.Id, nil // success! + } + + e = convertTtaeToAceWherePossible(e) + if !(errors.As(e, &ace) && ace.IsRetryable()) { + return "", e // return the fatal error + } + + err = errors.Join(err, e) // the error is retryable; stack it with the rest + } + + return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax)) } diff --git a/apstra/api_iba_widgets.go b/apstra/api_iba_widgets.go index 355b0713..136b0458 100644 --- a/apstra/api_iba_widgets.go +++ b/apstra/api_iba_widgets.go @@ -2,7 +2,9 @@ package apstra import ( "context" + "errors" "fmt" + "math/rand" "net/http" "time" ) @@ -233,18 +235,54 @@ func (o *Client) getIbaWidgetByLabel(ctx context.Context, bpId ObjectId, label s } func (o *Client) createIbaWidget(ctx context.Context, bpId ObjectId, widget *rawIbaWidget) (ObjectId, error) { - response := &objectIdResponse{} - + var response objectIdResponse err := o.talkToApstra(ctx, &talkToApstraIn{ method: http.MethodPost, urlStr: fmt.Sprintf(apiUrlIbaWidgets, bpId), apiInput: &widget, apiResponse: &response, }) - if err != nil { - return "", err + if err == nil { + return response.Id, nil } - return response.Id, nil + + err = convertTtaeToAceWherePossible(err) + + var ace ClientErr + if !(errors.As(err, &ace) && ace.IsRetryable()) { + return "", err // fatal error + } + + retryMax := o.GetTuningParam("createIbaWidgetMaxRetries") + retryInterval := time.Duration(o.GetTuningParam("createIbaWidgetRetryIntervalMs")) * time.Millisecond + + for i := 0; i < retryMax; i++ { + // Make a random wait, in case multiple threads are running + if rand.Int()%2 == 0 { + time.Sleep(retryInterval) + } + + time.Sleep(retryInterval * time.Duration(i)) + + e := o.talkToApstra(ctx, &talkToApstraIn{ + method: http.MethodPost, + urlStr: fmt.Sprintf(apiUrlIbaWidgets, bpId), + apiInput: &widget, + apiResponse: &response, + }) + if e == nil { + return response.Id, nil // success! + } + + e = convertTtaeToAceWherePossible(e) + if !(errors.As(e, &ace) && ace.IsRetryable()) { + return "", e // return the fatal error + } + + err = errors.Join(err, e) // the error is retryable; stack it with the rest + } + + return "", errors.Join(err, fmt.Errorf("reached retry limit %d", retryMax)) } func (o *Client) updateIbaWidget(ctx context.Context, bpId ObjectId, id ObjectId, widget *rawIbaWidget) error { diff --git a/apstra/client.go b/apstra/client.go index 71f6dc52..e132435b 100644 --- a/apstra/client.go +++ b/apstra/client.go @@ -37,10 +37,12 @@ const ( ErrLagHasAssignedStructrues ErrTimeout ErrAgentProfilePlatformRequired + ErrIbaCurrentMountConflictsWithExistingMount clientPollingIntervalMs = 1000 defaultTimerPollingIntervalMs = 1000 + defaultTimerRetryIntervalMs = 100 defaultTimerTimeoutSec = 10 defaultMaxRetries = 5 @@ -158,6 +160,8 @@ func (o *Client) GetTuningParam(name string) int { switch { case strings.Contains(name, "TimeoutSec"): return defaultTimerTimeoutSec + case strings.Contains(name, "RetryIntervalMs"): + return defaultTimerRetryIntervalMs case strings.Contains(name, "PollingIntervalMs"): return defaultTimerPollingIntervalMs case strings.Contains(name, "MaxRetries"): diff --git a/apstra/talk_to_apstra.go b/apstra/talk_to_apstra.go index c352265f..621af129 100644 --- a/apstra/talk_to_apstra.go +++ b/apstra/talk_to_apstra.go @@ -105,6 +105,8 @@ func convertTtaeToAceWherePossible(err error) error { case strings.Contains(ttae.Msg, "Error executing facade API GET /obj-policy-export") && strings.Contains(ttae.Msg, "'NoneType' object has no attribute 'id'"): return ClientErr{errType: ErrNotfound, err: errors.New(ttae.Msg)} + case strings.Contains(ttae.Msg, "The current mount is conflicting with an existing mount"): + return ClientErr{errType: ErrIbaCurrentMountConflictsWithExistingMount, retryable: true, err: errors.New(ttae.Msg)} } } }