Merge pull request #454 from stmcallister/extend-retry

Increase Retry Time on Data Sources
PagerDuty · Feb 10, 2022 · a0d1acf · a0d1acf
2 parents 661767b + c4169b1
commit a0d1acf
Show file tree

Hide file tree

Showing 19 changed files with 64 additions and 47 deletions.
diff --git a/pagerduty/data_source_pagerduty_business_service.go b/pagerduty/data_source_pagerduty_business_service.go
@@ -37,7 +37,7 @@ func dataSourcePagerDutyBusinessServiceRead(d *schema.ResourceData, meta interfa
 
 	searchName := d.Get("name").(string)
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.BusinessServices.List()
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_escalation_policy.go b/pagerduty/data_source_pagerduty_escalation_policy.go
@@ -37,7 +37,7 @@ func dataSourcePagerDutyEscalationPolicyRead(d *schema.ResourceData, meta interf
 		Query: searchName,
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.EscalationPolicies.List(o)
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_extension_schema.go b/pagerduty/data_source_pagerduty_extension_schema.go
@@ -38,7 +38,7 @@ func dataSourcePagerDutyExtensionSchemaRead(d *schema.ResourceData, meta interfa
 
 	searchName := d.Get("name").(string)
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.ExtensionSchemas.List(&pagerduty.ListExtensionSchemasOptions{Query: searchName})
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_priority.go b/pagerduty/data_source_pagerduty_priority.go
@@ -39,7 +39,7 @@ func dataSourcePagerDutyPriorityRead(d *schema.ResourceData, meta interface{}) e
 
 	searchTeam := d.Get("name").(string)
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Priorities.List()
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_ruleset.go b/pagerduty/data_source_pagerduty_ruleset.go
@@ -40,7 +40,7 @@ func dataSourcePagerDutyRulesetRead(d *schema.ResourceData, meta interface{}) er
 
 	searchName := d.Get("name").(string)
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Rulesets.List()
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_schedule.go b/pagerduty/data_source_pagerduty_schedule.go
@@ -37,7 +37,7 @@ func dataSourcePagerDutyScheduleRead(d *schema.ResourceData, meta interface{}) e
 		Query: searchName,
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Schedules.List(o)
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_service.go b/pagerduty/data_source_pagerduty_service.go
@@ -41,7 +41,7 @@ func dataSourcePagerDutyServiceRead(d *schema.ResourceData, meta interface{}) er
 		Query: searchName,
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Services.List(o)
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_service_integration.go b/pagerduty/data_source_pagerduty_service_integration.go
@@ -49,7 +49,7 @@ func dataSourcePagerDutyServiceIntegrationRead(d *schema.ResourceData, meta inte
 		Query: searchName,
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Services.List(o)
 		if err != nil {
 			return handleError(err)

diff --git a/pagerduty/data_source_pagerduty_tag.go b/pagerduty/data_source_pagerduty_tag.go
@@ -38,7 +38,7 @@ func dataSourcePagerDutyTagRead(d *schema.ResourceData, meta interface{}) error
 		Query: searchTag,
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Tags.List(o)
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_team.go b/pagerduty/data_source_pagerduty_team.go
@@ -46,7 +46,7 @@ func dataSourcePagerDutyTeamRead(d *schema.ResourceData, meta interface{}) error
 		Query: searchTeam,
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Teams.List(o)
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_user.go b/pagerduty/data_source_pagerduty_user.go
@@ -41,7 +41,7 @@ func dataSourcePagerDutyUserRead(d *schema.ResourceData, meta interface{}) error
 		Query: searchEmail,
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Users.List(o)
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/data_source_pagerduty_user_contact_method.go b/pagerduty/data_source_pagerduty_user_contact_method.go
@@ -70,7 +70,7 @@ func dataSourcePagerDutyUserContactMethodRead(d *schema.ResourceData, meta inter
 	searchLabel := d.Get("label").(string)
 	searchType := d.Get("type").(string)
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Users.ListContactMethods(userId)
 		if err != nil {
 			errResp := handleNotFoundError(err, d)

diff --git a/pagerduty/data_source_pagerduty_vendor.go b/pagerduty/data_source_pagerduty_vendor.go
@@ -42,7 +42,7 @@ func dataSourcePagerDutyVendorRead(d *schema.ResourceData, meta interface{}) err
 	o := &pagerduty.ListVendorsOptions{
 		Query: searchName,
 	}
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		resp, _, err := client.Vendors.List(o)
 		if err != nil {
 			if isErrCode(err, 429) {

diff --git a/pagerduty/resource_pagerduty_addon.go b/pagerduty/resource_pagerduty_addon.go
@@ -47,7 +47,7 @@ func fetchPagerDutyAddon(d *schema.ResourceData, meta interface{}, errCallback f
 		return err
 	}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		addon, _, err := client.Addons.Get(d.Id())
 		if err != nil {
 			log.Printf("[WARN] Service read error")

diff --git a/pagerduty/resource_pagerduty_business_service.go b/pagerduty/resource_pagerduty_business_service.go
@@ -98,7 +98,7 @@ func resourcePagerDutyBusinessServiceCreate(d *schema.ResourceData, meta interfa
 		return err
 	}
 
-	retryErr := resource.Retry(2*time.Minute, func() *resource.RetryError {
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 
 		businessService, err := buildBusinessServiceStruct(d)
 		if err != nil {
@@ -128,7 +128,7 @@ func resourcePagerDutyBusinessServiceRead(d *schema.ResourceData, meta interface
 
 	log.Printf("[INFO] Reading PagerDuty business service %s", d.Id())
 
-	retryErr := resource.Retry(2*time.Minute, func() *resource.RetryError {
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if businessService, _, err := client.BusinessServices.Get(d.Id()); err != nil {
 			return resource.RetryableError(err)
 		} else if businessService != nil {

diff --git a/pagerduty/resource_pagerduty_business_service_subscriber.go b/pagerduty/resource_pagerduty_business_service_subscriber.go
@@ -60,7 +60,7 @@ func resourcePagerDutyBusinessServiceSubscriberCreate(d *schema.ResourceData, me
 
 	businessServiceId := d.Get("business_service_id").(string)
 
-	retryErr := resource.Retry(2*time.Minute, func() *resource.RetryError {
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 
 		businessServiceSubscriber, err := buildBusinessServiceSubscriberStruct(d)
 		if err != nil {
@@ -96,7 +96,7 @@ func resourcePagerDutyBusinessServiceSubscriberRead(d *schema.ResourceData, meta
 
 	log.Printf("[INFO] Reading PagerDuty business service %s subscriber %s type %s", businessServiceId, businessServiceSubscriber.ID, businessServiceSubscriber.Type)
 
-	return resource.Retry(30*time.Second, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if subscriberResponse, _, err := client.BusinessServiceSubscribers.List(businessServiceId); err != nil {
 			time.Sleep(2 * time.Second)
 			return resource.RetryableError(err)

diff --git a/pagerduty/resource_pagerduty_escalation_policy.go b/pagerduty/resource_pagerduty_escalation_policy.go
@@ -110,19 +110,32 @@ func resourcePagerDutyEscalationPolicyCreate(d *schema.ResourceData, meta interf
 	if err != nil {
 		return err
 	}
+	var readErr error
 
 	escalationPolicy := buildEscalationPolicyStruct(d)
 
 	log.Printf("[INFO] Creating PagerDuty escalation policy: %s", escalationPolicy.Name)
 
-	escalationPolicy, _, err = client.EscalationPolicies.Create(escalationPolicy)
-	if err != nil {
-		return err
-	}
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
+		escalationPolicy, _, err := client.EscalationPolicies.Create(escalationPolicy)
+		if err != nil {
+			if isErrCode(err, 429) {
+				// Delaying retry by 30s as recommended by PagerDuty
+				// https://developer.pagerduty.com/docs/rest-api-v2/rate-limiting/#what-are-possible-workarounds-to-the-events-api-rate-limit
+				time.Sleep(30 * time.Second)
+				return resource.RetryableError(err)
+			}
 
-	d.SetId(escalationPolicy.ID)
+			return resource.NonRetryableError(err)
+		}
 
-	return resourcePagerDutyEscalationPolicyRead(d, meta)
+		d.SetId(escalationPolicy.ID)
+		readErr = resourcePagerDutyEscalationPolicyRead(d, meta)
+		if readErr != nil {
+			return resource.NonRetryableError(readErr)
+		}
+		return nil
+	})
 }
 
 func resourcePagerDutyEscalationPolicyRead(d *schema.ResourceData, meta interface{}) error {
@@ -135,7 +148,7 @@ func resourcePagerDutyEscalationPolicyRead(d *schema.ResourceData, meta interfac
 
 	o := &pagerduty.GetEscalationPolicyOptions{}
 
-	return resource.Retry(2*time.Minute, func() *resource.RetryError {
+	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		escalationPolicy, _, err := client.EscalationPolicies.Get(d.Id(), o)
 		if err != nil {
 			time.Sleep(2 * time.Second)
@@ -168,7 +181,7 @@ func resourcePagerDutyEscalationPolicyUpdate(d *schema.ResourceData, meta interf
 
 	log.Printf("[INFO] Updating PagerDuty escalation policy: %s", d.Id())
 
-	retryErr := resource.Retry(2*time.Minute, func() *resource.RetryError {
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if _, _, err := client.EscalationPolicies.Update(d.Id(), escalationPolicy); err != nil {
 			return resource.RetryableError(err)
 		}

diff --git a/pagerduty/resource_pagerduty_service_dependency.go b/pagerduty/resource_pagerduty_service_dependency.go
@@ -130,7 +130,7 @@ func resourcePagerDutyServiceDependencyAssociate(d *schema.ResourceData, meta in
 	log.Printf("[INFO] Associating PagerDuty dependency %s", serviceDependency.ID)
 
 	var dependencies *pagerduty.ListServiceDependencies
-	retryErr := resource.Retry(30*time.Second, func() *resource.RetryError {
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if dependencies, _, err = client.ServiceDependencies.AssociateServiceDependencies(&input); err != nil {
 			if isErrCode(err, 404) {
 				return resource.RetryableError(err)
@@ -166,26 +166,30 @@ func resourcePagerDutyServiceDependencyDisassociate(d *schema.ResourceData, meta
 
 	log.Printf("[INFO] Disassociating PagerDuty dependency %s", dependency.DependentService.ID)
 
-	// listServiceRelationships by calling get dependencies using the serviceDependency.DependentService.ID
-	depResp, _, err := client.ServiceDependencies.GetServiceDependenciesForType(dependency.DependentService.ID, dependency.DependentService.Type)
-	if err != nil {
-		return err
-	}
-
 	var foundDep *pagerduty.ServiceDependency
 
-	// loop serviceRelationships until relationship.IDs match
-	for _, rel := range depResp.Relationships {
-		if rel.ID == d.Id() {
-			foundDep = rel
-			break
+	// listServiceRelationships by calling get dependencies using the serviceDependency.DependentService.ID
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
+		if dependencies, _, err := client.ServiceDependencies.GetServiceDependenciesForType(dependency.DependentService.ID, dependency.DependentService.Type); err != nil {
+			if isErrCode(err, 404) || isErrCode(err, 500) || isErrCode(err, 429) {
+				return resource.RetryableError(err)
+			}
+			return resource.NonRetryableError(err)
+		} else if dependencies != nil {
+			for _, rel := range dependencies.Relationships {
+				if rel.ID == d.Id() {
+					foundDep = rel
+					break
+				}
+			}
 		}
-	}
-	// check if relationship not found
-	if foundDep == nil {
-		d.SetId("")
 		return nil
+	})
+	if retryErr != nil {
+		time.Sleep(5 * time.Second)
+		return retryErr
 	}
+
 	// convertType is needed because the PagerDuty API returns the 'reference' values in responses but wants the other
 	// values in requests
 	foundDep.SupportingService.Type = convertType(foundDep.SupportingService.Type)
@@ -199,17 +203,17 @@ func resourcePagerDutyServiceDependencyDisassociate(d *schema.ResourceData, meta
 	input := pagerduty.ListServiceDependencies{
 		Relationships: r,
 	}
-	retryErr := resource.Retry(30*time.Second, func() *resource.RetryError {
+	retryErr = resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if _, _, err = client.ServiceDependencies.DisassociateServiceDependencies(&input); err != nil {
-			if isErrCode(err, 404) {
+			if isErrCode(err, 404) || isErrCode(err, 429) {
 				return resource.RetryableError(err)
 			}
 			return resource.NonRetryableError(err)
 		}
 		return nil
 	})
 	if retryErr != nil {
-		time.Sleep(2 * time.Second)
+		time.Sleep(5 * time.Second)
 		return retryErr
 	}
 
@@ -273,7 +277,7 @@ func findDependencySetState(depID, serviceID, serviceType string, d *schema.Reso
 
 	// Pausing to let the PD API sync.
 	time.Sleep(1 * time.Second)
-	retryErr := resource.Retry(30*time.Second, func() *resource.RetryError {
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if dependencies, _, err := client.ServiceDependencies.GetServiceDependenciesForType(serviceID, serviceType); err != nil {
 			if isErrCode(err, 404) || isErrCode(err, 500) || isErrCode(err, 429) {
 				return resource.RetryableError(err)

diff --git a/pagerduty/resource_pagerduty_tag_assignment.go b/pagerduty/resource_pagerduty_tag_assignment.go
@@ -72,7 +72,7 @@ func resourcePagerDutyTagAssignmentCreate(d *schema.ResourceData, meta interface
 
 	log.Printf("[INFO] Creating PagerDuty tag assignment with tagID %s for %s entity with ID %s", assignment.TagID, assignment.EntityType, assignment.EntityID)
 
-	retryErr := resource.Retry(10*time.Second, func() *resource.RetryError {
+	retryErr := resource.Retry(5*time.Minute, func() *resource.RetryError {
 		if _, err := client.Tags.Assign(assignment.EntityType, assignment.EntityID, assignments); err != nil {
 			if isErrCode(err, 400) || isErrCode(err, 429) {
 				return resource.RetryableError(err)