Use paginated API for listing Security groups, ASGs, ELBs, and launch configrations (#632)

Author: hligit

resources/autoscaling-groups.go

@@ -12,20 +12,24 @@ func init() {
 
 func ListAutoscalingGroups(s *session.Session) ([]Resource, error) {
 	svc := autoscaling.New(s)
+	resources := make([]Resource, 0)
 
 	params := &autoscaling.DescribeAutoScalingGroupsInput{}
-	resp, err := svc.DescribeAutoScalingGroups(params)
+	err := svc.DescribeAutoScalingGroupsPages(params,
+		func(page *autoscaling.DescribeAutoScalingGroupsOutput, lastPage bool) bool {
+			for _, asg := range page.AutoScalingGroups {
+				resources = append(resources, &AutoScalingGroup{
+					svc:  svc,
+					name: asg.AutoScalingGroupName,
+				})
+			}
+			return !lastPage
+		})
+
 	if err != nil {
 		return nil, err
 	}
 
-	resources := make([]Resource, 0)
-	for _, asg := range resp.AutoScalingGroups {
-		resources = append(resources, &AutoScalingGroup{
-			svc:  svc,
-			name: asg.AutoScalingGroupName,
-		})
-	}
 	return resources, nil
 }
 

resources/autoscaling-launchconfigurations.go

@@ -10,21 +10,25 @@ func init() {
 }
 
 func ListLaunchConfigurations(s *session.Session) ([]Resource, error) {
+	resources := make([]Resource, 0)
 	svc := autoscaling.New(s)
 
 	params := &autoscaling.DescribeLaunchConfigurationsInput{}
-	resp, err := svc.DescribeLaunchConfigurations(params)
+	err := svc.DescribeLaunchConfigurationsPages(params,
+		func(page *autoscaling.DescribeLaunchConfigurationsOutput, lastPage bool) bool {
+			for _, launchconfig := range page.LaunchConfigurations {
+				resources = append(resources, &LaunchConfiguration{
+					svc:  svc,
+					name: launchconfig.LaunchConfigurationName,
+				})
+			}
+			return !lastPage
+		})
+
 	if err != nil {
 		return nil, err
 	}
 
-	resources := make([]Resource, 0)
-	for _, launchconfig := range resp.LaunchConfigurations {
-		resources = append(resources, &LaunchConfiguration{
-			svc:  svc,
-			name: launchconfig.LaunchConfigurationName,
-		})
-	}
 	return resources, nil
 }
 

resources/ec2-security-groups.go

@@ -23,25 +23,28 @@ func init() {
 
 func ListEC2SecurityGroups(sess *session.Session) ([]Resource, error) {
 	svc := ec2.New(sess)
+	resources := make([]Resource, 0)
 
 	params := &ec2.DescribeSecurityGroupsInput{}
-	resp, err := svc.DescribeSecurityGroups(params)
+	err := svc.DescribeSecurityGroupsPages(params,
+		func(page *ec2.DescribeSecurityGroupsOutput, lastPage bool) bool {
+			for _, group := range page.SecurityGroups {
+				resources = append(resources, &EC2SecurityGroup{
+					svc:     svc,
+					group:   group,
+					id:      group.GroupId,
+					name:    group.GroupName,
+					ingress: group.IpPermissions,
+					egress:  group.IpPermissionsEgress,
+				})
+			}
+			return !lastPage
+		})
+
 	if err != nil {
 		return nil, err
 	}
 
-	resources := make([]Resource, 0)
-	for _, group := range resp.SecurityGroups {
-		resources = append(resources, &EC2SecurityGroup{
-			svc:     svc,
-			group:   group,
-			id:      group.GroupId,
-			name:    group.GroupName,
-			ingress: group.IpPermissions,
-			egress:  group.IpPermissionsEgress,
-		})
-	}
-
 	return resources, nil
 }
 
@@ -86,11 +89,11 @@ func (sg *EC2SecurityGroup) Remove() error {
 
 func (sg *EC2SecurityGroup) Properties() types.Properties {
 	properties := types.NewProperties()
-    for _, tagValue := range sg.group.Tags {
-        properties.SetTag(tagValue.Key, tagValue.Value)
-    }
-    properties.Set("Name", sg.name)
-    return properties
+	for _, tagValue := range sg.group.Tags {
+		properties.SetTag(tagValue.Key, tagValue.Value)
+	}
+	properties.Set("Name", sg.name)
+	return properties
 }
 
 func (sg *EC2SecurityGroup) String() string {

resources/elb-elb.go

@@ -18,31 +18,43 @@ func init() {
 
 func ListELBLoadBalancers(sess *session.Session) ([]Resource, error) {
 	resources := make([]Resource, 0)
-
+	elbNames := make([]*string, 0)
 	svc := elb.New(sess)
 
-	elbResp, err := svc.DescribeLoadBalancers(nil)
+	err := svc.DescribeLoadBalancersPages(nil,
+		func(page *elb.DescribeLoadBalancersOutput, lastPage bool) bool {
+			for _, desc := range page.LoadBalancerDescriptions {
+				elbNames = append(elbNames, desc.LoadBalancerName)
+			}
+			return !lastPage
+		})
+
 	if err != nil {
 		return nil, err
 	}
 
-	for _, elbLoadBalancer := range elbResp.LoadBalancerDescriptions {
-		// Tags for ELBs need to be fetched separately
+	for len(elbNames) > 0 {
+		requestElements := len(elbNames)
+		if requestElements > 20 {
+			requestElements = 20
+		}
+
 		tagResp, err := svc.DescribeTags(&elb.DescribeTagsInput{
-			LoadBalancerNames: []*string{elbLoadBalancer.LoadBalancerName},
+			LoadBalancerNames: elbNames[:requestElements],
 		})
-
 		if err != nil {
 			return nil, err
 		}
-
 		for _, elbTagInfo := range tagResp.TagDescriptions {
 			resources = append(resources, &ELBLoadBalancer{
 				svc:  svc,
 				name: elbTagInfo.LoadBalancerName,
 				tags: elbTagInfo.Tags,
 			})
 		}
+
+		// Remove the elements that were queried
+		elbNames = elbNames[requestElements:]
 	}
 
 	return resources, nil

resources/elbv2-alb.go

@@ -19,19 +19,22 @@ func init() {
 
 func ListELBv2LoadBalancers(sess *session.Session) ([]Resource, error) {
 	svc := elbv2.New(sess)
+	var tagReqELBv2ARNs []*string
+	ELBv2ArnToName := make(map[string]*string)
+
+	err := svc.DescribeLoadBalancersPages(nil,
+		func(page *elbv2.DescribeLoadBalancersOutput, lastPage bool) bool {
+			for _, elbv2 := range page.LoadBalancers {
+				tagReqELBv2ARNs = append(tagReqELBv2ARNs, elbv2.LoadBalancerArn)
+				ELBv2ArnToName[*elbv2.LoadBalancerArn] = elbv2.LoadBalancerName
+			}
+			return !lastPage
+		})
 
-	elbResp, err := svc.DescribeLoadBalancers(nil)
 	if err != nil {
 		return nil, err
 	}
 
-	var tagReqELBv2ARNs []*string
-	ELBv2ArnToName := make(map[string]*string)
-	for _, elbv2 := range elbResp.LoadBalancers {
-		tagReqELBv2ARNs = append(tagReqELBv2ARNs, elbv2.LoadBalancerArn)
-		ELBv2ArnToName[*elbv2.LoadBalancerArn] = elbv2.LoadBalancerName
-	}
-
 	// Tags for ELBv2s need to be fetched separately
 	// We can only specify up to 20 in a single call
 	// See: https://github.com/aws/aws-sdk-go/blob/0e8c61841163762f870f6976775800ded4a789b0/service/elbv2/api.go#L5398

resources/elbv2-targetgroup.go

@@ -19,19 +19,21 @@ func init() {
 
 func ListELBv2TargetGroups(sess *session.Session) ([]Resource, error) {
 	svc := elbv2.New(sess)
+	var tagReqELBv2TargetGroupARNs []*string
+	targetGroupArnToName := make(map[string]*string)
 
-	resourceResp, err := svc.DescribeTargetGroups(nil)
+	err := svc.DescribeTargetGroupsPages(nil,
+		func(page *elbv2.DescribeTargetGroupsOutput, lastPage bool) bool {
+			for _, targetGroup := range page.TargetGroups {
+				tagReqELBv2TargetGroupARNs = append(tagReqELBv2TargetGroupARNs, targetGroup.TargetGroupArn)
+				targetGroupArnToName[*targetGroup.TargetGroupArn] = targetGroup.TargetGroupName
+			}
+			return !lastPage
+		})
 	if err != nil {
 		return nil, err
 	}
 
-	var tagReqELBv2TargetGroupARNs []*string
-	targetGroupArnToName := make(map[string]*string)
-	for _, targetGroup := range resourceResp.TargetGroups {
-		tagReqELBv2TargetGroupARNs = append(tagReqELBv2TargetGroupARNs, targetGroup.TargetGroupArn)
-		targetGroupArnToName[*targetGroup.TargetGroupArn] = targetGroup.TargetGroupName
-	}
-
 	// Tags for ELBv2 target groups need to be fetched separately
 	// We can only specify up to 20 in a single call
 	// See: https://github.com/aws/aws-sdk-go/blob/0e8c61841163762f870f6976775800ded4a789b0/service/elbv2/api.go#L5398