From 2e55beec569e5e4937f1a655dbc59e7c6c3a6804 Mon Sep 17 00:00:00 2001 From: Philip Snowberger Date: Tue, 21 Jun 2016 16:20:49 -0700 Subject: [PATCH] Add pagination to all resource type responses. --- lib/terraforming/resource/db_parameter_group.rb | 4 ++-- lib/terraforming/resource/db_security_group.rb | 2 +- lib/terraforming/resource/db_subnet_group.rb | 2 +- lib/terraforming/resource/ec2.rb | 4 ++-- lib/terraforming/resource/eip.rb | 2 +- .../resource/elasti_cache_cluster.rb | 2 +- .../resource/elasti_cache_subnet_group.rb | 2 +- lib/terraforming/resource/elb.rb | 2 +- lib/terraforming/resource/iam_group.rb | 2 +- .../resource/iam_group_membership.rb | 2 +- lib/terraforming/resource/iam_group_policy.rb | 8 ++++---- .../resource/iam_instance_profile.rb | 2 +- lib/terraforming/resource/iam_policy.rb | 2 +- .../resource/iam_policy_attachment.rb | 17 +++++++++++++++-- lib/terraforming/resource/iam_role.rb | 2 +- lib/terraforming/resource/iam_role_policy.rb | 8 ++++---- lib/terraforming/resource/iam_user.rb | 2 +- lib/terraforming/resource/iam_user_policy.rb | 8 ++++---- lib/terraforming/resource/internet_gateway.rb | 2 +- .../resource/launch_configuration.rb | 4 +--- lib/terraforming/resource/network_acl.rb | 2 +- lib/terraforming/resource/network_interface.rb | 2 +- .../resource/opsworks_custom_layer.rb | 2 +- lib/terraforming/resource/opsworks_stack.rb | 2 +- lib/terraforming/resource/rds.rb | 2 +- lib/terraforming/resource/redshift.rb | 2 +- lib/terraforming/resource/route53_record.rb | 2 +- lib/terraforming/resource/route53_zone.rb | 2 +- lib/terraforming/resource/route_table.rb | 2 +- .../resource/route_table_association.rb | 2 +- lib/terraforming/resource/s3.rb | 2 +- lib/terraforming/resource/security_group.rb | 2 +- lib/terraforming/resource/sqs.rb | 2 +- lib/terraforming/resource/subnet.rb | 2 +- lib/terraforming/resource/vpc.rb | 2 +- lib/terraforming/resource/vpn_gateway.rb | 2 +- lib/terraforming/util.rb | 1 + .../lib/terraforming/resource/iam_group_spec.rb | 10 +++++++++- 38 files changed, 71 insertions(+), 51 deletions(-) diff --git a/lib/terraforming/resource/db_parameter_group.rb b/lib/terraforming/resource/db_parameter_group.rb index 487436c9..8fccabd4 100644 --- a/lib/terraforming/resource/db_parameter_group.rb +++ b/lib/terraforming/resource/db_parameter_group.rb @@ -43,11 +43,11 @@ def tfstate private def db_parameter_groups - @client.describe_db_parameter_groups.db_parameter_groups + @client.describe_db_parameter_groups.map(&:db_parameter_groups).flatten end def db_parameters_in(parameter_group) - @client.describe_db_parameters(db_parameter_group_name: parameter_group.db_parameter_group_name).parameters + @client.describe_db_parameters(db_parameter_group_name: parameter_group.db_parameter_group_name).map(&:parameters).flatten end def module_name_of(parameter_group) diff --git a/lib/terraforming/resource/db_security_group.rb b/lib/terraforming/resource/db_security_group.rb index 55f7eaad..eac6d534 100644 --- a/lib/terraforming/resource/db_security_group.rb +++ b/lib/terraforming/resource/db_security_group.rb @@ -46,7 +46,7 @@ def ingresses_of(security_group) end def db_security_groups - @client.describe_db_security_groups.db_security_groups.select { |sg| ingresses_of(sg).length > 0 } + @client.describe_db_security_groups.map(&:db_security_groups).flatten.select { |sg| ingresses_of(sg).length > 0 } end def module_name_of(security_group) diff --git a/lib/terraforming/resource/db_subnet_group.rb b/lib/terraforming/resource/db_subnet_group.rb index 229c29a3..8d832692 100644 --- a/lib/terraforming/resource/db_subnet_group.rb +++ b/lib/terraforming/resource/db_subnet_group.rb @@ -41,7 +41,7 @@ def tfstate private def db_subnet_groups - @client.describe_db_subnet_groups.db_subnet_groups + @client.describe_db_subnet_groups.map(&:db_subnet_groups).flatten end def module_name_of(subnet_group) diff --git a/lib/terraforming/resource/ec2.rb b/lib/terraforming/resource/ec2.rb index f036363b..cb2608c9 100644 --- a/lib/terraforming/resource/ec2.rb +++ b/lib/terraforming/resource/ec2.rb @@ -73,7 +73,7 @@ def block_device_ids_of(instance) def block_devices_of(instance) return [] unless instance.block_device_mappings.length > 0 - @client.describe_volumes(volume_ids: block_device_ids_of(instance)).volumes + @client.describe_volumes(volume_ids: block_device_ids_of(instance)).map(&:volumes).flatten end def block_device_mapping_of(instance, volume_id) @@ -101,7 +101,7 @@ def monitoring_state(instance) end def instances - @client.describe_instances.reservations.map(&:instances).flatten.reject { |instance| instance.state.name == "terminated" } + @client.describe_instances.map(&:reservations).flatten.map(&:instances).flatten.reject { |instance| instance.state.name == "terminated" } end def module_name_of(instance) diff --git a/lib/terraforming/resource/eip.rb b/lib/terraforming/resource/eip.rb index 5f78a252..eb6f17c6 100644 --- a/lib/terraforming/resource/eip.rb +++ b/lib/terraforming/resource/eip.rb @@ -47,7 +47,7 @@ def tfstate private def eips - @client.describe_addresses.addresses + @client.describe_addresses.map(&:addresses).flatten end def is_vpc?(addr) diff --git a/lib/terraforming/resource/elasti_cache_cluster.rb b/lib/terraforming/resource/elasti_cache_cluster.rb index eb44aa55..eea48231 100644 --- a/lib/terraforming/resource/elasti_cache_cluster.rb +++ b/lib/terraforming/resource/elasti_cache_cluster.rb @@ -57,7 +57,7 @@ def tfstate private def cache_clusters - @client.describe_cache_clusters(show_cache_node_info: true).cache_clusters + @client.describe_cache_clusters(show_cache_node_info: true).map(&:cache_clusters).flatten end def cluster_in_vpc?(cache_cluster) diff --git a/lib/terraforming/resource/elasti_cache_subnet_group.rb b/lib/terraforming/resource/elasti_cache_subnet_group.rb index 57a923ff..f5d71d67 100644 --- a/lib/terraforming/resource/elasti_cache_subnet_group.rb +++ b/lib/terraforming/resource/elasti_cache_subnet_group.rb @@ -41,7 +41,7 @@ def tfstate private def cache_subnet_groups - @client.describe_cache_subnet_groups.cache_subnet_groups + @client.describe_cache_subnet_groups.map(&:cache_subnet_groups).flatten end def subnet_ids_of(cache_subnet_group) diff --git a/lib/terraforming/resource/elb.rb b/lib/terraforming/resource/elb.rb index f02852b5..eea3b4da 100644 --- a/lib/terraforming/resource/elb.rb +++ b/lib/terraforming/resource/elb.rb @@ -181,7 +181,7 @@ def listener_hashcode_of(listener) end def load_balancers - @client.describe_load_balancers.load_balancer_descriptions + @client.describe_load_balancers.map(&:load_balancer_descriptions).flatten end def load_balancer_attributes_of(load_balancer) diff --git a/lib/terraforming/resource/iam_group.rb b/lib/terraforming/resource/iam_group.rb index a9fde6ef..6fc05186 100644 --- a/lib/terraforming/resource/iam_group.rb +++ b/lib/terraforming/resource/iam_group.rb @@ -43,7 +43,7 @@ def tfstate private def iam_groups - @client.list_groups.collect {|r| r.groups}.flatten + @client.list_groups.map(&:groups).flatten end end end diff --git a/lib/terraforming/resource/iam_group_membership.rb b/lib/terraforming/resource/iam_group_membership.rb index e92f1426..3febaa2c 100644 --- a/lib/terraforming/resource/iam_group_membership.rb +++ b/lib/terraforming/resource/iam_group_membership.rb @@ -48,7 +48,7 @@ def group_members_of(group) end def iam_groups - @client.list_groups.groups + @client.list_groups.map(&:groups).flatten end def membership_name_of(group) diff --git a/lib/terraforming/resource/iam_group_policy.rb b/lib/terraforming/resource/iam_group_policy.rb index 667b8a52..ef63af38 100644 --- a/lib/terraforming/resource/iam_group_policy.rb +++ b/lib/terraforming/resource/iam_group_policy.rb @@ -45,14 +45,14 @@ def unique_name(policy) "#{policy.group_name}_#{policy.policy_name}" end - def iam_groups - @client.list_groups.groups - end - def iam_group_policy_id_of(policy) "#{policy.group_name}:#{policy.policy_name}" end + def iam_groups + @client.list_groups.map(&:groups).flatten + end + def iam_group_policy_names_in(group) @client.list_group_policies(group_name: group.group_name).policy_names end diff --git a/lib/terraforming/resource/iam_instance_profile.rb b/lib/terraforming/resource/iam_instance_profile.rb index 44ecbbd3..24e3e4ec 100644 --- a/lib/terraforming/resource/iam_instance_profile.rb +++ b/lib/terraforming/resource/iam_instance_profile.rb @@ -43,7 +43,7 @@ def tfstate private def iam_instance_profiles - @client.list_instance_profiles.instance_profiles + @client.list_instance_profiles.map(&:instance_profiles).flatten end end end diff --git a/lib/terraforming/resource/iam_policy.rb b/lib/terraforming/resource/iam_policy.rb index 64ffa979..1a566d9e 100644 --- a/lib/terraforming/resource/iam_policy.rb +++ b/lib/terraforming/resource/iam_policy.rb @@ -44,7 +44,7 @@ def tfstate private def iam_policies - @client.list_policies(scope: "Local").collect {|r| r.policies}.flatten + @client.list_policies(scope: "Local").map(&:policies).flatten end def iam_policy_description(policy) diff --git a/lib/terraforming/resource/iam_policy_attachment.rb b/lib/terraforming/resource/iam_policy_attachment.rb index 0b25a2e7..5db7a377 100644 --- a/lib/terraforming/resource/iam_policy_attachment.rb +++ b/lib/terraforming/resource/iam_policy_attachment.rb @@ -48,11 +48,24 @@ def attachment_name_from(policy) end def entities_for_policy(policy) - @client.list_entities_for_policy(policy_arn: policy.arn) + # list_entities_for_policy is a weird one: the response class + # has three different member variables that we need to + # paginate through altogether. + result = Aws::IAM::Types::ListEntitiesForPolicyResponse.new + result.policy_groups = [] + result.policy_users = [] + result.policy_roles = [] + @client.list_entities_for_policy(policy_arn: policy.arn).each do |resp| + result.policy_groups += resp.policy_groups + result.policy_users += resp.policy_users + result.policy_roles += resp.policy_roles + end + + result end def iam_policies - @client.list_policies(scope: "Local").policies + @client.list_policies(scope: "Local").map(&:policies).flatten end def iam_policy_attachments diff --git a/lib/terraforming/resource/iam_role.rb b/lib/terraforming/resource/iam_role.rb index 50edd7b6..dc8a0a8e 100644 --- a/lib/terraforming/resource/iam_role.rb +++ b/lib/terraforming/resource/iam_role.rb @@ -45,7 +45,7 @@ def tfstate private def iam_roles - @client.list_roles.collect {|r| r.roles}.flatten + @client.list_roles.map(&:roles).flatten end end end diff --git a/lib/terraforming/resource/iam_role_policy.rb b/lib/terraforming/resource/iam_role_policy.rb index ea9a3e21..6ac741d9 100644 --- a/lib/terraforming/resource/iam_role_policy.rb +++ b/lib/terraforming/resource/iam_role_policy.rb @@ -45,14 +45,14 @@ def unique_name(policy) "#{policy.role_name}_#{policy.policy_name}" end - def iam_roles - @client.list_roles.roles - end - def iam_role_policy_id_of(policy) "#{policy.role_name}:#{policy.policy_name}" end + def iam_roles + @client.list_roles.map(&:roles).flatten + end + def iam_role_policy_names_in(role) @client.list_role_policies(role_name: role.role_name).policy_names end diff --git a/lib/terraforming/resource/iam_user.rb b/lib/terraforming/resource/iam_user.rb index 8bcf525c..658a6c94 100644 --- a/lib/terraforming/resource/iam_user.rb +++ b/lib/terraforming/resource/iam_user.rb @@ -43,7 +43,7 @@ def tfstate private def iam_users - @client.list_users.collect {|r| r.users}.flatten + @client.list_users.map(&:users).flatten end def module_name_of(user) diff --git a/lib/terraforming/resource/iam_user_policy.rb b/lib/terraforming/resource/iam_user_policy.rb index 4b401f72..1a9eea98 100644 --- a/lib/terraforming/resource/iam_user_policy.rb +++ b/lib/terraforming/resource/iam_user_policy.rb @@ -45,14 +45,14 @@ def unique_name(policy) "#{policy.user_name}_#{policy.policy_name}" end - def iam_users - @client.list_users.users - end - def iam_user_policy_id_of(policy) "#{policy.user_name}:#{policy.policy_name}" end + def iam_users + @client.list_users.map(&:users).flatten + end + def iam_user_policy_names_in(user) @client.list_user_policies(user_name: user.user_name).policy_names end diff --git a/lib/terraforming/resource/internet_gateway.rb b/lib/terraforming/resource/internet_gateway.rb index f49180a9..6b62c172 100644 --- a/lib/terraforming/resource/internet_gateway.rb +++ b/lib/terraforming/resource/internet_gateway.rb @@ -43,7 +43,7 @@ def tfstate private def internet_gateways - @client.describe_internet_gateways.internet_gateways + @client.describe_internet_gateways.map(&:internet_gateways).flatten end def module_name_of(internet_gateway) diff --git a/lib/terraforming/resource/launch_configuration.rb b/lib/terraforming/resource/launch_configuration.rb index 429c6f9c..b71c41df 100644 --- a/lib/terraforming/resource/launch_configuration.rb +++ b/lib/terraforming/resource/launch_configuration.rb @@ -96,9 +96,7 @@ def hash_security_group(name) end def launch_configurations - @client.describe_launch_configurations.collect do |r| - r.launch_configurations - end.flatten + @client.describe_launch_configurations.map(&:launch_configurations).flatten end def module_name_of(launch_configuration) diff --git a/lib/terraforming/resource/network_acl.rb b/lib/terraforming/resource/network_acl.rb index df3b7b18..f3a2aa23 100644 --- a/lib/terraforming/resource/network_acl.rb +++ b/lib/terraforming/resource/network_acl.rb @@ -68,7 +68,7 @@ def module_name_of(network_acl) end def network_acls - @client.describe_network_acls.network_acls + @client.describe_network_acls.map(&:network_acls).flatten end def subnet_ids_of(network_acl) diff --git a/lib/terraforming/resource/network_interface.rb b/lib/terraforming/resource/network_interface.rb index e08f170e..ac6ca139 100644 --- a/lib/terraforming/resource/network_interface.rb +++ b/lib/terraforming/resource/network_interface.rb @@ -61,7 +61,7 @@ def module_name_of(network_interface) end def network_interfaces - @client.describe_network_interfaces.network_interfaces + @client.describe_network_interfaces.map(&:network_interfaces).flatten end end diff --git a/lib/terraforming/resource/opsworks_custom_layer.rb b/lib/terraforming/resource/opsworks_custom_layer.rb index 76f90b8a..03b5de7a 100644 --- a/lib/terraforming/resource/opsworks_custom_layer.rb +++ b/lib/terraforming/resource/opsworks_custom_layer.rb @@ -103,7 +103,7 @@ def tfstate private def stacks - @client.describe_stacks.stacks + @client.describe_stacks.map(&:stacks).flatten end def stack_layers(stack_id) diff --git a/lib/terraforming/resource/opsworks_stack.rb b/lib/terraforming/resource/opsworks_stack.rb index 085817f2..3cd63f04 100644 --- a/lib/terraforming/resource/opsworks_stack.rb +++ b/lib/terraforming/resource/opsworks_stack.rb @@ -69,7 +69,7 @@ def tfstate private def stacks - @client.describe_stacks.stacks + @client.describe_stacks.map(&:stacks).flatten end def module_name_of(stack) diff --git a/lib/terraforming/resource/rds.rb b/lib/terraforming/resource/rds.rb index 00f4d2b3..001583c7 100644 --- a/lib/terraforming/resource/rds.rb +++ b/lib/terraforming/resource/rds.rb @@ -63,7 +63,7 @@ def tfstate private def db_instances - @client.describe_db_instances.db_instances + @client.describe_db_instances.map(&:db_instances).flatten end def module_name_of(instance) diff --git a/lib/terraforming/resource/redshift.rb b/lib/terraforming/resource/redshift.rb index aaeb1d23..1f85f08b 100644 --- a/lib/terraforming/resource/redshift.rb +++ b/lib/terraforming/resource/redshift.rb @@ -56,7 +56,7 @@ def tfstate private def clusters - @client.describe_clusters.clusters + @client.describe_clusters.map(&:clusters).flatten end def module_name_of(cluster) diff --git a/lib/terraforming/resource/route53_record.rb b/lib/terraforming/resource/route53_record.rb index 5b727fe5..078f7ee2 100644 --- a/lib/terraforming/resource/route53_record.rb +++ b/lib/terraforming/resource/route53_record.rb @@ -52,7 +52,7 @@ def tfstate private def hosted_zones - @client.list_hosted_zones.hosted_zones + @client.list_hosted_zones.map(&:hosted_zones).flatten end def record_id_of(record, zone_id) diff --git a/lib/terraforming/resource/route53_zone.rb b/lib/terraforming/resource/route53_zone.rb index 4b8be343..e93ed5a7 100644 --- a/lib/terraforming/resource/route53_zone.rb +++ b/lib/terraforming/resource/route53_zone.rb @@ -49,7 +49,7 @@ def tfstate private def hosted_zones - @client.list_hosted_zones.hosted_zones.map { |hosted_zone| @client.get_hosted_zone(id: hosted_zone.id) } + @client.list_hosted_zones.map(&:hosted_zones).flatten.map { |hosted_zone| @client.get_hosted_zone(id: hosted_zone.id) } end def tags_of(hosted_zone) diff --git a/lib/terraforming/resource/route_table.rb b/lib/terraforming/resource/route_table.rb index 516ed98c..2f3d4288 100644 --- a/lib/terraforming/resource/route_table.rb +++ b/lib/terraforming/resource/route_table.rb @@ -57,7 +57,7 @@ def module_name_of(route_table) end def route_tables - @client.describe_route_tables.route_tables + @client.describe_route_tables.map(&:route_tables).flatten end def routes_attributes_of(route_table) diff --git a/lib/terraforming/resource/route_table_association.rb b/lib/terraforming/resource/route_table_association.rb index f294b982..a18d0084 100644 --- a/lib/terraforming/resource/route_table_association.rb +++ b/lib/terraforming/resource/route_table_association.rb @@ -52,7 +52,7 @@ def module_name_of(route_table, assoc) end def route_tables - @client.describe_route_tables.route_tables + @client.describe_route_tables.map(&:route_tables).flatten end end end diff --git a/lib/terraforming/resource/s3.rb b/lib/terraforming/resource/s3.rb index b9da8c6e..0cdd1132 100644 --- a/lib/terraforming/resource/s3.rb +++ b/lib/terraforming/resource/s3.rb @@ -53,7 +53,7 @@ def bucket_policy_of(bucket) end def buckets - @client.list_buckets.buckets.select { |bucket| same_region?(bucket) } + @client.list_buckets.map(&:buckets).flatten.select { |bucket| same_region?(bucket) } end def module_name_of(bucket) diff --git a/lib/terraforming/resource/security_group.rb b/lib/terraforming/resource/security_group.rb index 9ce42c70..dfc6ff65 100644 --- a/lib/terraforming/resource/security_group.rb +++ b/lib/terraforming/resource/security_group.rb @@ -153,7 +153,7 @@ def self_referenced_permission?(security_group, permission) end def security_groups - @client.describe_security_groups.security_groups + @client.describe_security_groups.map(&:security_groups).flatten end def security_groups_in(permission, security_group) diff --git a/lib/terraforming/resource/sqs.rb b/lib/terraforming/resource/sqs.rb index 77ce49cd..9c01e005 100644 --- a/lib/terraforming/resource/sqs.rb +++ b/lib/terraforming/resource/sqs.rb @@ -59,7 +59,7 @@ def queues end def queue_urls - @client.list_queues.queue_urls + @client.list_queues.map(&:queue_urls).flatten end def module_name_of(queue) diff --git a/lib/terraforming/resource/subnet.rb b/lib/terraforming/resource/subnet.rb index 067b66e3..c541eb5b 100644 --- a/lib/terraforming/resource/subnet.rb +++ b/lib/terraforming/resource/subnet.rb @@ -44,7 +44,7 @@ def tfstate private def subnets - @client.describe_subnets.subnets + @client.describe_subnets.map(&:subnets).flatten end def module_name_of(subnet) diff --git a/lib/terraforming/resource/vpc.rb b/lib/terraforming/resource/vpc.rb index 2db7ff8d..3346c515 100644 --- a/lib/terraforming/resource/vpc.rb +++ b/lib/terraforming/resource/vpc.rb @@ -56,7 +56,7 @@ def module_name_of(vpc) end def vpcs - @client.describe_vpcs.vpcs + @client.describe_vpcs.map(&:vpcs).flatten end def vpc_attribute(vpc, attribute) diff --git a/lib/terraforming/resource/vpn_gateway.rb b/lib/terraforming/resource/vpn_gateway.rb index 3e578624..abe5854a 100644 --- a/lib/terraforming/resource/vpn_gateway.rb +++ b/lib/terraforming/resource/vpn_gateway.rb @@ -44,7 +44,7 @@ def tfstate private def vpn_gateways - @client.describe_vpn_gateways.vpn_gateways + @client.describe_vpn_gateways.map(&:vpn_gateways).flatten end def module_name_of(vpn_gateway) diff --git a/lib/terraforming/util.rb b/lib/terraforming/util.rb index 055664a6..28846fd8 100644 --- a/lib/terraforming/util.rb +++ b/lib/terraforming/util.rb @@ -26,5 +26,6 @@ def prettify_policy(document, breakline: false, unescape: false) json.strip end end + end end diff --git a/spec/lib/terraforming/resource/iam_group_spec.rb b/spec/lib/terraforming/resource/iam_group_spec.rb index 9538b8c8..26301de8 100644 --- a/spec/lib/terraforming/resource/iam_group_spec.rb +++ b/spec/lib/terraforming/resource/iam_group_spec.rb @@ -27,7 +27,15 @@ module Resource end before do - client.stub_responses(:list_groups, groups: groups) + client.stub_responses(:list_groups, [{ + groups: [groups[0]], + is_truncated: true, + marker: 'marker' + }, { + groups: [groups[1]], + is_truncated: false, + marker: nil + }]) end describe ".tf" do