From 7b8ee13f1a65b07d17d5e20ce10c3e245b97afdf Mon Sep 17 00:00:00 2001 From: Shubham Ganar Date: Tue, 19 Jul 2022 20:39:19 +0530 Subject: [PATCH] Add support for deleting resources from multiple regions in AWS --- cloudwash/client.py | 5 +- cloudwash/providers/ec2.py | 126 ++++++++++++++++++++----------------- settings.yaml.template | 3 +- 3 files changed, 72 insertions(+), 62 deletions(-) diff --git a/cloudwash/client.py b/cloudwash/client.py index 8ef4b6f03..6d6553b2d 100644 --- a/cloudwash/client.py +++ b/cloudwash/client.py @@ -7,9 +7,8 @@ @contextmanager -def compute_client(compute_resource): +def compute_client(compute_resource, **kwargs): """The context manager for compute resource client to initiate and disconnect - :param str compute_resource: The compute resource name """ if compute_resource == "azure": @@ -33,7 +32,7 @@ def compute_client(compute_resource): client = wrapanapi.EC2System( username=settings.providers.ec2.username, password=settings.providers.ec2.password, - region=settings.providers.ec2.region, + region=kwargs['ec2_region'], ) else: raise ValueError( diff --git a/cloudwash/providers/ec2.py b/cloudwash/providers/ec2.py index 2b7fd6f86..1e82a7c90 100644 --- a/cloudwash/providers/ec2.py +++ b/cloudwash/providers/ec2.py @@ -10,67 +10,77 @@ def cleanup(**kwargs): is_dry_run = kwargs["dry_run"] - - with compute_client("ec2") as ec2_client: - # Dry Data Collection Defs - def dry_vms(): - all_vms = ec2_client.list_vms() - for vm in all_vms: - if vm.name in settings.providers.ec2.except_vm_list: - dry_data["VMS"]["skip"].append(vm.name) - continue - elif total_running_time(vm).minutes >= settings.sla_minutes: - if vm.name in settings.providers.ec2.except_vm_stop_list: - dry_data["VMS"]["stop"].append(vm.name) + data = ['VMS', 'DISCS', 'PIPS', 'RESOURCES'] + regions = settings.providers.ec2.regions + with compute_client("ec2", ec2_region="us-west-2") as client: + if "all" in regions: + regions = client.list_regions() + for region in regions: + dry_data['VMS']['stop'] = [] + dry_data['VMS']['skip'] = [] + for items in data: + dry_data[items]['delete'] = [] + with compute_client("ec2", ec2_region=region) as ec2_client: + # Dry Data Collection Defs + def dry_vms(): + all_vms = ec2_client.list_vms() + for vm in all_vms: + if vm.name in settings.providers.ec2.except_vm_list: + dry_data["VMS"]["skip"].append(vm.name) continue - elif vm.name.startswith(settings.delete_vm): - dry_data["VMS"]["delete"].append(vm.name) - return dry_data["VMS"] + elif total_running_time(vm).minutes >= settings.sla_minutes: + if vm.name in settings.providers.ec2.except_vm_stop_list: + dry_data["VMS"]["stop"].append(vm.name) + continue + elif vm.name.startswith(settings.delete_vm): + dry_data["VMS"]["delete"].append(vm.name) + return dry_data["VMS"] - def dry_nics(): - rnics = ec2_client.get_all_unused_network_interfaces() - [dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"]) for dnic in rnics] - return dry_data["NICS"]["delete"] + def dry_nics(): + rnics = ec2_client.get_all_unused_network_interfaces() + [dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"]) for dnic in rnics] + return dry_data["NICS"]["delete"] - def dry_discs(): - rdiscs = ec2_client.get_all_unattached_volumes() - [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs] - return dry_data["DISCS"]["delete"] + def dry_discs(): + rdiscs = ec2_client.get_all_unattached_volumes() + [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs] + return dry_data["DISCS"]["delete"] - def dry_pips(): - rpips = ec2_client.get_all_disassociated_addresses() - [dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips] - return dry_data["PIPS"]["delete"] + def dry_pips(): + rpips = ec2_client.get_all_disassociated_addresses() + [dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips] + return dry_data["PIPS"]["delete"] - # Remove / Stop VMs - def remove_vms(avms): - # Remove VMs - [ec2_client.get_vm(vm_name).delete() for vm_name in avms["delete"]] - # Stop VMs - [ec2_client.get_vm(vm_name).stop() for vm_name in avms["stop"]] + # Remove / Stop VMs + def remove_vms(avms): + # Remove VMs + [ec2_client.get_vm(vm_name).delete() for vm_name in avms["delete"]] + # Stop VMs + [ec2_client.get_vm(vm_name).stop() for vm_name in avms["stop"]] - # Actual Cleaning and dry execution - if kwargs["vms"] or kwargs["_all"]: - avms = dry_vms() - if not is_dry_run: - remove_vms(avms=avms) - logger.info(f"Stopped VMs: \n{avms['stop']}") - logger.info(f"Removed VMs: \n{avms['delete']}") - logger.info(f"Skipped VMs: \n{avms['skip']}") - if kwargs["nics"] or kwargs["_all"]: - rnics = dry_nics() - if not is_dry_run: - ec2_client.remove_all_unused_nics() - logger.info(f"Removed NICs: \n{rnics}") - if kwargs["discs"] or kwargs["_all"]: - rdiscs = dry_discs() - if not is_dry_run: - ec2_client.remove_all_unused_volumes() - logger.info(f"Removed Discs: \n{rdiscs}") - if kwargs["pips"] or kwargs["_all"]: - rpips = dry_pips() - if not is_dry_run: - ec2_client.remove_all_unused_ips() - logger.info(f"Removed PIPs: \n{rpips}") - if is_dry_run: - echo_dry(dry_data) + # Actual Cleaning and dry execution + logger.info(f"\nResources from the region: {region}") + if kwargs["vms"] or kwargs["_all"]: + avms = dry_vms() + if not is_dry_run: + remove_vms(avms=avms) + logger.info(f"Stopped VMs: \n{avms['stop']}") + logger.info(f"Removed VMs: \n{avms['delete']}") + logger.info(f"Skipped VMs: \n{avms['skip']}") + if kwargs["nics"] or kwargs["_all"]: + rnics = dry_nics() + if not is_dry_run: + ec2_client.remove_all_unused_nics() + logger.info(f"Removed NICs: \n{rnics}") + if kwargs["discs"] or kwargs["_all"]: + rdiscs = dry_discs() + if not is_dry_run: + ec2_client.remove_all_unused_volumes() + logger.info(f"Removed Discs: \n{rdiscs}") + if kwargs["pips"] or kwargs["_all"]: + rpips = dry_pips() + if not is_dry_run: + ec2_client.remove_all_unused_ips() + logger.info(f"Removed PIPs: \n{rpips}") + if is_dry_run: + echo_dry(dry_data) diff --git a/settings.yaml.template b/settings.yaml.template index e313cd7ed..1d8221590 100644 --- a/settings.yaml.template +++ b/settings.yaml.template @@ -28,7 +28,8 @@ PROVIDERS: EC2: USERNAME: PASSWORD: - REGION: + # Multiple regions can be added like ["ap-south-1", "us-west-2", "us-west-1"] or ["all"] for all regions + REGIONS: [] # VMs that would be skipped from cleanup EXCEPT_VM_LIST: [] # VMs that would be stopped from running states