Merge pull request #126 from jyejare/unified_cleanup_funcs

Unified/Simplified Cloud Provider Cleanup and Resource Types

cloudwash/cli.py

@@ -50,8 +50,9 @@ def cleanup_providers(ctx, dry, version):
         click.echo(f"Version: {cloudwash_version}")
         click.echo(f"Settings File: {settings.settings_file}")
     if ctx.invoked_subcommand:
+        settings.set('dry_run', dry)
         logger.info(
-            f"\n<<<<<<< Running the cleanup script in {'DRY' if dry else 'ACTION'} RUN mode >>>>>>>"
+            f"\n<<<<<<< Running the cleanup script in {'DRY' if dry else 'ACTION'} mode >>>>>>>"
         )
 
 

cloudwash/entities/providers.py

@@ -0,0 +1,37 @@
+from cloudwash.entities.resources.discs import CleanAWSDiscs
+from cloudwash.entities.resources.discs import CleanAzureDiscs
+from cloudwash.entities.resources.vms import CleanAWSVms
+from cloudwash.entities.resources.vms import CleanAzureVMs
+
+
+class providerCleanup:
+    def __init__(self, client):
+        self.client = client
+
+    @property
+    def vms(self):
+        providerclass = self.__class__.__name__
+        if 'Azure' in providerclass:
+            return CleanAzureVMs(client=self.client)
+        elif 'AWS' in providerclass:
+            return CleanAWSVms(client=self.client)
+
+    @property
+    def discs(self):
+        providerclass = self.__class__.__name__
+        if 'Azure' in providerclass:
+            return CleanAzureDiscs(client=self.client)
+        elif 'AWS' in providerclass:
+            return CleanAWSDiscs(client=self.client)
+
+
+class AzureCleanup(providerCleanup):
+    def __init__(self, client):
+        self.client = client
+        super().__init__(client)
+
+
+class AWSCleanup(providerCleanup):
+    def __init__(self, client):
+        self.client = client
+        super().__init__(client)

cloudwash/entities/base.py → cloudwash/entities/resources/base.py

@@ -42,6 +42,24 @@ def _set_dry(self):
         pass
 
 
+class DiscsCleanup(ResourceCleanup):
+    @abstractmethod
+    def list(self):
+        pass
+
+    @abstractmethod
+    def cleanup(self):
+        pass
+
+    @abstractmethod
+    def remove(self):
+        pass
+
+    @abstractmethod
+    def _set_dry(self):
+        pass
+
+
 class ResourceCleanupManager:
     def __init__(self):
         self.resources = []

cloudwash/entities/resources/discs.py

@@ -0,0 +1,51 @@
+from cloudwash.config import settings
+from cloudwash.entities.resources.base import DiscsCleanup
+from cloudwash.logger import logger
+from cloudwash.utils import dry_data
+
+
+class CleanDiscs(DiscsCleanup):
+    def __init__(self, client):
+        self.client = client
+        self._delete = []
+        self.list()
+
+    def _set_dry(self):
+        # VMsContainer = namedtuple('VMsCotainer', ['delete', 'stop', 'skip'])
+        # return VMsContainer(self._delete, self._stop, self._skip)
+        dry_data['DISCS']['delete'] = self._delete
+
+    def list(self):
+        pass
+
+    def remove(self):
+        pass
+
+    def cleanup(self):
+        if not settings.dry_run:
+            self.remove()
+
+
+class CleanAWSDiscs(CleanDiscs):
+    def list(self):
+        if settings.aws.criteria.disc.unassigned:
+            rdiscs = self.client.get_all_unattached_volumes()
+            rdiscs = [rdisc["VolumeId"] for rdisc in rdiscs]
+            self._delete.extend(rdiscs)
+        self._set_dry()
+
+    def remove(self):
+        self.client.remove_all_unused_volumes()
+        logger.info(f"Removed Discs: \n{self._delete}")
+
+
+class CleanAzureDiscs(CleanDiscs):
+    def list(self):
+        if settings.aws.criteria.disc.unassigned:
+            rdiscs = self.client.list_free_discs()
+            self._delete.extend(rdiscs)
+        self._set_dry()
+
+    def remove(self):
+        self.client.remove_discs_by_search()
+        logger.info(f"Removed Discs: \n{self._delete}")

cloudwash/entities/resources/aws.py → cloudwash/entities/resources/vms.py

@@ -1,13 +1,13 @@
 from cloudwash.config import settings
-from cloudwash.entities.base import VMsCleanup
+from cloudwash.entities.resources.base import VMsCleanup
 from cloudwash.logger import logger
 from cloudwash.utils import dry_data
 from cloudwash.utils import total_running_time
 
 
 class CleanVMs(VMsCleanup):
-    def __init__(self, awsclient):
-        self.client = awsclient
+    def __init__(self, client):
+        self.client = client
         self._delete = []
         self._stop = []
         self._skip = []
@@ -20,12 +20,36 @@ def _set_dry(self):
         dry_data['VMS']['stop'] = self._stop
         dry_data['VMS']['skip'] = self._skip
 
+    def list(self):
+        pass
+
+    def remove(self):
+        for vm_name in self._delete:
+            self.client.get_vm(vm_name).delete()
+        logger.info(f"Removed VMs: \n{self._delete}")
+
+    def stop(self):
+        for vm_name in self._stop:
+            self.client.get_vm(vm_name).stop()
+        logger.info(f"Stopped VMs: \n{self._stop}")
+
+    def skip(self):
+        logger.info(f"Skipped VMs: \n{self._skip}")
+
+    def cleanup(self):
+        if not settings.dry_run:
+            self.remove()
+            self.stop()
+            self.skip()
+
+
+class CleanAWSVms(CleanVMs):
     def list(self):
         all_vms = self.client.list_vms()
 
         for vm in all_vms:
             if vm.name in settings.aws.exceptions.vm.vm_list:
-                self._delete.append(vm.name)
+                self._skip.append(vm.name)
                 continue
 
             elif total_running_time(vm).minutes >= settings.aws.criteria.vm.sla_minutes:
@@ -37,20 +61,32 @@ def list(self):
                     self._delete.append(vm.name)
         self._set_dry()
 
-    def remove(self):
-        for vm_name in self._delete:
-            self.client.get_vm(vm_name).delete()
-        logger.info(f"Removed VMs: \n{self._delete}")
-
-    def stop(self):
-        for vm_name in self._stop:
-            self.client.get_vm(vm_name).stop()
-        logger.info(f"Stopped VMs: \n{self._stop}")
 
-    def skip(self):
-        logger.info(f"Skipped VMs: \n{self._skip}")
+class CleanAzureVMs(CleanVMs):
+    def list(self):
+        all_vms = self.client.list_vms()
 
-    def cleanup(self):
-        self.remove()
-        self.stop()
-        self.skip()
+        for vm in all_vms:
+            if not vm.exists:
+                self._delete.append(vm.name)
+                continue
+            # Don't assess the VM that is still in creating state
+            if vm.state.lower() == "vmstate.creating":
+                continue
+            # Match the user defined criteria in settings to delete the VM
+            if vm.name in settings.azure.exceptions.vm.vm_list:
+                self._skip.append(vm.name)
+            elif (
+                getattr(
+                    total_running_time(vm),
+                    "minutes",
+                    int(settings.azure.criteria.vm.sla_minutes) + 1,
+                )
+                >= settings.azure.criteria.vm.sla_minutes
+            ):
+                if vm.name in settings.azure.exceptions.vm.stop_list:
+                    self._stop.append(vm.name)
+                    continue
+                elif vm.name.startswith(settings.azure.criteria.vm.delete_vm):
+                    self._delete.append(vm.name)
+        self._set_dry()

cloudwash/providers/aws.py

@@ -1,7 +1,7 @@
 """ec2 CR Cleanup Utilities"""
 from cloudwash.client import compute_client
 from cloudwash.config import settings
-from cloudwash.entities.resources.aws import CleanVMs
+from cloudwash.entities.providers import AWSCleanup
 from cloudwash.logger import logger
 from cloudwash.utils import dry_data
 from cloudwash.utils import echo_dry
@@ -20,6 +20,8 @@ def cleanup(**kwargs):
         for items in data:
             dry_data[items]['delete'] = []
         with compute_client("aws", aws_region=region) as aws_client:
+            awscleanup = AWSCleanup(client=aws_client)
+
             # Dry Data Collection Defs
             def dry_nics():
                 rnics = []
@@ -31,13 +33,6 @@ def dry_nics():
                     ]
                 return rnics
 
-            def dry_discs():
-                rdiscs = []
-                if settings.aws.criteria.disc.unassigned:
-                    rdiscs = aws_client.get_all_unattached_volumes()
-                    [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs]
-                return rdiscs
-
             def dry_images():
                 rimages = []
                 if settings.aws.criteria.image.unassigned:
@@ -89,20 +84,14 @@ def remove_stacks(stacks):
             # Actual Cleaning and dry execution
             logger.info(f"\nResources from the region: {region}")
             if kwargs["vms"] or kwargs["_all"]:
-                vms_cleanup = CleanVMs(awsclient=aws_client)
-                if not is_dry_run:
-                    vms_cleanup.cleanup()
-
+                awscleanup.vms.cleanup()
             if kwargs["nics"] or kwargs["_all"]:
                 rnics = dry_nics()
                 if not is_dry_run and rnics:
                     aws_client.remove_all_unused_nics()
                     logger.info(f"Removed NICs: \n{rnics}")
             if kwargs["discs"] or kwargs["_all"]:
-                rdiscs = dry_discs()
-                if not is_dry_run and rdiscs:
-                    aws_client.remove_all_unused_volumes()
-                    logger.info(f"Removed Discs: \n{rdiscs}")
+                awscleanup.discs.cleanup()
             if kwargs["images"] or kwargs["_all"]:
                 rimages = dry_images()
                 if not is_dry_run and rimages:

cloudwash/providers/azure.py

@@ -1,39 +1,10 @@
 """Azure CR Cleanup Utilities"""
 from cloudwash.client import compute_client
 from cloudwash.config import settings
+from cloudwash.entities.providers import AzureCleanup
 from cloudwash.logger import logger
 from cloudwash.utils import dry_data
 from cloudwash.utils import echo_dry
-from cloudwash.utils import total_running_time
-
-
-def _dry_vms(all_vms):
-    """Filters and returns running VMs to be deleted from all VMs"""
-    _vms = {"stop": [], "delete": [], "skip": []}
-    for vm in all_vms:
-        # Remove the VM that's in Failed state and cant perform in assessments
-        if not vm.exists:
-            _vms["delete"].append(vm.name)
-            continue
-        # Don't assess the VM that is still in creating state
-        if vm.state.lower() == "vmstate.creating":
-            continue
-        # Match the user defined criteria in settings to delete the VM
-        if vm.name in settings.azure.exceptions.vm.vm_list:
-            _vms["skip"].append(vm.name)
-            continue
-        elif (
-            getattr(
-                total_running_time(vm), "minutes", int(settings.azure.criteria.vm.sla_minutes) + 1
-            )
-            >= settings.azure.criteria.vm.sla_minutes
-        ):
-            if vm.name in settings.azure.exceptions.vm.stop_list:
-                _vms["stop"].append(vm.name)
-                continue
-            elif vm.name.startswith(settings.azure.criteria.vm.delete_vm):
-                _vms["delete"].append(vm.name)
-    return _vms
 
 
 def cleanup(**kwargs):
@@ -57,17 +28,11 @@ def cleanup(**kwargs):
                 groups = azure_client.list_resource_groups()
 
         for group in groups:
-            dry_data['VMS']['stop'] = []
-            dry_data['VMS']['skip'] = []
             for items in data:
                 dry_data[items]['delete'] = []
 
             with compute_client("azure", azure_region=region, resource_group=group) as azure_client:
-                # Dry Data Collection Defs
-                def dry_vms():
-                    all_vms = azure_client.list_vms()
-                    dry_data["VMS"] = _dry_vms(all_vms)
-                    return dry_data["VMS"]
+                azurecleanup = AzureCleanup(client=azure_client)
 
                 def dry_nics():
                     rnics = []
@@ -76,13 +41,6 @@ def dry_nics():
                         [dry_data["NICS"]["delete"].append(dnic) for dnic in rnics]
                     return rnics
 
-                def dry_discs():
-                    rdiscs = []
-                    if settings.azure.criteria.disc.unassigned:
-                        rdiscs = azure_client.list_free_discs()
-                        [dry_data["DISCS"]["delete"].append(ddisc) for ddisc in rdiscs]
-                    return rdiscs
-
                 def dry_pips():
                     rpips = []
                     if settings.azure.criteria.public_ip.unassigned:
@@ -119,33 +77,18 @@ def dry_images():
                         dry_data["IMAGES"]["delete"].extend(remove_images)
                     return remove_images
 
-                # Remove / Stop VMs
-                def remove_vms(avms):
-                    # Remove VMs
-                    [azure_client.get_vm(vm_name).delete() for vm_name in avms["delete"]]
-                    # Stop VMs
-                    [azure_client.get_vm(vm_name).stop() for vm_name in avms["stop"]]
-
                 # Actual Cleaning and dry execution
                 logger.info(f"\nResources from the region and resource group: {region}/{group}")
 
                 if kwargs["vms"] or kwargs["_all"]:
-                    avms = dry_vms()
-                    if not is_dry_run:
-                        remove_vms(avms=avms)
-                        logger.info(f"Stopped VMs: \n{avms['stop']}")
-                        logger.info(f"Removed VMs: \n{avms['delete']}")
-                        logger.info(f"Skipped VMs: \n{avms['skip']}")
+                    azurecleanup.vms.cleanup()
                 if kwargs["nics"] or kwargs["_all"]:
                     rnics = dry_nics()
                     if not is_dry_run and rnics:
                         azure_client.remove_nics_by_search()
                         logger.info(f"Removed NICs: \n{rnics}")
                 if kwargs["discs"] or kwargs["_all"]:
-                    rdiscs = dry_discs()
-                    if not is_dry_run and rdiscs:
-                        azure_client.remove_discs_by_search()
-                        logger.info(f"Removed Discs: \n{rdiscs}")
+                    azurecleanup.discs.cleanup()
                 if kwargs["pips"] or kwargs["_all"]:
                     rpips = dry_pips()
                     if not is_dry_run and rpips: