Add support for collecting deletable OCP leftovers from AWS

cloudwash/cli.py

@@ -98,8 +98,13 @@ def azure(ctx, vms, discs, nics, images, pips, _all, _all_rg):
 @click.option("--images", is_flag=True, help="Remove only images from the provider")
 @click.option("--pips", is_flag=True, help="Remove only Public IPs from the provider")
 @click.option("--stacks", is_flag=True, help="Remove only CloudFormations from the provider")
+@click.option(
+    "--ocps",
+    is_flag=True,
+    help="Remove only unused OCP Cluster occupied resources from the provider",
+)
 @click.pass_context
-def aws(ctx, vms, discs, nics, images, pips, stacks, _all):
+def aws(ctx, vms, discs, nics, images, pips, stacks, ocps, _all):
     # Validate Amazon Settings
     validate_provider(ctx.command.name)
     is_dry_run = ctx.parent.params["dry"]
@@ -110,6 +115,7 @@ def aws(ctx, vms, discs, nics, images, pips, stacks, _all):
         images=images,
         pips=pips,
         stacks=stacks,
+        ocps=ocps,
         _all=_all,
         dry_run=is_dry_run,
     )

cloudwash/providers/aws.py

@@ -2,14 +2,19 @@
 from cloudwash.client import compute_client
 from cloudwash.config import settings
 from cloudwash.logger import logger
+from cloudwash.utils import calculate_time_threshold
+from cloudwash.utils import delete_ocp
 from cloudwash.utils import dry_data
 from cloudwash.utils import echo_dry
+from cloudwash.utils import filter_resources_by_time_modified
+from cloudwash.utils import group_ocps_by_cluster
+from cloudwash.utils import OCP_TAG_SUBSTR
 from cloudwash.utils import total_running_time
 
 
 def cleanup(**kwargs):
     is_dry_run = kwargs["dry_run"]
-    data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS']
+    data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS', 'OCPS']
     regions = settings.aws.auth.regions
     if "all" in regions:
         with compute_client("aws", aws_region="us-west-2") as client:
@@ -96,6 +101,39 @@ def dry_stacks():
 
                 return rstacks
 
+            def dry_ocps():
+                time_threshold = calculate_time_threshold(time_ref=settings.aws.criteria.ocps.sla)
+
+                query = " ".join([f"tag.key:{OCP_TAG_SUBSTR}*", f"region:{region}"])
+                resources = resource_explorer_client.list_resources(query=query)
+
+                # Prepare resources to be filtered before deletion
+                cluster_map = group_ocps_by_cluster(resources=resources)
+                for cluster_name in cluster_map.keys():
+                    cluster_resources = cluster_map[cluster_name].get("Resources")
+                    instances = cluster_map[cluster_name].get("Instances")
+
+                    if instances:
+                        # For resources with associated EC2 Instances, filter by Instances SLA
+                        if not filter_resources_by_time_modified(
+                            time_threshold,
+                            resources=instances,
+                        ):
+                            dry_data["OCPS"]["delete"].extend(cluster_resources)
+                    else:
+                        # For resources with no associated EC2 Instances, identify as leftovers
+                        dry_data["OCPS"]["delete"].extend(
+                            filter_resources_by_time_modified(
+                                time_threshold, resources=cluster_resources
+                            )
+                        )
+
+                # Sort resources by type
+                dry_data["OCPS"]["delete"] = sorted(
+                    dry_data["OCPS"]["delete"], key=lambda x: x.resource_type
+                )
+                return dry_data["OCPS"]["delete"]
+
             # Remove / Stop VMs
             def remove_vms(avms):
                 # Remove VMs
@@ -142,5 +180,15 @@ def remove_stacks(stacks):
                 if not is_dry_run:
                     remove_stacks(stacks=rstacks)
                     logger.info(f"Removed Stacks: \n{rstacks}")
+            if kwargs["ocps"] or kwargs["_all"]:
+                # Differentiate between the cleanup region and the Resource Explorer client region
+                ocp_client_region = settings.aws.criteria.ocps.ocp_client_region
+                with compute_client(
+                    "aws", aws_region=ocp_client_region
+                ) as resource_explorer_client:
+                    rocps = dry_ocps()
+                    if not is_dry_run:
+                        for ocp in rocps:
+                            delete_ocp(ocp)
             if is_dry_run:
                 echo_dry(dry_data)

cloudwash/utils.py

@@ -2,19 +2,25 @@
 from collections import namedtuple
 from datetime import datetime
 
+import dateparser
 import pytz
+from wrapanapi.systems.ec2 import ResourceExplorerResource
 
 from cloudwash.logger import logger
 
+OCP_TAG_SUBSTR = "kubernetes.io/cluster/"
+
 _vms_dict = {"VMS": {"delete": [], "stop": [], "skip": []}}
 dry_data = {
     "NICS": {"delete": []},
     "DISCS": {"delete": []},
     "PIPS": {"delete": []},
+    "OCPS": {"delete": []},
     "RESOURCES": {"delete": []},
     "STACKS": {"delete": []},
     "IMAGES": {"delete": []},
 }
+
 dry_data.update(_vms_dict)
 
 
@@ -32,13 +38,20 @@ def echo_dry(dry_data=None) -> None:
     deletable_nics = dry_data["NICS"]["delete"]
     deletable_images = dry_data["IMAGES"]["delete"]
     deletable_pips = dry_data["PIPS"]["delete"] if "PIPS" in dry_data else None
+    deletable_ocps = {
+        ocp.resource_type: [
+            r.name for r in dry_data["OCPS"]["delete"] if r.resource_type == ocp.resource_type
+        ]
+        for ocp in dry_data["OCPS"]["delete"]
+    }
     deletable_resources = dry_data["RESOURCES"]["delete"]
     deletable_stacks = dry_data["STACKS"]["delete"] if "STACKS" in dry_data else None
     if deletable_vms or stopable_vms or skipped_vms:
         logger.info(
             f"VMs:\n\tDeletable: {deletable_vms}\n\tStoppable: {stopable_vms}\n\t"
             f"Skip: {skipped_vms}"
         )
+
     if deletable_discs:
         logger.info(f"DISCs:\n\tDeletable: {deletable_discs}")
     if deletable_nics:
@@ -47,6 +60,8 @@ def echo_dry(dry_data=None) -> None:
         logger.info(f"IMAGES:\n\tDeletable: {deletable_images}")
     if deletable_pips:
         logger.info(f"PIPs:\n\tDeletable: {deletable_pips}")
+    if deletable_ocps:
+        logger.info(f"OCPs:\n\tDeletable: {deletable_ocps}")
     if deletable_resources:
         logger.info(f"RESOURCEs:\n\tDeletable: {deletable_resources}")
     if deletable_stacks:
@@ -61,6 +76,7 @@ def echo_dry(dry_data=None) -> None:
             deletable_resources,
             deletable_stacks,
             deletable_images,
+            deletable_ocps,
         ]
     ):
         logger.info("\nNo resources are eligible for cleanup!")
@@ -112,3 +128,81 @@ def gce_zones() -> list:
     _zones_combo = {**_bcds, **_abcfs, **_abcs}
     zones = [f"{loc}-{zone}" for loc, zones in _zones_combo.items() for zone in zones]
     return zones
+
+
+def group_ocps_by_cluster(resources: list = None) -> dict:
+    """Group different types of AWS resources under their original OCP clusters
+
+    :param list resources: AWS resources collected by defined region and sla
+    :return: A dictionary with the clusters as keys and the associated resources as values
+    """
+    if resources is None:
+        resources = []
+    clusters_map = {}
+
+    for resource in resources:
+        for key in resource.get_tags(regex=OCP_TAG_SUBSTR):
+            cluster_name = key.get("Key")
+            if OCP_TAG_SUBSTR in cluster_name:
+                cluster_name = cluster_name.split(OCP_TAG_SUBSTR)[1]
+                if cluster_name not in clusters_map.keys():
+                    clusters_map[cluster_name] = {"Resources": [], "Instances": []}
+
+                # Set cluster's EC2 instances
+                if hasattr(resource, 'ec2_instance'):
+                    clusters_map[cluster_name]["Instances"].append(resource)
+                # Set resource under cluster
+                else:
+                    clusters_map[cluster_name]["Resources"].append(resource)
+    return clusters_map
+
+
+def calculate_time_threshold(time_ref=""):
+    """Parses a time reference for data filtering
+
+    :param str time_ref: a relative time reference for indicating the filter value
+    of a relative time, given in a {time_value}{time_unit} format; default is "" (no filtering)
+    :return datetime time_threshold
+    """
+    if time_ref is None:
+        time_ref = ""
+
+    if time_ref.isnumeric():
+        # Use default time value as Minutes
+        time_ref += "m"
+
+    # Time Ref is Optional; if empty, time_threshold will be set as "now"
+    time_threshold = dateparser.parse(f"now-{time_ref}-UTC")
+    logger.debug(
+        f"\nAssociated OCP resources are filtered by last creation time of: {time_threshold}"
+    )
+    return time_threshold
+
+
+def filter_resources_by_time_modified(
+    time_threshold,
+    resources: list[ResourceExplorerResource] = None,
+) -> list:
+    """
+    Filter list of AWS resources by checking modification date ("LastReportedAt")
+    :param datetime time_threshold: Time filtering criteria
+    :param list resources: List of resources to be filtered out
+
+    :return: list of resources that last modified before time threshold
+
+    :Example:
+        Use the time_ref "1h" to collect resources that exist for more than an hour
+    """
+    filtered_resources = []
+
+    for resource in resources:
+        # Will not collect resources recorded during the SLA time
+        if resource.date_modified > time_threshold:
+            continue
+        filtered_resources.append(resource)
+    return filtered_resources
+
+
+def delete_ocp(ocp):
+    # WIP: add support for deletion
+    pass

settings.yaml.template

@@ -73,7 +73,7 @@ AWS:
         ACCESS_KEY:
         SECRET_KEY:
         # Multiple regions can be added like ["ap-south-1", "us-west-2", "us-west-1"] or ["all"] for all regions
-        REGIONS: []
+        REGIONS: [] # Cleanup regions
     CRITERIA:
         VM:
             # The VM to be deleted with prepend string, e.g VM name that starts with 'test'
@@ -97,6 +97,11 @@ AWS:
             DELETE_STACK: 'test'
             # Number of minutes the deletable CloudFormation should be allowed to live, e.g 120 minutes = 2 Hours
             SLA_MINUTES: 120
+        OCPS:
+            OCP_CLIENT_REGION: "us-east-1"
+            # Specified as {time_value}{time_unit} format, e.g. "7d" = 7 Days
+            # If a time unit is not specified (the value is numeric), it will be considered as Minutes
+            SLA: 7d
     EXCEPTIONS:
         VM:
             # VM names that would be skipped from cleanup