Clean Dedicated Hosts (#448)

.github/workflows/cleaner.yml → .github/workflows/clean_ami.yml

@@ -1,18 +1,19 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: MIT
 
-name: Monthly Resources Cleaner
+name: AMI Monthly Resources Cleaner
 
 on:
   schedule:
     - cron: "0 0 1 * *" # Run on the first day of every month
   workflow_dispatch:
 
 jobs:
-  clean-resources:
+  clean-ami:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v3
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -21,5 +22,5 @@ jobs:
           aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2
 
-      - name: Clean old aws resources
-        run: make integration-cleaner
+      - name: Clean old ami
+        run: go run ./integration/clean/clean_ami/clean_ami.go --tags=clean

.github/workflows/clean_dedicated_host.yml

@@ -0,0 +1,26 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+
+name: Dedicated Host Daily Resources Cleaner
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # Run Every Day At Midnight
+  workflow_dispatch:
+
+jobs:
+  clean-dedicated-hosts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v3
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Clean old dedicated host
+        run: go run ./integration/clean/clean_dedicated_host/clean_dedicated_host.go --tags=clean

.github/workflows/internal-pipeline-dedicated-host-cleaner.yml

@@ -0,0 +1,27 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+
+# @TODO delete this when we remove the internal cw agent integration pipeline
+name: Dedicated Host Daily Resources Cleaner
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # Run Every Day At Midnight
+  workflow_dispatch:
+
+jobs:
+  clean-dedicated-hosts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v3
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.INTERNAL_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.INTERNAL_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Clean old dedicated host
+        run: go run ./integration/clean/clean_dedicated_host/clean_dedicated_host.go --tags=clean

Makefile

@@ -142,9 +142,6 @@ fmt-sh: install-tools
 test:
 	CGO_ENABLED=0 go test -coverprofile coverage.txt -failfast ./awscsm/... ./cfg/... ./cmd/... ./handlers/... ./internal/... ./logger/... ./logs/... ./metric/... ./plugins/... ./profiler/... ./tool/... ./translator/...
 
-integration-cleaner:
-	go run ./integration/clean/clean_ami.go --tags=clean
-
 clean::
 	rm -rf release/ build/
 	rm -f CWAGENT_VERSION

integration/clean/README.md

@@ -1,10 +1,18 @@
 **What does the cleaner do?**
 
-Cleaner cleans out old ami (ami older than 60 days)
+###Cleaner cleans out old ami (ami older than 60 days)
 
 The cleaner first searches for ami names (these are the ami created by the pipeline for use int he integration tests)
 1. cloudwatch-agent-integration-test*
 
 Then checks to see if the creation date is greater than 60 days. (The aws sdk v2 gives creation date as a pointer to string. To convert to golang time we use the aws smithy go time. This allows us to compare to 60 days in past time)
 
-If the ami is older than 60 days old then we delete the ami
+If the ami is older than 60 days old then we delete the ami
+
+###Cleans dedicated hosts for mac
+
+The cleaner first searches for dedicated host tag Name:IntegrationTestMacDedicatedHost
+
+Then checks to see if the creation date is greater than 26 hours and host status is available
+
+Delete is true

integration/clean/clean_ami.go → integration/clean/clean_ami/clean_ami.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"time"
 
@@ -18,25 +19,24 @@ import (
 	smithyTime "github.com/aws/smithy-go/time"
 )
 
+const daysToKeep = 60
+const keepDuration = -1 * time.Hour * 24 * time.Duration(daysToKeep)
+
 func main() {
 	err := cleanAMI()
 	if err != nil {
 		log.Fatalf("errors cleaning %v", err)
 	}
 }
 
-const daysToKeep = 60
-const keepDuration = -1 * time.Hour * 24 * time.Duration(daysToKeep)
-
-var expirationDate = time.Now().UTC().Add(keepDuration)
-
-func cleanAMI() []error {
+func cleanAMI() error {
 	log.Print("Begin to clean EC2 AMI")
 
+	expirationDate := time.Now().UTC().Add(keepDuration)
 	cxt := context.Background()
 	defaultConfig, err := config.LoadDefaultConfig(cxt)
 	if err != nil {
-		return []error{err}
+		return err
 	}
 	ec2client := ec2.NewFromConfig(defaultConfig)
 
@@ -49,14 +49,14 @@ func cleanAMI() []error {
 	describeImagesInput := ec2.DescribeImagesInput{Filters: []types.Filter{nameFilter}}
 	describeImagesOutput, err := ec2client.DescribeImages(cxt, &describeImagesInput)
 	if err != nil {
-		return []error{err}
+		return err
 	}
 
-	var errors []error
+	var errList []error
 	for _, image := range describeImagesOutput.Images {
 		creationDate, err := smithyTime.ParseDateTime(*image.CreationDate)
 		if err != nil {
-			errors = append(errors, err)
+			errList = append(errList, err)
 			continue
 		}
 		log.Printf("image name %v image id %v experation date %v creation date parsed %v image creation date raw %v",
@@ -66,13 +66,13 @@ func cleanAMI() []error {
 			deregisterImageInput := ec2.DeregisterImageInput{ImageId: image.ImageId}
 			_, err := ec2client.DeregisterImage(cxt, &deregisterImageInput)
 			if err != nil {
-				errors = append(errors, err)
+				errList = append(errList, err)
 			}
 		}
 	}
 
-	if len(errors) != 0 {
-		return errors
+	if len(errList) != 0 {
+		return fmt.Errorf("%v", errList)
 	}
 
 	return nil

integration/clean/clean_dedicated_host/clean_dedicated_host.go

@@ -0,0 +1,80 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build clean
+// +build clean
+
+package main
+
+import (
+	"context"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	"log"
+	"time"
+)
+
+// Can't release a host if it was being used within the last 24 hr add 2 hr as a buffer
+const keepDurationDedicatedHost = -1 * time.Hour * 26
+const tagName = "tag:Name"
+const tagValue = "IntegrationTestMacDedicatedHost"
+
+func main() {
+	err := cleanDedicatedHost()
+	if err != nil {
+		log.Fatalf("errors cleaning %v", err)
+	}
+}
+
+func cleanDedicatedHost() error {
+	log.Print("Begin to clean EC2 Dedicated Host")
+
+	expirationDateDedicatedHost := time.Now().UTC().Add(keepDurationDedicatedHost)
+	cxt := context.Background()
+	defaultConfig, err := config.LoadDefaultConfig(cxt)
+	if err != nil {
+		return err
+	}
+	ec2client := ec2.NewFromConfig(defaultConfig)
+
+	dedicatedHosts, err := getDedicatedHost(cxt, ec2client)
+	if err != nil {
+		return err
+	}
+
+	dedicatedHostIds := make([]string, 0)
+	for _, dedicatedHost := range dedicatedHosts {
+		log.Printf("dedicated host id %v experation date %v dedicated host creation date raw %v",
+			*dedicatedHost.HostId, expirationDateDedicatedHost, *dedicatedHost.AllocationTime)
+		if expirationDateDedicatedHost.After(*dedicatedHost.AllocationTime) && dedicatedHost.State == types.AllocationStateAvailable {
+			log.Printf("Try to delete dedicated host %s tags %v launch-date %s", *dedicatedHost.HostId, dedicatedHost.Tags, *dedicatedHost.AllocationTime)
+			dedicatedHostIds = append(dedicatedHostIds, *dedicatedHost.HostId)
+		}
+	}
+
+	if len(dedicatedHostIds) == 0 {
+		log.Printf("No dedicated hosts to release")
+		return nil
+	}
+
+	log.Printf("Dedicated hosts to release %v", dedicatedHostIds)
+	releaseDedicatedHost := ec2.ReleaseHostsInput{HostIds: dedicatedHostIds}
+	_, err = ec2client.ReleaseHosts(cxt, &releaseDedicatedHost)
+	return err
+}
+
+func getDedicatedHost(cxt context.Context, ec2client *ec2.Client) ([]types.Host, error) {
+	// Get list of dedicated host
+	nameFilter := types.Filter{Name: aws.String(tagName), Values: []string{
+		tagValue,
+	}}
+
+	describeDedicatedHostInput := ec2.DescribeHostsInput{Filter: []types.Filter{nameFilter}}
+	describeDedicatedHostOutput, err := ec2client.DescribeHosts(cxt, &describeDedicatedHostInput)
+	if err != nil {
+		return nil, err
+	}
+	return describeDedicatedHostOutput.Hosts, nil
+}