Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Provide AutoScalingGroupName for webhook notifications #384

Merged
merged 1 commit into from
Mar 9, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions pkg/monitor/sqsevent/asg-lifecycle-event.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,13 @@ func (m SQSMonitor) asgTerminationToInterruptionEvent(event EventBridgeEvent, me
}

interruptionEvent := monitor.InterruptionEvent{
EventID: fmt.Sprintf("asg-lifecycle-term-%x", event.ID),
Kind: SQSTerminateKind,
StartTime: event.getTime(),
NodeName: nodeName,
InstanceID: lifecycleDetail.EC2InstanceID,
Description: fmt.Sprintf("ASG Lifecycle Termination event received. Instance will be interrupted at %s \n", event.getTime()),
EventID: fmt.Sprintf("asg-lifecycle-term-%x", event.ID),
Kind: SQSTerminateKind,
AutoScalingGroupName: lifecycleDetail.AutoScalingGroupName,
StartTime: event.getTime(),
NodeName: nodeName,
InstanceID: lifecycleDetail.EC2InstanceID,
Description: fmt.Sprintf("ASG Lifecycle Termination event received. Instance will be interrupted at %s \n", event.getTime()),
}

interruptionEvent.PostDrainTask = func(interruptionEvent monitor.InterruptionEvent, _ node.Node) error {
Expand Down
15 changes: 8 additions & 7 deletions pkg/monitor/sqsevent/ec2-state-change-event.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,15 @@ func (m SQSMonitor) ec2StateChangeToInterruptionEvent(event EventBridgeEvent, me
if err != nil {
return monitor.InterruptionEvent{}, err
}

asgName, err := m.retrieveAutoScalingGroupName(ec2StateChangeDetail.InstanceID)
interruptionEvent := monitor.InterruptionEvent{
EventID: fmt.Sprintf("ec2-state-change-event-%x", event.ID),
Kind: SQSTerminateKind,
StartTime: event.getTime(),
NodeName: nodeName,
InstanceID: ec2StateChangeDetail.InstanceID,
Description: fmt.Sprintf("EC2 State Change event received. Instance went into %s at %s \n", ec2StateChangeDetail.State, event.getTime()),
EventID: fmt.Sprintf("ec2-state-change-event-%x", event.ID),
Kind: SQSTerminateKind,
StartTime: event.getTime(),
NodeName: nodeName,
AutoScalingGroupName: asgName,
InstanceID: ec2StateChangeDetail.InstanceID,
Description: fmt.Sprintf("EC2 State Change event received. Instance went into %s at %s \n", ec2StateChangeDetail.State, event.getTime()),
}
interruptionEvent.PostDrainTask = func(interruptionEvent monitor.InterruptionEvent, n node.Node) error {
errs := m.deleteMessages([]*sqs.Message{message})
Expand Down
14 changes: 8 additions & 6 deletions pkg/monitor/sqsevent/rebalance-recommendation-event.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,16 @@ func (m SQSMonitor) rebalanceRecommendationToInterruptionEvent(event EventBridge
if err != nil {
return monitor.InterruptionEvent{}, err
}
asgName, err := m.retrieveAutoScalingGroupName(rebalanceRecDetail.InstanceID)

interruptionEvent := monitor.InterruptionEvent{
EventID: fmt.Sprintf("rebalance-recommendation-event-%x", event.ID),
Kind: SQSTerminateKind,
StartTime: event.getTime(),
NodeName: nodeName,
InstanceID: rebalanceRecDetail.InstanceID,
Description: fmt.Sprintf("Rebalance recommendation event received. Instance will be cordoned at %s \n", event.getTime()),
EventID: fmt.Sprintf("rebalance-recommendation-event-%x", event.ID),
Kind: SQSTerminateKind,
AutoScalingGroupName: asgName,
StartTime: event.getTime(),
NodeName: nodeName,
InstanceID: rebalanceRecDetail.InstanceID,
Description: fmt.Sprintf("Rebalance recommendation event received. Instance will be cordoned at %s \n", event.getTime()),
}
interruptionEvent.PostDrainTask = func(interruptionEvent monitor.InterruptionEvent, n node.Node) error {
errs := m.deleteMessages([]*sqs.Message{message})
Expand Down
15 changes: 8 additions & 7 deletions pkg/monitor/sqsevent/spot-itn-event.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,15 @@ func (m SQSMonitor) spotITNTerminationToInterruptionEvent(event EventBridgeEvent
if err != nil {
return monitor.InterruptionEvent{}, err
}

asgName, err := m.retrieveAutoScalingGroupName(spotInterruptionDetail.InstanceID)
interruptionEvent := monitor.InterruptionEvent{
EventID: fmt.Sprintf("spot-itn-event-%x", event.ID),
Kind: SQSTerminateKind,
StartTime: event.getTime(),
NodeName: nodeName,
InstanceID: spotInterruptionDetail.InstanceID,
Description: fmt.Sprintf("Spot Interruption event received. Instance will be interrupted at %s \n", event.getTime()),
EventID: fmt.Sprintf("spot-itn-event-%x", event.ID),
Kind: SQSTerminateKind,
AutoScalingGroupName: asgName,
StartTime: event.getTime(),
NodeName: nodeName,
InstanceID: spotInterruptionDetail.InstanceID,
Description: fmt.Sprintf("Spot Interruption event received. Instance will be interrupted at %s \n", event.getTime()),
}
interruptionEvent.PostDrainTask = func(interruptionEvent monitor.InterruptionEvent, n node.Node) error {
errs := m.deleteMessages([]*sqs.Message{message})
Expand Down
33 changes: 21 additions & 12 deletions pkg/monitor/sqsevent/sqs-monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,20 +222,11 @@ func (m SQSMonitor) isInstanceManaged(instanceID string) (bool, error) {
if instanceID == "" {
return false, fmt.Errorf("Instance ID was empty when calling isInstanceManaged")
}
asgDescribeInstanceInput := autoscaling.DescribeAutoScalingInstancesInput{
InstanceIds: []*string{&instanceID},
MaxRecords: aws.Int64(50),
}
asgs, err := m.ASG.DescribeAutoScalingInstances(&asgDescribeInstanceInput)
if err != nil {
asgName, err := m.retrieveAutoScalingGroupName(instanceID)
if asgName == "" {
return false, err
}
if len(asgs.AutoScalingInstances) == 0 {
log.Debug().Str("instance_id", instanceID).Msg("Did not find an Auto Scaling Group for the given instance id")
return false, nil
}
asgName := asgs.AutoScalingInstances[0].AutoScalingGroupName
asgFilter := autoscaling.Filter{Name: aws.String("auto-scaling-group"), Values: []*string{asgName}}
asgFilter := autoscaling.Filter{Name: aws.String("auto-scaling-group"), Values: []*string{aws.String(asgName)}}
asgDescribeTagsInput := autoscaling.DescribeTagsInput{
Filters: []*autoscaling.Filter{&asgFilter},
}
Expand All @@ -259,3 +250,21 @@ func (m SQSMonitor) isInstanceManaged(instanceID string) (bool, error) {
}
return isManaged, err
}

// retrieveAutoScalingGroupName returns the autoscaling group name for a given instanceID
func (m SQSMonitor) retrieveAutoScalingGroupName(instanceID string) (string, error) {
asgDescribeInstanceInput := autoscaling.DescribeAutoScalingInstancesInput{
InstanceIds: []*string{&instanceID},
MaxRecords: aws.Int64(50),
}
asgs, err := m.ASG.DescribeAutoScalingInstances(&asgDescribeInstanceInput)
if err != nil {
return "", err
}
if len(asgs.AutoScalingInstances) == 0 {
log.Debug().Str("instance_id", instanceID).Msg("Did not find an Auto Scaling Group for the given instance id")
return "", nil
}
asgName := asgs.AutoScalingInstances[0].AutoScalingGroupName
return *asgName, err
}
26 changes: 14 additions & 12 deletions pkg/monitor/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,20 @@ type DrainTask func(InterruptionEvent, node.Node) error

// InterruptionEvent gives more context of the interruption event
type InterruptionEvent struct {
EventID string
Kind string
Description string
State string
NodeName string
InstanceID string
StartTime time.Time
EndTime time.Time
Drained bool
InProgress bool
PreDrainTask DrainTask `json:"-"`
PostDrainTask DrainTask `json:"-"`
EventID string
Kind string
Description string
State string
AutoScalingGroupName string
NodeName string
NodeLabels map[string]string
InstanceID string
StartTime time.Time
EndTime time.Time
Drained bool
InProgress bool
PreDrainTask DrainTask `json:"-"`
PostDrainTask DrainTask `json:"-"`
}

// TimeUntilEvent returns the duration until the event start time
Expand Down