Skip to content

Commit

Permalink
converting aws provider to use aws-sdk-go-v2 (#1678)
Browse files Browse the repository at this point in the history 
Co-authored-by: chickencoding123 <8017298+chickencoding123@users.noreply.github.com>
  • Loading branch information
@chickencoding123 @amitrana16
chickencoding123 and amitrana16 authored Jan 15, 2025
1 parent 7d729b0 commit 914c9b2
Show file tree
Hide file tree
Showing 12 changed files with 609 additions and 540 deletions.
25 changes: 24 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ require (
github.com/Azure/go-autorest/autorest/azure/auth v0.5.3
github.com/Azure/go-autorest/autorest/to v0.4.0
github.com/UpCloudLtd/upcloud-go-api/v6 v6.5.0
github.com/aws/aws-sdk-go v1.55.2
github.com/bramvdbogaerde/go-scp v1.0.0
github.com/digitalocean/godo v1.57.0
github.com/distribution/reference v0.6.0
Expand Down Expand Up @@ -70,6 +69,30 @@ require (
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/aws/aws-sdk-go-v2 v1.32.8 // direct
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/config v1.28.10 // direct
github.com/aws/aws-sdk-go-v2/credentials v1.17.51 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 // direct
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.51.4 // direct
github.com/aws/aws-sdk-go-v2/service/ebs v1.27.10 // direct
github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.3 // direct
github.com/aws/aws-sdk-go-v2/service/iam v1.38.4 // direct
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 // indirect
github.com/aws/aws-sdk-go-v2/service/route53 v1.47.1 // direct
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 // direct
github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 // indirect
github.com/aws/smithy-go v1.22.1 // direct
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand Down
50 changes: 48 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,54 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.55.2 h1:/2OFM8uFfK9e+cqHTw9YPrvTzIXT2XkFGXRM7WbJb7E=
github.com/aws/aws-sdk-go v1.55.2/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.32.8 h1:cZV+NUS/eGxKXMtmyhtYPJ7Z4YLoI/V8bkTdRZfYhGo=
github.com/aws/aws-sdk-go-v2 v1.32.8/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
github.com/aws/aws-sdk-go-v2/config v1.28.10 h1:fKODZHfqQu06pCzR69KJ3GuttraRJkhlC8g80RZ0Dfg=
github.com/aws/aws-sdk-go-v2/config v1.28.10/go.mod h1:PvdxRYZ5Um9QMq9PQ0zHHNdtKK+he2NHtFCUFMXWXeg=
github.com/aws/aws-sdk-go-v2/credentials v1.17.51 h1:F/9Sm6Y6k4LqDesZDPJCLxQGXNNHd/ZtJiWd0lCZKRk=
github.com/aws/aws-sdk-go-v2/credentials v1.17.51/go.mod h1:TKbzCHm43AoPyA+iLGGcruXd4AFhF8tOmLex2R9jWNQ=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 h1:IBAoD/1d8A8/1aA8g4MBVtTRHhXRiNAgwdbo/xRM2DI=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23/go.mod h1:vfENuCM7dofkgKpYzuzf1VT1UKkA/YL3qanfBn7HCaA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 h1:XnXVe2zRyPf0+fAW5L05esmngvBpC6DQZK7oZB/z/Co=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48/go.mod h1:S3wey90OrS4f7kYxH6PT175YyEcHTORY07++HurMaRM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 h1:jSJjSBzw8VDIbWv+mmvBSP8ezsztMYJGH+eKqi9AmNs=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27/go.mod h1:/DAhLbFRgwhmvJdOfSm+WwikZrCuUJiA4WgJG0fTNSw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 h1:l+X4K77Dui85pIj5foXDhPlnqcNRG2QUyvca300lXh8=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27/go.mod h1:KvZXSFEXm6x84yE8qffKvT3x8J5clWnVFXphpohhzJ8=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 h1:AmB5QxnD+fBFrg9LcqzkgF/CaYvMyU/BTlejG4t1S7Q=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27/go.mod h1:Sai7P3xTiyv9ZUYO3IFxMnmiIP759/67iQbU4kdmkyU=
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.51.4 h1:w4Tdy9sQlJdcF5dZ9H5uRxradA9Mi2Hp4eOHQmxUJhA=
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.51.4/go.mod h1:6klY3glv/b/phmA0CUj38SWNBior8rKtVvAJrAXljis=
github.com/aws/aws-sdk-go-v2/service/ebs v1.27.10 h1:NhD9+pA7Lk6hK0UHh1F4LC8yYoGS8OOOi/X4qgWFhek=
github.com/aws/aws-sdk-go-v2/service/ebs v1.27.10/go.mod h1:Zn1yBXTLyeapYa3U9IU86oPhQkiTZoWD7qBCgOxajw8=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.3 h1:h5UPeMBMm29Vjk45QVnH2Qu2QMbzRrWUORwyGjzWQso=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.3/go.mod h1:WAFpTnWeO2BNfwpQ8LTTTx9l9/bTztMPrA8gkh41PvI=
github.com/aws/aws-sdk-go-v2/service/iam v1.38.4 h1:440YtmP8Cn6Qp7WHYfvz2/Xzmu1v1Vox/FJnzUDDQGM=
github.com/aws/aws-sdk-go-v2/service/iam v1.38.4/go.mod h1:oXqc4hmGhZpj06Zu8z+ahXhdbjq4Uw8pjN9flty0Ync=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 h1:iwYS40JnrBeA9e9aI5S6KKN4EB2zR4iUVYN0nwVivz4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8/go.mod h1:Fm9Mi+ApqmFiknZtGpohVcBGvpTu542VC4XO9YudRi0=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 h1:cWno7lefSH6Pp+mSznagKCgfDGeZRin66UvYUqAkyeA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8/go.mod h1:tPD+VjU3ABTBoEJ3nctu5Nyg4P4yjqSH5bJGGkY4+XE=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 h1:/Mn7gTedG86nbpjT4QEKsN1D/fThiYe1qvq7WsBGNHg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8/go.mod h1:Ae3va9LPmvjj231ukHB6UeT8nS7wTPfC3tMZSZMwNYg=
github.com/aws/aws-sdk-go-v2/service/route53 v1.47.1 h1:UpJqR435MxGZGRqIo4YZATcjC5OvQUYZy1gtU9Ee55o=
github.com/aws/aws-sdk-go-v2/service/route53 v1.47.1/go.mod h1:eI5iH9B3C6Ooj+PosK7FALYCZOGDVHyPEyX1gya5R04=
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 h1:a7aQ3RW+ug4IbhoQp29NZdc7vqrzKZZfWZSaQAXOZvQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 h1:YqtxripbjWb2QLyzRK9pByfEDvgg95gpC2AyDq4hFE8=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.9/go.mod h1:lV8iQpg6OLOfBnqbGMBKYjilBlf633qwHnBEiMSPoHY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 h1:6dBT1Lz8fK11m22R+AqfRsFn8320K0T5DTGxxOQBSMw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8/go.mod h1:/kiBvRQXBc6xeJTYzhSdGvJ5vm1tjaDEjH+MSeRJnlY=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 h1:VwhTrsTuVn52an4mXx29PqRzs2Dvu921NpGk7y43tAM=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.6/go.mod h1:+8h7PZb3yY5ftmVLD7ocEoE98hdc8PoKS0H3wfx1dlc=
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
Expand Down
96 changes: 30 additions & 66 deletions provider/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@
package aws

import (
"errors"
"context"
"fmt"
"strings"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ebs"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/route53"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/autoscaling"
"github.com/aws/aws-sdk-go-v2/service/ebs"
"github.com/aws/aws-sdk-go-v2/service/ec2"
awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/aws/aws-sdk-go-v2/service/iam"
"github.com/aws/aws-sdk-go-v2/service/route53"
"github.com/nanovms/ops/lepton"
"github.com/nanovms/ops/types"
)
Expand All @@ -22,16 +23,19 @@ const ProviderName = "aws"

// AWS Provider to interact with AWS cloud infrastructure
type AWS struct {
execCtx context.Context
Storage *S3
dnsService *route53.Route53
volumeService *ebs.EBS
session *session.Session
ec2 *ec2.EC2
dnsService *route53.Client
volumeService *ebs.Client
ec2 *ec2.Client
asg *autoscaling.Client
iam *iam.Client
}

// NewProvider AWS
func NewProvider() *AWS {
return &AWS{}
execCtx := context.Background()
return &AWS{execCtx: execCtx}
}

// strips any zone qualifier from 'zone' string
Expand All @@ -42,61 +46,21 @@ func stripZone(zone string) string {
return strings.TrimRight(zone, "abc")
}

func loadAWSCreds() (err error) {
foundValidCredentials := false

fileCreds := credentials.NewSharedCredentials("", "")

_, err = fileCreds.Get()
if err == nil {
foundValidCredentials = true
}

envCreds := credentials.NewEnvCredentials()

_, err = envCreds.Get()
if err == nil {
foundValidCredentials = true
}

if foundValidCredentials {
err = nil
}

return
}

// Initialize AWS related things
func (p *AWS) Initialize(config *types.ProviderConfig) error {
p.Storage = &S3{}

if config.Zone == "" {
return errors.New("zone missing")
}

err := loadAWSCreds()
awsSdkConfig, err := GetAwsSdkConfig(p.execCtx, &config.Zone)
if err != nil {
return err
}
p.dnsService = route53.NewFromConfig(*awsSdkConfig)
p.ec2 = ec2.NewFromConfig(*awsSdkConfig)
p.volumeService = ebs.NewFromConfig(*awsSdkConfig)
p.asg = autoscaling.NewFromConfig(*awsSdkConfig)
p.iam = iam.NewFromConfig(*awsSdkConfig)

session, err := session.NewSession(
&aws.Config{
Region: aws.String(stripZone(config.Zone)),
},
)
if err != nil {
return err
}

p.session = session
p.dnsService = route53.New(session)
p.ec2 = ec2.New(session)
p.volumeService = ebs.New(session,
aws.NewConfig().
WithRegion(stripZone(config.Zone)).
WithMaxRetries(7))

_, err = p.ec2.DescribeRegions(&ec2.DescribeRegionsInput{RegionNames: aws.StringSlice([]string{stripZone(config.Zone)})})
_, err = p.ec2.DescribeRegions(p.execCtx, &ec2.DescribeRegionsInput{RegionNames: []string{stripZone(config.Zone)}})
if err != nil {
return fmt.Errorf("region with name %v is invalid", config.Zone)
}
Expand All @@ -105,38 +69,38 @@ func (p *AWS) Initialize(config *types.ProviderConfig) error {
}

// buildAwsTags converts configuration tags to AWS tags and returns the resource name. The defaultName is overridden if there is a tag with key name
func buildAwsTags(configTags []types.Tag, defaultName string) ([]*ec2.Tag, string) {
tags := []*ec2.Tag{}
func buildAwsTags(configTags []types.Tag, defaultName string) ([]awsEc2Types.Tag, string) {
tags := []awsEc2Types.Tag{}
var nameSpecified bool
name := defaultName

for _, tag := range configTags {
tags = append(tags, &ec2.Tag{Key: aws.String(tag.Key), Value: aws.String(tag.Value)})
tags = append(tags, awsEc2Types.Tag{Key: aws.String(tag.Key), Value: aws.String(tag.Value)})
if tag.Key == "Name" {
nameSpecified = true
name = tag.Value
}
}

if !nameSpecified {
tags = append(tags, &ec2.Tag{
tags = append(tags, awsEc2Types.Tag{
Key: aws.String("Name"),
Value: aws.String(name),
})
}

tags = append(tags, &ec2.Tag{
tags = append(tags, awsEc2Types.Tag{
Key: aws.String("CreatedBy"),
Value: aws.String("ops"),
})

return tags, name
}

func (p *AWS) getNameTag(tags []*ec2.Tag) *ec2.Tag {
func (p *AWS) getNameTag(tags []awsEc2Types.Tag) *awsEc2Types.Tag {
for _, tag := range tags {
if *tag.Key == "Name" {
return tag
return &tag
}
}

Expand Down
38 changes: 20 additions & 18 deletions provider/aws/aws_dns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,20 @@ package aws

import (
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/route53"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/route53"
awsRoute53Types "github.com/aws/aws-sdk-go-v2/service/route53/types"
"github.com/nanovms/ops/lepton"
"github.com/nanovms/ops/types"
)

// FindOrCreateZoneIDByName searches for a DNS zone with the name passed by argument and if it doesn't exist it creates one
func (p *AWS) FindOrCreateZoneIDByName(config *types.Config, dnsName string) (string, error) {
var zoneID string
hostedZones, err := p.dnsService.ListHostedZonesByName(&route53.ListHostedZonesByNameInput{DNSName: &dnsName})
hostedZones, err := p.dnsService.ListHostedZonesByName(p.execCtx, &route53.ListHostedZonesByNameInput{DNSName: &dnsName})
if err == nil && hostedZones.HostedZones == nil {
reference := strconv.Itoa(int(time.Now().Unix()))

Expand All @@ -24,7 +26,7 @@ func (p *AWS) FindOrCreateZoneIDByName(config *types.Config, dnsName string) (st
Name: &dnsName,
}

hostedZone, err := p.dnsService.CreateHostedZone(createHostedZoneInput)
hostedZone, err := p.dnsService.CreateHostedZone(p.execCtx, createHostedZoneInput)
if err != nil {
return "", err
}
Expand All @@ -41,26 +43,26 @@ func (p *AWS) FindOrCreateZoneIDByName(config *types.Config, dnsName string) (st

// DeleteZoneRecordIfExists deletes a record from a DNS zone if it exists
func (p *AWS) DeleteZoneRecordIfExists(config *types.Config, zoneID string, recordName string) error {
records, err := p.dnsService.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{HostedZoneId: &zoneID})
records, err := p.dnsService.ListResourceRecordSets(p.execCtx, &route53.ListResourceRecordSetsInput{HostedZoneId: &zoneID})
if err != nil {
return err
}

for _, record := range records.ResourceRecordSets {
if *record.Name == recordName && *record.Type == "A" {
if *record.Name == recordName && record.Type == "A" {
input := &route53.ChangeResourceRecordSetsInput{
ChangeBatch: &route53.ChangeBatch{
Changes: []*route53.Change{
ChangeBatch: &awsRoute53Types.ChangeBatch{
Changes: []awsRoute53Types.Change{
{
Action: aws.String("DELETE"),
ResourceRecordSet: record,
Action: awsRoute53Types.ChangeActionDelete,
ResourceRecordSet: &record,
},
},
},
HostedZoneId: aws.String(zoneID),
}

_, err = p.dnsService.ChangeResourceRecordSets(input)
_, err = p.dnsService.ChangeResourceRecordSets(p.execCtx, input)
if err != nil {
return err
}
Expand All @@ -73,27 +75,27 @@ func (p *AWS) DeleteZoneRecordIfExists(config *types.Config, zoneID string, reco
// CreateZoneRecord creates a record in a DNS zone
func (p *AWS) CreateZoneRecord(config *types.Config, zoneID string, record *lepton.DNSRecord) error {
input := &route53.ChangeResourceRecordSetsInput{
ChangeBatch: &route53.ChangeBatch{
Changes: []*route53.Change{
ChangeBatch: &awsRoute53Types.ChangeBatch{
Changes: []awsRoute53Types.Change{
{
Action: aws.String("CREATE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Action: awsRoute53Types.ChangeActionCreate,
ResourceRecordSet: &awsRoute53Types.ResourceRecordSet{
Name: aws.String(record.Name),
ResourceRecords: []*route53.ResourceRecord{
ResourceRecords: []awsRoute53Types.ResourceRecord{
{
Value: aws.String(record.IP),
},
},
TTL: aws.Int64(int64(record.TTL)),
Type: aws.String(record.Type),
Type: awsRoute53Types.RRType(strings.ToUpper(record.Type)),
},
},
},
},
HostedZoneId: aws.String(zoneID),
}

_, err := p.dnsService.ChangeResourceRecordSets(input)
_, err := p.dnsService.ChangeResourceRecordSets(p.execCtx, input)
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit 914c9b2

Please sign in to comment.