diff --git a/go.mod b/go.mod index e3adfe82..a1a4d8ad 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/Azure/go-autorest/autorest/azure/auth v0.5.3 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/UpCloudLtd/upcloud-go-api/v6 v6.5.0 - github.com/aws/aws-sdk-go v1.55.2 github.com/bramvdbogaerde/go-scp v1.0.0 github.com/digitalocean/godo v1.57.0 github.com/distribution/reference v0.6.0 @@ -70,6 +69,30 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.8 // direct + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.28.10 // direct + github.com/aws/aws-sdk-go-v2/credentials v1.17.51 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 // direct + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 // indirect + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.51.4 // direct + github.com/aws/aws-sdk-go-v2/service/ebs v1.27.10 // direct + github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.3 // direct + github.com/aws/aws-sdk-go-v2/service/iam v1.38.4 // direct + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 // indirect + github.com/aws/aws-sdk-go-v2/service/route53 v1.47.1 // direct + github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 // direct + github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 // indirect + github.com/aws/smithy-go v1.22.1 // direct github.com/blang/semver/v4 v4.0.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index efeb9a71..88ec3a79 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,54 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.55.2 h1:/2OFM8uFfK9e+cqHTw9YPrvTzIXT2XkFGXRM7WbJb7E= -github.com/aws/aws-sdk-go v1.55.2/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.32.8 h1:cZV+NUS/eGxKXMtmyhtYPJ7Z4YLoI/V8bkTdRZfYhGo= +github.com/aws/aws-sdk-go-v2 v1.32.8/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.10 h1:fKODZHfqQu06pCzR69KJ3GuttraRJkhlC8g80RZ0Dfg= +github.com/aws/aws-sdk-go-v2/config v1.28.10/go.mod h1:PvdxRYZ5Um9QMq9PQ0zHHNdtKK+he2NHtFCUFMXWXeg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.51 h1:F/9Sm6Y6k4LqDesZDPJCLxQGXNNHd/ZtJiWd0lCZKRk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.51/go.mod h1:TKbzCHm43AoPyA+iLGGcruXd4AFhF8tOmLex2R9jWNQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 h1:IBAoD/1d8A8/1aA8g4MBVtTRHhXRiNAgwdbo/xRM2DI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23/go.mod h1:vfENuCM7dofkgKpYzuzf1VT1UKkA/YL3qanfBn7HCaA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 h1:XnXVe2zRyPf0+fAW5L05esmngvBpC6DQZK7oZB/z/Co= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48/go.mod h1:S3wey90OrS4f7kYxH6PT175YyEcHTORY07++HurMaRM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 h1:jSJjSBzw8VDIbWv+mmvBSP8ezsztMYJGH+eKqi9AmNs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27/go.mod h1:/DAhLbFRgwhmvJdOfSm+WwikZrCuUJiA4WgJG0fTNSw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 h1:l+X4K77Dui85pIj5foXDhPlnqcNRG2QUyvca300lXh8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27/go.mod h1:KvZXSFEXm6x84yE8qffKvT3x8J5clWnVFXphpohhzJ8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 h1:AmB5QxnD+fBFrg9LcqzkgF/CaYvMyU/BTlejG4t1S7Q= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27/go.mod h1:Sai7P3xTiyv9ZUYO3IFxMnmiIP759/67iQbU4kdmkyU= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.51.4 h1:w4Tdy9sQlJdcF5dZ9H5uRxradA9Mi2Hp4eOHQmxUJhA= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.51.4/go.mod h1:6klY3glv/b/phmA0CUj38SWNBior8rKtVvAJrAXljis= +github.com/aws/aws-sdk-go-v2/service/ebs v1.27.10 h1:NhD9+pA7Lk6hK0UHh1F4LC8yYoGS8OOOi/X4qgWFhek= +github.com/aws/aws-sdk-go-v2/service/ebs v1.27.10/go.mod h1:Zn1yBXTLyeapYa3U9IU86oPhQkiTZoWD7qBCgOxajw8= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.3 h1:h5UPeMBMm29Vjk45QVnH2Qu2QMbzRrWUORwyGjzWQso= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.198.3/go.mod h1:WAFpTnWeO2BNfwpQ8LTTTx9l9/bTztMPrA8gkh41PvI= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.4 h1:440YtmP8Cn6Qp7WHYfvz2/Xzmu1v1Vox/FJnzUDDQGM= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.4/go.mod h1:oXqc4hmGhZpj06Zu8z+ahXhdbjq4Uw8pjN9flty0Ync= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 h1:iwYS40JnrBeA9e9aI5S6KKN4EB2zR4iUVYN0nwVivz4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8/go.mod h1:Fm9Mi+ApqmFiknZtGpohVcBGvpTu542VC4XO9YudRi0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 h1:cWno7lefSH6Pp+mSznagKCgfDGeZRin66UvYUqAkyeA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8/go.mod h1:tPD+VjU3ABTBoEJ3nctu5Nyg4P4yjqSH5bJGGkY4+XE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 h1:/Mn7gTedG86nbpjT4QEKsN1D/fThiYe1qvq7WsBGNHg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8/go.mod h1:Ae3va9LPmvjj231ukHB6UeT8nS7wTPfC3tMZSZMwNYg= +github.com/aws/aws-sdk-go-v2/service/route53 v1.47.1 h1:UpJqR435MxGZGRqIo4YZATcjC5OvQUYZy1gtU9Ee55o= +github.com/aws/aws-sdk-go-v2/service/route53 v1.47.1/go.mod h1:eI5iH9B3C6Ooj+PosK7FALYCZOGDVHyPEyX1gya5R04= +github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 h1:a7aQ3RW+ug4IbhoQp29NZdc7vqrzKZZfWZSaQAXOZvQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 h1:YqtxripbjWb2QLyzRK9pByfEDvgg95gpC2AyDq4hFE8= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.9/go.mod h1:lV8iQpg6OLOfBnqbGMBKYjilBlf633qwHnBEiMSPoHY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 h1:6dBT1Lz8fK11m22R+AqfRsFn8320K0T5DTGxxOQBSMw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8/go.mod h1:/kiBvRQXBc6xeJTYzhSdGvJ5vm1tjaDEjH+MSeRJnlY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 h1:VwhTrsTuVn52an4mXx29PqRzs2Dvu921NpGk7y43tAM= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.6/go.mod h1:+8h7PZb3yY5ftmVLD7ocEoE98hdc8PoKS0H3wfx1dlc= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= diff --git a/provider/aws/aws.go b/provider/aws/aws.go index 3383813d..34c3df6b 100644 --- a/provider/aws/aws.go +++ b/provider/aws/aws.go @@ -3,16 +3,17 @@ package aws import ( - "errors" + "context" "fmt" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ebs" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/route53" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + "github.com/aws/aws-sdk-go-v2/service/ebs" + "github.com/aws/aws-sdk-go-v2/service/ec2" + awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/types" ) @@ -22,16 +23,19 @@ const ProviderName = "aws" // AWS Provider to interact with AWS cloud infrastructure type AWS struct { + execCtx context.Context Storage *S3 - dnsService *route53.Route53 - volumeService *ebs.EBS - session *session.Session - ec2 *ec2.EC2 + dnsService *route53.Client + volumeService *ebs.Client + ec2 *ec2.Client + asg *autoscaling.Client + iam *iam.Client } // NewProvider AWS func NewProvider() *AWS { - return &AWS{} + execCtx := context.Background() + return &AWS{execCtx: execCtx} } // strips any zone qualifier from 'zone' string @@ -42,61 +46,21 @@ func stripZone(zone string) string { return strings.TrimRight(zone, "abc") } -func loadAWSCreds() (err error) { - foundValidCredentials := false - - fileCreds := credentials.NewSharedCredentials("", "") - - _, err = fileCreds.Get() - if err == nil { - foundValidCredentials = true - } - - envCreds := credentials.NewEnvCredentials() - - _, err = envCreds.Get() - if err == nil { - foundValidCredentials = true - } - - if foundValidCredentials { - err = nil - } - - return -} - // Initialize AWS related things func (p *AWS) Initialize(config *types.ProviderConfig) error { p.Storage = &S3{} - if config.Zone == "" { - return errors.New("zone missing") - } - - err := loadAWSCreds() + awsSdkConfig, err := GetAwsSdkConfig(p.execCtx, &config.Zone) if err != nil { return err } + p.dnsService = route53.NewFromConfig(*awsSdkConfig) + p.ec2 = ec2.NewFromConfig(*awsSdkConfig) + p.volumeService = ebs.NewFromConfig(*awsSdkConfig) + p.asg = autoscaling.NewFromConfig(*awsSdkConfig) + p.iam = iam.NewFromConfig(*awsSdkConfig) - session, err := session.NewSession( - &aws.Config{ - Region: aws.String(stripZone(config.Zone)), - }, - ) - if err != nil { - return err - } - - p.session = session - p.dnsService = route53.New(session) - p.ec2 = ec2.New(session) - p.volumeService = ebs.New(session, - aws.NewConfig(). - WithRegion(stripZone(config.Zone)). - WithMaxRetries(7)) - - _, err = p.ec2.DescribeRegions(&ec2.DescribeRegionsInput{RegionNames: aws.StringSlice([]string{stripZone(config.Zone)})}) + _, err = p.ec2.DescribeRegions(p.execCtx, &ec2.DescribeRegionsInput{RegionNames: []string{stripZone(config.Zone)}}) if err != nil { return fmt.Errorf("region with name %v is invalid", config.Zone) } @@ -105,13 +69,13 @@ func (p *AWS) Initialize(config *types.ProviderConfig) error { } // buildAwsTags converts configuration tags to AWS tags and returns the resource name. The defaultName is overridden if there is a tag with key name -func buildAwsTags(configTags []types.Tag, defaultName string) ([]*ec2.Tag, string) { - tags := []*ec2.Tag{} +func buildAwsTags(configTags []types.Tag, defaultName string) ([]awsEc2Types.Tag, string) { + tags := []awsEc2Types.Tag{} var nameSpecified bool name := defaultName for _, tag := range configTags { - tags = append(tags, &ec2.Tag{Key: aws.String(tag.Key), Value: aws.String(tag.Value)}) + tags = append(tags, awsEc2Types.Tag{Key: aws.String(tag.Key), Value: aws.String(tag.Value)}) if tag.Key == "Name" { nameSpecified = true name = tag.Value @@ -119,13 +83,13 @@ func buildAwsTags(configTags []types.Tag, defaultName string) ([]*ec2.Tag, strin } if !nameSpecified { - tags = append(tags, &ec2.Tag{ + tags = append(tags, awsEc2Types.Tag{ Key: aws.String("Name"), Value: aws.String(name), }) } - tags = append(tags, &ec2.Tag{ + tags = append(tags, awsEc2Types.Tag{ Key: aws.String("CreatedBy"), Value: aws.String("ops"), }) @@ -133,10 +97,10 @@ func buildAwsTags(configTags []types.Tag, defaultName string) ([]*ec2.Tag, strin return tags, name } -func (p *AWS) getNameTag(tags []*ec2.Tag) *ec2.Tag { +func (p *AWS) getNameTag(tags []awsEc2Types.Tag) *awsEc2Types.Tag { for _, tag := range tags { if *tag.Key == "Name" { - return tag + return &tag } } diff --git a/provider/aws/aws_dns.go b/provider/aws/aws_dns.go index 62155df7..66157957 100644 --- a/provider/aws/aws_dns.go +++ b/provider/aws/aws_dns.go @@ -4,10 +4,12 @@ package aws import ( "strconv" + "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/route53" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/route53" + awsRoute53Types "github.com/aws/aws-sdk-go-v2/service/route53/types" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/types" ) @@ -15,7 +17,7 @@ import ( // FindOrCreateZoneIDByName searches for a DNS zone with the name passed by argument and if it doesn't exist it creates one func (p *AWS) FindOrCreateZoneIDByName(config *types.Config, dnsName string) (string, error) { var zoneID string - hostedZones, err := p.dnsService.ListHostedZonesByName(&route53.ListHostedZonesByNameInput{DNSName: &dnsName}) + hostedZones, err := p.dnsService.ListHostedZonesByName(p.execCtx, &route53.ListHostedZonesByNameInput{DNSName: &dnsName}) if err == nil && hostedZones.HostedZones == nil { reference := strconv.Itoa(int(time.Now().Unix())) @@ -24,7 +26,7 @@ func (p *AWS) FindOrCreateZoneIDByName(config *types.Config, dnsName string) (st Name: &dnsName, } - hostedZone, err := p.dnsService.CreateHostedZone(createHostedZoneInput) + hostedZone, err := p.dnsService.CreateHostedZone(p.execCtx, createHostedZoneInput) if err != nil { return "", err } @@ -41,26 +43,26 @@ func (p *AWS) FindOrCreateZoneIDByName(config *types.Config, dnsName string) (st // DeleteZoneRecordIfExists deletes a record from a DNS zone if it exists func (p *AWS) DeleteZoneRecordIfExists(config *types.Config, zoneID string, recordName string) error { - records, err := p.dnsService.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{HostedZoneId: &zoneID}) + records, err := p.dnsService.ListResourceRecordSets(p.execCtx, &route53.ListResourceRecordSetsInput{HostedZoneId: &zoneID}) if err != nil { return err } for _, record := range records.ResourceRecordSets { - if *record.Name == recordName && *record.Type == "A" { + if *record.Name == recordName && record.Type == "A" { input := &route53.ChangeResourceRecordSetsInput{ - ChangeBatch: &route53.ChangeBatch{ - Changes: []*route53.Change{ + ChangeBatch: &awsRoute53Types.ChangeBatch{ + Changes: []awsRoute53Types.Change{ { - Action: aws.String("DELETE"), - ResourceRecordSet: record, + Action: awsRoute53Types.ChangeActionDelete, + ResourceRecordSet: &record, }, }, }, HostedZoneId: aws.String(zoneID), } - _, err = p.dnsService.ChangeResourceRecordSets(input) + _, err = p.dnsService.ChangeResourceRecordSets(p.execCtx, input) if err != nil { return err } @@ -73,19 +75,19 @@ func (p *AWS) DeleteZoneRecordIfExists(config *types.Config, zoneID string, reco // CreateZoneRecord creates a record in a DNS zone func (p *AWS) CreateZoneRecord(config *types.Config, zoneID string, record *lepton.DNSRecord) error { input := &route53.ChangeResourceRecordSetsInput{ - ChangeBatch: &route53.ChangeBatch{ - Changes: []*route53.Change{ + ChangeBatch: &awsRoute53Types.ChangeBatch{ + Changes: []awsRoute53Types.Change{ { - Action: aws.String("CREATE"), - ResourceRecordSet: &route53.ResourceRecordSet{ + Action: awsRoute53Types.ChangeActionCreate, + ResourceRecordSet: &awsRoute53Types.ResourceRecordSet{ Name: aws.String(record.Name), - ResourceRecords: []*route53.ResourceRecord{ + ResourceRecords: []awsRoute53Types.ResourceRecord{ { Value: aws.String(record.IP), }, }, TTL: aws.Int64(int64(record.TTL)), - Type: aws.String(record.Type), + Type: awsRoute53Types.RRType(strings.ToUpper(record.Type)), }, }, }, @@ -93,7 +95,7 @@ func (p *AWS) CreateZoneRecord(config *types.Config, zoneID string, record *lept HostedZoneId: aws.String(zoneID), } - _, err := p.dnsService.ChangeResourceRecordSets(input) + _, err := p.dnsService.ChangeResourceRecordSets(p.execCtx, input) if err != nil { return err } diff --git a/provider/aws/aws_image.go b/provider/aws/aws_image.go index 9001c7db..534c5652 100644 --- a/provider/aws/aws_image.go +++ b/provider/aws/aws_image.go @@ -4,6 +4,7 @@ package aws import ( "bytes" + "context" "crypto/sha256" b64 "encoding/base64" "encoding/json" @@ -16,12 +17,12 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ebs" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ebs" + awsEbsTypes "github.com/aws/aws-sdk-go-v2/service/ebs/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + smithy "github.com/aws/smithy-go" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/log" "github.com/nanovms/ops/types" @@ -115,7 +116,7 @@ func (p *AWS) CreateImage(ctx *lepton.Context, imagePath string) error { key := c.CloudConfig.ImageName ctx.Logger().Info("Creating snapshot") - snapshotID, err := p.createSnapshot(imagePath, c.CloudConfig.KMS) + snapshotID, err := p.createSnapshot(&c.CloudConfig.Zone, imagePath, c.CloudConfig.KMS) if err != nil { return err } @@ -124,8 +125,8 @@ func (p *AWS) CreateImage(ctx *lepton.Context, imagePath string) error { tags, _ := buildAwsTags(c.CloudConfig.Tags, key) ctx.Logger().Info("Tagging snapshot") - _, err = p.ec2.CreateTags(&ec2.CreateTagsInput{ - Resources: aws.StringSlice([]string{snapshotID}), + _, err = p.ec2.CreateTags(p.execCtx, &ec2.CreateTagsInput{ + Resources: []string{snapshotID}, Tags: tags, }) if err != nil { @@ -140,14 +141,14 @@ func (p *AWS) CreateImage(ctx *lepton.Context, imagePath string) error { // register ami rinput := &ec2.RegisterImageInput{ Name: aws.String(amiName), - Architecture: aws.String(getArchitecture(c.CloudConfig.Flavor)), - BlockDeviceMappings: []*ec2.BlockDeviceMapping{ + Architecture: awsEc2Types.ArchitectureValues(getArchitecture(c.CloudConfig.Flavor)), + BlockDeviceMappings: []awsEc2Types.BlockDeviceMapping{ { DeviceName: aws.String("/dev/sda1"), - Ebs: &ec2.EbsBlockDevice{ + Ebs: &awsEc2Types.EbsBlockDevice{ DeleteOnTermination: aws.Bool(true), SnapshotId: aws.String(snapshotID), - VolumeType: aws.String("gp2"), + VolumeType: awsEc2Types.VolumeTypeGp2, }, }, }, @@ -158,15 +159,15 @@ func (p *AWS) CreateImage(ctx *lepton.Context, imagePath string) error { } ctx.Logger().Info("Registering image") - resreg, err := p.ec2.RegisterImage(rinput) + resreg, err := p.ec2.RegisterImage(p.execCtx, rinput) if err != nil { return err } // Add name tag to the created ami ctx.Logger().Info("Tagging image") - _, err = p.ec2.CreateTags(&ec2.CreateTagsInput{ - Resources: []*string{resreg.ImageId}, + _, err = p.ec2.CreateTags(p.execCtx, &ec2.CreateTagsInput{ + Resources: []string{aws.ToString(resreg.ImageId)}, Tags: tags, }) if err != nil { @@ -178,18 +179,7 @@ func (p *AWS) CreateImage(ctx *lepton.Context, imagePath string) error { // MirrorImage copies an image using its imageName from one region to another func (p *AWS) MirrorImage(ctx *lepton.Context, imageName, srcRegion, dstRegion string) (string, error) { - srcSession, err := session.NewSession( - &aws.Config{ - Region: aws.String(stripZone(srcRegion)), - }, - ) - if err != nil { - return "", err - } - - srcEc2 := ec2.New(srcSession) - - i, err := p.findImageByNameUsingSession(srcEc2, imageName) + i, err := p.findImageByNameUsingSession(p.ec2, imageName) if i == nil { return "", fmt.Errorf("no image with name %s found", imageName) } @@ -197,18 +187,7 @@ func (p *AWS) MirrorImage(ctx *lepton.Context, imageName, srcRegion, dstRegion s return "", fmt.Errorf("error while search for image: %s", err.Error()) } - dstSession, err := session.NewSession( - &aws.Config{ - Region: aws.String(stripZone(dstRegion)), - }, - ) - if err != nil { - return "", err - } - - dstEc2 := ec2.New(dstSession) - - output, err := dstEc2.CopyImage(&ec2.CopyImageInput{ + output, err := p.ec2.CopyImage(p.execCtx, &ec2.CopyImageInput{ Name: aws.String(imageName), SourceImageId: i.ImageId, SourceRegion: &srcRegion, @@ -219,8 +198,8 @@ func (p *AWS) MirrorImage(ctx *lepton.Context, imageName, srcRegion, dstRegion s tags, _ := buildAwsTags(ctx.Config().CloudConfig.Tags, imageName) - dstEc2.CreateTags(&ec2.CreateTagsInput{ - Resources: []*string{output.ImageId}, + _, err = p.ec2.CreateTags(p.execCtx, &ec2.CreateTagsInput{ + Resources: []string{aws.ToString(output.ImageId)}, Tags: tags, }) @@ -232,7 +211,7 @@ func (p *AWS) MirrorImage(ctx *lepton.Context, imageName, srcRegion, dstRegion s // createSnapshot process create Snapshot to EBS // Returns snapshotID and err -func (p *AWS) createSnapshot(imagePath string, kms string) (string, error) { +func (p *AWS) createSnapshot(zone *string, imagePath string, kms string) (string, error) { // Open file first f, err := os.Open(imagePath) if err != nil { @@ -256,7 +235,7 @@ func (p *AWS) createSnapshot(imagePath string, kms string) (string, error) { bar := progressbar.Default(maxBar) esi := &ebs.StartSnapshotInput{ - Tags: []*ebs.Tag{}, + Tags: []awsEbsTypes.Tag{}, VolumeSize: aws.Int64(sizeInGb), } @@ -268,7 +247,7 @@ func (p *AWS) createSnapshot(imagePath string, kms string) (string, error) { } } - snapshotOutput, err := p.volumeService.StartSnapshot(esi) + snapshotOutput, err := p.volumeService.StartSnapshot(p.execCtx, esi) if err != nil { return "", err } @@ -341,11 +320,11 @@ func (p *AWS) createSnapshot(imagePath string, kms string) (string, error) { h.Write(snapshotBlocksChecksums) snapshotChecksum := b64.StdEncoding.EncodeToString(h.Sum(nil)) - if _, err := p.volumeService.CompleteSnapshot(&ebs.CompleteSnapshotInput{ - ChangedBlocksCount: &blockIndex, + if _, err := p.volumeService.CompleteSnapshot(p.execCtx, &ebs.CompleteSnapshotInput{ + ChangedBlocksCount: aws.Int32(int32(blockIndex)), Checksum: aws.String(snapshotChecksum), - ChecksumAggregationMethod: aws.String("LINEAR"), - ChecksumAlgorithm: aws.String("SHA256"), + ChecksumAggregationMethod: awsEbsTypes.ChecksumAggregationMethodChecksumAggregationLinear, + ChecksumAlgorithm: awsEbsTypes.ChecksumAlgorithmChecksumAlgorithmSha256, SnapshotId: aws.String(snapshotID), }); err != nil { return snapshotID, err @@ -353,8 +332,8 @@ func (p *AWS) createSnapshot(imagePath string, kms string) (string, error) { bar.Add64(1) - if err := p.ec2.WaitUntilSnapshotCompleted(&ec2.DescribeSnapshotsInput{ - SnapshotIds: aws.StringSlice([]string{*snapshotOutput.SnapshotId}), + if err := WaitUntilEc2SnapshotCompleted(p.execCtx, zone, &ec2.DescribeSnapshotsInput{ + SnapshotIds: []string{*snapshotOutput.SnapshotId}, }); err != nil { return snapshotID, err } @@ -376,7 +355,7 @@ func (p *AWS) retryPutSnapshotBlocks(bar *progressbar.ProgressBar, f *os.File, s input, _ := buildSnapshotBlockInput(snapshotID, data.BlockIndex, block) log.Debug("RetryPutSnapshotBlock", data.BlockIndex, "PreviousErr", data.Error) - if _, err := p.volumeService.PutSnapshotBlock(input); err != nil { + if _, err := p.volumeService.PutSnapshotBlock(p.execCtx, input); err != nil { errs = append(errs, err) } @@ -392,10 +371,10 @@ func (p *AWS) retryPutSnapshotBlocks(bar *progressbar.ProgressBar, f *os.File, s func (p *AWS) writeToBlock(input *ebs.PutSnapshotBlockInput, wg *sync.WaitGroup, chanBlockResult chan PutSnapshotBlockResult) { defer wg.Done() - _, err := p.volumeService.PutSnapshotBlock(input) + _, err := p.volumeService.PutSnapshotBlock(p.execCtx, input) chanBlockResult <- PutSnapshotBlockResult{ Error: err, - BlockIndex: *input.BlockIndex, + BlockIndex: int64(*input.BlockIndex), } } @@ -406,11 +385,11 @@ func buildSnapshotBlockInput(snapshotID string, blockIndex int64, block []byte) checksum := b64.StdEncoding.EncodeToString(h.Sum(nil)) return &ebs.PutSnapshotBlockInput{ - BlockData: aws.ReadSeekCloser(bytes.NewReader(block)), - BlockIndex: aws.Int64(blockIndex), + BlockData: bytes.NewReader(block), + BlockIndex: aws.Int32(int32(blockIndex)), Checksum: aws.String(checksum), - ChecksumAlgorithm: aws.String("SHA256"), - DataLength: aws.Int64(SnapshotBlockDataLength), + ChecksumAlgorithm: awsEbsTypes.ChecksumAlgorithmChecksumAlgorithmSha256, + DataLength: aws.Int32(SnapshotBlockDataLength), SnapshotId: aws.String(snapshotID), }, h.Sum(nil) } @@ -516,20 +495,18 @@ func getArchitecture(flavor string) string { return "x86_64" } -func getAWSImages(ec2Service *ec2.EC2) (*ec2.DescribeImagesOutput, error) { - filters := []*ec2.Filter{{Name: aws.String("tag:CreatedBy"), Values: aws.StringSlice([]string{"ops"})}} +func getAWSImages(execCtx context.Context, ec2Service *ec2.Client) (*ec2.DescribeImagesOutput, error) { + filters := []awsEc2Types.Filter{{Name: aws.String("tag:CreatedBy"), Values: []string{"ops"}}} input := &ec2.DescribeImagesInput{ - Owners: []*string{ - aws.String("self"), - }, + Owners: []string{"self"}, Filters: filters, } - result, err := ec2Service.DescribeImages(input) + result, err := ec2Service.DescribeImages(execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: return nil, errors.New(aerr.Error()) } @@ -545,7 +522,7 @@ func getAWSImages(ec2Service *ec2.EC2) (*ec2.DescribeImagesOutput, error) { func (p *AWS) GetImages(ctx *lepton.Context, filter string) ([]lepton.CloudImage, error) { var cimages []lepton.CloudImage - result, err := getAWSImages(p.ec2) + result, err := getAWSImages(p.execCtx, p.ec2) if err != nil { return nil, err } @@ -555,7 +532,7 @@ func (p *AWS) GetImages(ctx *lepton.Context, filter string) ([]lepton.CloudImage tagName := p.getNameTag(image.Tags) if tagName == nil { - tagName = &ec2.Tag{Value: aws.String("n/a")} + tagName = &awsEc2Types.Tag{Value: aws.String("n/a")} } labels := []string{} @@ -569,7 +546,7 @@ func (p *AWS) GetImages(ctx *lepton.Context, filter string) ([]lepton.CloudImage Tag: *tagName.Value, Name: *image.Name, ID: *image.ImageId, - Status: *image.State, + Status: string(image.State), Labels: labels, Created: imageCreatedAt, } @@ -636,8 +613,8 @@ func (p *AWS) DeleteImage(ctx *lepton.Context, imagename string) error { return fmt.Errorf("error running deregister image operation: %s", err) } - amiID := aws.StringValue(image.ImageId) - snapID := aws.StringValue(image.BlockDeviceMappings[0].Ebs.SnapshotId) + amiID := aws.ToString(image.ImageId) + snapID := aws.ToString(image.BlockDeviceMappings[0].Ebs.SnapshotId) // grab snapshotid && grab image id @@ -645,7 +622,7 @@ func (p *AWS) DeleteImage(ctx *lepton.Context, imagename string) error { ImageId: aws.String(amiID), DryRun: aws.Bool(false), } - _, err = p.ec2.DeregisterImage(params) + _, err = p.ec2.DeregisterImage(p.execCtx, params) if err != nil { return fmt.Errorf("error running deregister image operation: %s", err) } @@ -655,7 +632,7 @@ func (p *AWS) DeleteImage(ctx *lepton.Context, imagename string) error { SnapshotId: aws.String(snapID), DryRun: aws.Bool(false), } - _, err = p.ec2.DeleteSnapshot(params2) + _, err = p.ec2.DeleteSnapshot(p.execCtx, params2) if err != nil { return fmt.Errorf("error running snapshot delete: %s", err) } @@ -663,24 +640,24 @@ func (p *AWS) DeleteImage(ctx *lepton.Context, imagename string) error { return nil } -func (p *AWS) findImageByName(name string) (*ec2.Image, error) { +func (p *AWS) findImageByName(name string) (*awsEc2Types.Image, error) { return p.findImageByNameUsingSession(p.ec2, name) } -func (p *AWS) findImageByNameUsingSession(ec2Session *ec2.EC2, name string) (*ec2.Image, error) { - ec2Filters := []*ec2.Filter{ - {Name: aws.String("tag:Name"), Values: []*string{&name}}, - {Name: aws.String("tag:CreatedBy"), Values: []*string{aws.String("ops")}}, +func (p *AWS) findImageByNameUsingSession(ec2Session *ec2.Client, name string) (*awsEc2Types.Image, error) { + ec2Filters := []awsEc2Types.Filter{ + {Name: aws.String("tag:Name"), Values: []string{name}}, + {Name: aws.String("tag:CreatedBy"), Values: []string{"ops"}}, } input := &ec2.DescribeImagesInput{ Filters: ec2Filters, } - result, err := ec2Session.DescribeImages(input) + result, err := ec2Session.DescribeImages(p.execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: log.Error(aerr) } @@ -694,7 +671,7 @@ func (p *AWS) findImageByNameUsingSession(ec2Session *ec2.EC2, name string) (*ec return nil, fmt.Errorf("image %v not found", name) } - return result.Images[0], nil + return &result.Images[0], nil } // SyncImage syncs image from provider to another provider @@ -717,10 +694,10 @@ func (p *AWS) getArchiveName(ctx *lepton.Context) string { func (p *AWS) waitSnapshotToBeReady(config *types.Config, importTaskID *string) (*string, error) { taskFilter := &ec2.DescribeImportSnapshotTasksInput{ - ImportTaskIds: []*string{importTaskID}, + ImportTaskIds: []string{aws.ToString(importTaskID)}, } - _, err := p.ec2.DescribeImportSnapshotTasks(taskFilter) + _, err := p.ec2.DescribeImportSnapshotTasks(p.execCtx, taskFilter) if err != nil { return nil, err } @@ -731,52 +708,7 @@ func (p *AWS) waitSnapshotToBeReady(config *types.Config, importTaskID *string) bar := progressbar.New(100) bar.RenderBlank() - ct := aws.BackgroundContext() - w := request.Waiter{ - Name: "DescribeImportSnapshotTasks", - Delay: request.ConstantWaiterDelay(15 * time.Second), - MaxAttempts: 120, - Acceptors: []request.WaiterAcceptor{ - { - State: request.SuccessWaiterState, - Matcher: request.PathAllWaiterMatch, - Argument: "ImportSnapshotTasks[].SnapshotTaskDetail.Status", - Expected: "completed", - }, - { - State: request.FailureWaiterState, - Matcher: request.PathAnyWaiterMatch, - Argument: "ImportSnapshotTasks[].SnapshotTaskDetail.Status", - Expected: "deleted", - }, - { - State: request.FailureWaiterState, - Matcher: request.PathAnyWaiterMatch, - Argument: "ImportSnapshotTasks[].SnapshotTaskDetail.Status", - Expected: "deleting", - }, - }, - NewRequest: func(opts []request.Option) (*request.Request, error) { - // update progress bar - snapshotTasksOutput, err := p.ec2.DescribeImportSnapshotTasks(taskFilter) - if err == nil && len(snapshotTasksOutput.ImportSnapshotTasks) > 0 { - snapshotProgress := (*snapshotTasksOutput.ImportSnapshotTasks[0]).SnapshotTaskDetail.Progress - - if snapshotProgress != nil { - progress, _ := strconv.Atoi(*snapshotProgress) - bar.Set(progress) - bar.RenderBlank() - } - } - - req, _ := p.ec2.DescribeImportSnapshotTasksRequest(taskFilter) - req.SetContext(ct) - req.ApplyOptions(opts...) - return req, nil - }, - } - - err = w.WaitWithContext(ct) + err = WaitUntilEc2SnapshotCompleted(p.execCtx, &config.CloudConfig.Zone, &ec2.DescribeSnapshotsInput{Filters: taskFilter.Filters}) bar.Set(100) bar.Finish() @@ -789,7 +721,7 @@ func (p *AWS) waitSnapshotToBeReady(config *types.Config, importTaskID *string) fmt.Printf("\nimport done - took %f minutes\n", time.Since(waitStartTime).Minutes()) - describeOutput, err := p.ec2.DescribeImportSnapshotTasks(taskFilter) + describeOutput, err := p.ec2.DescribeImportSnapshotTasks(p.execCtx, taskFilter) if err != nil { return nil, err } diff --git a/provider/aws/aws_instance.go b/provider/aws/aws_instance.go index ebb95bfe..34b17e1f 100644 --- a/provider/aws/aws_instance.go +++ b/provider/aws/aws_instance.go @@ -3,6 +3,7 @@ package aws import ( + "context" "encoding/base64" "encoding/json" "errors" @@ -11,61 +12,53 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + smithy "github.com/aws/smithy-go" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/log" "github.com/olekukonko/tablewriter" ) -func formalizeAWSInstance(instance *ec2.Instance) *lepton.CloudInstance { +func formalizeAWSInstance(instance *awsEc2Types.Instance) *lepton.CloudInstance { imageName := "unknown" instanceName := "unknown" for x := 0; x < len(instance.Tags); x++ { - if aws.StringValue(instance.Tags[x].Key) == "Name" { - instanceName = aws.StringValue(instance.Tags[x].Value) - } else if aws.StringValue(instance.Tags[x].Key) == "image" { - imageName = aws.StringValue(instance.Tags[x].Value) + if aws.ToString(instance.Tags[x].Key) == "Name" { + instanceName = aws.ToString(instance.Tags[x].Value) + } else if aws.ToString(instance.Tags[x].Key) == "image" { + imageName = aws.ToString(instance.Tags[x].Value) } } var privateIps, publicIps []string for _, ninterface := range instance.NetworkInterfaces { - privateIps = append(privateIps, aws.StringValue(ninterface.PrivateIpAddress)) + privateIps = append(privateIps, aws.ToString(ninterface.PrivateIpAddress)) if ninterface.Association != nil && ninterface.Association.PublicIp != nil { - publicIps = append(publicIps, aws.StringValue(ninterface.Association.PublicIp)) + publicIps = append(publicIps, aws.ToString(ninterface.Association.PublicIp)) } } return &lepton.CloudInstance{ - ID: aws.StringValue(instance.InstanceId), + ID: aws.ToString(instance.InstanceId), Name: instanceName, - Status: aws.StringValue(instance.State.Name), - Created: aws.TimeValue(instance.LaunchTime).String(), + Status: string(instance.State.Name), + Created: aws.ToTime(instance.LaunchTime).String(), PublicIps: publicIps, PrivateIps: privateIps, Image: imageName, } } -func getAWSInstances(region string, filter []*ec2.Filter) []lepton.CloudInstance { - svc, err := session.NewSession(&aws.Config{ - Region: aws.String(stripZone(region))}, - ) - if err != nil { - log.Fatalf("failed creation session: %s", err.Error()) - } - compute := ec2.New(svc) - - filter = append(filter, &ec2.Filter{Name: aws.String("tag:CreatedBy"), Values: aws.StringSlice([]string{"ops"})}) +func getAWSInstances(execCtx context.Context, ec2Client *ec2.Client, region string, filter []awsEc2Types.Filter) ([]lepton.CloudInstance, error) { + filter = append(filter, awsEc2Types.Filter{Name: aws.String("tag:CreatedBy"), Values: []string{"ops"}}) request := ec2.DescribeInstancesInput{ Filters: filter, } - result, err := compute.DescribeInstances(&request) + result, err := ec2Client.DescribeInstances(execCtx, &request, func(opts *ec2.Options) { opts.Region = region }) if err != nil { log.Fatalf("failed getting instances: ", err.Error()) } @@ -77,12 +70,12 @@ func getAWSInstances(region string, filter []*ec2.Filter) []lepton.CloudInstance for i := 0; i < len(reservation.Instances); i++ { instance := reservation.Instances[i] - cinstances = append(cinstances, *formalizeAWSInstance(instance)) + cinstances = append(cinstances, *formalizeAWSInstance(&instance)) } } - return cinstances + return cinstances, nil } // RebootInstance reboots the instance. @@ -103,21 +96,21 @@ func (p *AWS) StartInstance(ctx *lepton.Context, instanceName string) error { } input := &ec2.StartInstancesInput{ - InstanceIds: []*string{ - aws.String(*instance.InstanceId), + InstanceIds: []string{ + aws.ToString(instance.InstanceId), }, } - result, err := p.ec2.StartInstances(input) + result, err := p.ec2.StartInstances(p.execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: - return errors.New(aerr.Message()) + return errors.New(aerr.ErrorMessage()) } } else { - return errors.New(aerr.Message()) + return errors.New(aerr.ErrorMessage()) } } @@ -141,21 +134,19 @@ func (p *AWS) StopInstance(ctx *lepton.Context, instanceName string) error { } input := &ec2.StopInstancesInput{ - InstanceIds: []*string{ - aws.String(*instance.InstanceId), - }, + InstanceIds: []string{aws.ToString(instance.InstanceId)}, } - result, err := p.ec2.StopInstances(input) + result, err := p.ec2.StopInstances(p.execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: - return errors.New(aerr.Message()) + return errors.New(aerr.ErrorMessage()) } } else { - return errors.New(aerr.Message()) + return errors.New(aerr.ErrorMessage()) } } @@ -170,7 +161,7 @@ func (p *AWS) StopInstance(ctx *lepton.Context, instanceName string) error { // CreateInstance - Creates instance on AWS Platform func (p *AWS) CreateInstance(ctx *lepton.Context) error { ctx.Logger().Debug("getting aws images") - result, err := getAWSImages(p.ec2) + result, err := getAWSImages(p.execCtx, p.ec2) if err != nil { ctx.Logger().Errorf("failed getting images") return err @@ -181,7 +172,7 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { ami := "" var last time.Time layout := "2006-01-02T15:04:05.000Z" - var image *ec2.Image + var image *awsEc2Types.Image rv := ctx.Config().CloudConfig.RootVolume @@ -191,7 +182,7 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { if result.Images[i].Tags != nil { for _, tag := range result.Images[i].Tags { if *tag.Key == "Name" && *tag.Value == imgName { - image = result.Images[i] + image = &result.Images[i] snapID = *(result.Images[i].BlockDeviceMappings[0].Ebs.SnapshotId) break } @@ -203,9 +194,9 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { return fmt.Errorf("can't find ami with name %s", imgName) } - ami = aws.StringValue(image.ImageId) + ami = aws.ToString(image.ImageId) - ntime := aws.StringValue(image.CreationDate) + ntime := aws.ToString(image.CreationDate) t, err := time.Parse(layout, ntime) if err != nil { return err @@ -222,45 +213,45 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { // create security group - could take a potential 'RemotePort' from // config.json in future ctx.Logger().Debug("getting vpc") - vpc, err := p.GetVPC(ctx, svc) + vpc, err := p.GetVPC(p.execCtx, ctx, svc) if err != nil { return err } if vpc == nil { ctx.Logger().Debugf("creating vpc with name %s", cloudConfig.VPC) - vpc, err = p.CreateVPC(ctx, svc) + vpc, err = p.CreateVPC(p.execCtx, ctx, svc) if err != nil { return err } } - var sg *ec2.SecurityGroup + var sg *awsEc2Types.SecurityGroup if cloudConfig.SecurityGroup != "" && cloudConfig.VPC != "" { ctx.Logger().Debugf("getting security group with name %s", cloudConfig.SecurityGroup) - sg, err = p.GetSecurityGroup(ctx, svc, vpc) + sg, err = p.GetSecurityGroup(p.execCtx, ctx, svc, vpc) if err != nil { return err } } else { iname := ctx.Config().RunConfig.InstanceName ctx.Logger().Debugf("creating new security group in vpc %s", *vpc.VpcId) - sg, err = p.CreateSG(ctx, svc, iname, *vpc.VpcId) + sg, err = p.CreateSG(p.execCtx, ctx, svc, iname, *vpc.VpcId) if err != nil { return err } } ctx.Logger().Debug("getting subnet") - var subnet *ec2.Subnet - subnet, err = p.GetSubnet(ctx, svc, *vpc.VpcId) + var subnet *awsEc2Types.Subnet + subnet, err = p.GetSubnet(p.execCtx, ctx, svc, *vpc.VpcId) if err != nil { return err } if subnet == nil { - subnet, err = p.CreateSubnet(ctx, vpc) + subnet, err = p.CreateSubnet(p.execCtx, ctx, vpc) if err != nil { return err } @@ -272,35 +263,35 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { // Create tags to assign to the instance tags, tagInstanceName := buildAwsTags(cloudConfig.Tags, ctx.Config().RunConfig.InstanceName) - tags = append(tags, &ec2.Tag{Key: aws.String("image"), Value: &imgName}) + tags = append(tags, awsEc2Types.Tag{Key: aws.String("image"), Value: &imgName}) - instanceNIS := &ec2.InstanceNetworkInterfaceSpecification{ + instanceNIS := &awsEc2Types.InstanceNetworkInterfaceSpecification{ DeleteOnTermination: aws.Bool(true), - DeviceIndex: aws.Int64(0), - Groups: []*string{ - aws.String(*sg.GroupId), + DeviceIndex: aws.Int32(0), + Groups: []string{ + aws.ToString(sg.GroupId), }, SubnetId: aws.String(*subnet.SubnetId), } instanceInput := &ec2.RunInstancesInput{ ImageId: aws.String(ami), - InstanceType: aws.String(cloudConfig.Flavor), - MinCount: aws.Int64(1), - MaxCount: aws.Int64(1), - TagSpecifications: []*ec2.TagSpecification{ - {ResourceType: aws.String("instance"), Tags: tags}, - {ResourceType: aws.String("volume"), Tags: tags}, + InstanceType: awsEc2Types.InstanceType(cloudConfig.Flavor), + MinCount: aws.Int32(1), + MaxCount: aws.Int32(1), + TagSpecifications: []awsEc2Types.TagSpecification{ + {ResourceType: awsEc2Types.ResourceType("instance"), Tags: tags}, + {ResourceType: awsEc2Types.ResourceType("volume"), Tags: tags}, }, } if rv.IsCustom() { ctx.Logger().Debug("setting custom root settings") - ebs := &ec2.EbsBlockDevice{} + ebs := &awsEc2Types.EbsBlockDevice{} ebs.SnapshotId = aws.String(snapID) if rv.Typeof != "" { - ebs.VolumeType = aws.String(rv.Typeof) + ebs.VolumeType = awsEc2Types.VolumeType(rv.Typeof) } if rv.Iops != 0 { @@ -309,7 +300,7 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { os.Exit(1) } - ebs.Iops = aws.Int64(rv.Iops) + ebs.Iops = aws.Int32(int32(rv.Iops)) } if rv.Throughput != 0 { @@ -318,14 +309,14 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { os.Exit(1) } - ebs.Throughput = aws.Int64(rv.Throughput) + ebs.Throughput = aws.Int32(int32(rv.Throughput)) } if rv.Size != 0 { - ebs.VolumeSize = aws.Int64(rv.Size) + ebs.VolumeSize = aws.Int32(int32(rv.Size)) } - instanceInput.BlockDeviceMappings = []*ec2.BlockDeviceMapping{ + instanceInput.BlockDeviceMappings = []awsEc2Types.BlockDeviceMapping{ { DeviceName: aws.String("/dev/sda1"), Ebs: ebs, @@ -334,7 +325,7 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { } if ctx.Config().CloudConfig.DedicatedHostID != "" { - instanceInput.Placement = &ec2.Placement{ + instanceInput.Placement = &awsEc2Types.Placement{ HostId: aws.String(ctx.Config().CloudConfig.DedicatedHostID), } } @@ -346,19 +337,19 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { if ctx.Config().CloudConfig.EnableIPv6 { if ctx.Config().RunConfig.IPv6Address != "" { v6ad := ctx.Config().RunConfig.IPv6Address - addie := &ec2.InstanceIpv6Address{ + addie := &awsEc2Types.InstanceIpv6Address{ Ipv6Address: aws.String(v6ad), } - instanceNIS.Ipv6Addresses = []*ec2.InstanceIpv6Address{addie} + instanceNIS.Ipv6Addresses = []awsEc2Types.InstanceIpv6Address{*addie} } else { - instanceNIS.SetIpv6AddressCount(1) + instanceNIS.Ipv6AddressCount = aws.Int32(1) } } - instanceInput.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{instanceNIS} + instanceInput.NetworkInterfaces = []awsEc2Types.InstanceNetworkInterfaceSpecification{*instanceNIS} if cloudConfig.InstanceProfile != "" { - instanceInput.IamInstanceProfile = &ec2.IamInstanceProfileSpecification{ + instanceInput.IamInstanceProfile = &awsEc2Types.IamInstanceProfileSpecification{ Name: aws.String(cloudConfig.InstanceProfile), } } @@ -377,7 +368,7 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { return err } - instanceInput.LaunchTemplate = &ec2.LaunchTemplateSpecification{ + instanceInput.LaunchTemplate = &awsEc2Types.LaunchTemplateSpecification{ LaunchTemplateName: aws.String(ltInput.LaunchTemplateName), Version: aws.String("$Default"), } @@ -385,7 +376,7 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { // Specify the details of the instance that you want to create. ctx.Logger().Debugf("running instance with input %v", instanceInput) - _, err = svc.RunInstances(instanceInput) + _, err = svc.RunInstances(p.execCtx, instanceInput) if err != nil { log.Errorf("Could not create instance %v", err) return err @@ -414,9 +405,9 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { InstanceId: aws.String(instance.ID), PublicIp: aws.String(cloudConfig.StaticIP), } - result, err := svc.AssociateAddress(input) + result, err := svc.AssociateAddress(p.execCtx, input) if err != nil { - log.Errorf("Could not associate elastic IP: %v", err.(awserr.Error).Error()) + log.Errorf("Could not associate elastic IP: %v", err.(smithy.APIError).Error()) } else { log.Debugf("result: %v", result) } @@ -457,17 +448,21 @@ func (p *AWS) CreateInstance(ctx *lepton.Context) error { // GetInstances return all instances on AWS func (p *AWS) GetInstances(ctx *lepton.Context) ([]lepton.CloudInstance, error) { - cinstances := getAWSInstances(ctx.Config().CloudConfig.Zone, nil) - + cinstances, err := getAWSInstances(p.execCtx, p.ec2, ctx.Config().CloudConfig.Zone, nil) + if err != nil { + return nil, err + } return cinstances, nil } // GetInstanceByName returns instance with given name func (p *AWS) GetInstanceByName(ctx *lepton.Context, name string) (*lepton.CloudInstance, error) { - var filters []*ec2.Filter - filters = append(filters, &ec2.Filter{Name: aws.String("tag:Name"), Values: aws.StringSlice([]string{name})}) - instances := getAWSInstances(ctx.Config().CloudConfig.Zone, filters) - if len(instances) == 0 { + var filters []awsEc2Types.Filter + filters = append(filters, awsEc2Types.Filter{Name: aws.String("tag:Name"), Values: []string{name}}) + instances, err := getAWSInstances(p.execCtx, p.ec2, ctx.Config().CloudConfig.Zone, filters) + if err != nil { + return nil, err + } else if len(instances) == 0 { return nil, lepton.ErrInstanceNotFound(name) } return &instances[0], nil @@ -533,8 +528,8 @@ func (p *AWS) DeleteInstance(ctx *lepton.Context, instanceName string) error { } input := &ec2.TerminateInstancesInput{ - InstanceIds: []*string{ - aws.String(*instance.InstanceId), + InstanceIds: []string{ + aws.ToString(instance.InstanceId), }, } @@ -543,10 +538,10 @@ func (p *AWS) DeleteInstance(ctx *lepton.Context, instanceName string) error { return err } - _, err = p.ec2.TerminateInstances(input) + _, err = p.ec2.TerminateInstances(p.execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: log.Error(aerr) } @@ -560,17 +555,17 @@ func (p *AWS) DeleteInstance(ctx *lepton.Context, instanceName string) error { fmt.Println("waiting for sg to be removed") i2 := &ec2.DescribeInstancesInput{ - InstanceIds: []*string{ - aws.String(*instance.InstanceId), + InstanceIds: []string{ + aws.ToString(instance.InstanceId), }, } - err = p.ec2.WaitUntilInstanceTerminated(i2) + _, err = WaitUntilEc2InstanceTerminated(p.execCtx, p.ec2, i2) if err != nil { fmt.Println(err) } - p.DeleteSG(sg.GroupId) + p.DeleteSG(p.execCtx, sg.GroupId) } return nil @@ -608,10 +603,10 @@ func (p *AWS) GetInstanceLogs(ctx *lepton.Context, instanceName string) (string, InstanceId: aws.String(*instance.InstanceId), } - result, err := p.ec2.GetConsoleOutput(input) + result, err := p.ec2.GetConsoleOutput(p.execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: log.Error(aerr) } @@ -621,7 +616,7 @@ func (p *AWS) GetInstanceLogs(ctx *lepton.Context, instanceName string) (string, return "", err } - data, err := base64.StdEncoding.DecodeString(aws.StringValue(result.Output)) + data, err := base64.StdEncoding.DecodeString(aws.ToString(result.Output)) if err != nil { return "", err } @@ -631,17 +626,17 @@ func (p *AWS) GetInstanceLogs(ctx *lepton.Context, instanceName string) (string, return l, nil } -func (p *AWS) findInstanceByName(name string) (*ec2.Instance, error) { - filter := []*ec2.Filter{ - {Name: aws.String("tag:CreatedBy"), Values: aws.StringSlice([]string{"ops"})}, - {Name: aws.String("tag:Name"), Values: aws.StringSlice([]string{name})}, - {Name: aws.String("instance-state-name"), Values: aws.StringSlice([]string{"running", "pending", "shutting-down", "stopping", "stopped"})}, +func (p *AWS) findInstanceByName(name string) (*awsEc2Types.Instance, error) { + filter := []awsEc2Types.Filter{ + {Name: aws.String("tag:CreatedBy"), Values: []string{"ops"}}, + {Name: aws.String("tag:Name"), Values: []string{name}}, + {Name: aws.String("instance-state-name"), Values: []string{"running", "pending", "shutting-down", "stopping", "stopped"}}, } request := ec2.DescribeInstancesInput{ Filters: filter, } - result, err := p.ec2.DescribeInstances(&request) + result, err := p.ec2.DescribeInstances(p.execCtx, &request) if err != nil { return nil, fmt.Errorf("failed getting instances: %v", err) } @@ -650,25 +645,25 @@ func (p *AWS) findInstanceByName(name string) (*ec2.Instance, error) { return nil, fmt.Errorf("instance with name %s not found", name) } - return result.Reservations[0].Instances[0], nil + return &result.Reservations[0].Instances[0], nil } // bit of a hack here // can convert to explicit tag // currently only returns sgs created by ops -func (p *AWS) findSGByName(name string) (*ec2.SecurityGroup, error) { - filter := []*ec2.Filter{ - {Name: aws.String("tag:ops-created"), Values: aws.StringSlice([]string{"true"})}, - {Name: aws.String("description"), Values: aws.StringSlice([]string{"security group for " + name})}, +func (p *AWS) findSGByName(name string) (*awsEc2Types.SecurityGroup, error) { + filter := []awsEc2Types.Filter{ + {Name: aws.String("tag:ops-created"), Values: []string{"true"}}, + {Name: aws.String("description"), Values: []string{"security group for " + name}}, } request := ec2.DescribeSecurityGroupsInput{ Filters: filter, } - result, err := p.ec2.DescribeSecurityGroups(&request) + result, err := p.ec2.DescribeSecurityGroups(p.execCtx, &request) if err != nil { return nil, fmt.Errorf("failed getting security group: %v", err) } - return result.SecurityGroups[0], nil + return &result.SecurityGroups[0], nil } diff --git a/provider/aws/aws_instance_group.go b/provider/aws/aws_instance_group.go index ad631f90..69a13528 100644 --- a/provider/aws/aws_instance_group.go +++ b/provider/aws/aws_instance_group.go @@ -5,9 +5,11 @@ package aws import ( "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/autoscaling" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + awsAutoscalingTypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/nanovms/ops/log" ) @@ -15,21 +17,21 @@ import ( type LaunchTemplateInput struct { AutoScalingGroup string ImageID string - InstanceNetworkInterface *ec2.InstanceNetworkInterfaceSpecification + InstanceNetworkInterface *awsEc2Types.InstanceNetworkInterfaceSpecification InstanceProfileName string InstanceType string LaunchTemplateName string - Tags []*ec2.Tag + Tags []awsEc2Types.Tag } // launchTemplateInstanceNetworkInterfaceSpecificationRequest // convert from InstanceNetworkInterfaceSpecification to LaunchTemplateInstanceNetworkInterfaceSpecificationRequest -func (lti LaunchTemplateInput) launchTemplateInstanceNetworkInterfaceSpecificationRequest() *ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest { +func (lti LaunchTemplateInput) launchTemplateInstanceNetworkInterfaceSpecificationRequest() *awsEc2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest { if lti.InstanceNetworkInterface == nil { return nil } - req := &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + req := &awsEc2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ DeleteOnTermination: lti.InstanceNetworkInterface.DeleteOnTermination, DeviceIndex: lti.InstanceNetworkInterface.DeviceIndex, Groups: lti.InstanceNetworkInterface.Groups, @@ -38,7 +40,7 @@ func (lti LaunchTemplateInput) launchTemplateInstanceNetworkInterfaceSpecificati } for _, item := range lti.InstanceNetworkInterface.Ipv6Addresses { - req.Ipv6Addresses = append(req.Ipv6Addresses, &ec2.InstanceIpv6AddressRequest{ + req.Ipv6Addresses = append(req.Ipv6Addresses, awsEc2Types.InstanceIpv6AddressRequest{ Ipv6Address: item.Ipv6Address, }) } @@ -68,27 +70,27 @@ func (p *AWS) autoScalingLaunchTemplate(req *LaunchTemplateInput) error { // and call CreateLaunchTemplate API for Amazon Elastic Compute Cloud. func (p *AWS) createLaunchTemplate(req *LaunchTemplateInput) (*ec2.CreateLaunchTemplateOutput, error) { input := &ec2.CreateLaunchTemplateInput{ - LaunchTemplateData: &ec2.RequestLaunchTemplateData{ - BlockDeviceMappings: []*ec2.LaunchTemplateBlockDeviceMappingRequest{ + LaunchTemplateData: &awsEc2Types.RequestLaunchTemplateData{ + BlockDeviceMappings: []awsEc2Types.LaunchTemplateBlockDeviceMappingRequest{ { DeviceName: aws.String("/dev/sda1"), - Ebs: &ec2.LaunchTemplateEbsBlockDeviceRequest{ + Ebs: &awsEc2Types.LaunchTemplateEbsBlockDeviceRequest{ DeleteOnTermination: aws.Bool(true), - VolumeType: aws.String("gp2"), + VolumeType: awsEc2Types.VolumeTypeGp2, }, }, }, ImageId: aws.String(req.ImageID), - InstanceType: aws.String(req.InstanceType), - Monitoring: &ec2.LaunchTemplatesMonitoringRequest{ + InstanceType: awsEc2Types.InstanceType(req.InstanceType), + Monitoring: &awsEc2Types.LaunchTemplatesMonitoringRequest{ Enabled: aws.Bool(true), }, - NetworkInterfaces: []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ - req.launchTemplateInstanceNetworkInterfaceSpecificationRequest(), + NetworkInterfaces: []awsEc2Types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + *req.launchTemplateInstanceNetworkInterfaceSpecificationRequest(), }, - TagSpecifications: []*ec2.LaunchTemplateTagSpecificationRequest{ + TagSpecifications: []awsEc2Types.LaunchTemplateTagSpecificationRequest{ { - ResourceType: aws.String("instance"), + ResourceType: awsEc2Types.ResourceTypeInstance, Tags: req.Tags, }, }, @@ -98,12 +100,12 @@ func (p *AWS) createLaunchTemplate(req *LaunchTemplateInput) (*ec2.CreateLaunchT } if req.InstanceProfileName != "" { - input.LaunchTemplateData.IamInstanceProfile = &ec2.LaunchTemplateIamInstanceProfileSpecificationRequest{ + input.LaunchTemplateData.IamInstanceProfile = &awsEc2Types.LaunchTemplateIamInstanceProfileSpecificationRequest{ Name: aws.String(req.InstanceProfileName), } } - return p.ec2.CreateLaunchTemplate(input) + return p.ec2.CreateLaunchTemplate(p.execCtx, input) } // modifyLaunchTemplate from one version to DefaultVersion @@ -112,24 +114,22 @@ func (p *AWS) modifyLaunchTemplate(version string, ltName string) error { DefaultVersion: aws.String(version), LaunchTemplateName: aws.String(ltName), } - _, err := p.ec2.ModifyLaunchTemplate(params) + _, err := p.ec2.ModifyLaunchTemplate(p.execCtx, params) return err } // updateAutoScalingGroup // build UpdateAutoScalingGroupInput and Updates the configuration for the specified Auto Scaling group. func (p *AWS) updateAutoScalingGroup(asgName string, ltName string) error { - asgCli := autoscaling.New(p.session) - params := &autoscaling.UpdateAutoScalingGroupInput{ AutoScalingGroupName: aws.String(asgName), - LaunchTemplate: &autoscaling.LaunchTemplateSpecification{ + LaunchTemplate: &awsAutoscalingTypes.LaunchTemplateSpecification{ LaunchTemplateName: aws.String(ltName), Version: aws.String("$Default"), }, } - if _, err := asgCli.UpdateAutoScalingGroup(params); err != nil { + if _, err := p.asg.UpdateAutoScalingGroup(p.execCtx, params); err != nil { return err } diff --git a/provider/aws/aws_network.go b/provider/aws/aws_network.go index 97535c5d..373a4665 100644 --- a/provider/aws/aws_network.go +++ b/provider/aws/aws_network.go @@ -3,6 +3,7 @@ package aws import ( + "context" "errors" "fmt" "net" @@ -12,40 +13,39 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + smithy "github.com/aws/smithy-go" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/network" "github.com/nanovms/ops/types" ) // GetSecurityGroup checks whether the configuration security group exists and has the configuration VPC assigned -func (p *AWS) GetSecurityGroup(ctx *lepton.Context, svc *ec2.EC2, vpc *ec2.Vpc) (sg *ec2.SecurityGroup, err error) { +func (p *AWS) GetSecurityGroup(execCtx context.Context, ctx *lepton.Context, svc *ec2.Client, vpc *awsEc2Types.Vpc) (sg *awsEc2Types.SecurityGroup, err error) { sgName := ctx.Config().CloudConfig.SecurityGroup input := &ec2.DescribeSecurityGroupsInput{ - Filters: []*ec2.Filter{ + Filters: []awsEc2Types.Filter{ { Name: aws.String("group-name"), - Values: aws.StringSlice([]string{sgName}), + Values: aws.ToStringSlice([]*string{aws.String(sgName)}), }, }, } var result *ec2.DescribeSecurityGroupsOutput - result, err = svc.DescribeSecurityGroups(input) + result, err = svc.DescribeSecurityGroups(execCtx, input) if err != nil { return } else if len(result.SecurityGroups) == 0 { input := &ec2.DescribeSecurityGroupsInput{ - GroupIds: []*string{ - aws.String(sgName), - }, + GroupIds: aws.ToStringSlice([]*string{aws.String(sgName)}), } - result, err = svc.DescribeSecurityGroups(input) + result, err = svc.DescribeSecurityGroups(execCtx, input) if err != nil { err = fmt.Errorf("get security group with id '%s': %s", sg, err.Error()) return @@ -60,26 +60,26 @@ func (p *AWS) GetSecurityGroup(ctx *lepton.Context, svc *ec2.EC2, vpc *ec2.Vpc) return } - sg = result.SecurityGroups[0] + sg = &result.SecurityGroups[0] return } // GetSubnet returns a subnet with the context subnet name or the default subnet of vpc passed by argument -func (p *AWS) GetSubnet(ctx *lepton.Context, svc *ec2.EC2, vpcID string) (*ec2.Subnet, error) { +func (p *AWS) GetSubnet(execCtx context.Context, ctx *lepton.Context, svc *ec2.Client, vpcID string) (*awsEc2Types.Subnet, error) { subnetName := ctx.Config().CloudConfig.Subnet - var filters []*ec2.Filter + var filters []awsEc2Types.Filter var result *ec2.DescribeSubnetsOutput var err error - filters = append(filters, &ec2.Filter{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})}) + filters = append(filters, awsEc2Types.Filter{Name: aws.String("vpc-id"), Values: []string{vpcID}}) if subnetName != "" { subnetIDRegexp, _ := regexp.Compile("^subnet-.+") if subnetIDRegexp.Match([]byte(subnetName)) { - result, err = svc.DescribeSubnets(&ec2.DescribeSubnetsInput{ - SubnetIds: aws.StringSlice([]string{subnetName}), + result, err = svc.DescribeSubnets(execCtx, &ec2.DescribeSubnetsInput{ + SubnetIds: []string{subnetName}, Filters: filters, }) if err != nil { @@ -87,8 +87,8 @@ func (p *AWS) GetSubnet(ctx *lepton.Context, svc *ec2.EC2, vpcID string) (*ec2.S return nil, err } } else { - result, err = svc.DescribeSubnets(&ec2.DescribeSubnetsInput{ - Filters: append(filters, &ec2.Filter{Name: aws.String("tag:Name"), Values: aws.StringSlice([]string{subnetName})}), + result, err = svc.DescribeSubnets(execCtx, &ec2.DescribeSubnetsInput{ + Filters: append(filters, awsEc2Types.Filter{Name: aws.String("tag:Name"), Values: []string{subnetName}}), }) if err != nil { err = fmt.Errorf("unable to describe subnets, %v", err) @@ -97,7 +97,7 @@ func (p *AWS) GetSubnet(ctx *lepton.Context, svc *ec2.EC2, vpcID string) (*ec2.S } if len(result.Subnets) != 0 { - return result.Subnets[0], nil + return &result.Subnets[0], nil } } else { @@ -105,7 +105,7 @@ func (p *AWS) GetSubnet(ctx *lepton.Context, svc *ec2.EC2, vpcID string) (*ec2.S Filters: filters, } - result, err = svc.DescribeSubnets(input) + result, err = svc.DescribeSubnets(execCtx, input) if err != nil { err = fmt.Errorf("unable to describe subnets, %v", err) return nil, err @@ -120,23 +120,23 @@ func (p *AWS) GetSubnet(ctx *lepton.Context, svc *ec2.EC2, vpcID string) (*ec2.S for _, subnet := range result.Subnets { if *subnet.DefaultForAz { - return subnet, nil + return &subnet, nil } } - return result.Subnets[0], nil + return &result.Subnets[0], nil } // CreateSubnet creates a subnet on vpc -func (p *AWS) CreateSubnet(ctx *lepton.Context, vpc *ec2.Vpc) (subnet *ec2.Subnet, err error) { +func (p *AWS) CreateSubnet(execCtx context.Context, ctx *lepton.Context, vpc *awsEc2Types.Vpc) (subnet *awsEc2Types.Subnet, err error) { tags, _ := buildAwsTags([]types.Tag{}, ctx.Config().CloudConfig.Subnet) createSubnetInput := &ec2.CreateSubnetInput{ AvailabilityZone: aws.String(ctx.Config().CloudConfig.Zone), VpcId: vpc.VpcId, CidrBlock: vpc.CidrBlock, - TagSpecifications: []*ec2.TagSpecification{ - {Tags: tags, ResourceType: aws.String("subnet")}, + TagSpecifications: []awsEc2Types.TagSpecification{ + {Tags: tags, ResourceType: awsEc2Types.ResourceTypeSubnet}, }, } @@ -149,7 +149,7 @@ func (p *AWS) CreateSubnet(ctx *lepton.Context, vpc *ec2.Vpc) (subnet *ec2.Subne createSubnetInput.Ipv6CidrBlock = types.StringPtr(ipv6Addr.String() + "/64") } - result, err := p.ec2.CreateSubnet(createSubnetInput) + result, err := p.ec2.CreateSubnet(execCtx, createSubnetInput) if err != nil { return } @@ -160,10 +160,10 @@ func (p *AWS) CreateSubnet(ctx *lepton.Context, vpc *ec2.Vpc) (subnet *ec2.Subne } // GetVPC returns a vpc with the context vpc name or the default vpc -func (p *AWS) GetVPC(ctx *lepton.Context, svc *ec2.EC2) (*ec2.Vpc, error) { +func (p *AWS) GetVPC(execCtx context.Context, ctx *lepton.Context, svc *ec2.Client) (*awsEc2Types.Vpc, error) { vpcName := ctx.Config().CloudConfig.VPC - var vpc *ec2.Vpc + var vpc *awsEc2Types.Vpc var input *ec2.DescribeVpcsInput var result *ec2.DescribeVpcsOutput var err error @@ -175,22 +175,22 @@ func (p *AWS) GetVPC(ctx *lepton.Context, svc *ec2.EC2) (*ec2.Vpc, error) { ctx.Logger().Debugf("no vpcs with name %s found", vpcName) ctx.Logger().Debugf("getting vpcs filtered by id %s", vpcName) input = &ec2.DescribeVpcsInput{ - VpcIds: aws.StringSlice([]string{vpcName}), + VpcIds: []string{vpcName}, } - result, err = svc.DescribeVpcs(input) + result, err = svc.DescribeVpcs(execCtx, input) if err != nil { return nil, fmt.Errorf("unable to describe VPCs, %v", err) } } else { ctx.Logger().Debugf("getting vpcs filtered by name %s", vpcName) - var filters []*ec2.Filter + var filters []awsEc2Types.Filter - filters = append(filters, &ec2.Filter{Name: aws.String("tag:Name"), Values: aws.StringSlice([]string{vpcName})}) + filters = append(filters, awsEc2Types.Filter{Name: aws.String("tag:Name"), Values: []string{vpcName}}) input = &ec2.DescribeVpcsInput{ Filters: filters, } - result, err = svc.DescribeVpcs(input) + result, err = svc.DescribeVpcs(execCtx, input) if err != nil { return nil, fmt.Errorf("unable to describe VPCs, %v", err) } @@ -199,12 +199,12 @@ func (p *AWS) GetVPC(ctx *lepton.Context, svc *ec2.EC2) (*ec2.Vpc, error) { ctx.Logger().Debugf("found %d vpcs that match the criteria %s", len(result.Vpcs), vpcName) if len(result.Vpcs) != 0 { - return result.Vpcs[0], nil + return &result.Vpcs[0], nil } } else { ctx.Logger().Debug("no vpc name specified") ctx.Logger().Debug("getting all vpcs") - result, err = svc.DescribeVpcs(input) + result, err = svc.DescribeVpcs(execCtx, input) if err != nil { return nil, fmt.Errorf("unable to describe VPCs, %v", err) } @@ -214,14 +214,14 @@ func (p *AWS) GetVPC(ctx *lepton.Context, svc *ec2.EC2) (*ec2.Vpc, error) { isDefault := *s.IsDefault if isDefault { ctx.Logger().Debug("picking default vpc") - vpc = result.Vpcs[i] + vpc = &result.Vpcs[i] } } // if there is no default VPC select the first vpc of the list if vpc == nil && len(result.Vpcs) != 0 { ctx.Logger().Debug("no default vpc found") - vpc = result.Vpcs[0] + vpc = &result.Vpcs[0] ctx.Logger().Debugf("picking vpc %+v", vpc) } } @@ -229,7 +229,7 @@ func (p *AWS) GetVPC(ctx *lepton.Context, svc *ec2.EC2) (*ec2.Vpc, error) { return vpc, nil } -func (p *AWS) buildFirewallRule(protocol string, port string, ipv4, ipv6 bool) *ec2.IpPermission { +func (p *AWS) buildFirewallRule(protocol string, port string, ipv4, ipv6 bool) *awsEc2Types.IpPermission { fromPort := port toPort := port @@ -253,36 +253,36 @@ func (p *AWS) buildFirewallRule(protocol string, port string, ipv4, ipv6 bool) * os.Exit(1) } - var ec2Permission = new(ec2.IpPermission) - ec2Permission.SetIpProtocol(protocol) - ec2Permission.SetFromPort(int64(fromPortInt)) - ec2Permission.SetToPort(int64(toPortInt)) + var ec2Permission = new(awsEc2Types.IpPermission) + ec2Permission.IpProtocol = &protocol + ec2Permission.FromPort = aws.Int32(int32(fromPortInt)) + ec2Permission.ToPort = aws.Int32(int32(toPortInt)) if ipv4 { - ec2Permission.SetIpRanges([]*ec2.IpRange{ + ec2Permission.IpRanges = []awsEc2Types.IpRange{ {CidrIp: aws.String("0.0.0.0/0")}, - }) + } } if ipv6 { - ec2Permission.SetIpv6Ranges([]*ec2.Ipv6Range{ + ec2Permission.Ipv6Ranges = []awsEc2Types.Ipv6Range{ {CidrIpv6: aws.String("::/0")}, - }) + } } return ec2Permission } // DeleteSG deletes a security group -func (p *AWS) DeleteSG(groupID *string) { +func (p *AWS) DeleteSG(execCtx context.Context, groupID *string) { input := &ec2.DeleteSecurityGroupInput{ GroupId: groupID, } - _, err := p.ec2.DeleteSecurityGroup(input) + _, err := p.ec2.DeleteSecurityGroup(execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: fmt.Println(aerr.Error()) } @@ -296,27 +296,27 @@ func (p *AWS) DeleteSG(groupID *string) { // CreateSG - Create security group // instance specific -func (p *AWS) CreateSG(ctx *lepton.Context, svc *ec2.EC2, iname string, vpcID string) (sg *ec2.SecurityGroup, err error) { +func (p *AWS) CreateSG(execCtx context.Context, ctx *lepton.Context, svc *ec2.Client, iname string, vpcID string) (sg *awsEc2Types.SecurityGroup, err error) { t := time.Now().UnixNano() s := strconv.FormatInt(t, 10) sgName := iname + "-" + s // Create tags to assign to the instance - tags := []*ec2.Tag{} - tags = append(tags, &ec2.Tag{Key: aws.String("ops-created"), Value: aws.String("true")}) + tags := []awsEc2Types.Tag{} + tags = append(tags, awsEc2Types.Tag{Key: aws.String("ops-created"), Value: aws.String("true")}) - createRes, err := svc.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ + createRes, err := svc.CreateSecurityGroup(execCtx, &ec2.CreateSecurityGroupInput{ GroupName: aws.String(sgName), Description: aws.String("security group for " + iname), VpcId: aws.String(vpcID), - TagSpecifications: []*ec2.TagSpecification{ - {ResourceType: aws.String("security-group"), Tags: tags}, + TagSpecifications: []awsEc2Types.TagSpecification{ + {ResourceType: awsEc2Types.ResourceTypeSecurityGroup, Tags: tags}, }, }) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { case "InvalidVpcID.NotFound": errstr := fmt.Sprintf("Unable to find VPC with ID %q.", vpcID) err = errors.New(errstr) @@ -331,31 +331,30 @@ func (p *AWS) CreateSG(ctx *lepton.Context, svc *ec2.EC2, iname string, vpcID st err = errors.New(errstr) return } - fmt.Printf("Created security group %s with VPC %s.\n", - aws.StringValue(createRes.GroupId), vpcID) + fmt.Printf("Created security group %s with VPC %s.\n", createRes.GroupId, vpcID) - var ec2Permissions []*ec2.IpPermission + var ec2Permissions []awsEc2Types.IpPermission var ipv6 bool if ctx.Config().CloudConfig.EnableIPv6 { rule := p.buildFirewallRule("icmpv6", "-1", false, true) - ec2Permissions = append(ec2Permissions, rule) + ec2Permissions = append(ec2Permissions, *rule) ipv6 = true } for _, port := range ctx.Config().RunConfig.Ports { rule := p.buildFirewallRule("tcp", port, true, ipv6) - ec2Permissions = append(ec2Permissions, rule) + ec2Permissions = append(ec2Permissions, *rule) } for _, port := range ctx.Config().RunConfig.UDPPorts { rule := p.buildFirewallRule("udp", port, true, ipv6) - ec2Permissions = append(ec2Permissions, rule) + ec2Permissions = append(ec2Permissions, *rule) } // maybe have these ports specified from config.json in near future if len(ec2Permissions) != 0 { - _, err = svc.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ + _, err = svc.AuthorizeSecurityGroupIngress(execCtx, &ec2.AuthorizeSecurityGroupIngressInput{ GroupId: createRes.GroupId, IpPermissions: ec2Permissions, }) @@ -366,8 +365,8 @@ func (p *AWS) CreateSG(ctx *lepton.Context, svc *ec2.EC2, iname string, vpcID st } } - result, err := svc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ - GroupIds: aws.StringSlice([]string{*createRes.GroupId}), + result, err := svc.DescribeSecurityGroups(execCtx, &ec2.DescribeSecurityGroupsInput{ + GroupIds: []string{*createRes.GroupId}, }) if err != nil { return @@ -375,13 +374,13 @@ func (p *AWS) CreateSG(ctx *lepton.Context, svc *ec2.EC2, iname string, vpcID st err = errors.New("failed creating security group") } - sg = result.SecurityGroups[0] + sg = &result.SecurityGroups[0] return } // CreateVPC creates a virtual network -func (p *AWS) CreateVPC(ctx *lepton.Context, svc *ec2.EC2) (vpc *ec2.Vpc, err error) { +func (p *AWS) CreateVPC(execCtx context.Context, ctx *lepton.Context, svc *ec2.Client) (vpc *awsEc2Types.Vpc, err error) { vnetName := ctx.Config().CloudConfig.VPC if vnetName == "" { @@ -391,7 +390,7 @@ func (p *AWS) CreateVPC(ctx *lepton.Context, svc *ec2.EC2) (vpc *ec2.Vpc, err er tags, _ := buildAwsTags([]types.Tag{}, vnetName) - vpcs, err := svc.DescribeVpcs(&ec2.DescribeVpcsInput{}) + vpcs, err := svc.DescribeVpcs(execCtx, &ec2.DescribeVpcsInput{}) if err != nil { return } @@ -404,19 +403,19 @@ func (p *AWS) CreateVPC(ctx *lepton.Context, svc *ec2.EC2) (vpc *ec2.Vpc, err er createInput := &ec2.CreateVpcInput{ CidrBlock: aws.String(network.AllocateNewCidrBlock(cidrBlocks)), - TagSpecifications: []*ec2.TagSpecification{ - {Tags: tags, ResourceType: aws.String("vpc")}, + TagSpecifications: []awsEc2Types.TagSpecification{ + {Tags: tags, ResourceType: awsEc2Types.ResourceTypeVpc}, }, } if ctx.Config().CloudConfig.EnableIPv6 { - createInput.SetAmazonProvidedIpv6CidrBlock(true) + createInput.AmazonProvidedIpv6CidrBlock = aws.Bool(true) } - _, err = svc.CreateVpc(createInput) + _, err = svc.CreateVpc(execCtx, createInput) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: err = errors.New(aerr.Error()) } @@ -426,23 +425,19 @@ func (p *AWS) CreateVPC(ctx *lepton.Context, svc *ec2.EC2) (vpc *ec2.Vpc, err er return } - vpc, err = p.GetVPC(ctx, svc) + vpc, err = p.GetVPC(execCtx, ctx, svc) if err != nil { return } // Add routes to allow external traffic - var routeTable *ec2.RouteTable - rtFilters := []*ec2.Filter{{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{*vpc.VpcId})}} - err = svc.DescribeRouteTablesPages(&ec2.DescribeRouteTablesInput{Filters: rtFilters}, func(page *ec2.DescribeRouteTablesOutput, lastPage bool) bool { - if len(page.RouteTables) != 0 { - routeTable = page.RouteTables[0] - } - - return false - }) + var routeTable *awsEc2Types.RouteTable + rtFilters := []awsEc2Types.Filter{{Name: aws.String("vpc-id"), Values: []string{*vpc.VpcId}}} + describeRouteTablesOutput, err := svc.DescribeRouteTables(execCtx, &ec2.DescribeRouteTablesInput{Filters: rtFilters}) if err != nil { return + } else if len(describeRouteTablesOutput.RouteTables) != 0 { + routeTable = &describeRouteTablesOutput.RouteTables[0] } if routeTable == nil { @@ -452,34 +447,30 @@ func (p *AWS) CreateVPC(ctx *lepton.Context, svc *ec2.EC2) (vpc *ec2.Vpc, err er return } - var gw *ec2.InternetGateway - gwFilters := []*ec2.Filter{{Name: aws.String("attachment.vpc-id"), Values: aws.StringSlice([]string{*vpc.VpcId})}} - err = svc.DescribeInternetGatewaysPages(&ec2.DescribeInternetGatewaysInput{Filters: gwFilters}, func(page *ec2.DescribeInternetGatewaysOutput, lastPage bool) bool { - if len(page.InternetGateways) != 0 { - gw = page.InternetGateways[0] - } - - return false - }) + var gw *awsEc2Types.InternetGateway + gwFilters := []awsEc2Types.Filter{{Name: aws.String("attachment.vpc-id"), Values: []string{*vpc.VpcId}}} + describeGatewaysOutput, err := svc.DescribeInternetGateways(execCtx, &ec2.DescribeInternetGatewaysInput{Filters: gwFilters}) if err != nil { return + } else if len(describeGatewaysOutput.InternetGateways) != 0 { + gw = &describeGatewaysOutput.InternetGateways[0] } if gw == nil { gwInput := &ec2.CreateInternetGatewayInput{} - gwOutput, err := svc.CreateInternetGateway(gwInput) + gwOutput, err := svc.CreateInternetGateway(execCtx, gwInput) if err != nil { err = fmt.Errorf("failed creating an Internet Gateway: %v", err) return nil, err } - _, err = svc.AttachInternetGateway(&ec2.AttachInternetGatewayInput{VpcId: vpc.VpcId, InternetGatewayId: gwOutput.InternetGateway.InternetGatewayId}) + _, err = svc.AttachInternetGateway(execCtx, &ec2.AttachInternetGatewayInput{VpcId: vpc.VpcId, InternetGatewayId: gwOutput.InternetGateway.InternetGatewayId}) if err != nil { err = fmt.Errorf("failed attaching an Internet Gateway to a VPC: %v", err) return nil, err } - gw = &ec2.InternetGateway{InternetGatewayId: gwOutput.InternetGateway.InternetGatewayId} + gw = &awsEc2Types.InternetGateway{InternetGatewayId: gwOutput.InternetGateway.InternetGatewayId} } if ctx.Config().CloudConfig.EnableIPv6 { @@ -488,7 +479,7 @@ func (p *AWS) CreateVPC(ctx *lepton.Context, svc *ec2.EC2) (vpc *ec2.Vpc, err er RouteTableId: routeTable.RouteTableId, GatewayId: gw.InternetGatewayId, } - _, err = svc.CreateRoute(createRouteInput) + _, err = svc.CreateRoute(execCtx, createRouteInput) if err != nil { err = fmt.Errorf("failed creating ipv6 public route: %v", err) return @@ -500,7 +491,7 @@ func (p *AWS) CreateVPC(ctx *lepton.Context, svc *ec2.EC2) (vpc *ec2.Vpc, err er RouteTableId: routeTable.RouteTableId, GatewayId: gw.InternetGatewayId, } - _, err = svc.CreateRoute(createRouteInput) + _, err = svc.CreateRoute(execCtx, createRouteInput) if err != nil { err = fmt.Errorf("failed creating ipv4 public route: %v", err) return diff --git a/provider/aws/aws_store.go b/provider/aws/aws_store.go index 6e1853e7..a79256cb 100644 --- a/provider/aws/aws_store.go +++ b/provider/aws/aws_store.go @@ -3,15 +3,16 @@ package aws import ( + "context" "fmt" "math" "os" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/s3" + smithy "github.com/aws/smithy-go" "github.com/nanovms/ops/log" "github.com/nanovms/ops/types" ) @@ -23,13 +24,18 @@ type S3 struct{} func (s *S3) CopyToBucket(config *types.Config, archPath string) error { bucket := config.CloudConfig.BucketName - zone := config.CloudConfig.Zone + execCtx := context.Background() + awsSdkConfig, err := GetAwsSdkConfig(execCtx, &config.CloudConfig.Zone) // this verification/role creator can be skipped for users that // already have it setup but don't have rights to verify if !config.CloudConfig.SkipImportVerify { + if err != nil { + return err + } + iamClient := iam.NewFromConfig(*awsSdkConfig) // verify we can even use the vm importer - VerifyRole(zone, bucket) + VerifyRole(execCtx, iamClient, config.CloudConfig.Zone, bucket) } file, err := os.Open(archPath) @@ -38,18 +44,13 @@ func (s *S3) CopyToBucket(config *types.Config, archPath string) error { } defer file.Close() - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(stripZone(zone))}, - ) - if err != nil { - return err - } + s3Client := s3.NewFromConfig(*awsSdkConfig) fileStats, _ := file.Stat() log.Info("Uploading image with", fmt.Sprintf("%fMB", float64(fileStats.Size())/math.Pow(10, 6))) - uploader := s3manager.NewUploader(sess) - _, err = uploader.Upload(&s3manager.UploadInput{ + uploader := manager.NewUploader(s3Client) + _, err = uploader.Upload(execCtx, &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(config.CloudConfig.ImageName), Body: file, @@ -64,27 +65,24 @@ func (s *S3) CopyToBucket(config *types.Config, archPath string) error { } // DeleteFromBucket deletes key from config's bucket -func (s *S3) DeleteFromBucket(config *types.Config, key string) error { +func (s *S3) DeleteFromBucket(execCtx context.Context, config *types.Config, key string) error { bucket := config.CloudConfig.BucketName - zone := config.CloudConfig.Zone - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(stripZone(zone))}, - ) + awsSdkConfig, err := GetAwsSdkConfig(execCtx, &config.CloudConfig.Zone) if err != nil { return err } - svc := s3.New(sess) + s3Client := s3.NewFromConfig(*awsSdkConfig) input := &s3.DeleteObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), } - _, err = svc.DeleteObject(input) + _, err = s3Client.DeleteObject(execCtx, input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { + if aerr, ok := err.(smithy.APIError); ok { + switch aerr.ErrorCode() { default: log.Error(aerr) } diff --git a/provider/aws/aws_utils.go b/provider/aws/aws_utils.go new file mode 100644 index 00000000..8ec9de60 --- /dev/null +++ b/provider/aws/aws_utils.go @@ -0,0 +1,126 @@ +//go:build aws || !onlyprovider + +package aws + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + smithyTime "github.com/aws/smithy-go/time" + smithyWaiter "github.com/aws/smithy-go/waiter" +) + +// Create and return an aws-sdk-v2 configuration object +func GetAwsSdkConfig(execCtx context.Context, zone *string) (*aws.Config, error) { + awsSdkConfigOpts := []func(*awsConfig.LoadOptions) error{} + awsProfile := os.Getenv("AWS_PROFILE") + if awsProfile != "" { + awsSdkConfigOpts = append(awsSdkConfigOpts, awsConfig.WithSharedConfigProfile(awsProfile)) + } + if zone != nil { + awsSdkConfigOpts = append(awsSdkConfigOpts, awsConfig.WithRegion(*zone)) + } + awsSdkConfig, err := awsConfig.LoadDefaultConfig(execCtx, awsSdkConfigOpts...) + if err != nil { + fmt.Println("Couldn't load default configuration. Have you set up your AWS account?") + fmt.Println(err) + return nil, err + } + return &awsSdkConfig, nil +} + +// Wait until an EC2 instance is terminated. This is a blocking operation on the provided execution context. +// Ensure proper filters since this function expects a single EC2 given the input +// Derived from NewSnapshotCompletedWaiter impl. at https://github.com/aws/aws-sdk-go-v2/blob/v1.32.8/service/ec2/api_op_DescribeSnapshots.go#L348 +func WaitUntilEc2InstanceTerminated(execCtx context.Context, ec2Client *ec2.Client, describeInstancesInput *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { + minDelay := 15 * time.Second + maxDelay := minDelay + 1 + waitDur := 120 * maxDelay + ctx, cancelFn := context.WithTimeout(execCtx, waitDur) + defer cancelFn() + remainingTime := waitDur + + var attempt int64 + for { + attempt++ + start := time.Now() + + out, err := ec2Client.DescribeInstances(ctx, describeInstancesInput) + + if err != nil { + return nil, err + } else if len(out.Reservations) < 1 { + return nil, fmt.Errorf("error while waiting for an EC2 to terminate. No EC2 were found for this input, %v", describeInstancesInput) + } else if len(out.Reservations) != 1 { + return nil, fmt.Errorf("this function expects one reservation so please adjust your filter, %v", describeInstancesInput) + } else if len(out.Reservations[0].Instances) != 1 { + return nil, fmt.Errorf("this function expects one instance so please adjust your filter, %v", describeInstancesInput) + } + + instance := out.Reservations[0].Instances[0] + + if instance.State.Name == awsEc2Types.InstanceStateNameTerminated { + return out, nil + } + + remainingTime -= time.Since(start) + if remainingTime < minDelay || remainingTime <= 0 { + break + } + + // compute exponential backoff between waiter retries + delay, err := smithyWaiter.ComputeDelay( + attempt, minDelay, maxDelay+1, remainingTime, + ) + if err != nil { + return nil, fmt.Errorf("error computing a waiter delay, %w", err) + } + + remainingTime -= delay + // sleep for the delay amount before invoking a request + if err := smithyTime.SleepWithContext(ctx, delay); err != nil { + return nil, fmt.Errorf("request cancelled while waiting, %w", err) + } + } + return nil, fmt.Errorf("exceeded max wait time for an EC2 instance to terminate") +} + +// Waits for snapshot completion. This is a blocking operation on the provided execution context. +// Ensure proper filters since this function expects a single snapshot given the input +func WaitUntilEc2SnapshotCompleted(execCtx context.Context, zone *string, describeSnapshotsInput *ec2.DescribeSnapshotsInput) error { + // Retry 120 times every 15-16 seconds + maxDuration := time.Duration(16 * time.Second * 120) + snapshotsCtx, cancel := context.WithTimeout(execCtx, maxDuration) + defer cancel() + awsSdkConfig, err := GetAwsSdkConfig(execCtx, zone) + if err != nil { + return err + } + describeSnapshotsClient := ec2.NewFromConfig(*awsSdkConfig) + waiter := *ec2.NewSnapshotCompletedWaiter(describeSnapshotsClient, func(opts *ec2.SnapshotCompletedWaiterOptions) { + // Max delay of 120 seconds between each attempt is too much because we're defining a custom retry function + // This will wait 15 seconds between each attempt + opts.MaxDelay = opts.MinDelay + 1 + opts.Retryable = func(_ context.Context, _ *ec2.DescribeSnapshotsInput, output *ec2.DescribeSnapshotsOutput, err error) (bool, error) { + // Total failure, stop trying + if err != nil { + fmt.Printf("Failed to read snapshot state. error: %v", err) + return false, err + } + // Success + if output.Snapshots[0].State == awsEc2Types.SnapshotStateCompleted { + return false, nil + } + // Retry, if possible + return true, nil + } + }) + waiter.WaitForOutput(snapshotsCtx, describeSnapshotsInput, maxDuration) + return nil +} diff --git a/provider/aws/aws_volume.go b/provider/aws/aws_volume.go index 54a2f164..a50e3eb2 100644 --- a/provider/aws/aws_volume.go +++ b/provider/aws/aws_volume.go @@ -9,8 +9,9 @@ import ( "strconv" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + awsEc2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/nanovms/ops/lepton" "github.com/nanovms/ops/types" ) @@ -18,7 +19,7 @@ import ( // CreateVolume creates a snapshot and use it to create a volume func (a *AWS) CreateVolume(ctx *lepton.Context, cv types.CloudVolume, data string, provider string) (lepton.NanosVolume, error) { config := ctx.Config() - var sizeInGb int64 + var sizeInGb int32 var vol lepton.NanosVolume if config.BaseVolumeSz != "" { size, err := lepton.GetSizeInGb(config.BaseVolumeSz) @@ -26,7 +27,7 @@ func (a *AWS) CreateVolume(ctx *lepton.Context, cv types.CloudVolume, data strin return vol, fmt.Errorf("cannot get volume size: %v", err) } config.BaseVolumeSz = "" // create minimum-sized local volume - sizeInGb = int64(size) + sizeInGb = int32(size) } // Create volume @@ -48,17 +49,17 @@ func (a *AWS) CreateVolume(ctx *lepton.Context, cv types.CloudVolume, data strin input := &ec2.ImportSnapshotInput{ Description: aws.String("name"), - DiskContainer: &ec2.SnapshotDiskContainer{ + DiskContainer: &awsEc2Types.SnapshotDiskContainer{ Description: aws.String("snapshot imported"), Format: aws.String("raw"), - UserBucket: &ec2.UserBucket{ - S3Bucket: aws.String(bucket), - S3Key: aws.String(key), + UserBucket: &awsEc2Types.UserBucket{ + S3Bucket: &bucket, + S3Key: &key, }, }, } - res, err := a.ec2.ImportSnapshot(input) + res, err := a.ec2.ImportSnapshot(a.execCtx, input) if err != nil { return vol, fmt.Errorf("import snapshot: %v", err) } @@ -69,7 +70,7 @@ func (a *AWS) CreateVolume(ctx *lepton.Context, cv types.CloudVolume, data strin } // delete the tmp s3 volume - err = a.Storage.DeleteFromBucket(config, key) + err = a.Storage.DeleteFromBucket(a.execCtx, config, key) if err != nil { return vol, err } @@ -81,16 +82,16 @@ func (a *AWS) CreateVolume(ctx *lepton.Context, cv types.CloudVolume, data strin createVolumeInput := &ec2.CreateVolumeInput{ AvailabilityZone: aws.String(config.CloudConfig.Zone), SnapshotId: snapshotID, - TagSpecifications: []*ec2.TagSpecification{ + TagSpecifications: []awsEc2Types.TagSpecification{ { - ResourceType: aws.String("volume"), + ResourceType: awsEc2Types.ResourceTypeVolume, Tags: tags, }, }, } if cv.Typeof != "" { - createVolumeInput.VolumeType = aws.String(cv.Typeof) + createVolumeInput.VolumeType = awsEc2Types.VolumeType(cv.Typeof) } if cv.Iops != 0 { @@ -99,7 +100,7 @@ func (a *AWS) CreateVolume(ctx *lepton.Context, cv types.CloudVolume, data strin os.Exit(1) } - createVolumeInput.Iops = aws.Int64(cv.Iops) + createVolumeInput.Iops = aws.Int32(int32(cv.Iops)) } if cv.Throughput != 0 { @@ -108,13 +109,13 @@ func (a *AWS) CreateVolume(ctx *lepton.Context, cv types.CloudVolume, data strin os.Exit(1) } - createVolumeInput.Throughput = aws.Int64(cv.Throughput) + createVolumeInput.Throughput = aws.Int32(int32(cv.Throughput)) } if sizeInGb != 0 { createVolumeInput.Size = &sizeInGb } - _, err = a.ec2.CreateVolume(createVolumeInput) + _, err = a.ec2.CreateVolume(a.execCtx, createVolumeInput) if err != nil { return vol, fmt.Errorf("create aws volume: %v", err) } @@ -127,12 +128,12 @@ func (a *AWS) GetAllVolumes(ctx *lepton.Context) (*[]lepton.NanosVolume, error) vols := &[]lepton.NanosVolume{} input := &ec2.DescribeVolumesInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("tag:CreatedBy"), Values: []*string{aws.String("ops")}}, + Filters: []awsEc2Types.Filter{ + {Name: aws.String("tag:CreatedBy"), Values: []string{"ops"}}, }, } - output, err := a.ec2.DescribeVolumes(input) + output, err := a.ec2.DescribeVolumes(a.execCtx, input) if err != nil { return nil, err } @@ -154,7 +155,7 @@ func (a *AWS) GetAllVolumes(ctx *lepton.Context) (*[]lepton.NanosVolume, error) vol := lepton.NanosVolume{ ID: *volume.VolumeId, Name: name, - Status: *volume.State, + Status: string(volume.State), Size: strconv.Itoa(int(*volume.Size)), Path: "", CreatedAt: volume.CreateTime.String(), @@ -177,7 +178,7 @@ func (a *AWS) DeleteVolume(ctx *lepton.Context, name string) error { input := &ec2.DeleteVolumeInput{ VolumeId: aws.String(*vol.VolumeId), } - _, err = a.ec2.DeleteVolume(input) + _, err = a.ec2.DeleteVolume(a.execCtx, input) if err != nil { return err } @@ -230,7 +231,7 @@ func (a *AWS) AttachVolume(ctx *lepton.Context, instanceName, name string, attac InstanceId: aws.String(*instance.InstanceId), VolumeId: aws.String(*vol.VolumeId), } - _, err = a.ec2.AttachVolume(input) + _, err = a.ec2.AttachVolume(a.execCtx, input) if err != nil { return err } @@ -255,7 +256,7 @@ func (a *AWS) DetachVolume(ctx *lepton.Context, instanceName, name string) error VolumeId: aws.String(*vol.VolumeId), } - _, err = a.ec2.DetachVolume(input) + _, err = a.ec2.DetachVolume(a.execCtx, input) if err != nil { return err } @@ -263,25 +264,25 @@ func (a *AWS) DetachVolume(ctx *lepton.Context, instanceName, name string) error return nil } -func (a *AWS) findVolumeByName(name string) (*ec2.Volume, error) { +func (a *AWS) findVolumeByName(name string) (*awsEc2Types.Volume, error) { input := &ec2.DescribeVolumesInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("tag:CreatedBy"), Values: []*string{aws.String("ops")}}, + Filters: []awsEc2Types.Filter{ + {Name: aws.String("tag:CreatedBy"), Values: []string{"ops"}}, }, } - output, err := a.ec2.DescribeVolumes(input) + output, err := a.ec2.DescribeVolumes(a.execCtx, input) if err != nil { return nil, err } for _, volume := range output.Volumes { if *volume.VolumeId == name { - return volume, nil + return &volume, nil } for _, tag := range volume.Tags { if (*tag.Key == "Name") && (*tag.Value == name) { - return volume, nil + return &volume, nil } } } diff --git a/provider/aws/role.go b/provider/aws/role.go index c3d39cfc..4518edcc 100644 --- a/provider/aws/role.go +++ b/provider/aws/role.go @@ -3,6 +3,7 @@ package aws import ( + "context" "encoding/json" "errors" "fmt" @@ -10,9 +11,8 @@ import ( "os" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/nanovms/ops/log" ) @@ -150,22 +150,22 @@ func roleError(bucket string, err error) { fmt.Printf("role :\n%+v\n", vmieDoc) } -func findBucketInPolicy(svc *iam.IAM, bucket string) (string, error) { +func findBucketInPolicy(execCtx context.Context, svc *iam.Client, bucket string) (string, error) { // find the policy lpi := &iam.ListAttachedRolePoliciesInput{ RoleName: aws.String(vmiName), } - pl, err := svc.ListAttachedRolePolicies(lpi) + pl, err := svc.ListAttachedRolePolicies(execCtx, lpi) if err != nil { return "", err } pid := "" for i := 0; i < len(pl.AttachedPolicies); i++ { - if aws.StringValue(pl.AttachedPolicies[i].PolicyName) == vmiName { - pid = aws.StringValue(pl.AttachedPolicies[i].PolicyArn) + if aws.ToString(pl.AttachedPolicies[i].PolicyName) == vmiName { + pid = aws.ToString(pl.AttachedPolicies[i].PolicyArn) break } } @@ -178,7 +178,7 @@ func findBucketInPolicy(svc *iam.IAM, bucket string) (string, error) { PolicyArn: aws.String(pid), } - po, err := svc.GetPolicy(gpi) + po, err := svc.GetPolicy(execCtx, gpi) if err != nil { return "", err } @@ -188,12 +188,12 @@ func findBucketInPolicy(svc *iam.IAM, bucket string) (string, error) { VersionId: po.Policy.DefaultVersionId, } - pv, err := svc.GetPolicyVersion(gpvi) + pv, err := svc.GetPolicyVersion(execCtx, gpvi) if err != nil { return "", err } - s := aws.StringValue(pv.PolicyVersion.Document) + s := aws.ToString(pv.PolicyVersion.Document) return url.QueryUnescape(s) @@ -201,17 +201,8 @@ func findBucketInPolicy(svc *iam.IAM, bucket string) (string, error) { // VerifyRole ensures we have a role and attached policy for the vmie service to hit our // bucket. -func VerifyRole(zone string, bucket string) { - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(zone)}, - ) - if err != nil { - log.Fatalf("error while creation session: %s", err.Error()) - } - - svc := iam.New(sess) - - resp, err := svc.ListRoles(&iam.ListRolesInput{}) +func VerifyRole(execCtx context.Context, iamClient *iam.Client, zone string, bucket string) { + resp, err := iamClient.ListRoles(execCtx, &iam.ListRolesInput{}, func(opts *iam.Options) { opts.Region = zone }) if err != nil { roleError(bucket, err) os.Exit(1) @@ -220,9 +211,9 @@ func VerifyRole(zone string, bucket string) { // this is probably a good candidate to cache in a metadata file // somewhere - having to do this on each upload is insane for _, role := range resp.Roles { - if aws.StringValue(role.RoleName) == vmiName { + if aws.ToString(role.RoleName) == vmiName { - dval, err := findBucketInPolicy(svc, bucket) + dval, err := findBucketInPolicy(execCtx, iamClient, bucket) if err != nil { roleError(bucket, err) os.Exit(1) @@ -240,7 +231,7 @@ func VerifyRole(zone string, bucket string) { PolicyDocument: aws.String(s), } - _, err = svc.PutRolePolicy(uri) + _, err = iamClient.PutRolePolicy(execCtx, uri) if err != nil { roleError(bucket, err) os.Exit(1) @@ -250,7 +241,7 @@ func VerifyRole(zone string, bucket string) { } } - err = createRole(svc, bucket) + err = createRole(execCtx, iamClient, bucket) if err != nil { roleError(bucket, err) os.Exit(1) @@ -258,7 +249,7 @@ func VerifyRole(zone string, bucket string) { } -func createRole(svc *iam.IAM, bucket string) error { +func createRole(execCtx context.Context, svc *iam.Client, bucket string) error { log.Info("creating a vmimport role for bucket " + bucket) ri := &iam.CreateRoleInput{ @@ -266,7 +257,7 @@ func createRole(svc *iam.IAM, bucket string) error { RoleName: aws.String(vmiName), } - _, err := svc.CreateRole(ri) + _, err := svc.CreateRole(execCtx, ri) if err != nil { return err } @@ -278,7 +269,7 @@ func createRole(svc *iam.IAM, bucket string) error { PolicyName: aws.String(vmiName), } - policyOut, err := svc.CreatePolicy(cpi) + policyOut, err := svc.CreatePolicy(execCtx, cpi) if err != nil { return err } @@ -288,7 +279,7 @@ func createRole(svc *iam.IAM, bucket string) error { RoleName: aws.String(vmiName), } - _, err = svc.AttachRolePolicy(ari) + _, err = svc.AttachRolePolicy(execCtx, ari) if err != nil { return err }