From c1d6ca2f137a7d1745c69ea63d1c07fab79a8c0d Mon Sep 17 00:00:00 2001 From: Angus Lees Date: Fri, 31 Jul 2020 15:32:25 +1000 Subject: [PATCH] Refactor EC2 Metadata (IMDS) code Goal: Make it clearer what is cached, and not cached. --- pkg/awsutils/awsutils.go | 363 ++++++++----------- pkg/awsutils/awsutils_test.go | 396 ++++++++------------- pkg/awsutils/imds.go | 237 ++++++++++++ pkg/awsutils/imds_test.go | 221 ++++++++++++ pkg/awsutils/mocks/awsutils_mocks.go | 5 +- pkg/ec2metadata/client.go | 34 -- pkg/ec2metadata/generate_mocks.go | 16 - pkg/ec2metadata/mocks/ec2metadata_mocks.go | 78 ---- pkg/ipamd/ipamd.go | 2 +- pkg/ipamd/ipamd_test.go | 2 +- 10 files changed, 753 insertions(+), 601 deletions(-) create mode 100644 pkg/awsutils/imds.go create mode 100644 pkg/awsutils/imds_test.go delete mode 100644 pkg/ec2metadata/client.go delete mode 100644 pkg/ec2metadata/generate_mocks.go delete mode 100644 pkg/ec2metadata/mocks/ec2metadata_mocks.go diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 43cf45ab062..062b833390f 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -18,42 +18,40 @@ import ( "encoding/json" "fmt" "math/rand" + "net" "os" "regexp" - "strconv" "strings" "sync" "time" - "github.com/pkg/errors" - - "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - "github.com/prometheus/client_golang/prometheus" - - "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadata" "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/retry" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" ) const ( metadataMACPath = "network/interfaces/macs/" - metadataAZ = "placement/availability-zone/" + metadataAZ = "placement/availability-zone" metadataLocalIP = "local-ipv4" metadataInstanceID = "instance-id" metadataInstanceType = "instance-type" metadataMAC = "mac" - metadataSGs = "/security-group-ids/" - metadataSubnetID = "/subnet-id/" - metadataVPCcidrs = "/vpc-ipv4-cidr-blocks/" - metadataDeviceNum = "/device-number/" - metadataInterface = "/interface-id/" + metadataSGs = "/security-group-ids" + metadataSubnetID = "/subnet-id" + metadataVPCcidrs = "/vpc-ipv4-cidr-blocks" + metadataDeviceNum = "/device-number" + metadataInterface = "/interface-id" metadataSubnetCIDR = "/subnet-ipv4-cidr-block" metadataIPv4s = "/local-ipv4s" maxENIEC2APIRetries = 12 @@ -98,7 +96,7 @@ var ( Name: "awscni_aws_api_latency_ms", Help: "AWS API call latency in ms", }, - []string{"api", "error"}, + []string{"api", "error", "status"}, ) awsAPIErr = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -147,7 +145,7 @@ type APIs interface { GetVPCIPv4CIDRs() []string // GetLocalIPv4 returns the primary IP address on the primary ENI interface - GetLocalIPv4() string + GetLocalIPv4() net.IP // GetPrimaryENI returns the primary ENI GetPrimaryENI() string @@ -176,7 +174,7 @@ type EC2InstanceMetadataCache struct { // metadata info securityGroups StringSet subnetID string - localIPv4 string + localIPv4 net.IP instanceID string instanceType string vpcIPv4CIDRs StringSet @@ -187,8 +185,8 @@ type EC2InstanceMetadataCache struct { unmanagedENIs StringSet useCustomNetworking bool - ec2Metadata ec2metadata.EC2Metadata - ec2SVC ec2wrapper.EC2 + imds TypedIMDS + ec2SVC ec2wrapper.EC2 } // ENIMetadata contains information about an ENI @@ -280,6 +278,37 @@ func (ss *StringSet) Has(item string) bool { return ss.data.Has(item) } +type instrumentedIMDS struct { + EC2MetadataIface +} + +func awsReqStatus(err error) string { + if err == nil { + return "200" + } + var aerr awserr.RequestFailure + if errors.As(err, &aerr) { + return fmt.Sprint(aerr.StatusCode()) + } + return "" // Unknown HTTP status code +} + +func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string) (string, error) { + start := time.Now() + result, err := i.EC2MetadataIface.GetMetadataWithContext(ctx, p) + duration := msSince(start) + + awsAPILatency.WithLabelValues("GetMetadata", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(duration) + + if err != nil { + awsAPIErrInc("GetMetadata", err) + log.Warnf("Failed to retrieve %s from instance metadata %v", err) + return "", err + } + + return result, nil +} + // New creates an EC2InstanceMetadataCache func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { //ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run @@ -288,10 +317,15 @@ func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { // Initializes prometheus metrics prometheusRegister() + awsSession := session.Must(session.NewSession(aws.NewConfig(). + WithMaxRetries(10), + )) + ec2Metadata := ec2metadata.New(awsSession) + cache := &EC2InstanceMetadataCache{} - cache.ec2Metadata = ec2metadata.New() + cache.imds = TypedIMDS{instrumentedIMDS{ec2Metadata}} - region, err := cache.ec2Metadata.Region() + region, err := ec2Metadata.Region() if err != nil { log.Errorf("Failed to retrieve region data from instance metadata %v", err) return nil, errors.Wrap(err, "instance metadata: failed to retrieve region data") @@ -323,64 +357,54 @@ func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { // InitWithEC2metadata initializes the EC2InstanceMetadataCache with the data retrieved from EC2 metadata service func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context) error { + var err error + // retrieve availability-zone - az, err := cache.ec2Metadata.GetMetadata(metadataAZ) + cache.availabilityZone, err = cache.imds.GetAZ(ctx) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve availability zone data from EC2 metadata service, %v", err) - return errors.Wrapf(err, "get instance metadata: failed to retrieve availability zone data") + return err } - cache.availabilityZone = az log.Debugf("Found availability zone: %s ", cache.availabilityZone) // retrieve eth0 local-ipv4 - cache.localIPv4, err = cache.ec2Metadata.GetMetadata(metadataLocalIP) + cache.localIPv4, err = cache.imds.GetLocalIPv4(ctx) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve instance's primary ip address data from instance metadata service %v", err) - return errors.Wrap(err, "get instance metadata: failed to retrieve the instance primary ip address data") + return err } log.Debugf("Discovered the instance primary ip address: %s", cache.localIPv4) // retrieve instance-id - cache.instanceID, err = cache.ec2Metadata.GetMetadata(metadataInstanceID) + cache.instanceID, err = cache.imds.GetInstanceID(ctx) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve instance-id from instance metadata %v", err) - return errors.Wrap(err, "get instance metadata: failed to retrieve instance-id") + return err } log.Debugf("Found instance-id: %s ", cache.instanceID) // retrieve instance-type - cache.instanceType, err = cache.ec2Metadata.GetMetadata(metadataInstanceType) + cache.instanceType, err = cache.imds.GetInstanceType(ctx) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve instance-type from instance metadata %v", err) - return errors.Wrap(err, "get instance metadata: failed to retrieve instance-type") + return err } log.Debugf("Found instance-type: %s ", cache.instanceType) // retrieve primary interface's mac - mac, err := cache.ec2Metadata.GetMetadata(metadataMAC) + mac, err := cache.imds.GetMac(ctx) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve primary interface MAC address from instance metadata service %v", err) - return errors.Wrap(err, "get instance metadata: failed to retrieve primary interface MAC address") + return err } cache.primaryENImac = mac log.Debugf("Found primary interface's MAC address: %s", mac) - err = cache.setPrimaryENI() + cache.primaryENI, err = cache.imds.GetInterfaceID(ctx, mac) if err != nil { return errors.Wrap(err, "get instance metadata: failed to find primary ENI") } + log.Debugf("%s is the primary ENI of this instance", cache.primaryENI) // retrieve sub-id - cache.subnetID, err = cache.ec2Metadata.GetMetadata(metadataMACPath + mac + metadataSubnetID) + cache.subnetID, err = cache.imds.GetSubnetID(ctx, mac) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve subnet-ids from instance metadata %v", err) - return errors.Wrap(err, "get instance metadata: failed to retrieve subnet-ids") + return err } log.Debugf("Found subnet-id: %s ", cache.subnetID) @@ -412,15 +436,13 @@ func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context) // refreshSGIDs retrieves security groups func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { - metadataSGIDs, err := cache.ec2Metadata.GetMetadata(metadataMACPath + mac + metadataSGs) + ctx := context.TODO() + + sgIDs, err := cache.imds.GetSecurityGroupIDs(ctx, mac) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve security-group-ids data from instance metadata service") - return errors.Wrap(err, "get instance metadata: failed to retrieve security-group-ids") + return err } - sgIDs := strings.Fields(metadataSGIDs) - newSGs := StringSet{} newSGs.Set(sgIDs) addedSGs := newSGs.Difference(&cache.securityGroups) @@ -465,7 +487,7 @@ func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { } start := time.Now() _, err = cache.ec2SVC.ModifyNetworkInterfaceAttributeWithContext(context.Background(), attributeInput, userAgent) - awsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("ModifyNetworkInterfaceAttribute", err) return errors.Wrap(err, "refreshSGIDs: unable to update the ENI's SG") @@ -477,14 +499,18 @@ func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { // refreshVPCIPv4CIDRs retrieves VPC IPv4 CIDR blocks func (cache *EC2InstanceMetadataCache) refreshVPCIPv4CIDRs(mac string) error { - metadataVPCIPv4CIDRs, err := cache.ec2Metadata.GetMetadata(metadataMACPath + mac + metadataVPCcidrs) + ctx := context.TODO() + + ipnets, err := cache.imds.GetVPCIPv4CIDRBlocks(ctx, mac) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve vpc-ipv4-cidr-blocks from instance metadata service") - return errors.Wrap(err, "get instance metadata: failed to retrieve vpc-ipv4-cidr-blocks data") + return err } - vpcIPv4CIDRs := strings.Fields(metadataVPCIPv4CIDRs) + // TODO: keep as net.IPNet and remove this round-trip to/from string + vpcIPv4CIDRs := make([]string, len(ipnets)) + for i, ipnet := range ipnets { + vpcIPv4CIDRs[i] = ipnet.String() + } newVpcIPv4CIDRs := StringSet{} newVpcIPv4CIDRs.Set(vpcIPv4CIDRs) @@ -501,179 +527,81 @@ func (cache *EC2InstanceMetadataCache) refreshVPCIPv4CIDRs(mac string) error { return nil } -func (cache *EC2InstanceMetadataCache) setPrimaryENI() error { - if cache.primaryENI != "" { - return nil - } - - // retrieve number of interfaces - metadataENImacs, err := cache.ec2Metadata.GetMetadata(metadataMACPath) - if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve interfaces data from instance metadata service, %v", err) - return errors.Wrap(err, "set primary ENI: failed to retrieve interfaces data") - } - eniMACs := strings.Fields(metadataENImacs) - log.Debugf("Discovered %d interfaces.", len(eniMACs)) - - // retrieve the attached ENIs - for _, eniMAC := range eniMACs { - // get device-number - device, err := cache.ec2Metadata.GetMetadata(metadataMACPath + eniMAC + metadataDeviceNum) - if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve the device-number data of ENI %s, %v", eniMAC, err) - return errors.Wrapf(err, "set primary ENI: failed to retrieve the device-number data of ENI %s", eniMAC) - } - - deviceNum, err := strconv.ParseInt(device, 0, 32) - if err != nil { - return errors.Wrapf(err, "set primary ENI: invalid device %s", device) - } - log.Debugf("Found device-number: %d ", deviceNum) - - eni, err := cache.ec2Metadata.GetMetadata(metadataMACPath + eniMAC + metadataInterface) - if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve interface-id data from instance metadata %v", err) - return errors.Wrap(err, "set primary ENI: failed to retrieve interface-id") - } - - log.Debugf("Found eni: %s ", eni) - result := strings.Split(eniMAC, "/") - if cache.primaryENImac == result[0] { - //primary interface - cache.primaryENI = eni - log.Debugf("%s is the primary ENI of this instance", eni) - return nil - } - } - return errors.New("set primary ENI: primary ENI not found") -} - // GetAttachedENIs retrieves ENI information from meta data service func (cache *EC2InstanceMetadataCache) GetAttachedENIs() (eniList []ENIMetadata, err error) { + ctx := context.TODO() + // retrieve number of interfaces - macs, err := cache.ec2Metadata.GetMetadata(metadataMACPath) + macs, err := cache.imds.GetMacs(ctx) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve interfaces data from instance metadata %v", err) - return nil, errors.Wrap(err, "get attached ENIs: failed to retrieve interfaces data") + return nil, err } - macsStrs := strings.Fields(macs) - log.Debugf("Total number of interfaces found: %d ", len(macsStrs)) + log.Debugf("Total number of interfaces found: %d ", len(macs)) - var enis []ENIMetadata + enis := make([]ENIMetadata, len(macs)) // retrieve the attached ENIs - for _, macStr := range macsStrs { - eniMetadata, err := cache.getENIMetadata(macStr) + for i, mac := range macs { + enis[i], err = cache.getENIMetadata(mac) if err != nil { - return nil, errors.Wrapf(err, "get attached ENIs: failed to retrieve ENI metadata for ENI: %s", macStr) + return nil, errors.Wrapf(err, "get attached ENIs: failed to retrieve ENI metadata for ENI: %s", mac) } - enis = append(enis, eniMetadata) } return enis, nil } -func (cache *EC2InstanceMetadataCache) getENIMetadata(macStr string) (ENIMetadata, error) { - eniMACList := strings.Split(macStr, "/") - eniMAC := eniMACList[0] +func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadata, error) { + ctx := context.TODO() + log.Debugf("Found ENI MAC address: %s", eniMAC) - eni, deviceNum, err := cache.getENIDeviceNumber(eniMAC) + eniID, err := cache.imds.GetInterfaceID(ctx, eniMAC) if err != nil { - return ENIMetadata{}, errors.Wrapf(err, "get ENI metadata: failed to retrieve device and ENI from metadata service: %s", eniMAC) + return ENIMetadata{}, err } - log.Debugf("Found ENI: %s, MAC %s, device %d", eni, eniMAC, deviceNum) - imdsIPv4s, cidr, err := cache.getIPsAndCIDR(eniMAC) + deviceNum, err := cache.imds.GetDeviceNumber(ctx, eniMAC) if err != nil { - return ENIMetadata{}, errors.Wrapf(err, "get ENI metadata: failed to retrieve IPs and CIDR for ENI: %s", eniMAC) + return ENIMetadata{}, err } - return ENIMetadata{ - ENIID: eni, - MAC: eniMAC, - DeviceNumber: deviceNum, - SubnetIPv4CIDR: cidr, - IPv4Addresses: imdsIPv4s, - }, nil -} - -// getIPsAndCIDR return list of IPs, CIDR, error -func (cache *EC2InstanceMetadataCache) getIPsAndCIDR(eniMAC string) ([]*ec2.NetworkInterfacePrivateIpAddress, string, error) { - start := time.Now() - cidr, err := cache.ec2Metadata.GetMetadata(metadataMACPath + eniMAC + metadataSubnetCIDR) - awsAPILatency.WithLabelValues("GetMetadata", fmt.Sprint(err != nil)).Observe(msSince(start)) + primaryMAC, err := cache.imds.GetMac(ctx) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve subnet-ipv4-cidr-block data from instance metadata %v", err) - return nil, "", errors.Wrapf(err, "failed to retrieve subnet-ipv4-cidr-block for ENI %s", eniMAC) + return ENIMetadata{}, err } - log.Debugf("Found CIDR %s for ENI %s", cidr, eniMAC) - - start = time.Now() - ipv4sAsString, err := cache.ec2Metadata.GetMetadata(metadataMACPath + eniMAC + metadataIPv4s) - awsAPILatency.WithLabelValues("GetMetadata", fmt.Sprint(err != nil)).Observe(msSince(start)) - if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve ENI %s local-ipv4s from instance metadata service, %v", eniMAC, err) - return nil, "", errors.Wrapf(err, "failed to retrieve ENI %s local-ipv4s", eniMAC) - } - - ipv4Strs := strings.Fields(ipv4sAsString) - log.Debugf("Found IP addresses %v on ENI %s", ipv4Strs, eniMAC) - ipv4s := make([]*ec2.NetworkInterfacePrivateIpAddress, 0, len(ipv4Strs)) - // network/interfaces/macs/mac/public-ipv4s The public IP address or Elastic IP addresses associated with the interface. - // There may be multiple IPv4 addresses on an instance. https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html - isFirst := true - for _, ipv4 := range ipv4Strs { - // The first IP in the list is always the primary IP of that ENI - primary := isFirst - ip := ipv4 - ipv4s = append(ipv4s, &ec2.NetworkInterfacePrivateIpAddress{PrivateIpAddress: &ip, Primary: &primary}) - isFirst = false - } - return ipv4s, cidr, nil -} - -// getENIDeviceNumber returns ENI ID, device number, error -func (cache *EC2InstanceMetadataCache) getENIDeviceNumber(eniMAC string) (eniID string, deviceNumber int, err error) { - // get device-number - start := time.Now() - device, err := cache.ec2Metadata.GetMetadata(metadataMACPath + eniMAC + metadataDeviceNum) - awsAPILatency.WithLabelValues("GetMetadata", fmt.Sprint(err != nil)).Observe(msSince(start)) - if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve the device-number of ENI %s, %v", eniMAC, err) - return "", -1, errors.Wrapf(err, "failed to retrieve device-number for ENI %s", eniMAC) + if eniMAC == primaryMAC && deviceNum != 0 { + // Can this even happen? To be backwards compatible, we will always use 0 here and log an error. + log.Errorf("Device number of primary ENI is %d! Forcing it to be 0 as expected", deviceNum) + deviceNum = 0 } - deviceNumber, err = strconv.Atoi(device) + log.Debugf("Found ENI: %s, MAC %s, device %d", eniID, eniMAC, deviceNum) + + cidr, err := cache.imds.GetSubnetIPv4CIDRBlock(ctx, eniMAC) if err != nil { - return "", -1, errors.Wrapf(err, "invalid device %s for ENI %s", device, eniMAC) + return ENIMetadata{}, err } - start = time.Now() - eniID, err = cache.ec2Metadata.GetMetadata(metadataMACPath + eniMAC + metadataInterface) - awsAPILatency.WithLabelValues("GetMetadata", fmt.Sprint(err != nil)).Observe(msSince(start)) + imdsIPv4s, err := cache.imds.GetLocalIPv4s(ctx, eniMAC) if err != nil { - awsAPIErrInc("GetMetadata", err) - log.Errorf("Failed to retrieve the interface-id data from instance metadata service, %v", err) - return "", -1, errors.Wrapf(err, "get attached ENIs: failed to retrieve interface-id for ENI %s", eniMAC) + return ENIMetadata{}, err } - if cache.primaryENI == eniID { - log.Debugf("Using device number 0 for primary ENI: %s", eniID) - if deviceNumber != 0 { - // Can this even happen? To be backwards compatible, we will always return 0 here and log an error. - log.Errorf("Device number of primary ENI is %d! It was expected to be 0", deviceNumber) - return eniID, 0, nil + // TODO: return a simpler data structure. + ec2ip4s := make([]*ec2.NetworkInterfacePrivateIpAddress, len(imdsIPv4s)) + for i, ip4 := range imdsIPv4s { + ec2ip4s[i] = &ec2.NetworkInterfacePrivateIpAddress{ + Primary: aws.Bool(i == 0), + PrivateIpAddress: aws.String(ip4.String()), } - return eniID, deviceNumber, nil } - // 0 is reserved for primary ENI - return eniID, deviceNumber, nil + + return ENIMetadata{ + ENIID: eniID, + MAC: eniMAC, + DeviceNumber: deviceNum, + SubnetIPv4CIDR: cidr.String(), + IPv4Addresses: ec2ip4s, + }, nil } // awsGetFreeDeviceNumber calls EC2 API DescribeInstances to get the next free device index @@ -684,7 +612,7 @@ func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber() (int, error) { start := time.Now() result, err := cache.ec2SVC.DescribeInstancesWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("DescribeInstances", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("DescribeInstances", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("DescribeInstances", err) log.Errorf("awsGetFreeDeviceNumber: Unable to retrieve instance data from EC2 control plane %v", err) @@ -750,7 +678,7 @@ func (cache *EC2InstanceMetadataCache) AllocENI(useCustomCfg bool, sg []*string, start := time.Now() _, err = cache.ec2SVC.ModifyNetworkInterfaceAttributeWithContext(context.Background(), attributeInput, userAgent) - awsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("ModifyNetworkInterfaceAttribute", err) err := cache.FreeENI(eniID) @@ -779,7 +707,7 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string) (string, error) { } start := time.Now() attachOutput, err := cache.ec2SVC.AttachNetworkInterfaceWithContext(context.Background(), attachInput, userAgent) - awsAPILatency.WithLabelValues("AttachNetworkInterface", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("AttachNetworkInterface", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("AttachNetworkInterface", err) log.Errorf("Failed to attach ENI %s: %v", eniID, err) @@ -827,7 +755,7 @@ func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string log.Infof("Creating ENI with security groups: %v in subnet: %s", sgs, *input.SubnetId) start := time.Now() result, err := cache.ec2SVC.CreateNetworkInterfaceWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("CreateNetworkInterface", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("CreateNetworkInterface", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("CreateNetworkInterface", err) log.Errorf("Failed to CreateNetworkInterface %v", err) @@ -880,7 +808,7 @@ func (cache *EC2InstanceMetadataCache) tagENI(eniID string, maxBackoffDelay time _ = retry.NWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxBackoffDelay, 0.3, 2), 5, func() error { start := time.Now() _, err := cache.ec2SVC.CreateTagsWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("CreateTags", err) log.Warnf("Failed to tag the newly created ENI %s:", eniID) @@ -970,7 +898,7 @@ func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDe err = retry.NWithBackoff(retry.NewSimpleBackoff(time.Millisecond*200, maxBackoffDelay, 0.15, 2.0), maxENIEC2APIRetries, func() error { start := time.Now() _, ec2Err := cache.ec2SVC.DetachNetworkInterfaceWithContext(context.Background(), detachInput, userAgent) - awsAPILatency.WithLabelValues("DetachNetworkInterface", fmt.Sprint(ec2Err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("DetachNetworkInterface", fmt.Sprint(ec2Err != nil), awsReqStatus(ec2Err)).Observe(msSince(start)) if ec2Err != nil { awsAPIErrInc("DetachNetworkInterface", ec2Err) log.Errorf("Failed to detach ENI %s %v", eniName, ec2Err) @@ -1005,7 +933,7 @@ func (cache *EC2InstanceMetadataCache) getENIAttachmentID(eniID string) (*string start := time.Now() result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { if aerr, ok := err.(awserr.Error); ok { if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" { @@ -1040,7 +968,7 @@ func (cache *EC2InstanceMetadataCache) deleteENI(eniName string, maxBackoffDelay err := retry.NWithBackoff(retry.NewSimpleBackoff(time.Millisecond*500, maxBackoffDelay, 0.15, 2.0), maxENIEC2APIRetries, func() error { start := time.Now() _, ec2Err := cache.ec2SVC.DeleteNetworkInterfaceWithContext(context.Background(), deleteInput, userAgent) - awsAPILatency.WithLabelValues("DeleteNetworkInterface", fmt.Sprint(ec2Err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("DeleteNetworkInterface", fmt.Sprint(ec2Err != nil), awsReqStatus(ec2Err)).Observe(msSince(start)) if ec2Err != nil { if aerr, ok := ec2Err.(awserr.Error); ok { // If already deleted, we are good @@ -1067,7 +995,7 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [ start := time.Now() result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { if aerr, ok := err.(awserr.Error); ok { if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" { @@ -1088,8 +1016,8 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [ return firstNI.PrivateIpAddresses, nil } -// DescribeAllENIs calls EC2 to refrech the ENIMetadata and tags for all attached ENIs -func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (eniMetadata []ENIMetadata, tagMap map[string]TagMap, trunkENI string, err error) { +// DescribeAllENIs calls EC2 to refresh the ENIMetadata and tags for all attached ENIs +func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[string]TagMap, string, error) { // Fetch all local ENI info from metadata allENIs, err := cache.GetAttachedENIs() if err != nil { @@ -1109,7 +1037,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (eniMetadata []ENIMetad input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: aws.StringSlice(eniIDs)} start := time.Now() ec2Response, err = cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err == nil { // No error, exit the loop break @@ -1148,7 +1076,8 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (eniMetadata []ENIMetad } // Collect ENI response into ENI metadata and tags. - tagMap = make(map[string]TagMap, len(ec2Response.NetworkInterfaces)) + var trunkENI string + tagMap := make(map[string]TagMap, len(ec2Response.NetworkInterfaces)) for _, ec2res := range ec2Response.NetworkInterfaces { if ec2res.Attachment != nil && aws.Int64Value(ec2res.Attachment.DeviceIndex) == 0 && !aws.BoolValue(ec2res.Attachment.DeleteOnTermination) { log.Warn("Primary ENI will not get deleted when node terminates because 'delete_on_termination' is set to false") @@ -1240,7 +1169,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddress(eniID string) error { start := time.Now() output, err := cache.ec2SVC.AssignPrivateIpAddressesWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("AssignPrivateIpAddresses", err) log.Errorf("Failed to allocate a private IP address %v", err) @@ -1318,7 +1247,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int start := time.Now() output, err := cache.ec2SVC.AssignPrivateIpAddressesWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { if containsPrivateIPAddressLimitExceededError(err) { log.Debug("AssignPrivateIpAddresses returned PrivateIpAddressLimitExceeded. This can happen if the data store is out of sync." + @@ -1372,7 +1301,7 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want log.Debugf("Not able to find the right ENI yet (attempt %d/%d)", attempt, maxENIEC2APIRetries) return ErrENINotFound }) - awsAPILatency.WithLabelValues("waitForENIAndIPsAttached", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("waitForENIAndIPsAttached", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { // If we have at least 1 Secondary IP, by now return what we have without an error if err == ErrAllSecondaryIPsNotFound { @@ -1403,7 +1332,7 @@ func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []st start := time.Now() _, err := cache.ec2SVC.UnassignPrivateIpAddressesWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("UnassignPrivateIpAddresses", err) log.Errorf("Failed to deallocate a private IP address %v", err) @@ -1461,7 +1390,7 @@ func (cache *EC2InstanceMetadataCache) tagENIcreateTS(eniID string, maxBackoffDe _ = retry.NWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxBackoffDelay, 0.3, 2), 5, func() error { start := time.Now() _, err := cache.ec2SVC.CreateTagsWithContext(context.Background(), input, userAgent) - awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil)).Observe(msSince(start)) + awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) if err != nil { awsAPIErrInc("CreateTags", err) log.Warnf("Failed to add tag to ENI %s: %v", eniID, err) @@ -1543,7 +1472,7 @@ func (cache *EC2InstanceMetadataCache) GetVPCIPv4CIDRs() []string { } // GetLocalIPv4 returns the primary IP address on the primary interface -func (cache *EC2InstanceMetadataCache) GetLocalIPv4() string { +func (cache *EC2InstanceMetadataCache) GetLocalIPv4() net.IP { return cache.localIPv4 } diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index f5af5228eef..1315950e491 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -31,242 +31,135 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" - mock_ec2metadata "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadata/mocks" mock_ec2wrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper/mocks" ) const ( - az = "us-east-1a" - localIP = "10.0.0.10" - instanceID = "i-0e1f3b9eb950e4980" - instanceType = "c1.medium" - primaryMAC = "12:ef:2a:98:e5:5a" - eni2MAC = "12:ef:2a:98:e5:5b" - sg1 = "sg-2e080f50" - sg2 = "sg-2e080f51" - sgs = sg1 + " " + sg2 - subnetID = "subnet-6b245523" - vpcCIDR = "10.0.0.0/16" - subnetCIDR = "10.0.1.0/24" - primaryeniID = "eni-00000000" - eniID = "eni-5731da78" - eniAttachID = "eni-attach-beb21856" - eni1Device = "0" - eni1PrivateIP = "10.0.0.1" - eni2Device = "1" - eni2PrivateIP = "10.0.0.2" - eni2AttachID = "eni-attach-fafdfafd" - eni2ID = "eni-12341234" + az = "us-east-1a" + localIP = "10.0.0.10" + instanceID = "i-0e1f3b9eb950e4980" + instanceType = "c1.medium" + primaryMAC = "12:ef:2a:98:e5:5a" + eni2MAC = "12:ef:2a:98:e5:5b" + sg1 = "sg-2e080f50" + sg2 = "sg-2e080f51" + sgs = sg1 + " " + sg2 + subnetID = "subnet-6b245523" + vpcCIDR = "10.0.0.0/16" + subnetCIDR = "10.0.1.0/24" + primaryeniID = "eni-00000000" + eniID = primaryeniID + eniAttachID = "eni-attach-beb21856" + eni1Device = "0" + eni1PrivateIP = "10.0.0.1" + eni2Device = "1" + eni2PrivateIP = "10.0.0.2" + eni2AttachID = "eni-attach-fafdfafd" + eni2ID = "eni-12341234" + metadataVPCIPv4CIDRs = "192.168.0.0/16 100.66.0.0/1" ) +func testMetadata(overrides map[string]interface{}) FakeIMDS { + data := map[string]interface{}{ + metadataAZ: az, + metadataLocalIP: localIP, + metadataInstanceID: instanceID, + metadataInstanceType: instanceType, + metadataMAC: primaryMAC, + metadataMACPath: primaryMAC, + metadataMACPath + primaryMAC + metadataDeviceNum: eni1Device, + metadataMACPath + primaryMAC + metadataInterface: primaryeniID, + metadataMACPath + primaryMAC + metadataSGs: sgs, + metadataMACPath + primaryMAC + metadataIPv4s: eni1PrivateIP, + metadataMACPath + primaryMAC + metadataSubnetID: subnetID, + metadataMACPath + primaryMAC + metadataSubnetCIDR: subnetCIDR, + metadataMACPath + primaryMAC + metadataVPCcidrs: metadataVPCIPv4CIDRs, + } + + for k, v := range overrides { + data[k] = v + } + + return FakeIMDS(data) +} + func setup(t *testing.T) (*gomock.Controller, - *mock_ec2metadata.MockEC2Metadata, *mock_ec2wrapper.MockEC2) { ctrl := gomock.NewController(t) return ctrl, - mock_ec2metadata.NewMockEC2Metadata(ctrl), mock_ec2wrapper.NewMockEC2(ctrl) } func TestInitWithEC2metadata(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) defer cancel() - ctrl, mockMetadata, mockEC2 := setup(t) - defer ctrl.Finish() - - metadataVPCIPv4CIDRs := "192.168.0.0/16 100.66.0.0/1" - - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceID).Return(instanceID, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceType).Return(instanceType, nil) - mockMetadata.EXPECT().GetMetadata(metadataMAC).Return(primaryMAC, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return(primaryMAC, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Return(eni1Device, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Return(primaryMAC, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSGs).Return(sgs, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetID).Return(subnetID, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetCIDR).Return(subnetCIDR, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataVPCcidrs).Return(metadataVPCIPv4CIDRs, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataIPv4s).Return("", nil) - - mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata, ec2SVC: mockEC2} - err := ins.initWithEC2Metadata(ctx) - assert.NoError(t, err) - assert.Equal(t, az, ins.availabilityZone) - assert.Equal(t, localIP, ins.localIPv4) - assert.Equal(t, ins.instanceID, instanceID) - assert.Equal(t, ins.primaryENImac, primaryMAC) - assert.Equal(t, len(ins.securityGroups.SortedList()), 2) - assert.Equal(t, subnetID, ins.subnetID) - assert.Equal(t, len(ins.vpcIPv4CIDRs.SortedList()), 2) -} - -func TestInitWithEC2metadataSubnetErr(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - ctrl, mockMetadata, _ := setup(t) - defer ctrl.Finish() - - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceID).Return(instanceID, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceType).Return(instanceType, nil) - mockMetadata.EXPECT().GetMetadata(metadataMAC).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Return(eni1Device, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetID).Return(subnetID, errors.New("Error on Subnet")) - - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} - err := ins.initWithEC2Metadata(ctx) - assert.Error(t, err) -} - -func TestInitWithEC2metadataSGErr(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - ctrl, mockMetadata, _ := setup(t) - defer ctrl.Finish() - - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceID).Return(instanceID, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceType).Return(instanceType, nil) - mockMetadata.EXPECT().GetMetadata(metadataMAC).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Return(eni1Device, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetID).Return(subnetID, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSGs).Return(sgs, errors.New("Error on SG")) - - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} - err := ins.initWithEC2Metadata(ctx) - assert.Error(t, err) -} - -func TestInitWithEC2metadataENIErrs(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - ctrl, mockMetadata, _ := setup(t) - defer ctrl.Finish() - - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceID).Return(instanceID, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceType).Return(instanceType, nil) - mockMetadata.EXPECT().GetMetadata(metadataMAC).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return("", errors.New("err on ENIs")) - - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} - err := ins.initWithEC2Metadata(ctx) - assert.Error(t, err) -} - -func TestInitWithEC2metadataMACErr(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - ctrl, mockMetadata, _ := setup(t) - defer ctrl.Finish() - - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceID).Return(instanceID, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceType).Return(instanceType, nil) - mockMetadata.EXPECT().GetMetadata(metadataMAC).Return(primaryMAC, errors.New("error on MAC")) - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} - err := ins.initWithEC2Metadata(ctx) - assert.Error(t, err) -} - -func TestInitWithEC2metadataLocalIPErr(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - ctrl, mockMetadata, _ := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() + mockMetadata := testMetadata(nil) - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, errors.New("error on localIP")) + mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} + ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2} err := ins.initWithEC2Metadata(ctx) - assert.Error(t, err) + if assert.NoError(t, err) { + assert.Equal(t, az, ins.availabilityZone) + assert.Equal(t, localIP, ins.localIPv4.String()) + assert.Equal(t, ins.instanceID, instanceID) + assert.Equal(t, ins.primaryENImac, primaryMAC) + assert.Equal(t, ins.primaryENI, primaryeniID) + assert.Equal(t, len(ins.securityGroups.SortedList()), 2) + assert.Equal(t, subnetID, ins.subnetID) + assert.Equal(t, len(ins.vpcIPv4CIDRs.SortedList()), 2) + } } -func TestInitWithEC2metadataInstanceErr(t *testing.T) { +func TestInitWithEC2metadataErr(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) defer cancel() - ctrl, mockMetadata, _ := setup(t) - defer ctrl.Finish() - - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceID).Return(instanceID, errors.New("error on instanceID")) - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} - err := ins.initWithEC2Metadata(ctx) - assert.Error(t, err) -} - -func TestInitWithEC2metadataAZErr(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - ctrl, mockMetadata, _ := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, errors.New("error on metadata AZ")) - - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} - err := ins.initWithEC2Metadata(ctx) - assert.Error(t, err) -} + mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) -func TestSetPrimaryENs(t *testing.T) { - ctrl, mockMetadata, _ := setup(t) - defer ctrl.Finish() + var keys []string + for k := range testMetadata(nil) { + keys = append(keys, k) + } - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return(primaryMAC+" "+eni2MAC, nil) + for _, key := range keys { + mockMetadata := testMetadata(map[string]interface{}{ + key: fmt.Errorf("An error with %s", key), + }) - gomock.InOrder( - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Return(eni1Device, nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Return(primaryeniID, nil), - ) + ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2} - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} - ins.primaryENImac = primaryMAC - err := ins.setPrimaryENI() - assert.NoError(t, err) - assert.Equal(t, ins.primaryENI, primaryeniID) + // This test is a bit silly. We expect broken metadata to result in an err return here. But if the code is resilient and _succeeds_, then of course that's ok too. Mostly we just want it not to panic. + assert.NotPanics(t, func() { + _ = ins.initWithEC2Metadata(ctx) + }, "Broken metadata %s resulted in panic", key) + } } func TestGetAttachedENIs(t *testing.T) { - ctrl, mockMetadata, _ := setup(t) - defer ctrl.Finish() - - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return(primaryMAC+" "+eni2MAC, nil) - - gomock.InOrder( - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Return(eni1Device, nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Return(eniID, nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetCIDR).Return(subnetCIDR, nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataIPv4s).Return("", nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataDeviceNum).Return(eni2Device, nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataInterface).Return(eni2ID, nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataSubnetCIDR).Return(subnetCIDR, nil), - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataIPv4s).Return("", nil), - ) + mockMetadata := testMetadata(map[string]interface{}{ + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, + metadataMACPath + eni2MAC + metadataInterface: eni2ID, + metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, + metadataMACPath + eni2MAC + metadataIPv4s: eni2PrivateIP, + }) - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} + ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}} ens, err := ins.GetAttachedENIs() - assert.NoError(t, err) - assert.Equal(t, len(ens), 2) + if assert.NoError(t, err) { + assert.Equal(t, len(ens), 2) + } } func TestAWSGetFreeDeviceNumberOnErr(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() // test error handling @@ -278,7 +171,7 @@ func TestAWSGetFreeDeviceNumberOnErr(t *testing.T) { } func TestAWSGetFreeDeviceNumberNoDevice(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() // test no free index @@ -301,7 +194,7 @@ func TestAWSGetFreeDeviceNumberNoDevice(t *testing.T) { } func TestGetENIAttachmentID(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() attachmentID := aws.String("foo-attach") @@ -363,7 +256,7 @@ func TestGetENIAttachmentID(t *testing.T) { } func TestDescribeAllENIs(t *testing.T) { - ctrl, mockMetadata, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() result := &ec2.DescribeNetworkInterfacesOutput{ @@ -391,15 +284,11 @@ func TestDescribeAllENIs(t *testing.T) { {"Other error", nil, maxENIEC2APIRetries, err, err}, } - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Times(len(testCases)).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Times(len(testCases)).Return(eni1Device, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Times(len(testCases)).Return(eniID, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetCIDR).Times(len(testCases)).Return(subnetCIDR, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataIPv4s).Times(len(testCases)).Return("", nil) + mockMetadata := testMetadata(nil) for _, tc := range testCases { mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(tc.n).Return(result, tc.awsErr) - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata, ec2SVC: mockEC2} + ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2} _, tags, _, err := ins.DescribeAllENIs() assert.Equal(t, tc.expErr, err, tc.name) assert.Equal(t, tc.exptags, tags, tc.name) @@ -409,24 +298,14 @@ func TestDescribeAllENIs(t *testing.T) { func TestTagEni(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) defer cancel() - ctrl, mockMetadata, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() - mockMetadata.EXPECT().GetMetadata(metadataAZ).Return(az, nil) - mockMetadata.EXPECT().GetMetadata(metadataLocalIP).Return(localIP, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceID).Return(instanceID, nil) - mockMetadata.EXPECT().GetMetadata(metadataInstanceType).Return(instanceType, nil) - mockMetadata.EXPECT().GetMetadata(metadataMAC).Return(primaryMAC, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return(primaryMAC, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Return(eni1Device, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Return(primaryMAC, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSGs).Return(sgs, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetID).Return(subnetID, nil) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetCIDR).Return(subnetCIDR, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataVPCcidrs).Return(vpcCIDR, nil).AnyTimes() - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataIPv4s).Return("", nil) + mockMetadata := testMetadata(nil) + mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata, ec2SVC: mockEC2} + ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2} + err := ins.initWithEC2Metadata(ctx) assert.NoError(t, err) mockEC2.EXPECT().CreateTagsWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("tagging failed")) @@ -440,7 +319,7 @@ func TestTagEni(t *testing.T) { } func TestAdditionalTagsEni(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() _ = os.Setenv(additionalEniTagsEnvVar, `{"testKey": "testing"}`) currentENIID := eniID @@ -489,9 +368,11 @@ func TestMapToTags(t *testing.T) { } func TestAllocENI(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() + mockMetadata := testMetadata(nil) + cureniID := eniID eni := ec2.CreateNetworkInterfaceOutput{NetworkInterface: &ec2.NetworkInterface{NetworkInterfaceId: &cureniID}} mockEC2.EXPECT().CreateNetworkInterfaceWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(&eni, nil) @@ -517,15 +398,20 @@ func TestAllocENI(t *testing.T) { mockEC2.EXPECT().CreateTagsWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2} + ins := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + } _, err := ins.AllocENI(false, nil, "") assert.NoError(t, err) } func TestAllocENINoFreeDevice(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() + mockMetadata := testMetadata(nil) + cureniID := eniID eni := ec2.CreateNetworkInterfaceOutput{NetworkInterface: &ec2.NetworkInterface{NetworkInterfaceId: &cureniID}} mockEC2.EXPECT().CreateNetworkInterfaceWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(&eni, nil) @@ -545,15 +431,20 @@ func TestAllocENINoFreeDevice(t *testing.T) { mockEC2.EXPECT().DescribeInstancesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) mockEC2.EXPECT().DeleteNetworkInterfaceWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2} + ins := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + } _, err := ins.AllocENI(false, nil, "") assert.Error(t, err) } func TestAllocENIMaxReached(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() + mockMetadata := testMetadata(nil) + cureniID := eniID eni := ec2.CreateNetworkInterfaceOutput{NetworkInterface: &ec2.NetworkInterface{NetworkInterfaceId: &cureniID}} mockEC2.EXPECT().CreateNetworkInterfaceWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(&eni, nil) @@ -575,13 +466,16 @@ func TestAllocENIMaxReached(t *testing.T) { mockEC2.EXPECT().AttachNetworkInterfaceWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("AttachmentLimitExceeded")) mockEC2.EXPECT().DeleteNetworkInterfaceWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2} + ins := &EC2InstanceMetadataCache{ + ec2SVC: mockEC2, + imds: TypedIMDS{mockMetadata}, + } _, err := ins.AllocENI(false, nil, "") assert.Error(t, err) } func TestFreeENI(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() attachmentID := eniAttachID @@ -598,7 +492,7 @@ func TestFreeENI(t *testing.T) { } func TestFreeENIRetry(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() attachmentID := eniAttachID @@ -618,7 +512,7 @@ func TestFreeENIRetry(t *testing.T) { } func TestFreeENIRetryMax(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() attachmentID := eniAttachID @@ -638,7 +532,7 @@ func TestFreeENIRetryMax(t *testing.T) { } func TestFreeENIDescribeErr(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("Error on DescribeNetworkInterfacesWithContext")) @@ -649,7 +543,7 @@ func TestFreeENIDescribeErr(t *testing.T) { } func TestDescribeInstanceTypes(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() mockEC2.EXPECT().DescribeInstanceTypesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: []*ec2.InstanceTypeInfo{ @@ -672,7 +566,7 @@ func TestDescribeInstanceTypes(t *testing.T) { } func TestAllocIPAddress(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() mockEC2.EXPECT().AssignPrivateIpAddressesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(&ec2.AssignPrivateIpAddressesOutput{}, nil) @@ -683,7 +577,7 @@ func TestAllocIPAddress(t *testing.T) { } func TestAllocIPAddressOnErr(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() mockEC2.EXPECT().AssignPrivateIpAddressesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("Error on AssignPrivateIpAddressesWithContext")) @@ -694,7 +588,7 @@ func TestAllocIPAddressOnErr(t *testing.T) { } func TestAllocIPAddresses(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() // when required IP numbers(5) is below ENI's limit(30) @@ -725,7 +619,7 @@ func TestAllocIPAddresses(t *testing.T) { } func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_OneResult(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() attachmentID := eniAttachID @@ -759,7 +653,7 @@ func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_OneResult(t } func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_NoResult(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() result := &ec2.DescribeNetworkInterfacesOutput{ @@ -773,7 +667,7 @@ func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_NoResult(t } func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_Error(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("dummy error")) @@ -850,23 +744,21 @@ func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctrl, mockMetadata, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() eniIPs := eni2PrivateIP for i := 0; i < tt.args.foundSecondaryIPs; i++ { eniIPs += " " + eni2PrivateIP + strconv.Itoa(i) } fmt.Println("eniips", eniIPs) - mockMetadata.EXPECT().GetMetadata(metadataMACPath).Return(primaryMAC+" "+eni2MAC, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataDeviceNum).Return(eni1Device, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataInterface).Return(primaryeniID, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataSubnetCIDR).Return(subnetCIDR, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+primaryMAC+metadataIPv4s).Return(eni1PrivateIP, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataDeviceNum).Return(eni2Device, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataInterface).Return(eni2ID, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataSubnetCIDR).Return(subnetCIDR, nil).Times(tt.args.times) - mockMetadata.EXPECT().GetMetadata(metadataMACPath+eni2MAC+metadataIPv4s).Return(eniIPs, nil).Times(tt.args.times) - cache := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata, ec2SVC: mockEC2} + mockMetadata := testMetadata(map[string]interface{}{ + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, + metadataMACPath + eni2MAC + metadataInterface: eni2ID, + metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, + metadataMACPath + eni2MAC + metadataIPv4s: eniIPs, + }) + cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2} gotEniMetadata, err := cache.waitForENIAndIPsAttached(tt.args.eni, tt.args.wantedSecondaryIPs, tt.args.maxBackoffDelay) if (err != nil) != tt.wantErr { t.Errorf("waitForENIAndIPsAttached() error = %v, wantErr %v", err, tt.wantErr) @@ -880,8 +772,8 @@ func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { } func TestEC2InstanceMetadataCache_SetUnmanagedENIs(t *testing.T) { - _, mockMetadata, _ := setup(t) - ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} + mockMetadata := testMetadata(nil) + ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}} ins.SetUnmanagedENIs(nil) assert.False(t, ins.IsUnmanagedENI("eni-1")) ins.SetUnmanagedENIs([]string{"eni-1", "eni-2"}) @@ -892,7 +784,7 @@ func TestEC2InstanceMetadataCache_SetUnmanagedENIs(t *testing.T) { } func TestEC2InstanceMetadataCache_cleanUpLeakedENIsInternal(t *testing.T) { - ctrl, _, mockEC2 := setup(t) + ctrl, mockEC2 := setup(t) defer ctrl.Finish() description := eniDescriptionPrefix + "test" diff --git a/pkg/awsutils/imds.go b/pkg/awsutils/imds.go new file mode 100644 index 00000000000..5cf697c2745 --- /dev/null +++ b/pkg/awsutils/imds.go @@ -0,0 +1,237 @@ +package awsutils + +import ( + "context" + "fmt" + "net" + "net/http" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/pkg/errors" +) + +// EC2MetadataIface is a subset of the EC2Metadata API. +type EC2MetadataIface interface { + GetMetadataWithContext(ctx context.Context, p string) (string, error) +} + +// typedIMDS is a typed wrapper around raw untyped IMDS SDK API. +type TypedIMDS struct { + EC2MetadataIface +} + +func (imds TypedIMDS) getList(ctx context.Context, key string) ([]string, error) { + data, err := imds.GetMetadataWithContext(ctx, key) + if err != nil { + return nil, err + } + + return strings.Fields(data), nil +} + +// GetAZ returns the Availability Zone in which the instance launched. +func (imds TypedIMDS) GetAZ(ctx context.Context) (string, error) { + return imds.GetMetadataWithContext(ctx, "placement/availability-zone") +} + +// GetInstanceType returns the type of this instance. +func (imds TypedIMDS) GetInstanceType(ctx context.Context) (string, error) { + return imds.GetMetadataWithContext(ctx, "instance-type") +} + +// GetLocalIPv4 returns the private (primary) IPv4 address of the instance. +func (imds TypedIMDS) GetLocalIPv4(ctx context.Context) (net.IP, error) { + return imds.getIP(ctx, "local-ipv4") +} + +// GetInstanceID returns the ID of this instance. +func (imds TypedIMDS) GetInstanceID(ctx context.Context) (string, error) { + return imds.GetMetadataWithContext(ctx, "instance-id") +} + +// GetMac returns the first/primary network interface mac address. +func (imds TypedIMDS) GetMac(ctx context.Context) (string, error) { + return imds.GetMetadataWithContext(ctx, "mac") +} + +// GetMacs returns the interface addresses attached to the instance. +func (imds TypedIMDS) GetMacs(ctx context.Context) ([]string, error) { + list, err := imds.getList(ctx, "network/interfaces/macs") + if err != nil { + return nil, err + } + // Remove trailing / + for i, item := range list { + list[i] = strings.TrimSuffix(item, "/") + } + return list, nil +} + +// GetInterfaceID returns the ID of the network interface. +func (imds TypedIMDS) GetInterfaceID(ctx context.Context, mac string) (string, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/interface-id", mac) + return imds.GetMetadataWithContext(ctx, key) +} + +func (imds TypedIMDS) getInt(ctx context.Context, key string) (int, error) { + data, err := imds.GetMetadataWithContext(ctx, key) + if err != nil { + return 0, err + } + return strconv.Atoi(data) +} + +// GetDeviceNumber returns the unique device number associated with an interface. The primary interface is 0. +func (imds TypedIMDS) GetDeviceNumber(ctx context.Context, mac string) (int, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/device-number", mac) + return imds.getInt(ctx, key) +} + +// GetSubnetID returns the ID of the subnet in which the interface resides. +func (imds TypedIMDS) GetSubnetID(ctx context.Context, mac string) (string, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/subnet-id", mac) + return imds.GetMetadataWithContext(ctx, key) +} + +// GetSecurityGroupIDs returns the IDs of the security groups to which the network interface belongs. +func (imds TypedIMDS) GetSecurityGroupIDs(ctx context.Context, mac string) ([]string, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/security-group-ids", mac) + return imds.getList(ctx, key) +} + +func (imds TypedIMDS) getIP(ctx context.Context, key string) (net.IP, error) { + data, err := imds.GetMetadataWithContext(ctx, key) + if err != nil { + return nil, err + } + + ip := net.ParseIP(data) + if ip == nil { + return nil, &net.ParseError{Type: "IP address", Text: data} + } + return ip, nil +} + +func (imds TypedIMDS) getIPs(ctx context.Context, key string) ([]net.IP, error) { + list, err := imds.getList(ctx, key) + if err != nil { + return nil, err + } + + ips := make([]net.IP, len(list)) + for i, item := range list { + ip := net.ParseIP(item) + if ip == nil { + return nil, &net.ParseError{Type: "IP address", Text: item} + } + ips[i] = ip + } + return ips, nil +} + +func (imds TypedIMDS) getCIDR(ctx context.Context, key string) (net.IPNet, error) { + data, err := imds.GetMetadataWithContext(ctx, key) + if err != nil { + return net.IPNet{}, err + } + + ip, network, err := net.ParseCIDR(data) + if err != nil { + return net.IPNet{}, err + } + // Why doesn't net.ParseCIDR just return values in this form? + cidr := net.IPNet{IP: ip, Mask: network.Mask} + return cidr, nil +} + +func (imds TypedIMDS) getCIDRs(ctx context.Context, key string) ([]net.IPNet, error) { + list, err := imds.getList(ctx, key) + if err != nil { + return nil, err + } + + cidrs := make([]net.IPNet, len(list)) + for i, item := range list { + ip, network, err := net.ParseCIDR(item) + if err != nil { + return nil, err + } + // Why doesn't net.ParseCIDR just return values in this form? + cidrs[i] = net.IPNet{IP: ip, Mask: network.Mask} + } + return cidrs, nil +} + +// GetLocalIPv4s returns the private IPv4 addresses associated with the interface. First returned address is the primary address. +func (imds TypedIMDS) GetLocalIPv4s(ctx context.Context, mac string) ([]net.IP, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac) + return imds.getIPs(ctx, key) +} + +// GetIPv6s returns the IPv6 addresses associated with the interface. +func (imds TypedIMDS) GetIPv6s(ctx context.Context, mac string) ([]net.IP, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac) + ips, err := imds.getIPs(ctx, key) + if IsNotFound(err) { + // No IPv6. Not an error, just a disappointment :( + return nil, nil + } + return ips, err +} + +// GetSubnetIPv4CIDRBlocks returns the IPv4 CIDR block for the subnet in which the interface resides. +func (imds TypedIMDS) GetSubnetIPv4CIDRBlock(ctx context.Context, mac string) (net.IPNet, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac) + return imds.getCIDR(ctx, key) +} + +// GetVPCIPv4CIDRBlocks returns the IPv4 CIDR blocks for the VPC. +func (imds TypedIMDS) GetVPCIPv4CIDRBlocks(ctx context.Context, mac string) ([]net.IPNet, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv4-cidr-blocks", mac) + return imds.getCIDRs(ctx, key) +} + +// GetVPCIPv6CIDRBlocks returns the IPv6 CIDR blocks for the VPC. +func (imds TypedIMDS) GetVPCIPv6CIDRBlocks(ctx context.Context, mac string) ([]net.IPNet, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv6-cidr-blocks", mac) + ipnets, err := imds.getCIDRs(ctx, key) + if IsNotFound(err) { + // No IPv6. Not an error, just a disappointment :( + return nil, nil + } + return ipnets, err +} + +// IsNotFound returns true if the error was caused by an AWS API 404 response. +func IsNotFound(err error) bool { + if err != nil { + var aerr awserr.RequestFailure + if errors.As(err, &aerr) { + return aerr.StatusCode() == http.StatusNotFound + } + } + return false +} + +type FakeIMDS map[string]interface{} + +func (f FakeIMDS) GetMetadataWithContext(ctx context.Context, p string) (string, error) { + result, ok := f[p] + if !ok { + result, ok = f[p+"/"] // Metadata API treats foo/ as foo + } + if !ok { + notFoundErr := awserr.NewRequestFailure(awserr.New("NotFound", "not found", nil), http.StatusNotFound, "dummy-reqid") + return "", fmt.Errorf("no test data for metadata path %s: %w", p, notFoundErr) + } + switch v := result.(type) { + case string: + return v, nil + case error: + return "", v + default: + panic(fmt.Sprintf("unknown test metadata value type %T for %s", result, p)) + } +} diff --git a/pkg/awsutils/imds_test.go b/pkg/awsutils/imds_test.go new file mode 100644 index 00000000000..221577c518f --- /dev/null +++ b/pkg/awsutils/imds_test.go @@ -0,0 +1,221 @@ +package awsutils + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetAZ(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "placement/availability-zone": "us-west-2b", + })} + + az, err := f.GetAZ(context.TODO()) + if assert.NoError(t, err) { + assert.Equal(t, az, "us-west-2b") + } +} + +func TestGetInstanceType(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "instance-type": "t3.medium", + })} + + ty, err := f.GetInstanceType(context.TODO()) + if assert.NoError(t, err) { + assert.Equal(t, ty, "t3.medium") + } +} + +func TestGetLocalIPv4(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "local-ipv4": "10.0.88.3", + })} + + ip, err := f.GetLocalIPv4(context.TODO()) + if assert.NoError(t, err) { + assert.Equal(t, ip, net.IPv4(10, 0, 88, 3)) + } +} + +func TestGetInstanceID(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "instance-id": "i-084abd1f69f27d987", + })} + + id, err := f.GetInstanceID(context.TODO()) + if assert.NoError(t, err) { + assert.Equal(t, id, "i-084abd1f69f27d987") + } +} + +func TestGetMac(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "mac": "02:68:f3:f6:c7:ef", + })} + + mac, err := f.GetMac(context.TODO()) + if assert.NoError(t, err) { + assert.Equal(t, mac, "02:68:f3:f6:c7:ef") + } +} + +func TestGetMacs(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs": `02:68:f3:f6:c7:ef/ +02:c5:f8:3e:6b:27/`, + })} + + macs, err := f.GetMacs(context.TODO()) + if assert.NoError(t, err) { + assert.Equal(t, macs, []string{"02:68:f3:f6:c7:ef", "02:c5:f8:3e:6b:27"}) + } +} + +func TestGetInterfaceID(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/interface-id": "eni-0c0fde533492c9df5", + })} + + id, err := f.GetInterfaceID(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, id, "eni-0c0fde533492c9df5") + } +} + +func TestGetDeviceNumber(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/device-number": "1", + })} + + n, err := f.GetDeviceNumber(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, n, 1) + } + + _, err = f.GetDeviceNumber(context.TODO(), "00:00:de:ad:be:ef") + if assert.Error(t, err) { + assert.True(t, IsNotFound(err)) + } +} + +func TestGetSubnetID(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/subnet-id": "subnet-0afaed81bf542db37", + })} + + id, err := f.GetSubnetID(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, id, "subnet-0afaed81bf542db37") + } +} + +func TestGetSecurityGroupIDs(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/security-group-ids": "sg-00581e028df71bda8", + })} + + list, err := f.GetSecurityGroupIDs(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, list, []string{"sg-00581e028df71bda8"}) + } +} + +func TestGetLocalIPv4s(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/local-ipv4s": `10.0.114.236 +10.0.120.181`, + })} + + ips, err := f.GetLocalIPv4s(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, ips, []net.IP{net.IPv4(10, 0, 114, 236), net.IPv4(10, 0, 120, 181)}) + } + + _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef") + if assert.Error(t, err) { + assert.True(t, IsNotFound(err)) + } +} + +func TestGetIPv6s(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/ipv6s": `2001:db8::1 +2001:db8::2`, + })} + + ips, err := f.GetIPv6s(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, ips, []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")}) + } + + nov6 := TypedIMDS{FakeIMDS(map[string]interface{}{ + // NB: IMDS returns 404, not empty string :( + })} + + ips, err = nov6.GetIPv6s(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.ElementsMatch(t, ips, []net.IP{}) + } + + // Note can't tell the difference between bad mac and no ipv6 :( + ips, err = f.GetIPv6s(context.TODO(), "00:00:de:ad:be:ef") + if assert.NoError(t, err) { + assert.ElementsMatch(t, ips, []net.IP{}) + } +} + +func TestGetSubnetIPv4CIDRBlock(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/subnet-ipv4-cidr-block": "10.0.64.0/18", + })} + + ip, err := f.GetSubnetIPv4CIDRBlock(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, ip, net.IPNet{IP: net.IPv4(10, 0, 64, 0), Mask: net.CIDRMask(18, 32)}) + } +} + +func TestGetVPCIPv4CIDRBlocks(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/vpc-ipv4-cidr-blocks": "10.0.0.0/16", + })} + + ips, err := f.GetVPCIPv4CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, ips, []net.IPNet{{IP: net.IPv4(10, 0, 0, 0), Mask: net.CIDRMask(16, 32)}}) + } + + _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef") + if assert.Error(t, err) { + assert.True(t, IsNotFound(err)) + } +} + +func TestGetVPCIPv6CIDRBlocks(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/vpc-ipv6-cidr-blocks": "2001:db8::/64", + })} + + ips, err := f.GetVPCIPv6CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, ips, []net.IPNet{{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(64, 128)}}) + } + + nov6 := TypedIMDS{FakeIMDS(map[string]interface{}{ + // NB: IMDS returns 404, not empty string :( + })} + + ips, err = nov6.GetVPCIPv6CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.ElementsMatch(t, ips, []net.IPNet{}) + } + + _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef") + if assert.Error(t, err) { + assert.True(t, IsNotFound(err)) + } +} diff --git a/pkg/awsutils/mocks/awsutils_mocks.go b/pkg/awsutils/mocks/awsutils_mocks.go index a641bf2b2d1..d9789bae0ea 100644 --- a/pkg/awsutils/mocks/awsutils_mocks.go +++ b/pkg/awsutils/mocks/awsutils_mocks.go @@ -19,6 +19,7 @@ package mock_awsutils import ( + net "net" reflect "reflect" awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" @@ -198,10 +199,10 @@ func (mr *MockAPIsMockRecorder) GetIPv4sFromEC2(arg0 interface{}) *gomock.Call { } // GetLocalIPv4 mocks base method -func (m *MockAPIs) GetLocalIPv4() string { +func (m *MockAPIs) GetLocalIPv4() net.IP { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetLocalIPv4") - ret0, _ := ret[0].(string) + ret0, _ := ret[0].(net.IP) return ret0 } diff --git a/pkg/ec2metadata/client.go b/pkg/ec2metadata/client.go deleted file mode 100644 index f5f805aa72e..00000000000 --- a/pkg/ec2metadata/client.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package ec2metadata - -import ( - "github.com/aws/aws-sdk-go/aws" - ec2metadatasvc "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" -) - -// EC2Metadata wraps the methods from the amazon-sdk-go's ec2metadata package -type EC2Metadata interface { - GetMetadata(path string) (string, error) - Region() (string, error) -} - -// New creates a new EC2Metadata object -func New() EC2Metadata { - awsSession := session.Must(session.NewSession(aws.NewConfig(). - WithMaxRetries(10), - )) - return ec2metadatasvc.New(awsSession) -} diff --git a/pkg/ec2metadata/generate_mocks.go b/pkg/ec2metadata/generate_mocks.go deleted file mode 100644 index d6bc1c6982b..00000000000 --- a/pkg/ec2metadata/generate_mocks.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package ec2metadata - -//go:generate go run github.com/golang/mock/mockgen -destination mocks/ec2metadata_mocks.go -copyright_file ../../scripts/copyright.txt . EC2Metadata diff --git a/pkg/ec2metadata/mocks/ec2metadata_mocks.go b/pkg/ec2metadata/mocks/ec2metadata_mocks.go deleted file mode 100644 index 8f4dbc39bd3..00000000000 --- a/pkg/ec2metadata/mocks/ec2metadata_mocks.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. -// - -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadata (interfaces: EC2Metadata) - -// Package mock_ec2metadata is a generated GoMock package. -package mock_ec2metadata - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// MockEC2Metadata is a mock of EC2Metadata interface -type MockEC2Metadata struct { - ctrl *gomock.Controller - recorder *MockEC2MetadataMockRecorder -} - -// MockEC2MetadataMockRecorder is the mock recorder for MockEC2Metadata -type MockEC2MetadataMockRecorder struct { - mock *MockEC2Metadata -} - -// NewMockEC2Metadata creates a new mock instance -func NewMockEC2Metadata(ctrl *gomock.Controller) *MockEC2Metadata { - mock := &MockEC2Metadata{ctrl: ctrl} - mock.recorder = &MockEC2MetadataMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockEC2Metadata) EXPECT() *MockEC2MetadataMockRecorder { - return m.recorder -} - -// GetMetadata mocks base method -func (m *MockEC2Metadata) GetMetadata(arg0 string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetMetadata", arg0) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetMetadata indicates an expected call of GetMetadata -func (mr *MockEC2MetadataMockRecorder) GetMetadata(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetadata", reflect.TypeOf((*MockEC2Metadata)(nil).GetMetadata), arg0) -} - -// Region mocks base method -func (m *MockEC2Metadata) Region() (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Region") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Region indicates an expected call of Region -func (mr *MockEC2MetadataMockRecorder) Region() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Region", reflect.TypeOf((*MockEC2Metadata)(nil).Region)) -} diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 230adabff6f..4c85a07b8d5 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -333,7 +333,7 @@ func (c *IPAMContext) nodeInit() error { } vpcCIDRs := c.awsClient.GetVPCIPv4CIDRs() - primaryIP := net.ParseIP(c.awsClient.GetLocalIPv4()) + primaryIP := c.awsClient.GetLocalIPv4() err = c.networkClient.SetupHostNetwork(vpcCIDRs, c.awsClient.GetPrimaryENImac(), &primaryIP, c.enablePodENI) if err != nil { return errors.Wrap(err, "ipamd init: failed to set up host network") diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index eb830ecf662..e523c6b2ca1 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -122,7 +122,7 @@ func TestNodeInit(t *testing.T) { m.awsutils.EXPECT().DescribeAllENIs().Return(eniMetadataSlice, map[string]awsutils.TagMap{}, "", nil) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - m.awsutils.EXPECT().GetLocalIPv4().Return(ipaddr01) + m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP) var rules []netlink.Rule m.network.EXPECT().GetRuleList().Return(rules, nil)