Add vlan support to ipamd

cmd/routed-eni-cni-plugin/cni.go

@@ -190,7 +190,6 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 		// build hostVethName
 		// Note: the maximum length for linux interface name is 15
 		hostVethName := generateHostVethName(conf.VethPrefix, conf.Name, args.ContainerID, args.IfName)
-
 		err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu, log)
 	}
 
@@ -308,6 +307,13 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	log.Infof("Received del network response for pod %s namespace %s sandbox %s: %+v", string(k8sArgs.K8S_POD_NAME),
 		string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), r)
 
+	log.Infof("Received del network response for pod %s namespace %s sandbox %s: %+v", string(k8sArgs.K8S_POD_NAME),
+		string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), r)
+
+	if r.IPv4Addr == "" {
+		log.Info("Pod not found, or IPv4Addr is empty. Skipping TeardownNS.")
+		return nil
+	}
 	deletedPodIP := net.ParseIP(r.IPv4Addr)
 	if deletedPodIP != nil {
 		addr := &net.IPNet{

cmd/routed-eni-cni-plugin/driver/driver.go

@@ -440,7 +440,6 @@ func tearDownNS(addr *net.IPNet, table int, netLink netlinkwrapper.NetLink, log
 	toContainerRule.Dst = addr
 	toContainerRule.Priority = toContainerRulePriority
 	err := netLink.RuleDel(toContainerRule)
-
 	if err != nil {
 		log.Errorf("Failed to delete toContainer rule for %s err %v", addr.String(), err)
 	} else {

config/v1.6/aws-k8s-cni.yaml

@@ -13,9 +13,13 @@ rules:
   - apiGroups: [""]
     resources:
       - pods
-      - nodes
       - namespaces
     verbs: ["list", "watch", "get"]
+  - apiGroups: [""]
+    resources:
+      - nodes
+    # Needs "update" to be able to label the node it is running on
+    verbs: ["list", "watch", "get", "update"]
   - apiGroups: ["extensions"]
     resources:
       - daemonsets

pkg/awsutils/awsutils.go

@@ -126,7 +126,7 @@ type APIs interface {
 	GetIPv4sFromEC2(eniID string) (addrList []*ec2.NetworkInterfacePrivateIpAddress, err error)
 
 	// DescribeAllENIs calls EC2 and returns the ENIMetadata and a tag map for each ENI
-	DescribeAllENIs() ([]ENIMetadata, map[string]TagMap, error)
+	DescribeAllENIs() (eniMetadata []ENIMetadata, tagMap map[string]TagMap, trunkENI string, err error)
 
 	// AllocIPAddress allocates an IP address for an ENI
 	AllocIPAddress(eniID string) error
@@ -729,6 +729,7 @@ func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string
 	} else {
 		log.Info("Using same config as the primary interface for the new ENI")
 	}
+
 	var sgs []string
 	for i := range input.Groups {
 		sgs = append(sgs, *input.Groups[i])
@@ -998,11 +999,11 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [
 }
 
 // DescribeAllENIs calls EC2 to refrech the ENIMetadata and tags for all attached ENIs
-func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[string]TagMap, error) {
+func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (eniMetadata []ENIMetadata, tagMap map[string]TagMap, trunkENI string, err error) {
 	// Fetch all local ENI info from metadata
 	allENIs, err := cache.GetAttachedENIs()
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata")
+		return nil, nil, "", errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata")
 	}
 
 	eniMap := make(map[string]ENIMetadata, len(allENIs))
@@ -1047,7 +1048,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
 	}
 
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, "", err
 	}
 
 	// Collect the verified ENIs
@@ -1057,13 +1058,19 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
 	}
 
 	// Collect ENI response into ENI metadata and tags.
-	tagMap := make(map[string]TagMap, len(ec2Response.NetworkInterfaces))
+	tagMap = make(map[string]TagMap, len(ec2Response.NetworkInterfaces))
 	for _, ec2res := range ec2Response.NetworkInterfaces {
 		if ec2res.Attachment != nil && aws.Int64Value(ec2res.Attachment.DeviceIndex) == 0 && !aws.BoolValue(ec2res.Attachment.DeleteOnTermination) {
 			log.Warn("Primary ENI will not get deleted when node terminates because 'delete_on_termination' is set to false")
 		}
 		eniID := aws.StringValue(ec2res.NetworkInterfaceId)
 		eniMetadata := eniMap[eniID]
+		interfaceType := aws.StringValue(ec2res.InterfaceType)
+		log.Infof("XXXXXXX Got interface type: %q", interfaceType) // TODO Debug only...
+		// TODO: This assumes we only have one trunk attached to the node..
+		if interfaceType == "trunk" || interfaceType == "vlan" {
+			trunkENI = eniID
+		}
 		// Check IPv4 addresses
 		logOutOfSyncState(eniID, eniMetadata.IPv4Addresses, ec2res.PrivateIpAddresses)
 		tags := make(map[string]string, len(ec2res.TagSet))
@@ -1078,7 +1085,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
 			tagMap[eniMetadata.ENIID] = tags
 		}
 	}
-	return verifiedENIs, tagMap, nil
+	return verifiedENIs, tagMap, trunkENI, nil
 }
 
 var eniErrorMessageRegex = regexp.MustCompile("'([a-zA-Z0-9-]+)'")

pkg/awsutils/awsutils_test.go

@@ -393,7 +393,7 @@ func TestDescribeAllENIs(t *testing.T) {
 	for _, tc := range testCases {
 		mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(tc.n).Return(result, tc.awsErr)
 		ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata, ec2SVC: mockEC2}
-		_, tags, err := ins.DescribeAllENIs()
+		_, tags, _, err := ins.DescribeAllENIs()
 		assert.Equal(t, tc.expErr, err, tc.name)
 		assert.Equal(t, tc.exptags, tags, tc.name)
 	}

pkg/eniconfig/eniconfig.go

@@ -126,12 +126,17 @@ func (h *Handler) Handle(ctx context.Context, event sdk.Event) error {
 					val = eniConfigDefault
 				}
 			}
-
+			// If value changes
 			if h.controller.myENI != val {
 				h.controller.eniLock.Lock()
 				defer h.controller.eniLock.Unlock()
 				h.controller.myENI = val
 				log.Debugf("Setting myENI to: %s", val)
+				if val != eniConfigDefault {
+					labels := o.GetLabels()
+					labels["vpc.amazonaws.com/eniConfig"] = val
+					o.SetLabels(labels)
+				}
 			}
 		}
 	}

pkg/eniconfig/eniconfig_test.go

@@ -46,7 +46,8 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string,
 	node := corev1.Node{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
-			Name: nodeName,
+			Name:   nodeName,
+			Labels: make(map[string]string),
 		},
 	}
 	accessor, err := meta.Accessor(&node)

pkg/ipamd/datastore/data_store.go

@@ -115,13 +115,15 @@ func (k IPAMKey) String() string {
 	return fmt.Sprintf("%s/%s/%s", k.NetworkName, k.ContainerID, k.IfName)
 }
 
-// ENI represents a single ENI.
+// ENI represents a single ENI. Exported fields will be marshaled for introspection.
 type ENI struct {
 	// AWS ENI ID
 	ID         string
 	createTime time.Time
 	// IsPrimary indicates whether ENI is a primary ENI
 	IsPrimary bool
+	// IsTrunk indicates whether this ENI is used to provide pods with dedicated ENIs
+	IsTrunk bool
 	// DeviceNumber is the device number of ENI (0 means the primary ENI)
 	DeviceNumber int
 	// IPv4Addresses shows whether each address is assigned, the key is IP address, which must
@@ -355,7 +357,7 @@ func (ds *DataStore) writeBackingStoreUnsafe() error {
 }
 
 // AddENI add ENI to data store
-func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) error {
+func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk bool) error {
 	ds.lock.Lock()
 	defer ds.lock.Unlock()
 
@@ -368,6 +370,7 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) erro
 	ds.eniPool[eniID] = &ENI{
 		createTime:    time.Now(),
 		IsPrimary:     isPrimary,
+		IsTrunk:       isTrunk,
 		ID:            eniID,
 		DeviceNumber:  deviceNumber,
 		IPv4Addresses: make(map[string]*AddressInfo)}
@@ -505,6 +508,17 @@ func (ds *DataStore) GetStats() (int, int) {
 	return ds.total, ds.assigned
 }
 
+func (ds *DataStore) GetTrunkENI() string {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	for _, eni := range ds.eniPool {
+		if eni.IsTrunk {
+			return eni.ID
+		}
+	}
+	return ""
+}
+
 // IsRequiredForWarmIPTarget determines if this ENI has warm IPs that are required to fulfill whatever WARM_IP_TARGET is
 // set to.
 func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool {
@@ -561,6 +575,11 @@ func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI
 			continue
 		}
 
+		if eni.IsTrunk {
+			ds.log.Debugf("ENI %s cannot be deleted because it is a trunk ENI", eni.ID)
+			continue
+		}
+
 		ds.log.Debugf("getDeletableENI: found a deletable ENI %s", eni.ID)
 		return eni
 	}

pkg/ipamd/datastore/data_store_test.go

@@ -34,13 +34,13 @@ var log = logger.New(&logConfig)
 func TestAddENI(t *testing.T) {
 	ds := NewDataStore(log, NullCheckpoint{})
 
-	err := ds.AddENI("eni-1", 1, true)
+	err := ds.AddENI("eni-1", 1, true, false)
 	assert.NoError(t, err)
 
-	err = ds.AddENI("eni-1", 1, true)
+	err = ds.AddENI("eni-1", 1, true, false)
 	assert.Error(t, err)
 
-	err = ds.AddENI("eni-2", 2, false)
+	err = ds.AddENI("eni-2", 2, false, false)
 	assert.NoError(t, err)
 
 	assert.Equal(t, len(ds.eniPool), 2)
@@ -52,13 +52,13 @@ func TestAddENI(t *testing.T) {
 func TestDeleteENI(t *testing.T) {
 	ds := NewDataStore(log, NullCheckpoint{})
 
-	err := ds.AddENI("eni-1", 1, true)
+	err := ds.AddENI("eni-1", 1, true, false)
 	assert.NoError(t, err)
 
-	err = ds.AddENI("eni-2", 2, false)
+	err = ds.AddENI("eni-2", 2, false, false)
 	assert.NoError(t, err)
 
-	err = ds.AddENI("eni-3", 3, false)
+	err = ds.AddENI("eni-3", 3, false, false)
 	assert.NoError(t, err)
 
 	eniInfos := ds.GetENIInfos()
@@ -95,10 +95,10 @@ func TestDeleteENI(t *testing.T) {
 func TestAddENIIPv4Address(t *testing.T) {
 	ds := NewDataStore(log, NullCheckpoint{})
 
-	err := ds.AddENI("eni-1", 1, true)
+	err := ds.AddENI("eni-1", 1, true, false)
 	assert.NoError(t, err)
 
-	err = ds.AddENI("eni-2", 2, false)
+	err = ds.AddENI("eni-2", 2, false, false)
 	assert.NoError(t, err)
 
 	err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
@@ -133,10 +133,10 @@ func TestAddENIIPv4Address(t *testing.T) {
 func TestGetENIIPs(t *testing.T) {
 	ds := NewDataStore(log, NullCheckpoint{})
 
-	err := ds.AddENI("eni-1", 1, true)
+	err := ds.AddENI("eni-1", 1, true, false)
 	assert.NoError(t, err)
 
-	err = ds.AddENI("eni-2", 2, false)
+	err = ds.AddENI("eni-2", 2, false, false)
 	assert.NoError(t, err)
 
 	err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
@@ -165,7 +165,7 @@ func TestGetENIIPs(t *testing.T) {
 
 func TestDelENIIPv4Address(t *testing.T) {
 	ds := NewDataStore(log, NullCheckpoint{})
-	err := ds.AddENI("eni-1", 1, true)
+	err := ds.AddENI("eni-1", 1, true, false)
 	assert.NoError(t, err)
 
 	err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
@@ -218,11 +218,14 @@ func TestPodIPv4Address(t *testing.T) {
 	checkpoint := NewTestCheckpoint(struct{}{})
 	ds := NewDataStore(log, checkpoint)
 
-	ds.AddENI("eni-1", 1, true)
+	err := ds.AddENI("eni-1", 1, true, false)
+	assert.NoError(t, err)
 
-	ds.AddENI("eni-2", 2, false)
+	err = ds.AddENI("eni-2", 2, false, false)
+	assert.NoError(t, err)
 
-	ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
+	err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
+	assert.NoError(t, err)
 
 	key1 := IPAMKey{"net0", "sandbox-1", "eth0"}
 	ip, _, err := ds.AssignPodIPv4Address(key1)
@@ -242,7 +245,8 @@ func TestPodIPv4Address(t *testing.T) {
 	podsInfos := ds.AllocatedIPs()
 	assert.Equal(t, len(podsInfos), 1)
 
-	ds.AddIPv4AddressToStore("eni-2", "1.1.2.2")
+	err = ds.AddIPv4AddressToStore("eni-2", "1.1.2.2")
+	assert.NoError(t, err)
 
 	// duplicate add
 	ip, _, err = ds.AssignPodIPv4Address(key1) // same id
@@ -280,7 +284,8 @@ func TestPodIPv4Address(t *testing.T) {
 	podsInfos = ds.AllocatedIPs()
 	assert.Equal(t, len(podsInfos), 2)
 
-	ds.AddIPv4AddressToStore("eni-1", "1.1.1.2")
+	err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2")
+	assert.NoError(t, err)
 
 	key3 := IPAMKey{"net0", "sandbox-3", "eth0"}
 	ip, _, err = ds.AssignPodIPv4Address(key3)
@@ -328,23 +333,23 @@ func TestPodIPv4Address(t *testing.T) {
 func TestWarmENIInteractions(t *testing.T) {
 	ds := NewDataStore(log, NullCheckpoint{})
 
-	ds.AddENI("eni-1", 1, true)
-	ds.AddENI("eni-2", 2, false)
-	ds.AddENI("eni-3", 3, false)
+	_ = ds.AddENI("eni-1", 1, true, false)
+	_ = ds.AddENI("eni-2", 2, false, false)
+	_ = ds.AddENI("eni-3", 3, false, false)
 
-	ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
+	_ = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
 	key1 := IPAMKey{"net0", "sandbox-1", "eth0"}
 	_, _, err := ds.AssignPodIPv4Address(key1)
 	assert.NoError(t, err)
 
-	ds.AddIPv4AddressToStore("eni-1", "1.1.1.2")
+	_ = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2")
 	key2 := IPAMKey{"net0", "sandbox-2", "eth0"}
 	_, _, err = ds.AssignPodIPv4Address(key2)
 	assert.NoError(t, err)
 
-	ds.AddIPv4AddressToStore("eni-2", "1.1.2.1")
-	ds.AddIPv4AddressToStore("eni-2", "1.1.2.2")
-	ds.AddIPv4AddressToStore("eni-3", "1.1.3.1")
+	_ = ds.AddIPv4AddressToStore("eni-2", "1.1.2.1")
+	_ = ds.AddIPv4AddressToStore("eni-2", "1.1.2.2")
+	_ = ds.AddIPv4AddressToStore("eni-3", "1.1.3.1")
 
 	noWarmIPTarget := 0