Add ipamd changes for sg support

cmd/cni-metrics-helper/README.md

@@ -15,10 +15,10 @@ The following diagram shows how `cni-metrics-helper` works in a cluster:
 
 ### Installing the cni-metrics-helper
 ```
-kubectl apply -f v1.5/cni-metrics-helper.yaml
+kubectl apply -f v1.6/cni-metrics-helper.yaml
 ```
 
-Adding this will publish the following metrics to CloudWatch:
+Adding the CNI metrics helper will publish the following metrics to CloudWatch:
 ```
 "addReqCount",
 "assignIPAddresses",
@@ -31,6 +31,7 @@ Adding this will publish the following metrics to CloudWatch:
 "ipamdActionInProgress",
 "ipamdErr",
 "maxIPAddresses",
+"podENIErr",
 "reconcileCount",
 "totalIPAddresses",
 ```

cmd/cni-metrics-helper/metrics/cni_metrics.go

@@ -128,6 +128,13 @@ var InterestingCNIMetrics = map[string]metricsConvert{
 				actionFunc: metricsAdd,
 				data:       &dataPoints{},
 				logToFile:  true}}},
+	"awscni_pod_eni_error_count": {
+		actions: []metricsAction{
+			{cwMetricName: "podENIErr",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
 }
 
 // CNIMetricsTarget defines data structure for kube-state-metric target

config/master/aws-k8s-cni-cn.yaml

@@ -29,12 +29,20 @@
   - ""
   "resources":
   - "pods"
-  - "nodes"
   - "namespaces"
   "verbs":
   - "list"
   - "watch"
   - "get"
+- "apiGroups":
+  - ""
+  "resources":
+  - "nodes"
+  "verbs":
+  - "list"
+  - "watch"
+  - "get"
+  - "update"
 - "apiGroups":
   - "extensions"
   "resources":
@@ -135,6 +143,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "ENABLE_POD_ENI"
+          "value": "false"
         - "name": "MY_NODE_NAME"
           "valueFrom":
             "fieldRef":

config/master/aws-k8s-cni-us-gov-east-1.yaml

@@ -29,12 +29,20 @@
   - ""
   "resources":
   - "pods"
-  - "nodes"
   - "namespaces"
   "verbs":
   - "list"
   - "watch"
   - "get"
+- "apiGroups":
+  - ""
+  "resources":
+  - "nodes"
+  "verbs":
+  - "list"
+  - "watch"
+  - "get"
+  - "update"
 - "apiGroups":
   - "extensions"
   "resources":
@@ -135,6 +143,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "ENABLE_POD_ENI"
+          "value": "false"
         - "name": "MY_NODE_NAME"
           "valueFrom":
             "fieldRef":

config/master/aws-k8s-cni-us-gov-west-1.yaml

@@ -29,12 +29,20 @@
   - ""
   "resources":
   - "pods"
-  - "nodes"
   - "namespaces"
   "verbs":
   - "list"
   - "watch"
   - "get"
+- "apiGroups":
+  - ""
+  "resources":
+  - "nodes"
+  "verbs":
+  - "list"
+  - "watch"
+  - "get"
+  - "update"
 - "apiGroups":
   - "extensions"
   "resources":
@@ -135,6 +143,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "ENABLE_POD_ENI"
+          "value": "false"
         - "name": "MY_NODE_NAME"
           "valueFrom":
             "fieldRef":

config/master/aws-k8s-cni.yaml

@@ -29,12 +29,20 @@
   - ""
   "resources":
   - "pods"
-  - "nodes"
   - "namespaces"
   "verbs":
   - "list"
   - "watch"
   - "get"
+- "apiGroups":
+  - ""
+  "resources":
+  - "nodes"
+  "verbs":
+  - "list"
+  - "watch"
+  - "get"
+  - "update"
 - "apiGroups":
   - "extensions"
   "resources":
@@ -135,6 +143,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "ENABLE_POD_ENI"
+          "value": "false"
         - "name": "MY_NODE_NAME"
           "valueFrom":
             "fieldRef":

config/master/manifests.jsonnet

@@ -40,9 +40,14 @@ local awsnode = {
       },
       {
         apiGroups: [""],
-        resources: ["pods", "nodes", "namespaces"],
+        resources: ["pods", "namespaces"],
         verbs: ["list", "watch", "get"],
       },
+      {
+        apiGroups: [""],
+        resources: ["nodes"],
+        verbs: ["list", "watch", "get", "update"],
+      },
       {
         apiGroups: ["extensions"],
         resources: ["*"],
@@ -155,26 +160,27 @@ local awsnode = {
                 initialDelaySeconds: 60,
               },
               env_:: {
+                ADDITIONAL_ENI_TAGS: "{}",
                 AWS_VPC_CNI_NODE_PORT_SUPPORT: "true",
-                AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG: "false",
                 AWS_VPC_ENI_MTU: "9001",
+                AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER: "false",
+                AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG: "false",
                 AWS_VPC_K8S_CNI_EXTERNALSNAT: "false",
-                AWS_VPC_K8S_CNI_RANDOMIZESNAT: "prng",
-                WARM_ENI_TARGET: "1",
                 AWS_VPC_K8S_CNI_LOGLEVEL: "DEBUG",
                 AWS_VPC_K8S_CNI_LOG_FILE: "/host/var/log/aws-routed-eni/ipamd.log",
+                AWS_VPC_K8S_CNI_RANDOMIZESNAT: "prng",
+                AWS_VPC_K8S_CNI_VETHPREFIX: "eni",
                 AWS_VPC_K8S_PLUGIN_LOG_FILE: "/var/log/aws-routed-eni/plugin.log",
                 AWS_VPC_K8S_PLUGIN_LOG_LEVEL: "DEBUG",
                 DISABLE_INTROSPECTION: "false",
                 DISABLE_METRICS: "false",
-                AWS_VPC_K8S_CNI_VETHPREFIX: "eni",
-                ADDITIONAL_ENI_TAGS: "{}",
-                AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER: "false",
+                ENABLE_POD_ENI: "false",
                 MY_NODE_NAME: {
                   valueFrom: {
                     fieldRef: {fieldPath: "spec.nodeName"},
                   },
                 },
+                WARM_ENI_TARGET: "1",
               },
               env: [
                 {name: kv[0]} + if std.isObject(kv[1]) then kv[1] else {value: kv[1]}

pkg/awsutils/awsutils.go

@@ -126,7 +126,7 @@ type APIs interface {
 	GetIPv4sFromEC2(eniID string) (addrList []*ec2.NetworkInterfacePrivateIpAddress, err error)
 
 	// DescribeAllENIs calls EC2 and returns the ENIMetadata and a tag map for each ENI
-	DescribeAllENIs() ([]ENIMetadata, map[string]TagMap, error)
+	DescribeAllENIs() (eniMetadata []ENIMetadata, tagMap map[string]TagMap, trunkENI string, err error)
 
 	// AllocIPAddress allocates an IP address for an ENI
 	AllocIPAddress(eniID string) error
@@ -729,6 +729,7 @@ func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string
 	} else {
 		log.Info("Using same config as the primary interface for the new ENI")
 	}
+
 	var sgs []string
 	for i := range input.Groups {
 		sgs = append(sgs, *input.Groups[i])
@@ -998,11 +999,11 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [
 }
 
 // DescribeAllENIs calls EC2 to refrech the ENIMetadata and tags for all attached ENIs
-func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[string]TagMap, error) {
+func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (eniMetadata []ENIMetadata, tagMap map[string]TagMap, trunkENI string, err error) {
 	// Fetch all local ENI info from metadata
 	allENIs, err := cache.GetAttachedENIs()
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata")
+		return nil, nil, "", errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata")
 	}
 
 	eniMap := make(map[string]ENIMetadata, len(allENIs))
@@ -1047,7 +1048,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
 	}
 
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, "", err
 	}
 
 	// Collect the verified ENIs
@@ -1057,13 +1058,18 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
 	}
 
 	// Collect ENI response into ENI metadata and tags.
-	tagMap := make(map[string]TagMap, len(ec2Response.NetworkInterfaces))
+	tagMap = make(map[string]TagMap, len(ec2Response.NetworkInterfaces))
 	for _, ec2res := range ec2Response.NetworkInterfaces {
 		if ec2res.Attachment != nil && aws.Int64Value(ec2res.Attachment.DeviceIndex) == 0 && !aws.BoolValue(ec2res.Attachment.DeleteOnTermination) {
 			log.Warn("Primary ENI will not get deleted when node terminates because 'delete_on_termination' is set to false")
 		}
 		eniID := aws.StringValue(ec2res.NetworkInterfaceId)
 		eniMetadata := eniMap[eniID]
+		interfaceType := aws.StringValue(ec2res.InterfaceType)
+		// This assumes we only have one trunk attached to the node..
+		if interfaceType == "trunk" {
+			trunkENI = eniID
+		}
 		// Check IPv4 addresses
 		logOutOfSyncState(eniID, eniMetadata.IPv4Addresses, ec2res.PrivateIpAddresses)
 		tags := make(map[string]string, len(ec2res.TagSet))
@@ -1078,7 +1084,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
 			tagMap[eniMetadata.ENIID] = tags
 		}
 	}
-	return verifiedENIs, tagMap, nil
+	return verifiedENIs, tagMap, trunkENI, nil
 }
 
 var eniErrorMessageRegex = regexp.MustCompile("'([a-zA-Z0-9-]+)'")

pkg/awsutils/awsutils_test.go

@@ -393,7 +393,7 @@ func TestDescribeAllENIs(t *testing.T) {
 	for _, tc := range testCases {
 		mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(tc.n).Return(result, tc.awsErr)
 		ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata, ec2SVC: mockEC2}
-		_, tags, err := ins.DescribeAllENIs()
+		_, tags, _, err := ins.DescribeAllENIs()
 		assert.Equal(t, tc.expErr, err, tc.name)
 		assert.Equal(t, tc.exptags, tags, tc.name)
 	}

pkg/eniconfig/eniconfig.go

@@ -126,12 +126,17 @@ func (h *Handler) Handle(ctx context.Context, event sdk.Event) error {
 					val = eniConfigDefault
 				}
 			}
-
+			// If value changes
 			if h.controller.myENI != val {
 				h.controller.eniLock.Lock()
 				defer h.controller.eniLock.Unlock()
 				h.controller.myENI = val
 				log.Debugf("Setting myENI to: %s", val)
+				if val != eniConfigDefault {
+					labels := o.GetLabels()
+					labels["vpc.amazonaws.com/eniConfig"] = val
+					o.SetLabels(labels)
+				}
 			}
 		}
 	}

pkg/eniconfig/eniconfig_test.go

@@ -46,7 +46,8 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string,
 	node := corev1.Node{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
-			Name: nodeName,
+			Name:   nodeName,
+			Labels: make(map[string]string),
 		},
 	}
 	accessor, err := meta.Accessor(&node)

pkg/ipamd/datastore/data_store.go

@@ -115,13 +115,15 @@ func (k IPAMKey) String() string {
 	return fmt.Sprintf("%s/%s/%s", k.NetworkName, k.ContainerID, k.IfName)
 }
 
-// ENI represents a single ENI.
+// ENI represents a single ENI. Exported fields will be marshaled for introspection.
 type ENI struct {
 	// AWS ENI ID
 	ID         string
 	createTime time.Time
 	// IsPrimary indicates whether ENI is a primary ENI
 	IsPrimary bool
+	// IsTrunk indicates whether this ENI is used to provide pods with dedicated ENIs
+	IsTrunk bool
 	// DeviceNumber is the device number of ENI (0 means the primary ENI)
 	DeviceNumber int
 	// IPv4Addresses shows whether each address is assigned, the key is IP address, which must
@@ -355,7 +357,7 @@ func (ds *DataStore) writeBackingStoreUnsafe() error {
 }
 
 // AddENI add ENI to data store
-func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) error {
+func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk bool) error {
 	ds.lock.Lock()
 	defer ds.lock.Unlock()
 
@@ -368,6 +370,7 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) erro
 	ds.eniPool[eniID] = &ENI{
 		createTime:    time.Now(),
 		IsPrimary:     isPrimary,
+		IsTrunk:       isTrunk,
 		ID:            eniID,
 		DeviceNumber:  deviceNumber,
 		IPv4Addresses: make(map[string]*AddressInfo)}
@@ -505,6 +508,17 @@ func (ds *DataStore) GetStats() (int, int) {
 	return ds.total, ds.assigned
 }
 
+func (ds *DataStore) GetTrunkENI() string {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	for _, eni := range ds.eniPool {
+		if eni.IsTrunk {
+			return eni.ID
+		}
+	}
+	return ""
+}
+
 // IsRequiredForWarmIPTarget determines if this ENI has warm IPs that are required to fulfill whatever WARM_IP_TARGET is
 // set to.
 func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool {
@@ -561,6 +575,11 @@ func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI
 			continue
 		}
 
+		if eni.IsTrunk {
+			ds.log.Debugf("ENI %s cannot be deleted because it is a trunk ENI", eni.ID)
+			continue
+		}
+
 		ds.log.Debugf("getDeletableENI: found a deletable ENI %s", eni.ID)
 		return eni
 	}