Merge branch 'master' into strict_na

.github/workflows/deps.yml

@@ -26,14 +26,14 @@ jobs:
       - id: govulncheck
         uses: ./.github/actions/govulncheck
         with:
-          go-version-input: 1.21.6
+          go-version-input: 1.21.7
           go-version-file: go.mod
           cache: false
           repo-checkout: false
       - id: govulncheck-tests-agent
         uses: ./.github/actions/govulncheck
         with:
-          go-version-input: 1.21.6
+          go-version-input: 1.21.7
           go-version-file: test/agent/go.mod
           cache: false
           repo-checkout: false

.github/workflows/weekly-cron-tests.yaml

@@ -44,7 +44,6 @@ jobs:
           RUN_CNI_INTEGRATION_TESTS: false
           PERFORMANCE_TEST_S3_BUCKET_NAME: cni-performance-tests
           RUN_PERFORMANCE_TESTS: true
-          RUN_TESTER_LB_ADDONS: true
         run: |
           ./scripts/run-integration-tests.sh
       - name: Run kops tests
@@ -54,9 +53,8 @@ jobs:
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
           RUN_CNI_INTEGRATION_TESTS: false
           RUN_KOPS_TEST: true
-          RUN_TESTER_LB_ADDONS: true
-          K8S_VERSION: 1.29.0-alpha.2
-          KOPS_VERSION: v1.29.0-alpha.2
+          K8S_VERSION: 1.29.0-alpha.3
+          KOPS_VERSION: v1.29.0-alpha.3
         run: |
           ./scripts/run-integration-tests.sh
         if: always()
@@ -67,7 +65,6 @@ jobs:
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
           RUN_CNI_INTEGRATION_TESTS: false
           RUN_BOTTLEROCKET_TEST: true
-          RUN_TESTER_LB_ADDONS: true
         run: |
           ./scripts/run-integration-tests.sh
         if: always()

CHANGELOG.md

@@ -1,13 +1,22 @@
 # Changelog
 
+## v1.16.3
+
+* Dependency - [Dependabot updates](https://github.com/aws/amazon-vpc-cni-k8s/pull/2775) (@jdn5126 )
+* Dependency - [Upgrade Golang version to 1.21.6](https://github.com/aws/amazon-vpc-cni-k8s/pull/2755) (@jdn5126 )
+* Improvement - [Enable ENABLE_V6_EGRESS on Clusters with Mixed IPv6/IPv4 Subnets](https://github.com/aws/amazon-vpc-cni-k8s/pull/2754) (@sergeylanzman )
+* Improvement - [cni-metrics-helper add podAnnotation value](https://github.com/aws/amazon-vpc-cni-k8s/pull/2748) (@prysmakou )
+* Improvement - [Track max pods, simplify warm IP pool management](https://github.com/aws/amazon-vpc-cni-k8s/pull/2745) (@jdn5126 )
+* Improvement - [Faster eni scaleup](https://github.com/aws/amazon-vpc-cni-k8s/pull/2744) (@jchen6585 )
+
 ## v1.16.2
 
 * Bug - [Refactor IPTable Rules](https://github.com/aws/amazon-vpc-cni-k8s/pull/2697) (@jchen6585 )
 * Bug - [log for DelNetworkReply now differentiates between IPv4 and IPv6 addr…](https://github.com/aws/amazon-vpc-cni-k8s/pull/2742) (@zachdorame )
 * Dependency - [revert CNI spec to 0.4.0](https://github.com/aws/amazon-vpc-cni-k8s/pull/2757) (@jdn5126 )
 * Dependency - [update crypto to patch CVE-2023-48795](https://github.com/aws/amazon-vpc-cni-k8s/pull/2740) (@haouc )
 * Dependency - [Dependabot updates: aws-sdk-go, containernetworking/plugins, go-logr, grpc, k8s.io/cli-runtime](https://github.com/aws/amazon-vpc-cni-k8s/pull/2738) (@jdn5126 )
-* Enhancement - [Iptables mock](https://github.com/aws/amazon-vpc-cni-k8s/pull/2721) (@jchen6585 )
+* Improvement - [Iptables mock](https://github.com/aws/amazon-vpc-cni-k8s/pull/2721) (@jchen6585 )
 
 ## v1.16.0
 

Makefile

@@ -22,7 +22,7 @@
 # VERSION is the source revision that executables and images are built from.
 VERSION ?= $(shell git describe --tags --always --dirty || echo "unknown")
 # GOLANG_IMAGE is the building golang container image used.
-GOLANG_IMAGE ?= public.ecr.aws/eks-distro-build-tooling/golang:1.21.6-7-gcc-al2
+GOLANG_IMAGE ?= public.ecr.aws/eks-distro-build-tooling/golang:1.21.7-8-gcc-al2
 # BASE_IMAGE_CNI is the base layer image for the primary AWS VPC CNI plugin container
 BASE_IMAGE_CNI ?= public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-iptables:latest.2
 # BASE_IMAGE_CNI_INIT is the base layer image for the AWS VPC CNI init container

README.md

@@ -226,7 +226,7 @@ Type: Integer as a String
 
 Default: 9001
 
-Used to configure the MTU size for attached ENIs. The valid range is from `576` to `9001`.
+Used to configure the MTU size for attached ENIs. The valid range for IPv4 is from `576` to `9001`, while the valid range for IPv6 is from `1280` to `9001`.
 
 #### `AWS_VPC_K8S_CNI_EXTERNALSNAT`
 
@@ -267,14 +267,14 @@ Default: empty
 Specify a comma-separated list of IPv4 CIDRs to exclude from SNAT. For every item in the list an `iptables` rule and off\-VPC
 IP rule will be applied. If an item is not a valid ipv4 range it will be skipped. This should be used when `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`.
 
-#### `POD_MTU` (v1.x.x+)
+#### `POD_MTU` (v1.16.4+)
 
 Type: Integer as a String
 
-*Note*: The default value is set to AWS_VPC_ENI_MTU, which defaults to 9001 if unset.
+*Note*: If unset, the default value is derived from `AWS_VPC_ENI_MTU`, which defaults to `9001`.
 Default: 9001
 
-Used to configure the MTU size for pod virtual interfaces. The valid range is from `576` to `9001`.
+Used to configure the MTU size for pod virtual interfaces. The valid range for IPv4 is from `576` to `9001`, while the valid range for IPv6 is from `1280` to `9001`.
 
 #### `WARM_ENI_TARGET`
 
@@ -598,7 +598,7 @@ Setting `ANNOTATE_POD_IP` to `true` will allow IPAMD to add an annotation `vpc.a
 
 There is a known [issue](https://github.com/kubernetes/kubernetes/issues/39113) with kubelet taking time to update `Pod.Status.PodIP` leading to calico being blocked on programming the policy. Setting `ANNOTATE_POD_IP` to `true` will enable AWS VPC CNI plugin to add Pod IP as an annotation to the pod spec to address this race condition.
 
-To annotate the pod with pod IP, you will have to add "patch" permission for pods resource in aws-node clusterrole. You can use the below command -
+To annotate the pod with pod IP, you will have to add `patch` permission for pods resource in aws-node clusterrole. You can use the below command -
 
 ```
 cat << EOF > append.yaml
@@ -615,6 +615,8 @@ EOF
 kubectl apply -f <(cat <(kubectl get clusterrole aws-node -o yaml) append.yaml)
 ```
 
+NOTE: Adding `patch` permissions to the `aws-node` Daemonset increases the security scope for the plugin, so add this permission only after performing a proper security assessment of the tradeoffs.
+
 #### `ENABLE_IPv4` (v1.10.0+)
 
 Type: Boolean as a String

charts/aws-vpc-cni/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: aws-vpc-cni
-version: 1.16.2
-appVersion: "v1.16.2"
+version: 1.16.3
+appVersion: "v1.16.3"
 description: A Helm chart for the AWS VPC CNI
 icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png
 home: https://github.com/aws/amazon-vpc-cni-k8s

charts/aws-vpc-cni/README.md

@@ -48,15 +48,15 @@ The following table lists the configurable parameters for this chart and their d
 | `minimumWindowsIPTarget`| Minimum IP target value for Windows prefix delegation   | `3`                                 |
 | `branchENICooldown`     | Number of seconds that branch ENIs remain in cooldown   | `60`                                |
 | `fullnameOverride`      | Override the fullname of the chart                      | `aws-node`                          |
-| `image.tag`             | Image tag                                               | `v1.16.2`                           |
+| `image.tag`             | Image tag                                               | `v1.16.3`                           |
 | `image.domain`          | ECR repository domain                                   | `amazonaws.com`                     |
 | `image.region`          | ECR repository region to use. Should match your cluster | `us-west-2`                         |
 | `image.endpoint`        | ECR repository endpoint to use.                         | `ecr`                               |
 | `image.account`         | ECR repository account number                           | `602401143452`                      |
 | `image.pullPolicy`      | Container pull policy                                   | `IfNotPresent`                      |
 | `image.override`        | A custom docker image to use                            | `nil`                               |
 | `imagePullSecrets`      | Docker registry pull secret                             | `[]`                                |
-| `init.image.tag`        | Image tag                                               | `v1.16.2`                           |
+| `init.image.tag`        | Image tag                                               | `v1.16.3`                           |
 | `init.image.domain`     | ECR repository domain                                   | `amazonaws.com`                     |
 | `init.image.region`     | ECR repository region to use. Should match your cluster | `us-west-2`                         |
 | `init.image.endpoint`   | ECR repository endpoint to use.                         | `ecr`                               |
@@ -69,7 +69,7 @@ The following table lists the configurable parameters for this chart and their d
 | `originalMatchLabels`   | Use the original daemonset matchLabels                  | `false`                             |
 | `nameOverride`          | Override the name of the chart                          | `aws-node`                          |
 | `nodeAgent.enabled`     | If the Node Agent container should be created           | `true`                              |
-| `nodeAgent.image.tag`   | Image tag for Node Agent                                | `v1.0.7`                            |
+| `nodeAgent.image.tag`   | Image tag for Node Agent                                | `v1.0.8`                            |
 | `nodeAgent.image.domain`| ECR repository domain                                   | `amazonaws.com`                     |
 | `nodeAgent.image.region`| ECR repository region to use. Should match your cluster | `us-west-2`                         |
 | `nodeAgent.image.endpoint`   | ECR repository endpoint to use.                    | `ecr`                               |
@@ -108,25 +108,27 @@ $ helm install aws-vpc-cni --namespace kube-system eks/aws-vpc-cni --values valu
 
 ## Adopting the existing aws-node resources in an EKS cluster
 
-If you do not want to delete the existing aws-node resources in your cluster that run the aws-vpc-cni and then install this helm chart, you can adopt the resources into a release instead. Refer to the script below to import existing resources into helm. Once you have annotated and labeled all the resources this chart specifies, enable the `originalMatchLabels` flag. If you have been careful this should not diff and leave all the resources unmodified and now under management of helm.
+If you do not want to delete the existing aws-node resources in your cluster that run the aws-vpc-cni and then install this helm chart, you can adopt the resources into a release instead. Refer to the script below to import existing resources into helm. Once you have annotated and labeled all the resources this chart specifies, enable the `originalMatchLabels` flag. If you have been careful, this should not diff and leave all the resources unmodified and now under management of helm.
 
-WARNING: Substitute YOUR_HELM_RELEASE_NAME_HERE with the name of your helm release.
 ```
 #!/usr/bin/env bash
 
 set -euo pipefail
 
 for kind in daemonSet clusterRole clusterRoleBinding serviceAccount; do
   echo "setting annotations and labels on $kind/aws-node"
-  kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-name=YOUR_HELM_RELEASE_NAME_HERE
+  kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-name=aws-vpc-cni
   kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-namespace=kube-system
   kubectl -n kube-system label --overwrite $kind aws-node app.kubernetes.io/managed-by=Helm
 done
 
-kubectl -n kube-system annotate --overwrite configmap amazon-vpc-cni meta.helm.sh/release-name=YOUR_HELM_RELEASE_NAME_HERE
+kubectl -n kube-system annotate --overwrite configmap amazon-vpc-cni meta.helm.sh/release-name=aws-vpc-cni
 kubectl -n kube-system annotate --overwrite configmap amazon-vpc-cni meta.helm.sh/release-namespace=kube-system
 kubectl -n kube-system label --overwrite configmap amazon-vpc-cni app.kubernetes.io/managed-by=Helm
 
+Kubernetes recommends using server-side apply for more control over the field manager. After adopting the chart resources, you can run the following command to apply the chart:
+```
+helm template aws-vpc-cni --include-crds --namespace kube-system eks/aws-vpc-cni --set originalMatchLabels=true | kubectl apply --server-side --force-conflicts --field-manager Helm -f -
 ```
 
 ## Migrate from Helm v2 to Helm v3

charts/aws-vpc-cni/values.yaml

@@ -8,7 +8,7 @@ nameOverride: aws-node
 
 init:
   image:
-    tag: v1.16.2
+    tag: v1.16.3
     domain: amazonaws.com
     region: us-west-2
     endpoint: ecr
@@ -27,7 +27,7 @@ init:
 nodeAgent:
   enabled: true
   image:
-    tag: v1.0.7
+    tag: v1.0.8
     domain: amazonaws.com
     region: us-west-2
     endpoint: ecr
@@ -50,7 +50,7 @@ nodeAgent:
   resources: {}
 
 image:
-  tag: v1.16.2
+  tag: v1.16.3
   domain: amazonaws.com
   region: us-west-2
   endpoint: ecr
@@ -83,7 +83,7 @@ env:
   DISABLE_NETWORK_RESOURCE_PROVISIONING: "false"
   ENABLE_IPv4: "true"
   ENABLE_IPv6: "false"
-  VPC_CNI_VERSION: "v1.16.2"
+  VPC_CNI_VERSION: "v1.16.3"
 
 # this flag enables you to use the match label that was present in the original daemonset deployed by EKS
 # You can then annotate and label the original aws-node resources and 'adopt' them into a helm release

charts/cni-metrics-helper/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: cni-metrics-helper
-version: 1.16.2
-appVersion: v1.16.2
+version: 1.16.3
+appVersion: v1.16.3
 description: A Helm chart for the AWS VPC CNI Metrics Helper
 icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png
 home: https://github.com/aws/amazon-vpc-cni-k8s

charts/cni-metrics-helper/README.md

@@ -47,7 +47,7 @@ The following table lists the configurable parameters for this chart and their d
 |------------------------------|---------------------------------------------------------------|--------------------|
 | fullnameOverride             | Override the fullname of the chart                            | cni-metrics-helper |
 | image.region                 | ECR repository region to use. Should match your cluster       | us-west-2          |
-| image.tag                    | Image tag                                                     | v1.16.2            |
+| image.tag                    | Image tag                                                     | v1.16.3            |
 | image.account                | ECR repository account number                                 | 602401143452       |
 | image.domain                 | ECR repository domain                                         | amazonaws.com      |
 | env.USE_CLOUDWATCH           | Whether to export CNI metrics to CloudWatch                   | true               |

charts/cni-metrics-helper/values.yaml

@@ -4,7 +4,7 @@ nameOverride: cni-metrics-helper
 
 image:
   region: us-west-2
-  tag: v1.16.2
+  tag: v1.16.3
   account: "602401143452"
   domain: "amazonaws.com"
   # Set to use custom image

cmd/aws-vpc-cni/main.go

@@ -66,7 +66,9 @@ const (
 	defaultAWSconflistFile       = "/app/10-aws.conflist"
 	tmpAWSconflistFile           = "/tmp/10-aws.conflist"
 	defaultVethPrefix            = "eni"
-	defaultMTU                   = "9001"
+	defaultMTU                   = 9001
+	minMTUv4                     = 576
+	minMTUv6                     = 1280
 	defaultEnablePodEni          = false
 	defaultPodSGEnforcingMode    = "strict"
 	defaultPluginLogFile         = "/var/log/aws-routed-eni/plugin.log"
@@ -279,8 +281,8 @@ func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool)
 		}
 	}
 	vethPrefix := utils.GetEnv(envVethPrefix, defaultVethPrefix)
-	// Derive pod MTU from ENI MTU by default
-	eniMTU := utils.GetEnv(envEniMTU, defaultMTU)
+	// Derive pod MTU from ENI MTU by default (note that values have already been validated)
+	eniMTU := utils.GetEnv(envEniMTU, strconv.Itoa(defaultMTU))
 	// If pod MTU environment variable is set, overwrite ENI MTU.
 	podMTU := utils.GetEnv(envPodMTU, eniMTU)
 	podSGEnforcingMode := utils.GetEnv(envPodSGEnforcingMode, defaultPodSGEnforcingMode)
@@ -389,6 +391,11 @@ func validateEnvVars() bool {
 		return false
 	}
 
+	// Validate MTU value for ENIs and pods
+	if !validateMTU(envEniMTU) || !validateMTU(envPodMTU) {
+		return false
+	}
+
 	prefixDelegationEn := utils.GetBoolAsStringEnvVar(envEnPrefixDelegation, defaultEnPrefixDelegation)
 	warmIPTarget := utils.GetEnv(envWarmIPTarget, "0")
 	warmPrefixTarget := utils.GetEnv(envWarmPrefixTarget, "0")
@@ -402,6 +409,29 @@ func validateEnvVars() bool {
 	return true
 }
 
+func validateMTU(envVar string) bool {
+	// Validate MTU range based on IP address family
+	enabledIPv6 := utils.GetBoolAsStringEnvVar(envEnIPv6, defaultEnableIPv6)
+
+	mtu, err, input := utils.GetIntFromStringEnvVar(envVar, defaultMTU)
+	if err != nil {
+		log.Errorf("%s MUST be a valid integer. %s is invalid", envVar, input)
+		return false
+	}
+	if enabledIPv6 {
+		if mtu < minMTUv6 || mtu > defaultMTU {
+			log.Errorf("%s cannot be less than 1280 or greater than 9001 in IPv6. %s is invalid", envVar, input)
+			return false
+		}
+	} else {
+		if mtu < minMTUv4 || mtu > defaultMTU {
+			log.Errorf("%s cannot be less than 576 or greater than 9001 in IPv4. %s is invalid", envVar, input)
+			return false
+		}
+	}
+	return true
+}
+
 func main() {
 	os.Exit(_main())
 }

cmd/aws-vpc-cni/main_test.go

@@ -47,3 +47,40 @@ func TestGenerateJSONPlusBandwidthAndTuning(t *testing.T) {
 	err := generateJSON(awsConflist, devNull, getPrimaryIPMock)
 	assert.NoError(t, err)
 }
+
+func TestMTUValidation(t *testing.T) {
+	// By default, ENI MTU and pod MTU should be valid
+	assert.True(t, validateMTU(envEniMTU))
+	assert.True(t, validateMTU(envPodMTU))
+
+	// Non-integer values should fail
+	_ = os.Setenv(envEniMTU, "true")
+	_ = os.Setenv(envPodMTU, "abc")
+	assert.False(t, validateMTU(envEniMTU))
+	assert.False(t, validateMTU(envPodMTU))
+
+	// Integer values within IPv4 range should succeed
+	_ = os.Setenv(envEniMTU, "5000")
+	_ = os.Setenv(envPodMTU, "3000")
+	assert.True(t, validateMTU(envEniMTU))
+	assert.True(t, validateMTU(envPodMTU))
+
+	// Integer values outside IPv4 range should fail
+	_ = os.Setenv(envEniMTU, "10000")
+	_ = os.Setenv(envPodMTU, "500")
+	assert.False(t, validateMTU(envEniMTU))
+	assert.False(t, validateMTU(envPodMTU))
+
+	// Integer values within IPv6 range should succeed
+	_ = os.Setenv(envEnIPv6, "true")
+	_ = os.Setenv(envEniMTU, "5000")
+	_ = os.Setenv(envPodMTU, "3000")
+	assert.True(t, validateMTU(envEniMTU))
+	assert.True(t, validateMTU(envPodMTU))
+
+	// Integer values outside IPv6 range should fail
+	_ = os.Setenv(envEniMTU, "10000")
+	_ = os.Setenv(envPodMTU, "1200")
+	assert.False(t, validateMTU(envEniMTU))
+	assert.False(t, validateMTU(envPodMTU))
+}

cmd/routed-eni-cni-plugin/cni.go

@@ -147,7 +147,8 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 		return errors.Wrap(err, "add cmd: failed to load k8s config from arg")
 	}
 
-	mtu := networkutils.GetEthernetMTU(conf.MTU)
+	// Derive pod MTU. Note that the value has already been validated.
+	mtu := networkutils.GetPodMTU(conf.MTU)
 	log.Debugf("MTU value set is %d:", mtu)
 
 	// Set up a connection to the ipamD server.