Support routing to external IPs behind service

README.md

@@ -586,7 +586,6 @@ VPC CNI can operate in either IPv4 or IPv6 mode. Setting `ENABLE_IPv6` to `true`
 will configure it in IPv6 mode. IPv6 is only supported in Prefix Delegation mode, so `ENABLE_PREFIX_DELEGATION` needs to be set to `true` if VPC CNI is 
 configured to operate in IPv6 mode. Prefix delegation is only supported on nitro instances. 
 
-
 **Note:** Please make sure that the required IPv6 IAM policy is applied (Refer to [IAM Policy](https://github.com/aws/amazon-vpc-cni-k8s#iam-policy) section above). Dual stack mode isn't yet supported. So, enabling both IPv4 and IPv6 will be treated as invalid configuration. Please refer to the [VPC CNI Feature Matrix](https://github.com/aws/amazon-vpc-cni-k8s#vpc-cni-feature-matrix) section below for additional information.
 
 ---
@@ -602,6 +601,15 @@ VPC CNI uses `iptables-legacy` by default. Setting `ENABLE_NFTABLES` to `true` w
 **Note:** VPC CNI image contains `iptables-legacy` and `iptables-nft`. Switching between them is done via `update-alternatives`. It is *strongly* recommended that the iptables mode matches that which is used by the base OS and `kube-proxy`.
 Switching modes while pods are running or rules are installed will not trigger reconciliation. It is recommended that rules are manually updated or nodes are drained and cordoned before updating. If reloading node, ensure that previous rules are not set to be persisted.
 
+#### `AWS_EXTERNAL_SERVICE_CIDRS` (v1.13.0+)
+
+Type: String
+
+Default: empty
+
+Specify a comma-separated list of IPv4 CIDRs that *must* be routed via main routing table. This is required for secondary ENIs to reach endpoints outside of VPC that are backed by a service.
+For every item in the list, an `ip rule` will be created with a priority greater than the `ip rule` capturing egress traffic from the container. If an item is not a valid IPv4 CIDR, it will be skipped.
+
 ### VPC CNI Feature Matrix
 
 IP Mode | Secondary IP Mode | Prefix Delegation | Security Groups Per Pod | WARM & MIN IP/Prefix Targets | External SNAT

pkg/ipamd/ipamd.go

@@ -632,6 +632,16 @@ func (c *IPAMContext) configureIPRulesForPods() error {
 			log.Warnf("UpdateRuleListBySrc in nodeInit() failed for IP %s: %v", info.IP, err)
 		}
 	}
+
+	// Program IP rules for external service CIDRs and cleanup stale rules.
+	// Note that we can reuse rule list despite it being modified by UpdateRuleListBySrc, as the
+	// modifications touched rules that this function ignores.
+	extServiceCIDRs := c.networkClient.GetExternalServiceCIDRs()
+	err = c.networkClient.UpdateExternalServiceIpRules(rules, extServiceCIDRs)
+	if err != nil {
+		log.Warnf("UpdateExternalServiceIpRules in nodeInit() failed")
+	}
+
 	return nil
 }
 

pkg/ipamd/ipamd_test.go

@@ -170,8 +170,9 @@ func TestNodeInit(t *testing.T) {
 
 	var rules []netlink.Rule
 	m.network.EXPECT().GetRuleList().Return(rules, nil)
-
 	m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any())
+	m.network.EXPECT().GetExternalServiceCIDRs().Return(nil)
+	m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any())
 
 	fakeNode := v1.Node{
 		TypeMeta:   metav1.TypeMeta{Kind: "Node"},
@@ -255,9 +256,9 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) {
 
 	var rules []netlink.Rule
 	m.network.EXPECT().GetRuleList().Return(rules, nil)
-
-	//m.network.EXPECT().UseExternalSNAT().Return(false)
 	m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any())
+	m.network.EXPECT().GetExternalServiceCIDRs().Return(nil)
+	m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any())
 
 	fakeNode := v1.Node{
 		TypeMeta:   metav1.TypeMeta{Kind: "Node"},