Add IPv6 egress support to eks IPv4 cluster

README.md

@@ -105,13 +105,23 @@ The following environment variables are available, and all of them are optional.
 
 ---
 
+#### `ENABLE_V6_EGRESS` (v1.13.0+)
+
+Type: Boolean as a String
+
+Default: `false`
+
+Specifies whether PODs in v4 cluster support IPv6 egress. If env is set to `true`, range `fd00::ac:00/118` is reserved for IPv6 egress.
+
+---
+
 #### `AWS_MANAGE_ENIS_NON_SCHEDULABLE` (v1.12.6+)
 
 Type: Boolean as a String
 
 Default: `false`
 
-Specifies whether IPAMD should allocate or deallocate ENIs on a non-schedulable node. 
+Specifies whether IPAMD should allocate or deallocate ENIs on a non-schedulable node.
 
 ---
 

cmd/aws-vpc-cni-init/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/aws/amazon-vpc-cni-k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/utils/cp"
 	"github.com/aws/amazon-vpc-cni-k8s/utils/imds"
+
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -33,10 +34,12 @@ const (
 	metadataMAC                     = "mac"
 	defaultDisableIPv4TcpEarlyDemux = false
 	defaultEnableIPv6               = false
+	defaultEnableIPv6Egress         = false
 
 	envDisableIPv4TcpEarlyDemux = "DISABLE_TCP_EARLY_DEMUX"
 	envEnableIPv6               = "ENABLE_IPv6"
 	envHostCniBinPath           = "HOST_CNI_BIN_PATH"
+	envEgressV6                 = "ENABLE_V6_EGRESS"
 )
 
 func getNodePrimaryIF() (string, error) {
@@ -110,19 +113,22 @@ func configureIPv6Settings(procSys procsyswrapper.ProcSys, primaryIF string) err
 		}
 		val, _ := procSys.Get(entry)
 		log.Infof("Updated %s to %s", entry, val)
-
-		entry = "net/ipv6/conf/all/forwarding"
+	}
+	// Check if IPv6 egress supporting is enabled in IPv4 cluster
+	ipv6EgressEnabled := utils.GetBoolAsStringEnvVar(envEgressV6, defaultEnableIPv6Egress)
+	if enableIPv6 || ipv6EgressEnabled {
+		entry := "net/ipv6/conf/all/forwarding"
 		err = procSys.Set(entry, "1")
 		if err != nil {
-			return errors.Wrap(err, "Failed to enable ipv6 forwarding")
+			return errors.Wrap(err, "Failed to enable IPv6 forwarding")
 		}
-		val, _ = procSys.Get(entry)
+		val, _ := procSys.Get(entry)
 		log.Infof("Updated %s to %s", entry, val)
 
 		entry = "net/ipv6/conf/" + primaryIF + "/accept_ra"
 		err = procSys.Set(entry, "2")
 		if err != nil {
-			return errors.Wrap(err, "Failed to enable ipv6 accept_ra")
+			return errors.Wrap(err, "Failed to enable IPv6 accept_ra")
 		}
 		val, _ = procSys.Get(entry)
 		log.Infof("Updated %s to %s", entry, val)

cmd/aws-vpc-cni/main.go

@@ -41,6 +41,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 
@@ -49,15 +50,18 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/cniutils"
 	"github.com/aws/amazon-vpc-cni-k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/utils/cp"
-	"github.com/aws/amazon-vpc-cni-k8s/utils/imds"
 )
 
 const (
 	egressPluginIpamSubnetV4     = "169.254.172.0/22"
+	egressPluginIpamSubnetV6     = "fd00::ac:00/118"
 	egressPluginIpamDstV4        = "0.0.0.0/0"
+	egressPluginIpamDstV6        = "::/0"
 	egressPluginIpamDataDirV4    = "/run/cni/v6pd/egress-v4-ipam"
+	egressPluginIpamDataDirV6    = "/run/cni/v4pd/egress-v6-ipam"
 	defaultHostCNIBinPath        = "/host/opt/cni/bin"
 	defaultHostCNIConfDirPath    = "/host/etc/cni/net.d"
 	defaultAWSconflistFile       = "/app/10-aws.conflist"
@@ -68,8 +72,10 @@ const (
 	defaultPodSGEnforcingMode    = "strict"
 	defaultPluginLogFile         = "/var/log/aws-routed-eni/plugin.log"
 	defaultEgressV4PluginLogFile = "/var/log/aws-routed-eni/egress-v4-plugin.log"
+	defaultEgressV6PluginLogFile = "/var/log/aws-routed-eni/egress-v6-plugin.log"
 	defaultPluginLogLevel        = "Debug"
-	defaultEnableIPv6            = "false"
+	defaultEnableIPv6            = false
+	defaultEnableIPv6Egress      = false
 	defaultRandomizeSNAT         = "prng"
 	defaultEnableNftables        = false
 	awsConflistFile              = "/10-aws.conflist"
@@ -86,12 +92,14 @@ const (
 	envPluginLogFile         = "AWS_VPC_K8S_PLUGIN_LOG_FILE"
 	envPluginLogLevel        = "AWS_VPC_K8S_PLUGIN_LOG_LEVEL"
 	envEgressV4PluginLogFile = "AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE"
+	envEgressV6PluginLogFile = "AWS_VPC_K8S_EGRESS_V6_PLUGIN_LOG_FILE"
 	envEnPrefixDelegation    = "ENABLE_PREFIX_DELEGATION"
 	envWarmIPTarget          = "WARM_IP_TARGET"
 	envMinIPTarget           = "MINIMUM_IP_TARGET"
 	envWarmPrefixTarget      = "WARM_PREFIX_TARGET"
 	envEnBandwidthPlugin     = "ENABLE_BANDWIDTH_PLUGIN"
 	envEnIPv6                = "ENABLE_IPv6"
+	envEnIPv6Egress          = "ENABLE_V6_EGRESS"
 	envRandomizeSNAT         = "AWS_VPC_K8S_CNI_RANDOMIZESNAT"
 	envEnableNftables        = "ENABLE_NFTABLES"
 )
@@ -196,39 +204,79 @@ func waitForInit() error {
 	}
 }
 
-func getNodePrimaryV4Address() (string, error) {
+func getPrimaryIP(ipv4 bool) (string, error) {
 	var hostIP string
 	var err error
-	for {
-		hostIP, err = imds.GetMetaData("local-ipv4")
-		if err != nil {
-			log.WithError(err).Fatalf("aws-vpc-cni failed")
-			return "", err
-		}
-		if hostIP != "" {
-			return hostIP, nil
+	imdsKey := "local-ipv4"
+	if !ipv4 {
+		imdsKey = "ipv6"
+	}
+
+	hostIP, err = cniutils.GetNodeMetadata(imdsKey)
+	if err != nil {
+		if ipv4 {
+			log.WithError(err).Fatalf("failed to retrieve local-ipv4 address in imds metadata")
+		} else {
+			log.WithError(err).Debugf("failed to retrieve ipv6 address in imds metadata")
 		}
+		return "", err
 	}
+	return hostIP, nil
 }
 
 func isValidJSON(inFile string) error {
 	var result map[string]interface{}
 	return json.Unmarshal([]byte(inFile), &result)
 }
-
-func generateJSON(jsonFile string, outFile string, nodeIP string) error {
+func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool) (string, error)) error {
 	byteValue, err := os.ReadFile(jsonFile)
 	if err != nil {
 		return err
 	}
 
+	// enabledIPv6 is to determine if EKS cluster is IPv4 or IPv6 cluster
+	// if this EKS cluster is IPv6 cluster, egress-cni-plugin will enable IPv4 egress by default
+	// if this EKS cluster is IPv4 cluster, egress-cni-plugin will only enable IPv6 egress if env var "ENABLE_V6_EGRESS" is "true"
+	enabledIPv6 := utils.GetBoolAsStringEnvVar(envEnIPv6, defaultEnableIPv6)
+	var egressIPAMSubnet string
+	var egressIPAMDst string
+	var egressIPAMDataDir string
+	var egressEnabled bool
+	var egressPluginLogFile string
+	var nodeIP = ""
+	if enabledIPv6 {
+		// EKS IPv6 cluster
+		egressIPAMSubnet = egressPluginIpamSubnetV4
+		egressIPAMDst = egressPluginIpamDstV4
+		egressIPAMDataDir = egressPluginIpamDataDirV4
+		egressEnabled = true // enable IPv4 egress by default of IPv6 cluster
+		egressPluginLogFile = utils.GetEnv(envEgressV4PluginLogFile, defaultEgressV4PluginLogFile)
+		nodeIP, err = getPrimaryIP(true)
+		// Node should have a IPv4 address even in IPv6 cluster
+		if err != nil {
+			log.Errorf("Failed to get Node IP, error: %v", err)
+			return err
+		}
+	} else {
+		// EKS IPv4 cluster
+		egressIPAMSubnet = egressPluginIpamSubnetV6
+		egressIPAMDst = egressPluginIpamDstV6
+		egressIPAMDataDir = egressPluginIpamDataDirV6
+		egressPluginLogFile = utils.GetEnv(envEgressV6PluginLogFile, defaultEgressV6PluginLogFile)
+		egressEnabled = utils.GetBoolAsStringEnvVar(envEnIPv6Egress, defaultEnableIPv6Egress)
+		if egressEnabled {
+			nodeIP, err = getPrimaryIP(false)
+			if err != nil {
+				log.Errorf("To support IPv6 egress, node primary ENI must have a global IPv6 address, error: %v", err)
+				return err
+			}
+		}
+	}
 	vethPrefix := utils.GetEnv(envVethPrefix, defaultVethPrefix)
 	mtu := utils.GetEnv(envEniMTU, defaultMTU)
 	podSGEnforcingMode := utils.GetEnv(envPodSGEnforcingMode, defaultPodSGEnforcingMode)
 	pluginLogFile := utils.GetEnv(envPluginLogFile, defaultPluginLogFile)
 	pluginLogLevel := utils.GetEnv(envPluginLogLevel, defaultPluginLogLevel)
-	egressV4pluginLogFile := utils.GetEnv(envEgressV4PluginLogFile, defaultEgressV4PluginLogFile)
-	enabledIPv6 := utils.GetEnv(envEnIPv6, defaultEnableIPv6)
 	randomizeSNAT := utils.GetEnv(envRandomizeSNAT, defaultRandomizeSNAT)
 
 	netconf := string(byteValue)
@@ -237,11 +285,11 @@ func generateJSON(jsonFile string, outFile string, nodeIP string) error {
 	netconf = strings.Replace(netconf, "__PODSGENFORCINGMODE__", podSGEnforcingMode, -1)
 	netconf = strings.Replace(netconf, "__PLUGINLOGFILE__", pluginLogFile, -1)
 	netconf = strings.Replace(netconf, "__PLUGINLOGLEVEL__", pluginLogLevel, -1)
-	netconf = strings.Replace(netconf, "__EGRESSPLUGINLOGFILE__", egressV4pluginLogFile, -1)
-	netconf = strings.Replace(netconf, "__EGRESSPLUGINENABLED__", enabledIPv6, -1)
-	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMSUBNET__", egressPluginIpamSubnetV4, -1)
-	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDST__", egressPluginIpamDstV4, -1)
-	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDATADIR__", egressPluginIpamDataDirV4, -1)
+	netconf = strings.Replace(netconf, "__EGRESSPLUGINLOGFILE__", egressPluginLogFile, -1)
+	netconf = strings.Replace(netconf, "__EGRESSPLUGINENABLED__", strconv.FormatBool(egressEnabled), -1)
+	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMSUBNET__", egressIPAMSubnet, -1)
+	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDST__", egressIPAMDst, -1)
+	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDATADIR__", egressIPAMDataDir, -1)
 	netconf = strings.Replace(netconf, "__RANDOMIZESNAT__", randomizeSNAT, -1)
 	netconf = strings.Replace(netconf, "__NODEIP__", nodeIP, -1)
 
@@ -386,16 +434,8 @@ func _main() int {
 	//	return 1
 	//}
 
-	// Get node IP for conflist
-	var nodeIP string
-	nodeIP, err = getNodePrimaryV4Address()
-	if err != nil {
-		log.Errorf("Failed to get Node IP, error: %v", err)
-		return 1
-	}
-
 	log.Infof("Copying config file... ")
-	err = generateJSON(defaultAWSconflistFile, tmpAWSconflistFile, nodeIP)
+	err = generateJSON(defaultAWSconflistFile, tmpAWSconflistFile, getPrimaryIP)
 	if err != nil {
 		log.WithError(err).Errorf("Failed to generate 10-awsconflist")
 		return 1

cmd/aws-vpc-cni/main_test.go

@@ -13,15 +13,22 @@ const (
 	nodeIP      = "10.0.0.0"
 )
 
+var getPrimaryIPMock = func(ipv4 bool) (string, error) {
+	if ipv4 {
+		return "10.0.0.0", nil
+	}
+	return "2600::", nil
+}
+
 // Validate that generateJSON runs against default conflist without error
 func TestGenerateJSON(t *testing.T) {
-	err := generateJSON(awsConflist, devNull, nodeIP)
+	err := generateJSON(awsConflist, devNull, getPrimaryIPMock)
 	assert.NoError(t, err)
 }
 
 // Validate that generateJSON runs without error when bandwidth plugin is added to default conflist
 func TestGenerateJSONPlusBandwidth(t *testing.T) {
 	_ = os.Setenv(envEnBandwidthPlugin, "true")
-	err := generateJSON(awsConflist, devNull, nodeIP)
+	err := generateJSON(awsConflist, devNull, getPrimaryIPMock)
 	assert.NoError(t, err)
 }