Skip to content

Commit

Permalink
Add IPv6 egress support to EKS IPv4 cluster
Browse files Browse the repository at this point in the history
  • Loading branch information
@wanyufe
wanyufe committed May 2, 2023
1 parent 62dbe0d commit 36e44ac
Show file tree
Hide file tree
Showing 13 changed files with 839 additions and 240 deletions.
16 changes: 11 additions & 5 deletions cmd/aws-vpc-cni-init/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/aws/amazon-vpc-cni-k8s/utils"
"github.com/aws/amazon-vpc-cni-k8s/utils/cp"
"github.com/aws/amazon-vpc-cni-k8s/utils/imds"

"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
Expand All @@ -33,10 +34,12 @@ const (
metadataMAC = "mac"
defaultDisableIPv4TcpEarlyDemux = false
defaultEnableIPv6 = false
defaultEnableIPv6Egress = false

envDisableIPv4TcpEarlyDemux = "DISABLE_TCP_EARLY_DEMUX"
envEnableIPv6 = "ENABLE_IPv6"
envHostCniBinPath = "HOST_CNI_BIN_PATH"
envEgressV6 = "ENABLE_V6_EGRESS"
)

func getNodePrimaryIF() (string, error) {
Expand Down Expand Up @@ -110,19 +113,22 @@ func configureIPv6Settings(procSys procsyswrapper.ProcSys, primaryIF string) err
}
val, _ := procSys.Get(entry)
log.Infof("Updated %s to %s", entry, val)

entry = "net/ipv6/conf/all/forwarding"
}
// Check if IPv6 egress supporting is enabled in IPv4 cluster
ipv6EgressEnabled := utils.GetBoolAsStringEnvVar(envEgressV6, defaultEnableIPv6Egress)
if enableIPv6 || ipv6EgressEnabled {
entry := "net/ipv6/conf/all/forwarding"
err = procSys.Set(entry, "1")
if err != nil {
return errors.Wrap(err, "Failed to enable ipv6 forwarding")
return errors.Wrap(err, "Failed to enable IPv6 forwarding")
}
val, _ = procSys.Get(entry)
val, _ := procSys.Get(entry)
log.Infof("Updated %s to %s", entry, val)

entry = "net/ipv6/conf/" + primaryIF + "/accept_ra"
err = procSys.Set(entry, "2")
if err != nil {
return errors.Wrap(err, "Failed to enable ipv6 accept_ra")
return errors.Wrap(err, "Failed to enable IPv6 accept_ra")
}
val, _ = procSys.Get(entry)
log.Infof("Updated %s to %s", entry, val)
Expand Down
98 changes: 69 additions & 29 deletions cmd/aws-vpc-cni/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
"net"
"os"
"os/exec"
"strconv"
"strings"
"time"

Expand All @@ -49,15 +50,18 @@ import (

"github.com/containernetworking/cni/pkg/types"

"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/cniutils"
"github.com/aws/amazon-vpc-cni-k8s/utils"
"github.com/aws/amazon-vpc-cni-k8s/utils/cp"
"github.com/aws/amazon-vpc-cni-k8s/utils/imds"
)

const (
egressPluginIpamSubnetV4 = "169.254.172.0/22"
egressPluginIpamSubnetV6 = "fd00::/8"
egressPluginIpamDstV4 = "0.0.0.0/0"
egressPluginIpamDstV6 = "::/0"
egressPluginIpamDataDirV4 = "/run/cni/v6pd/egress-v4-ipam"
egressPluginIpamDataDirV6 = "/run/cni/v4pd/egress-v6-ipam"
defaultHostCNIBinPath = "/host/opt/cni/bin"
defaultHostCNIConfDirPath = "/host/etc/cni/net.d"
defaultAWSconflistFile = "/app/10-aws.conflist"
Expand All @@ -68,8 +72,10 @@ const (
defaultPodSGEnforcingMode = "strict"
defaultPluginLogFile = "/var/log/aws-routed-eni/plugin.log"
defaultEgressV4PluginLogFile = "/var/log/aws-routed-eni/egress-v4-plugin.log"
defaultEgressV6PluginLogFile = "/var/log/aws-routed-eni/egress-v6-plugin.log"
defaultPluginLogLevel = "Debug"
defaultEnableIPv6 = "false"
defaultEnableIPv6 = false
defaultEnableIPv6Egress = false
defaultRandomizeSNAT = "prng"
defaultEnableNftables = false
awsConflistFile = "/10-aws.conflist"
Expand All @@ -86,12 +92,14 @@ const (
envPluginLogFile = "AWS_VPC_K8S_PLUGIN_LOG_FILE"
envPluginLogLevel = "AWS_VPC_K8S_PLUGIN_LOG_LEVEL"
envEgressV4PluginLogFile = "AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE"
envEgressV6PluginLogFile = "AWS_VPC_K8S_EGRESS_V6_PLUGIN_LOG_FILE"
envEnPrefixDelegation = "ENABLE_PREFIX_DELEGATION"
envWarmIPTarget = "WARM_IP_TARGET"
envMinIPTarget = "MINIMUM_IP_TARGET"
envWarmPrefixTarget = "WARM_PREFIX_TARGET"
envEnBandwidthPlugin = "ENABLE_BANDWIDTH_PLUGIN"
envEnIPv6 = "ENABLE_IPv6"
envEnIPv6Egress = "ENABLE_V6_EGRESS"
envRandomizeSNAT = "AWS_VPC_K8S_CNI_RANDOMIZESNAT"
envEnableNftables = "ENABLE_NFTABLES"
)
Expand Down Expand Up @@ -196,39 +204,79 @@ func waitForInit() error {
}
}

func getNodePrimaryV4Address() (string, error) {
func getPrimaryIP(ipv4 bool) (string, error) {
var hostIP string
var err error
for {
hostIP, err = imds.GetMetaData("local-ipv4")
if err != nil {
log.WithError(err).Fatalf("aws-vpc-cni failed")
return "", err
}
if hostIP != "" {
return hostIP, nil
imdsKey := "local-ipv4"
if !ipv4 {
imdsKey = "ipv6"
}

hostIP, err = cniutils.GetNodeMetadata(imdsKey)
if err != nil {
if ipv4 {
log.WithError(err).Fatalf("failed to retrieve local-ipv4 address in imds metadata")
} else {
log.WithError(err).Debugf("failed to retrieve ipv6 address in imds metadata")
}
return "", err
}
return hostIP, nil
}

func isValidJSON(inFile string) error {
var result map[string]interface{}
return json.Unmarshal([]byte(inFile), &result)
}

func generateJSON(jsonFile string, outFile string, nodeIP string) error {
func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool) (string, error)) error {
byteValue, err := os.ReadFile(jsonFile)
if err != nil {
return err
}

// enabledIPv6 is to determine if EKS cluster is IPv4 or IPv6 cluster
// if this EKS cluster is IPv6 cluster, egress-cni-plugin will enable IPv4 egress by default
// if this EKS cluster is IPv4 cluster, egress-cni-plugin will only enable IPv6 egress if env var "ENABLE_V6_EGRESS" is "true"
enabledIPv6 := utils.GetBoolAsStringEnvVar(envEnIPv6, defaultEnableIPv6)
var egressIPAMSubnet string
var egressIPAMDst string
var egressIPAMDataDir string
var egressEnabled bool
var egressPluginLogFile string
var nodeIP = ""
if enabledIPv6 {
// EKS IPv6 cluster
egressIPAMSubnet = egressPluginIpamSubnetV4
egressIPAMDst = egressPluginIpamDstV4
egressIPAMDataDir = egressPluginIpamDataDirV4
egressEnabled = true // enable IPv4 egress by default of IPv6 cluster
egressPluginLogFile = utils.GetEnv(envEgressV4PluginLogFile, defaultEgressV4PluginLogFile)
nodeIP, err = getPrimaryIP(true)
// Node should have a IPv4 address even in IPv6 cluster
if err != nil {
log.Errorf("Failed to get Node IP, error: %v", err)
return err
}
} else {
// EKS IPv4 cluster
egressIPAMSubnet = egressPluginIpamSubnetV6
egressIPAMDst = egressPluginIpamDstV6
egressIPAMDataDir = egressPluginIpamDataDirV6
egressPluginLogFile = utils.GetEnv(envEgressV6PluginLogFile, defaultEgressV6PluginLogFile)
egressEnabled = utils.GetBoolAsStringEnvVar(envEnIPv6Egress, defaultEnableIPv6Egress)
if egressEnabled {
nodeIP, err = getPrimaryIP(false)
if err != nil {
log.Errorf("To support IPv6 egress, node primary ENI must have a global IPv6 address, error: %v", err)
return err
}
}
}
vethPrefix := utils.GetEnv(envVethPrefix, defaultVethPrefix)
mtu := utils.GetEnv(envEniMTU, defaultMTU)
podSGEnforcingMode := utils.GetEnv(envPodSGEnforcingMode, defaultPodSGEnforcingMode)
pluginLogFile := utils.GetEnv(envPluginLogFile, defaultPluginLogFile)
pluginLogLevel := utils.GetEnv(envPluginLogLevel, defaultPluginLogLevel)
egressV4pluginLogFile := utils.GetEnv(envEgressV4PluginLogFile, defaultEgressV4PluginLogFile)
enabledIPv6 := utils.GetEnv(envEnIPv6, defaultEnableIPv6)
randomizeSNAT := utils.GetEnv(envRandomizeSNAT, defaultRandomizeSNAT)

netconf := string(byteValue)
Expand All @@ -237,11 +285,11 @@ func generateJSON(jsonFile string, outFile string, nodeIP string) error {
netconf = strings.Replace(netconf, "__PODSGENFORCINGMODE__", podSGEnforcingMode, -1)
netconf = strings.Replace(netconf, "__PLUGINLOGFILE__", pluginLogFile, -1)
netconf = strings.Replace(netconf, "__PLUGINLOGLEVEL__", pluginLogLevel, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINLOGFILE__", egressV4pluginLogFile, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINENABLED__", enabledIPv6, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMSUBNET__", egressPluginIpamSubnetV4, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDST__", egressPluginIpamDstV4, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDATADIR__", egressPluginIpamDataDirV4, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINLOGFILE__", egressPluginLogFile, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINENABLED__", strconv.FormatBool(egressEnabled), -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMSUBNET__", egressIPAMSubnet, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDST__", egressIPAMDst, -1)
netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDATADIR__", egressIPAMDataDir, -1)
netconf = strings.Replace(netconf, "__RANDOMIZESNAT__", randomizeSNAT, -1)
netconf = strings.Replace(netconf, "__NODEIP__", nodeIP, -1)

Expand Down Expand Up @@ -386,16 +434,8 @@ func _main() int {
// return 1
//}

// Get node IP for conflist
var nodeIP string
nodeIP, err = getNodePrimaryV4Address()
if err != nil {
log.Errorf("Failed to get Node IP, error: %v", err)
return 1
}

log.Infof("Copying config file... ")
err = generateJSON(defaultAWSconflistFile, tmpAWSconflistFile, nodeIP)
err = generateJSON(defaultAWSconflistFile, tmpAWSconflistFile, getPrimaryIP)
if err != nil {
log.WithError(err).Errorf("Failed to generate 10-awsconflist")
return 1
Expand Down
11 changes: 9 additions & 2 deletions cmd/aws-vpc-cni/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,22 @@ const (
nodeIP = "10.0.0.0"
)

var getPrimaryIPMock = func(ipv4 bool) (string, error) {
if ipv4 {
return "10.0.0.0", nil
}
return "2600::", nil
}

// Validate that generateJSON runs against default conflist without error
func TestGenerateJSON(t *testing.T) {
err := generateJSON(awsConflist, devNull, nodeIP)
err := generateJSON(awsConflist, devNull, getPrimaryIPMock)
assert.NoError(t, err)
}

// Validate that generateJSON runs without error when bandwidth plugin is added to default conflist
func TestGenerateJSONPlusBandwidth(t *testing.T) {
_ = os.Setenv(envEnBandwidthPlugin, "true")
err := generateJSON(awsConflist, devNull, nodeIP)
err := generateJSON(awsConflist, devNull, getPrimaryIPMock)
assert.NoError(t, err)
}
Loading

0 comments on commit 36e44ac

Please sign in to comment.