aws · mogren · Oct 28, 2019 · Oct 23, 2019
@@ -250,6 +250,18 @@ is not used, and the maximum number of ENIs is always equal to the maximum numbe
 
 ---
 
+`AWS_VPC_K8S_CNI_LOGLEVEL`
+
+Type: String
+
+Default: `DEBUG`
+
+Valid Values: `trace`, `debug`, `info`, `warn`, `error`, `critical` or `off`. (Not case sensitive)
+
+Specifies the loglevel for ipamd.
+
+---
+
 `AWS_VPC_K8S_CNI_LOG_FILE`
 
 Type: String

@@ -5,7 +5,8 @@
     {
       "name": "aws-cni",
       "type": "aws-cni",
-      "vethPrefix": "__VETHPREFIX__"
+      "vethPrefix": "__VETHPREFIX__",
+      "mtu": "__MTU__"
     },
     {
       "type": "portmap",

@@ -163,7 +163,7 @@ func New() NetworkAPIs {
 		typeOfSNAT:             typeOfSNAT(),
 		nodePortSupportEnabled: nodePortSupportEnabled(),
 		mainENIMark:            getConnmark(),
-		mtu:                    GetEthernetMTU(),
+		mtu:                    GetEthernetMTU(""),
 
 		netLink: netlinkwrapper.NewNetLink(),
 		ns:      nswrapper.NewNS(),
@@ -247,6 +247,14 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDR *net.IPNet, vpcCIDRs []*string,
 		}
 	}
 
+	link, err := LinkByMac(primaryMAC, n.netLink, retryLinkByMacInterval)
+	if err != nil {
+		return errors.Wrapf(err, "setupHostNetwork: failed to find the link primary ENI with MAC address %s", primaryMAC)
+	}
+	if err = n.netLink.LinkSetMTU(link, n.mtu); err != nil {
+		return errors.Wrapf(err, "setupHostNetwork: failed to set MTU to %d for %s", n.mtu, primaryIntf)
+	}
+
 	// If node port support is enabled, add a rule that will force force marked traffic out of the main ENI.  We then
 	// add iptables rules below that will mark traffic that needs this special treatment.  In particular NodePort
 	// traffic always comes in via the main ENI but response traffic would go out of the pod's assigned ENI if we
@@ -936,9 +944,13 @@ func (n *linuxNetwork) UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNe
 	return nil
 }
 
-// GetEthernetMTU gets the MTU setting from AWS_VPC_ENI_MTU, or defaults to 9001 if not set.
-func GetEthernetMTU() int {
-	if envMTUValue := os.Getenv(envMTU); envMTUValue != "" {
+// GetEthernetMTU gets the MTU setting from AWS_VPC_ENI_MTU if set, or takes the passed in string. Defaults to 9001 if not set.
+func GetEthernetMTU(envMTUValue string) int {
+	inputStr, found := os.LookupEnv(envMTU)
+	if found {
+		envMTUValue = inputStr
+	}
+	if envMTUValue != "" {
 		mtu, err := strconv.Atoi(envMTUValue)
 		if err != nil {
 			log.Errorf("Failed to parse %s will use %d: %v", envMTU, maximumMTU, err.Error())

@@ -38,6 +38,7 @@ import (
 )
 
 const (
+	loopback      = ""
 	testMAC1      = "01:23:45:67:89:a0"
 	testMAC2      = "01:23:45:67:89:a1"
 	testTable     = 10
@@ -72,17 +73,14 @@ func TestSetupENINetwork(t *testing.T) {
 
 	hwAddr, err := net.ParseMAC(testMAC1)
 	assert.NoError(t, err)
-
 	mockLinkAttrs1 := &netlink.LinkAttrs{
 		HardwareAddr: hwAddr,
 	}
 	hwAddr, err = net.ParseMAC(testMAC2)
 	assert.NoError(t, err)
-
 	mockLinkAttrs2 := &netlink.LinkAttrs{
 		HardwareAddr: hwAddr,
 	}
-
 	lo := mock_netlink.NewMockLink(ctrl)
 	eth1 := mock_netlink.NewMockLink(ctrl)
 	// Emulate a delay attaching the ENI so a retry is necessary
@@ -94,14 +92,11 @@ func TestSetupENINetwork(t *testing.T) {
 	lo.EXPECT().Attrs().Return(mockLinkAttrs1)
 	eth1.EXPECT().Attrs().Return(mockLinkAttrs2)
 	gomock.InOrder(firstlistSet, secondlistSet)
-
 	mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
 	mockNetLink.EXPECT().LinkSetUp(gomock.Any()).Return(nil)
-
 	// eth1's device
 	eth1.EXPECT().Attrs().Return(mockLinkAttrs2)
 	eth1.EXPECT().Attrs().Return(mockLinkAttrs2)
-
 	// eth1's IP address
 	testeniAddr := &net.IPNet{
 		IP:   net.ParseIP(testeniIP),
@@ -150,14 +145,16 @@ func TestSetupHostNetworkNodePortDisabled(t *testing.T) {
 
 	ln := &linuxNetwork{
 		mainENIMark: 0x80,
-
-		netLink: mockNetLink,
-		ns:      mockNS,
+		mtu:         testMTU,
+		netLink:     mockNetLink,
+		ns:          mockNS,
 		newIptables: func() (iptablesIface, error) {
 			return mockIptables, nil
 		},
 	}
+	mockPrimaryInterfaceLookup(ctrl, mockNetLink)
 
+	mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
 	var hostRule netlink.Rule
 	mockNetLink.EXPECT().NewRule().Return(&hostRule)
 	mockNetLink.EXPECT().RuleDel(&hostRule)
@@ -166,10 +163,19 @@ func TestSetupHostNetworkNodePortDisabled(t *testing.T) {
 	mockNetLink.EXPECT().RuleDel(&mainENIRule)
 
 	var vpcCIDRs []*string
-	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
+	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, loopback, &testENINetIP)
 	assert.NoError(t, err)
 }
 
+func mockPrimaryInterfaceLookup(ctrl *gomock.Controller, mockNetLink *mock_netlinkwrapper.MockNetLink) {
+	lo := mock_netlink.NewMockLink(ctrl)
+	mockLinkAttrs1 := &netlink.LinkAttrs{
+		HardwareAddr: net.HardwareAddr{},
+	}
+	mockNetLink.EXPECT().LinkList().Return([]netlink.Link{lo}, nil)
+	lo.EXPECT().Attrs().AnyTimes().Return(mockLinkAttrs1)
+}
+
 func TestUpdateRuleListBySrc(t *testing.T) {
 	ctrl, mockNetLink, _, _, _ := setup(t)
 	defer ctrl.Finish()
@@ -267,6 +273,7 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {
 		useExternalSNAT:        true,
 		nodePortSupportEnabled: true,
 		mainENIMark:            defaultConnmark,
+		mtu:                    testMTU,
 
 		netLink: mockNetLink,
 		ns:      mockNS,
@@ -278,6 +285,9 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {
 		},
 	}
 
+	mockPrimaryInterfaceLookup(ctrl, mockNetLink)
+	mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
+
 	var hostRule netlink.Rule
 	mockNetLink.EXPECT().NewRule().Return(&hostRule)
 	mockNetLink.EXPECT().RuleDel(&hostRule)
@@ -288,11 +298,7 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {
 
 	var vpcCIDRs []*string
 
-	// loopback for primary device is a little bit hacky. But the test is stable and it should be
-	// OK for test purpose.
-	LoopBackMac := ""
-
-	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, LoopBackMac, &testENINetIP)
+	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, loopback, &testENINetIP)
 	assert.NoError(t, err)
 
 	assert.Equal(t, map[string]map[string][][]string{
@@ -316,17 +322,17 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {
 
 func TestLoadMTUFromEnvTooLow(t *testing.T) {
 	_ = os.Setenv(envMTU, "1")
-	assert.Equal(t, GetEthernetMTU(), minimumMTU)
+	assert.Equal(t, GetEthernetMTU(""), minimumMTU)
 }
 
 func TestLoadMTUFromEnv1500(t *testing.T) {
 	_ = os.Setenv(envMTU, "1500")
-	assert.Equal(t, GetEthernetMTU(), 1500)
+	assert.Equal(t, GetEthernetMTU(""), 1500)
 }
 
 func TestLoadMTUFromEnvTooHigh(t *testing.T) {
 	_ = os.Setenv(envMTU, "65536")
-	assert.Equal(t, GetEthernetMTU(), maximumMTU)
+	assert.Equal(t, GetEthernetMTU(""), maximumMTU)
 }
 
 func TestLoadExcludeSNATCIDRsFromEnv(t *testing.T) {
@@ -347,6 +353,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) {
 		excludeSNATCIDRs:       []string{"10.12.0.0/16", "10.13.0.0/16"},
 		nodePortSupportEnabled: true,
 		mainENIMark:            defaultConnmark,
+		mtu:                    testMTU,
 
 		netLink: mockNetLink,
 		ns:      mockNS,
@@ -358,6 +365,9 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) {
 		},
 	}
 
+	mockPrimaryInterfaceLookup(ctrl, mockNetLink)
+
+	mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
 	var hostRule netlink.Rule
 	mockNetLink.EXPECT().NewRule().Return(&hostRule)
 	mockNetLink.EXPECT().RuleDel(&hostRule)
@@ -368,7 +378,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) {
 
 	var vpcCIDRs []*string
 	vpcCIDRs = []*string{aws.String("10.10.0.0/16"), aws.String("10.11.0.0/16")}
-	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
+	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, loopback, &testENINetIP)
 	assert.NoError(t, err)
 	assert.Equal(t,
 		map[string]map[string][][]string{
@@ -398,6 +408,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) {
 		excludeSNATCIDRs:       nil,
 		nodePortSupportEnabled: true,
 		mainENIMark:            defaultConnmark,
+		mtu:                    testMTU,
 
 		netLink: mockNetLink,
 		ns:      mockNS,
@@ -408,7 +419,9 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) {
 			return &mockRPFilter, nil
 		},
 	}
+	mockPrimaryInterfaceLookup(ctrl, mockNetLink)
 
+	mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
 	var hostRule netlink.Rule
 	mockNetLink.EXPECT().NewRule().Return(&hostRule)
 	mockNetLink.EXPECT().RuleDel(&hostRule)
@@ -426,7 +439,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) {
 	_ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5")
 	_ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0")
 
-	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
+	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, loopback, &testENINetIP)
 	assert.NoError(t, err)
 
 	assert.Equal(t,
@@ -457,6 +470,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) {
 		excludeSNATCIDRs:       []string{"10.12.0.0/16", "10.13.0.0/16"},
 		nodePortSupportEnabled: true,
 		mainENIMark:            defaultConnmark,
+		mtu:                    testMTU,
 
 		netLink: mockNetLink,
 		ns:      mockNS,
@@ -467,7 +481,9 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) {
 			return &mockRPFilter, nil
 		},
 	}
+	mockPrimaryInterfaceLookup(ctrl, mockNetLink)
 
+	mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
 	var hostRule netlink.Rule
 	mockNetLink.EXPECT().NewRule().Return(&hostRule)
 	mockNetLink.EXPECT().RuleDel(&hostRule)
@@ -485,7 +501,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) {
 
 	// remove exclusions
 	vpcCIDRs := []*string{aws.String("10.10.0.0/16"), aws.String("10.11.0.0/16")}
-	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
+	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, loopback, &testENINetIP)
 	assert.NoError(t, err)
 
 	assert.Equal(t,
@@ -515,6 +531,7 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) {
 		useExternalSNAT:        true,
 		nodePortSupportEnabled: true,
 		mainENIMark:            defaultConnmark,
+		mtu:                    testMTU,
 
 		netLink: mockNetLink,
 		ns:      mockNS,
@@ -525,7 +542,9 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) {
 			return &mockRPFilter, nil
 		},
 	}
+	mockPrimaryInterfaceLookup(ctrl, mockNetLink)
 
+	mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
 	var hostRule netlink.Rule
 	mockNetLink.EXPECT().NewRule().Return(&hostRule)
 	mockNetLink.EXPECT().RuleDel(&hostRule)
@@ -536,7 +555,7 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) {
 
 	var vpcCIDRs []*string
 	vpcCIDRs = []*string{aws.String("10.10.0.0/16"), aws.String("10.11.0.0/16")}
-	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
+	err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, loopback, &testENINetIP)
 	assert.NoError(t, err)
 }
 

@@ -22,6 +22,8 @@ import (
 	"os"
 	"runtime"
 
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
+
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 
@@ -67,6 +69,9 @@ type NetConf struct {
 	// veth device name. It should be no more than four characters, and
 	// defaults to 'eni'.
 	VethPrefix string `json:"vethPrefix"`
+
+	// MTU for eth0
+	MTU string `json:"mtu"`
 }
 
 // K8sArgs is the valid CNI_ARGS used for Kubernetes
@@ -121,6 +126,13 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 		return errors.New("conf.VethPrefix can be at most 4 characters long")
 	}
 
+	// MTU
+	if conf.MTU == "" {
+		log.Debug("MTU not set, defaulting to 9001")
+		conf.MTU = "9001"
+	}
+	mtu := networkutils.GetEthernetMTU(conf.MTU)
+
 	cniVersion := conf.CNIVersion
 
 	// Set up a connection to the ipamD server.
@@ -175,7 +187,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	// Note: the maximum length for linux interface name is 15
 	hostVethName := generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 
-	err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT)
+	err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu)
 
 	if err != nil {
 		log.Errorf("Failed SetupPodNetwork for pod %s namespace %s container %s: %v",