Skip to content

Commit

Permalink
Add MTU to the plugin config
Browse files Browse the repository at this point in the history
  • Loading branch information
Claes Mogren committed Oct 28, 2019
1 parent b8a8b33 commit 2daee65
Show file tree
Hide file tree
Showing 10 changed files with 120 additions and 63 deletions.
12 changes: 12 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,18 @@ is not used, and the maximum number of ENIs is always equal to the maximum numbe

---

`AWS_VPC_K8S_CNI_LOGLEVEL`

Type: String

Default: `DEBUG`

Valid Values: `trace`, `debug`, `info`, `warn`, `error`, `critical` or `off`. (Not case sensitive)

Specifies the loglevel for ipamd.

---

`AWS_VPC_K8S_CNI_LOG_FILE`

Type: String
Expand Down
3 changes: 2 additions & 1 deletion misc/10-aws.conflist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
{
"name": "aws-cni",
"type": "aws-cni",
"vethPrefix": "__VETHPREFIX__"
"vethPrefix": "__VETHPREFIX__",
"mtu": "__MTU__"
},
{
"type": "portmap",
Expand Down
20 changes: 16 additions & 4 deletions pkg/networkutils/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func New() NetworkAPIs {
typeOfSNAT: typeOfSNAT(),
nodePortSupportEnabled: nodePortSupportEnabled(),
mainENIMark: getConnmark(),
mtu: GetEthernetMTU(),
mtu: GetEthernetMTU(""),

netLink: netlinkwrapper.NewNetLink(),
ns: nswrapper.NewNS(),
Expand Down Expand Up @@ -247,6 +247,14 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDR *net.IPNet, vpcCIDRs []*string,
}
}

link, err := LinkByMac(primaryMAC, n.netLink, retryLinkByMacInterval)
if err != nil {
return errors.Wrapf(err, "setupHostNetwork: failed to find the link primary ENI with MAC address %s", primaryMAC)
}
if err = n.netLink.LinkSetMTU(link, n.mtu); err != nil {
return errors.Wrapf(err, "setupHostNetwork: failed to set MTU to %d for %s", n.mtu, primaryIntf)
}

// If node port support is enabled, add a rule that will force force marked traffic out of the main ENI. We then
// add iptables rules below that will mark traffic that needs this special treatment. In particular NodePort
// traffic always comes in via the main ENI but response traffic would go out of the pod's assigned ENI if we
Expand Down Expand Up @@ -936,9 +944,13 @@ func (n *linuxNetwork) UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNe
return nil
}

// GetEthernetMTU gets the MTU setting from AWS_VPC_ENI_MTU, or defaults to 9001 if not set.
func GetEthernetMTU() int {
if envMTUValue := os.Getenv(envMTU); envMTUValue != "" {
// GetEthernetMTU gets the MTU setting from AWS_VPC_ENI_MTU if set, or takes the passed in string. Defaults to 9001 if not set.
func GetEthernetMTU(envMTUValue string) int {
inputStr, found := os.LookupEnv(envMTU)
if found {
envMTUValue = inputStr
}
if envMTUValue != "" {
mtu, err := strconv.Atoi(envMTUValue)
if err != nil {
log.Errorf("Failed to parse %s will use %d: %v", envMTU, maximumMTU, err.Error())
Expand Down
74 changes: 48 additions & 26 deletions pkg/networkutils/network_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
)

const (
dockerFirst = "02:42:ac:11:00:02"
testMAC1 = "01:23:45:67:89:a0"
testMAC2 = "01:23:45:67:89:a1"
testTable = 10
Expand Down Expand Up @@ -72,17 +73,14 @@ func TestSetupENINetwork(t *testing.T) {

hwAddr, err := net.ParseMAC(testMAC1)
assert.NoError(t, err)

mockLinkAttrs1 := &netlink.LinkAttrs{
HardwareAddr: hwAddr,
}
hwAddr, err = net.ParseMAC(testMAC2)
assert.NoError(t, err)

mockLinkAttrs2 := &netlink.LinkAttrs{
HardwareAddr: hwAddr,
}

lo := mock_netlink.NewMockLink(ctrl)
eth1 := mock_netlink.NewMockLink(ctrl)
// Emulate a delay attaching the ENI so a retry is necessary
Expand All @@ -94,14 +92,11 @@ func TestSetupENINetwork(t *testing.T) {
lo.EXPECT().Attrs().Return(mockLinkAttrs1)
eth1.EXPECT().Attrs().Return(mockLinkAttrs2)
gomock.InOrder(firstlistSet, secondlistSet)

mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
mockNetLink.EXPECT().LinkSetUp(gomock.Any()).Return(nil)

// eth1's device
eth1.EXPECT().Attrs().Return(mockLinkAttrs2)
eth1.EXPECT().Attrs().Return(mockLinkAttrs2)

// eth1's IP address
testeniAddr := &net.IPNet{
IP: net.ParseIP(testeniIP),
Expand Down Expand Up @@ -150,14 +145,16 @@ func TestSetupHostNetworkNodePortDisabled(t *testing.T) {

ln := &linuxNetwork{
mainENIMark: 0x80,

netLink: mockNetLink,
ns: mockNS,
mtu: testMTU,
netLink: mockNetLink,
ns: mockNS,
newIptables: func() (iptablesIface, error) {
return mockIptables, nil
},
}
err := mockPrimaryInterfaceLookup(t, ctrl, mockNetLink, testMAC1)

mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
var hostRule netlink.Rule
mockNetLink.EXPECT().NewRule().Return(&hostRule)
mockNetLink.EXPECT().RuleDel(&hostRule)
Expand All @@ -166,10 +163,22 @@ func TestSetupHostNetworkNodePortDisabled(t *testing.T) {
mockNetLink.EXPECT().RuleDel(&mainENIRule)

var vpcCIDRs []*string
err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
err = ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, testMAC1, &testENINetIP)
assert.NoError(t, err)
}

func mockPrimaryInterfaceLookup(t *testing.T, ctrl *gomock.Controller, mockNetLink *mock_netlinkwrapper.MockNetLink, mac string) error {
hwAddr, err := net.ParseMAC(mac)
assert.NoError(t, err)
mockLinkAttrs1 := &netlink.LinkAttrs{
HardwareAddr: hwAddr,
}
lo := mock_netlink.NewMockLink(ctrl)
mockNetLink.EXPECT().LinkList().Return([]netlink.Link{lo}, nil)
lo.EXPECT().Attrs().AnyTimes().Return(mockLinkAttrs1)
return err
}

func TestUpdateRuleListBySrc(t *testing.T) {
ctrl, mockNetLink, _, _, _ := setup(t)
defer ctrl.Finish()
Expand Down Expand Up @@ -267,6 +276,7 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {
useExternalSNAT: true,
nodePortSupportEnabled: true,
mainENIMark: defaultConnmark,
mtu: testMTU,

netLink: mockNetLink,
ns: mockNS,
Expand All @@ -278,6 +288,9 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {
},
}

err := mockPrimaryInterfaceLookup(t, ctrl, mockNetLink, dockerFirst)
mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)

var hostRule netlink.Rule
mockNetLink.EXPECT().NewRule().Return(&hostRule)
mockNetLink.EXPECT().RuleDel(&hostRule)
Expand All @@ -288,19 +301,15 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {

var vpcCIDRs []*string

// loopback for primary device is a little bit hacky. But the test is stable and it should be
// OK for test purpose.
LoopBackMac := ""

err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, LoopBackMac, &testENINetIP)
err = ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, dockerFirst, &testENINetIP)
assert.NoError(t, err)

assert.Equal(t, map[string]map[string][][]string{
"mangle": {
"PREROUTING": [][]string{
{
"-m", "comment", "--comment", "AWS, primary ENI",
"-i", "lo",
"-i", "eth0",
"-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in",
"-j", "CONNMARK", "--set-mark", "0x80/0x80",
},
Expand All @@ -316,17 +325,17 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) {

func TestLoadMTUFromEnvTooLow(t *testing.T) {
_ = os.Setenv(envMTU, "1")
assert.Equal(t, GetEthernetMTU(), minimumMTU)
assert.Equal(t, GetEthernetMTU(""), minimumMTU)
}

func TestLoadMTUFromEnv1500(t *testing.T) {
_ = os.Setenv(envMTU, "1500")
assert.Equal(t, GetEthernetMTU(), 1500)
assert.Equal(t, GetEthernetMTU(""), 1500)
}

func TestLoadMTUFromEnvTooHigh(t *testing.T) {
_ = os.Setenv(envMTU, "65536")
assert.Equal(t, GetEthernetMTU(), maximumMTU)
assert.Equal(t, GetEthernetMTU(""), maximumMTU)
}

func TestLoadExcludeSNATCIDRsFromEnv(t *testing.T) {
Expand All @@ -347,6 +356,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) {
excludeSNATCIDRs: []string{"10.12.0.0/16", "10.13.0.0/16"},
nodePortSupportEnabled: true,
mainENIMark: defaultConnmark,
mtu: testMTU,

netLink: mockNetLink,
ns: mockNS,
Expand All @@ -358,6 +368,9 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) {
},
}

err := mockPrimaryInterfaceLookup(t, ctrl, mockNetLink, dockerFirst)

mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
var hostRule netlink.Rule
mockNetLink.EXPECT().NewRule().Return(&hostRule)
mockNetLink.EXPECT().RuleDel(&hostRule)
Expand All @@ -368,7 +381,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) {

var vpcCIDRs []*string
vpcCIDRs = []*string{aws.String("10.10.0.0/16"), aws.String("10.11.0.0/16")}
err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
err = ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, dockerFirst, &testENINetIP)
assert.NoError(t, err)
assert.Equal(t,
map[string]map[string][][]string{
Expand All @@ -381,7 +394,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) {
"POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}},
"mangle": {
"PREROUTING": [][]string{
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"},
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eth0", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"},
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"},
},
},
Expand All @@ -398,6 +411,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) {
excludeSNATCIDRs: nil,
nodePortSupportEnabled: true,
mainENIMark: defaultConnmark,
mtu: testMTU,

netLink: mockNetLink,
ns: mockNS,
Expand All @@ -408,7 +422,9 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) {
return &mockRPFilter, nil
},
}
err := mockPrimaryInterfaceLookup(t, ctrl, mockNetLink, dockerFirst)

mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
var hostRule netlink.Rule
mockNetLink.EXPECT().NewRule().Return(&hostRule)
mockNetLink.EXPECT().RuleDel(&hostRule)
Expand All @@ -426,7 +442,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) {
_ = mockIptables.NewChain("nat", "AWS-SNAT-CHAIN-5")
_ = mockIptables.Append("nat", "POSTROUTING", "-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0")

err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
err = ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, dockerFirst, &testENINetIP)
assert.NoError(t, err)

assert.Equal(t,
Expand All @@ -440,7 +456,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) {
"POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}},
"mangle": {
"PREROUTING": [][]string{
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"},
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eth0", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"},
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"},
},
},
Expand All @@ -457,6 +473,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) {
excludeSNATCIDRs: []string{"10.12.0.0/16", "10.13.0.0/16"},
nodePortSupportEnabled: true,
mainENIMark: defaultConnmark,
mtu: testMTU,

netLink: mockNetLink,
ns: mockNS,
Expand All @@ -467,7 +484,9 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) {
return &mockRPFilter, nil
},
}
err := mockPrimaryInterfaceLookup(t, ctrl, mockNetLink, dockerFirst)

mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
var hostRule netlink.Rule
mockNetLink.EXPECT().NewRule().Return(&hostRule)
mockNetLink.EXPECT().RuleDel(&hostRule)
Expand All @@ -485,7 +504,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) {

// remove exclusions
vpcCIDRs := []*string{aws.String("10.10.0.0/16"), aws.String("10.11.0.0/16")}
err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
err = ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, dockerFirst, &testENINetIP)
assert.NoError(t, err)

assert.Equal(t,
Expand All @@ -499,7 +518,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) {
"POSTROUTING": [][]string{{"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"}}},
"mangle": {
"PREROUTING": [][]string{
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"},
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eth0", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80"},
{"-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"},
},
},
Expand All @@ -515,6 +534,7 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) {
useExternalSNAT: true,
nodePortSupportEnabled: true,
mainENIMark: defaultConnmark,
mtu: testMTU,

netLink: mockNetLink,
ns: mockNS,
Expand All @@ -525,7 +545,9 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) {
return &mockRPFilter, nil
},
}
err := mockPrimaryInterfaceLookup(t, ctrl, mockNetLink, dockerFirst)

mockNetLink.EXPECT().LinkSetMTU(gomock.Any(), testMTU).Return(nil)
var hostRule netlink.Rule
mockNetLink.EXPECT().NewRule().Return(&hostRule)
mockNetLink.EXPECT().RuleDel(&hostRule)
Expand All @@ -536,7 +558,7 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) {

var vpcCIDRs []*string
vpcCIDRs = []*string{aws.String("10.10.0.0/16"), aws.String("10.11.0.0/16")}
err := ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, "", &testENINetIP)
err = ln.SetupHostNetwork(testENINetIPNet, vpcCIDRs, dockerFirst, &testENINetIP)
assert.NoError(t, err)
}

Expand Down
14 changes: 13 additions & 1 deletion plugins/routed-eni/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"os"
"runtime"

"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"

"golang.org/x/net/context"
"google.golang.org/grpc"

Expand Down Expand Up @@ -67,6 +69,9 @@ type NetConf struct {
// veth device name. It should be no more than four characters, and
// defaults to 'eni'.
VethPrefix string `json:"vethPrefix"`

// MTU for eth0
Mtu string `json:"mtu"`
}

// K8sArgs is the valid CNI_ARGS used for Kubernetes
Expand Down Expand Up @@ -121,6 +126,13 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
return errors.New("conf.VethPrefix can be at most 4 characters long")
}

// MTU
if conf.Mtu == "" {
log.Debug("MTU not set, defaulting to 9001")
conf.Mtu = "9001"
}
mtu := networkutils.GetEthernetMTU(conf.Mtu)

cniVersion := conf.CNIVersion

// Set up a connection to the ipamD server.
Expand Down Expand Up @@ -175,7 +187,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
// Note: the maximum length for linux interface name is 15
hostVethName := generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))

err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT)
err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu)

if err != nil {
log.Errorf("Failed SetupPodNetwork for pod %s namespace %s container %s: %v",
Expand Down
Loading

0 comments on commit 2daee65

Please sign in to comment.