From d3272b49779b9f5266b114231c2552ba9d4c6953 Mon Sep 17 00:00:00 2001 From: Date Huang Date: Thu, 6 Jan 2022 15:34:11 +0800 Subject: [PATCH] bridge: add vlan trunk support add vlan trunk support for veth vlan trunk only support L2 only mode without any IPAM refer ovs-cni design https://github.com/k8snetworkplumbingwg/ovs-cni/blob/main/pkg/plugin/plugin.go design: origin "vlan" option will be PVID or untagged vlan for the network. "vlanTrunk" will setup tagged vlan for veth. entry type: `{ "id": 100 }` will specify only tagged vlan 100 `{ "minID": 100, "maxID": 120 }` will specify tagged vlan from 100 to 120 (include 100 and 120) vlanTrunk is a list of above entry type, so you can use this to add tagged vlan `[ { "id": 100 }, { "minID": 1000, "maxID": 2000 } ]` complete config will be like this { "cniVersion": "0.3.1", "name": "mynet", "type": "bridge", "bridge": "mynet0", "vlan": 100, "vlanTrunk": [ { "id": 101 }, { "minID": 1000, "maxID": 2000 }, { "minID": 3000, "maxID": 4000 } ], "ipam": {} } Signed-off-by: Date Huang --- plugins/main/bridge/bridge.go | 112 +++++++++++++++++++++++---- plugins/main/bridge/bridge_test.go | 118 ++++++++++++++++++++++++++++- 2 files changed, 211 insertions(+), 19 deletions(-) diff --git a/plugins/main/bridge/bridge.go b/plugins/main/bridge/bridge.go index 23c1256ff..82f631508 100644 --- a/plugins/main/bridge/bridge.go +++ b/plugins/main/bridge/bridge.go @@ -21,6 +21,7 @@ import ( "net" "os" "runtime" + "sort" "syscall" "time" @@ -46,17 +47,18 @@ const defaultBrName = "cni0" type NetConf struct { types.NetConf - BrName string `json:"bridge"` - IsGW bool `json:"isGateway"` - IsDefaultGW bool `json:"isDefaultGateway"` - ForceAddress bool `json:"forceAddress"` - IPMasq bool `json:"ipMasq"` - MTU int `json:"mtu"` - HairpinMode bool `json:"hairpinMode"` - PromiscMode bool `json:"promiscMode"` - Vlan int `json:"vlan"` - MacSpoofChk bool `json:"macspoofchk,omitempty"` - EnableDad bool `json:"enabledad,omitempty"` + BrName string `json:"bridge"` + IsGW bool `json:"isGateway"` + IsDefaultGW bool `json:"isDefaultGateway"` + ForceAddress bool `json:"forceAddress"` + IPMasq bool `json:"ipMasq"` + MTU int `json:"mtu"` + HairpinMode bool `json:"hairpinMode"` + PromiscMode bool `json:"promiscMode"` + Vlan int `json:"vlan"` + VlanTrunk []*VlanTrunk `json:"vlanTrunk,omitempty"` + MacSpoofChk bool `json:"macspoofchk,omitempty"` + EnableDad bool `json:"enabledad,omitempty"` Args struct { Cni BridgeArgs `json:"cni,omitempty"` @@ -65,7 +67,14 @@ type NetConf struct { Mac string `json:"mac,omitempty"` } `json:"runtimeConfig,omitempty"` - mac string + mac string + vlans []int +} + +type VlanTrunk struct { + MinID *int `json:"minID,omitempty"` + MaxID *int `json:"maxID,omitempty"` + ID *int `json:"id,omitempty"` } type BridgeArgs struct { @@ -101,6 +110,11 @@ func loadNetConf(bytes []byte, envArgs string) (*NetConf, string, error) { if n.Vlan < 0 || n.Vlan > 4094 { return nil, "", fmt.Errorf("invalid VLAN ID %d (must be between 0 and 4094)", n.Vlan) } + var err error + if n.vlans, err = collectVlanTrunk(n.VlanTrunk); err != nil { + // fail to parsing + return nil, "", err + } if envArgs != "" { e := MacEnvArgs{} @@ -124,6 +138,59 @@ func loadNetConf(bytes []byte, envArgs string) (*NetConf, string, error) { return n, n.CNIVersion, nil } +// This method is copied from https://github.com/k8snetworkplumbingwg/ovs-cni/blob/main/pkg/plugin/plugin.go +func collectVlanTrunk(vlanTrunk []*VlanTrunk) ([]int, error) { + if vlanTrunk == nil { + return nil, nil + } + + vlanMap := make(map[int]bool) + for _, item := range vlanTrunk { + var minID int = 0 + var maxID int = 0 + var ID int = 0 + if item.MinID != nil { + minID = *item.MinID + if minID < 0 || minID > 4094 { + return nil, errors.New("incorrect trunk minID parameter") + } + } + if item.MaxID != nil { + maxID = *item.MaxID + if maxID < 0 || maxID > 4094 { + return nil, errors.New("incorrect trunk maxID parameter") + } + if maxID < minID { + return nil, errors.New("minID is greater than maxID in trunk parameter") + } + } + if minID > 0 && maxID > 0 { + for v := minID; v <= maxID; v++ { + vlanMap[v] = true + } + } + + // single vid + if item.ID != nil { + ID = *item.ID + if ID < 0 || ID > 4094 { + return nil, errors.New("incorrect trunk id parameter") + } + vlanMap[ID] = true + } + } + + if len(vlanMap) == 0 { + return nil, nil + } + vlans := make([]int, 0, len(vlanMap)) + for k := range vlanMap { + vlans = append(vlans, k) + } + sort.Slice(vlans, func(i int, j int) bool { return vlans[i] < vlans[j] }) + return vlans, nil +} + // calcGateways processes the results from the IPAM plugin and does the // following for each IP family: // - Calculates and compiles a list of gateway addresses @@ -314,7 +381,7 @@ func ensureVlanInterface(br *netlink.Bridge, vlanId int) (netlink.Link, error) { return nil, fmt.Errorf("faild to find host namespace: %v", err) } - _, brGatewayIface, err := setupVeth(hostNS, br, name, br.MTU, false, vlanId, "") + _, brGatewayIface, err := setupVeth(hostNS, br, name, br.MTU, false, vlanId, nil, "") if err != nil { return nil, fmt.Errorf("faild to create vlan gateway %q: %v", name, err) } @@ -333,7 +400,7 @@ func ensureVlanInterface(br *netlink.Bridge, vlanId int) (netlink.Link, error) { return brGatewayVeth, nil } -func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool, vlanID int, mac string) (*current.Interface, *current.Interface, error) { +func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool, vlanID int, vlans []int, mac string) (*current.Interface, *current.Interface, error) { contIface := ¤t.Interface{} hostIface := ¤t.Interface{} @@ -377,6 +444,15 @@ func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairp } } + if vlans != nil { + for _, v := range vlans { + err = netlink.BridgeVlanAdd(hostVeth, uint16(v), false, false, false, true) + if err != nil { + return nil, nil, fmt.Errorf("failed to setup vlan tag on interface %q: %w", hostIface.Name, err) + } + } + } + return hostIface, contIface, nil } @@ -387,7 +463,7 @@ func calcGatewayIP(ipn *net.IPNet) net.IP { func setupBridge(n *NetConf) (*netlink.Bridge, *current.Interface, error) { vlanFiltering := false - if n.Vlan != 0 { + if n.Vlan != 0 || n.VlanTrunk != nil { vlanFiltering = true } // create bridge if necessary @@ -427,6 +503,10 @@ func cmdAdd(args *skel.CmdArgs) error { return fmt.Errorf("cannot set hairpin mode and promiscuous mode at the same time.") } + if n.vlans != nil && len(n.vlans) > 0 && isLayer3 { + return fmt.Errorf("cannot set vlanTrunk and IPAM at the same time.") + } + br, brInterface, err := setupBridge(n) if err != nil { return err @@ -438,7 +518,7 @@ func cmdAdd(args *skel.CmdArgs) error { } defer netns.Close() - hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode, n.Vlan, n.mac) + hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode, n.Vlan, n.vlans, n.mac) if err != nil { return err } diff --git a/plugins/main/bridge/bridge_test.go b/plugins/main/bridge/bridge_test.go index 118074ed6..3a811f7df 100644 --- a/plugins/main/bridge/bridge_test.go +++ b/plugins/main/bridge/bridge_test.go @@ -74,6 +74,7 @@ type testCase struct { isLayer2 bool expGWCIDRs []string // Expected gateway addresses in CIDR form vlan int + vlanTrunk []*VlanTrunk ipMasq bool macspoofchk bool AddErr020 string @@ -129,6 +130,23 @@ const ( vlan = `, "vlan": %d` + vlanTrunkStartStr = `, + "vlanTrunk": [` + + vlanTrunk = ` + { + "id": %d + }` + + vlanTrunkRange = ` + { + "minID": %d, + "maxID": %d + }` + + vlanTrunkEndStr = ` + ]` + netDefault = `, "isDefaultGateway": true` @@ -189,6 +207,23 @@ func (tc testCase) netConfJSON(dataDir string) string { if tc.vlan != 0 { conf += fmt.Sprintf(vlan, tc.vlan) } + + if tc.isLayer2 && tc.vlanTrunk != nil { + conf += vlanTrunkStartStr + for i, vlan := range tc.vlanTrunk { + if i > 0 { + conf += "," + } + if vlan.ID != nil { + conf += fmt.Sprintf(vlanTrunk, *vlan.ID) + } + if vlan.MinID != nil && vlan.MaxID != nil { + conf += fmt.Sprintf(vlanTrunkRange, *vlan.MinID, *vlan.MaxID) + } + } + conf += vlanTrunkEndStr + } + if tc.ipMasq { conf += tc.ipMasqConfig() } @@ -503,7 +538,7 @@ func (tester *testerV10x) cmdAddTest(tc testCase, dataDir string) (types.Result, } // Check the bridge vlan filtering equals true - if tc.vlan != 0 { + if tc.vlan != 0 || tc.vlanTrunk != nil { Expect(*link.(*netlink.Bridge).VlanFiltering).To(Equal(true)) } else { Expect(*link.(*netlink.Bridge).VlanFiltering).To(Equal(false)) @@ -557,6 +592,25 @@ func (tester *testerV10x) cmdAddTest(tc testCase, dataDir string) (types.Result, Expect(checkVlan(tc.vlan, vlans)).To(BeTrue()) } + // check VlanTrunks exist on the veth interface + if tc.vlanTrunk != nil { + interfaceMap, err := netlink.BridgeVlanList() + Expect(err).NotTo(HaveOccurred()) + vlans, isExist := interfaceMap[int32(link.Attrs().Index)] + Expect(isExist).To(BeTrue()) + + for _, vlan_entry := range tc.vlanTrunk { + if vlan_entry.ID != nil { + Expect(checkVlan(*vlan_entry.ID, vlans)).To(BeTrue()) + } + if vlan_entry.MinID != nil && vlan_entry.MaxID != nil { + for vid := *vlan_entry.MinID; vid <= *vlan_entry.MaxID; vid++ { + Expect(checkVlan(vid, vlans)).To(BeTrue()) + } + } + } + } + // Check that the bridge has a different mac from the veth // If not, it means the bridge has an unstable mac and will change // as ifs are added and removed @@ -803,7 +857,7 @@ func (tester *testerV04x) cmdAddTest(tc testCase, dataDir string) (types.Result, } // Check the bridge vlan filtering equals true - if tc.vlan != 0 { + if tc.vlan != 0 || tc.vlanTrunk != nil { Expect(*link.(*netlink.Bridge).VlanFiltering).To(Equal(true)) } else { Expect(*link.(*netlink.Bridge).VlanFiltering).To(Equal(false)) @@ -857,6 +911,25 @@ func (tester *testerV04x) cmdAddTest(tc testCase, dataDir string) (types.Result, Expect(checkVlan(tc.vlan, vlans)).To(BeTrue()) } + // check VlanTrunks exist on the veth interface + if tc.vlanTrunk != nil { + interfaceMap, err := netlink.BridgeVlanList() + Expect(err).NotTo(HaveOccurred()) + vlans, isExist := interfaceMap[int32(link.Attrs().Index)] + Expect(isExist).To(BeTrue()) + + for _, vlan_entry := range tc.vlanTrunk { + if vlan_entry.ID != nil { + Expect(checkVlan(*vlan_entry.ID, vlans)).To(BeTrue()) + } + if vlan_entry.MinID != nil && vlan_entry.MaxID != nil { + for vid := *vlan_entry.MinID; vid <= *vlan_entry.MaxID; vid++ { + Expect(checkVlan(vid, vlans)).To(BeTrue()) + } + } + } + } + // Check that the bridge has a different mac from the veth // If not, it means the bridge has an unstable mac and will change // as ifs are added and removed @@ -1103,7 +1176,7 @@ func (tester *testerV03x) cmdAddTest(tc testCase, dataDir string) (types.Result, } // Check the bridge vlan filtering equals true - if tc.vlan != 0 { + if tc.vlan != 0 || tc.vlanTrunk != nil { Expect(*link.(*netlink.Bridge).VlanFiltering).To(Equal(true)) } else { Expect(*link.(*netlink.Bridge).VlanFiltering).To(Equal(false)) @@ -1157,6 +1230,25 @@ func (tester *testerV03x) cmdAddTest(tc testCase, dataDir string) (types.Result, Expect(checkVlan(tc.vlan, vlans)).To(BeTrue()) } + // check VlanTrunks exist on the veth interface + if tc.vlanTrunk != nil { + interfaceMap, err := netlink.BridgeVlanList() + Expect(err).NotTo(HaveOccurred()) + vlans, isExist := interfaceMap[int32(link.Attrs().Index)] + Expect(isExist).To(BeTrue()) + + for _, vlan_entry := range tc.vlanTrunk { + if vlan_entry.ID != nil { + Expect(checkVlan(*vlan_entry.ID, vlans)).To(BeTrue()) + } + if vlan_entry.MinID != nil && vlan_entry.MaxID != nil { + for vid := *vlan_entry.MinID; vid <= *vlan_entry.MaxID; vid++ { + Expect(checkVlan(vid, vlans)).To(BeTrue()) + } + } + } + } + // Check that the bridge has a different mac from the veth // If not, it means the bridge has an unstable mac and will change // as ifs are added and removed @@ -1736,6 +1828,26 @@ var _ = Describe("bridge Operations", func() { cmdAddDelTest(originalNS, targetNS, tc, dataDir) }) + // TODO find some way to put pointer + It(fmt.Sprintf("[%s] configures and deconfigures a l2 bridge with vlan id 100, vlanTrunk 101,200~210 using ADD/DEL", ver), func() { + id, minID, maxID := 101, 200, 210 + tc := testCase{ + cniVersion: ver, + isLayer2: true, + vlan: 100, + vlanTrunk: []*VlanTrunk{ + {ID: &id}, + { + MinID: &minID, + MaxID: &maxID, + }, + }, + AddErr020: "cannot convert: no valid IP addresses", + AddErr010: "cannot convert: no valid IP addresses", + } + cmdAddDelTest(originalNS, targetNS, tc, dataDir) + }) + for i, tc := range []testCase{ { // IPv4 only