Skip to content

Commit

Permalink
support new SGPP standard mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@M00nF1sh
M00nF1sh committed Mar 9, 2022
1 parent 1de5da4 commit f9a0435
Show file tree
Hide file tree
Showing 14 changed files with 5,694 additions and 838 deletions.
121 changes: 74 additions & 47 deletions cmd/routed-eni-cni-plugin/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (
"strconv"
"strings"

"github.com/aws/amazon-vpc-cni-k8s/pkg/sgpp"

"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/types/current"
Expand All @@ -45,7 +47,6 @@ import (

const ipamdAddress = "127.0.0.1:50051"

const vlanInterfacePrefix = "vlan"
const dummyVlanInterfacePrefix = "dummy"

var version string
Expand All @@ -62,6 +63,9 @@ type NetConf struct {
// MTU for eth0
MTU string `json:"mtu"`

// PodSGEnforcingMode is the enforcing mode for Security groups for pods feature
PodSGEnforcingMode sgpp.EnforcingMode `json:"podSGEnforcingMode"`

PluginLogFile string `json:"pluginLogFile"`

PluginLogLevel string `json:"pluginLogLevel"`
Expand Down Expand Up @@ -91,8 +95,9 @@ func init() {
func LoadNetConf(bytes []byte) (*NetConf, logger.Logger, error) {
// Default config
conf := NetConf{
MTU: "9001",
VethPrefix: "eni",
MTU: "9001",
VethPrefix: "eni",
PodSGEnforcingMode: sgpp.DefaultEnforcingMode,
}

if err := json.Unmarshal(bytes, &conf); err != nil {
Expand Down Expand Up @@ -208,9 +213,10 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap

// Non-zero value means pods are using branch ENI
if r.PodVlanId != 0 {
hostVethName = generateHostVethName(vlanInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC,
r.PodENISubnetGW, int(r.ParentIfIndex), mtu, log)
hostVethNamePrefix := sgpp.BuildHostVethNamePrefix(conf.VethPrefix, conf.PodSGEnforcingMode)
hostVethName = generateHostVethName(hostVethNamePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
err = driverClient.SetupBranchENIPodNetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC,
r.PodENISubnetGW, int(r.ParentIfIndex), mtu, conf.PodSGEnforcingMode, log)

// This is a dummyVlanInterfaceName generated to identify dummyVlanInterface
// which will be created for PPSG scenario to pass along the vlanId information
Expand All @@ -226,7 +232,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
// build hostVethName
// Note: the maximum length for linux interface name is 15
hostVethName = generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), r.VPCv4CIDRs, r.UseExternalSNAT, mtu, log)
err = driverClient.SetupPodNetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), mtu, log)
}

if err != nil {
Expand Down Expand Up @@ -322,37 +328,14 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
log.Info("Netns() is empty, so network already cleanedup. Nothing to do")
return nil
}
prevResult, ok := conf.PrevResult.(*current.Result)

// Try to use prevResult if available
// prevResult might not be availabe, if we are still using older cni spec < 0.4.0.
// So we should fallback to the old clean up method
if ok {
dummyVlanInterfaceName := generateHostVethName(dummyVlanInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
for _, iface := range prevResult.Interfaces {
if iface.Name == dummyVlanInterfaceName {
podVlanId, err := strconv.Atoi(iface.Mac)
if err != nil {
log.Errorf("Failed to parse vlanId from prevResult: %v", err)
return errors.Wrap(err, "del cmd: failed to parse vlanId from prevResult")
}

// podVlanID can not be 0 as we add dummyVlanInterface only for ppsg
// if it is 0 then we should return an error
if podVlanId == 0 {
log.Errorf("Found SG pod:%s namespace:%s with 0 vlanID", k8sArgs.K8S_POD_NAME, k8sArgs.K8S_POD_NAMESPACE)
return errors.Wrap(err, "del cmd: found Incorrect 0 vlandId for ppsg")
}

err = cleanUpPodENI(podVlanId, log, args.ContainerID, driverClient)
if err != nil {
return err
}
log.Infof("Received del network response for pod %s namespace %s sandbox %s with vlanId: %v", string(k8sArgs.K8S_POD_NAME),
string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), podVlanId)
return nil
}
}
handled, err := tryDelWithPrevResult(driverClient, conf, k8sArgs, args.IfName, log)
if err != nil {
return errors.Wrap(err, "del cmd: failed to delete with prevResult")
}
if handled {
log.Infof("Handled CNI del request with prevResult: ContainerID(%s) Netns(%s) IfName(%s) PodNamespace(%s) PodName(%s)",
args.ContainerID, args.Netns, args.IfName, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
return nil
}

// notify local IP address manager to free secondary IP
Expand Down Expand Up @@ -418,9 +401,9 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
}

if r.PodVlanId != 0 {
err = driverClient.TeardownPodENINetwork(int(r.PodVlanId), log)
err = driverClient.TeardownBranchENIPodNetwork(addr, int(r.PodVlanId), conf.PodSGEnforcingMode, log)
} else {
err = driverClient.TeardownNS(addr, int(r.DeviceNumber), log)
err = driverClient.TeardownPodNetwork(addr, int(r.DeviceNumber), log)
}

if err != nil {
Expand All @@ -434,14 +417,58 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
return nil
}

func cleanUpPodENI(podVlanId int, log logger.Logger, containerId string, driverClient driver.NetworkAPIs) error {
err := driverClient.TeardownPodENINetwork(podVlanId, log)
if err != nil {
log.Errorf("Failed on TeardownPodNetwork for container ID %s: %v",
containerId, err)
return errors.Wrap(err, "del cmd: failed on tear down pod network")
// tryDelWithPrevResult will try to process CNI delete request without IPAMD.
// returns true if the del request is handled.
func tryDelWithPrevResult(driverClient driver.NetworkAPIs, conf *NetConf, k8sArgs K8sArgs, contVethName string, log logger.Logger) (bool, error) {
// prevResult might not be available, if we are still using older cni spec < 0.4.0.
prevResult, ok := conf.PrevResult.(*current.Result)
if !ok {
return false, nil
}
return nil

dummyIfaceName := generateHostVethName(dummyVlanInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
_, dummyIface, found := findCNIInterfaceByName(prevResult.Interfaces, dummyIfaceName)
if !found {
return false, nil
}
podVlanID, err := strconv.Atoi(dummyIface.Mac)
if err != nil || podVlanID == 0 {
return true, errors.Errorf("malformed vlanID from prevResult: %s", dummyIface.Mac)
}

containerIfaceIndex, _, found := findCNIInterfaceByName(prevResult.Interfaces, contVethName)
if !found {
return false, nil
}
containerIPs := findCNIIPConfigsByIfaceIndex(prevResult.IPs, containerIfaceIndex)
if len(containerIPs) != 1 {
return false, nil
}
containerIP := containerIPs[0].Address

if err := driverClient.TeardownBranchENIPodNetwork(&containerIP, podVlanID, conf.PodSGEnforcingMode, log); err != nil {
return true, err
}
return true, nil
}

func findCNIInterfaceByName(ifaceList []*current.Interface, ifaceName string) (ifaceIndex int, iface *current.Interface, found bool) {
for ifaceIndex, iface := range ifaceList {
if iface.Name == ifaceName {
return ifaceIndex, iface, true
}
}
return 0, nil, false
}

func findCNIIPConfigsByIfaceIndex(ipConfigs []*current.IPConfig, ifaceIndex int) []*current.IPConfig {
var matchedIPConfigs []*current.IPConfig
for _, ipConfig := range ipConfigs {
if ipConfig.Interface != nil && *ipConfig.Interface == ifaceIndex {
matchedIPConfigs = append(matchedIPConfigs, ipConfig)
}
}
return matchedIPConfigs
}

func main() {
Expand Down
Loading

0 comments on commit f9a0435

Please sign in to comment.