Skip to content

Commit

Permalink
Add HostNetworking Test for PPSG in test agent (aws#1720)
Browse files Browse the repository at this point in the history 
* Add HostNetworking Test for PPSG in test agent

* Updated PPSG test to validate vlan.eth link
  • Loading branch information
@cgchinmay
cgchinmay authored Dec 9, 2021
1 parent f32af68 commit 86ece93
Show file tree
Hide file tree
Showing 2 changed files with 225 additions and 1 deletion.
17 changes: 16 additions & 1 deletion test/agent/cmd/networking/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,13 @@ func main() {
var podNetworkingValidationInputString string
var shouldTestSetup bool
var shouldTestCleanup bool
// per pod security group
var ppsg bool

flag.StringVar(&podNetworkingValidationInputString, "pod-networking-validation-input", "", "json string containing the array of pods whose networking needs to be validated")
flag.BoolVar(&shouldTestCleanup, "test-cleanup", false, "bool flag when set to true tests that networking is teared down after pod has been deleted")
flag.BoolVar(&shouldTestSetup, "test-setup", false, "bool flag when set to true tests the networking is setup correctly after pod is running")
flag.BoolVar(&ppsg, "test-ppsg", false, "bool flag when set to true tests the networking setup for pods using security groups")

flag.Parse()

Expand All @@ -47,7 +50,19 @@ func main() {

log.Printf("list of pod against which test will be run %v", podNetworkingValidationInput.PodList)

if shouldTestSetup {
if ppsg && shouldTestSetup {
log.Print("testing networking is setup for pods using security groups")
err := tester.TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput)
if err != nil {
log.Fatalf("found 1 or more pod setup validation failure: %v", err)
}
} else if ppsg && shouldTestCleanup {
log.Print("testing networking is teared down for pods using security groups")
err := tester.TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput)
if err != nil {
log.Fatalf("found 1 or more pod setup validation failure: %v", err)
}
} else if !ppsg && shouldTestSetup {
log.Print("testing networking is setup for regular pods")
err := tester.TestNetworkingSetupForRegularPod(podNetworkingValidationInput)
if err != nil {
Expand Down
209 changes: 209 additions & 0 deletions test/agent/cmd/networking/tester/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,133 @@ func TestNetworkingSetupForRegularPod(podNetworkingValidationInput input.PodNetw
return validationErrors
}

// TestNetworkingSetupForPods using security groups
func TestNetworkingSetupForPodsUsingSecurityGroup(podNetworkingValidationInput input.PodNetworkingValidationInput) []error {
var validationErrors []error
var podIP net.IP
interfaceToVlanTableMap := make(map[string]int)
vlanTableToBranchENIMap := make(map[int]string)

ipFamily := netlink.FAMILY_V4
if podNetworkingValidationInput.IPFamily == "IPv6" {
ipFamily = netlink.FAMILY_V6
}

// Get the list of IP rules
ruleList, err := netlink.RuleList(ipFamily)
if err != nil {
log.Fatalf("failed to list ip rules %v", err)
}

for _, rule := range ruleList {
if strings.HasPrefix(rule.IifName, "vlan.eth") && rule.Table != 0 {
vlanTableToBranchENIMap[rule.Table] = rule.IifName
} else if rule.IifName != "" && rule.Table != 0 {
interfaceToVlanTableMap[rule.IifName] = rule.Table
}
}

for _, pod := range podNetworkingValidationInput.PodList {
podIP = net.ParseIP(pod.PodIPv4Address)
if podNetworkingValidationInput.IPFamily == "IPv6" {
podIP = net.ParseIP(pod.PodIPv6Address)
}

log.Printf("testing for Pod name: %s Namespace: %s, IP: %s",
pod.PodName, pod.PodNamespace, podIP)

// Get the veth pair for pod in host network namespace
hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix)
link, err := netlink.LinkByName(hostVethName)
if err != nil {
validationErrors = append(validationErrors,
fmt.Errorf("failed to find netlink %s: %v", hostVethName, err))
continue
}

vlanTable := 0
if table, ok := interfaceToVlanTableMap[hostVethName]; !ok {
validationErrors = append(validationErrors,
fmt.Errorf("Missing Rule for pod: %s, podNamespace: %s, veth: %s", pod.PodName, pod.PodNamespace, hostVethName))
} else {
vlanTable = table
}

// Check if branch ENI exists for given pod
if branchENI, ok := vlanTableToBranchENIMap[vlanTable]; !ok {
validationErrors = append(validationErrors,
fmt.Errorf("Missing Branch ENI for pod: %s, podNamespace: %s", pod.PodName, pod.PodNamespace))
} else {
// Check if branchENI is in UP state
eniLink, err := netlink.LinkByName(branchENI)
if err != nil {
validationErrors = append(validationErrors,
fmt.Errorf("failed to find netlink %s: %v", branchENI, err))
} else {
isENILinkUp := strings.Contains(eniLink.Attrs().Flags.String(), "up")
if !isENILinkUp {
validationErrors = append(validationErrors,
fmt.Errorf("branch eni %s is not up %s", branchENI, link.Attrs().Flags.String()))
} else {
log.Printf("Found Branch ENI: %s for Pod: %s Namespace: %s, IP: %s in UP state", branchENI,
pod.PodName, pod.PodNamespace, podIP)
}
}
}

// Validate MTU value if it is set to true
if podNetworkingValidationInput.ValidateMTU {
if link.Attrs().MTU != podNetworkingValidationInput.MTU {
validationErrors = append(validationErrors,
fmt.Errorf("MTU value %v for pod: %s on veth pair: %s failed to match the expected value: %v", link.Attrs().MTU, pod.PodName, hostVethName, podNetworkingValidationInput.MTU))
} else {
log.Printf("Found Valid MTU value:%d for pod: %s on veth Pair: %s\n", link.Attrs().MTU, pod.PodName, hostVethName)
}
}

// Verify IP Link for the Pod is UP
isLinkUp := strings.Contains(link.Attrs().Flags.String(), "up")
if !isLinkUp {
validationErrors = append(validationErrors,
fmt.Errorf("veth pair on host side is not up %s", link.Attrs().Flags.String()))
continue
}

log.Printf("found veth pair %s in host network namespace in up state with index %d",
hostVethName, link.Attrs().Index)

routes, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{
Table: vlanTable,
}, netlink.RT_FILTER_TABLE)

if err != nil {
validationErrors = append(validationErrors,
fmt.Errorf("Failed to list routes for table %d", vlanTable))
}

if len(routes) != 3 {
validationErrors = append(validationErrors,
fmt.Errorf("Some Routes missing for vlanTable: %d", vlanTable))
} else {
log.Printf("Found correct number(3) of routes in vlanTable: %d for podIP: %s", vlanTable, podIP)
}

routeExistsForPodIP := false
for _, route := range routes {
if route.Dst != nil && route.Dst.IP.String() == podIP.String() {
routeExistsForPodIP = true
break
}
}

if !routeExistsForPodIP {
validationErrors = append(validationErrors,
fmt.Errorf("Missing Route for PodIP: %s in Table: %d", podIP, vlanTable))
}
}
return validationErrors
}

// TestNetworkTearedDownForRegularPods test pod networking is correctly teared down by the CNI Plugin
// The test assumes that the IP assigned to the older Pod is not assigned to a new Pod while this test
// is being executed
Expand Down Expand Up @@ -270,6 +397,88 @@ func TestNetworkTearedDownForRegularPods(podNetworkingValidationInput input.PodN
return validationError
}

// TestNetworkingForPods using security groups is teared down correctly
func TestNetworkTearedDownForPodsUsingSecurityGroup(podNetworkingValidationInput input.PodNetworkingValidationInput) []error {
var validationErrors []error
var podIP net.IP
interfaceToVlanTableMap := make(map[string]int)
vlanTableToBranchENIMap := make(map[int]string)

ipFamily := netlink.FAMILY_V4
if podNetworkingValidationInput.IPFamily == "IPv6" {
ipFamily = netlink.FAMILY_V6
}
// Get the list of IP rules
ruleList, err := netlink.RuleList(ipFamily)
if err != nil {
log.Fatalf("failed to list ip rules %v", err)
}

for _, rule := range ruleList {
if strings.HasPrefix(rule.IifName, "vlan.eth") && rule.Table != 0 {
vlanTableToBranchENIMap[rule.Table] = rule.IifName
} else if rule.IifName != "" && rule.Table != 0 {
interfaceToVlanTableMap[rule.IifName] = rule.Table
}
}

// Check if branchENI's are cleanup
if len(vlanTableToBranchENIMap) != 0 {
validationErrors = append(validationErrors,
fmt.Errorf("found leaked branch ENI"))
for _, eni := range vlanTableToBranchENIMap {
log.Printf("Leaked branch ENI: %s", eni)
}
}

for _, pod := range podNetworkingValidationInput.PodList {
podIP = net.ParseIP(pod.PodIPv4Address)
if podNetworkingValidationInput.IPFamily == "IPv6" {
podIP = net.ParseIP(pod.PodIPv6Address)
}

log.Printf("testing for Pod name: %s Namespace: %s, IP: %s",
pod.PodName, pod.PodNamespace, pod.PodIPv4Address)

// Make sure the veth pair doesn't exist anymore
hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix)
link, err := netlink.LinkByName(hostVethName)
if err == nil {
// Found leaked veth pair
validationErrors = append(validationErrors,
fmt.Errorf("found an existing veth pair for the pod %s: %v", pod.PodName, link))

// check if vlanTable rule exists for leaked veth pair
if table, ok := interfaceToVlanTableMap[hostVethName]; ok {
validationErrors = append(validationErrors,
fmt.Errorf("Rule exists for pod: %s, podNamespace: %s, veth: %s with target vlanTable:%d", pod.PodName, pod.PodNamespace, hostVethName, table))

routes, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{
Table: table,
}, netlink.RT_FILTER_TABLE)

if err != nil {
validationErrors = append(validationErrors,
fmt.Errorf("Failed to list routes for table %d", table))
} else {
if len(routes) != 0 {
validationErrors = append(validationErrors,
fmt.Errorf("Some Routes exist in vlanTable: %d for podIP: %s", table, podIP))
} else {
log.Printf("No Routes routes found in vlanTable: %d for podIP: %s", table, podIP)
}
}
}
continue
}
log.Printf("veth pair %s not found for the pod: %v", hostVethName, err)

log.Printf("no leaked resource found for the pod %s/%s", pod.PodNamespace, pod.PodName)
}

return validationErrors
}

func isRuleToOrFromIP(rule netlink.Rule, ip net.IP) bool {
if (rule.Src != nil && rule.Src.IP.Equal(ip)) ||
(rule.Dst != nil && rule.Dst.IP.Equal(ip)) {
Expand Down

0 comments on commit 86ece93

Please sign in to comment.