From 10d4829d99eb04a5a0c723c8410fa415fa9931b6 Mon Sep 17 00:00:00 2001 From: Yohei Ueda Date: Sat, 13 Jul 2024 02:18:32 +0900 Subject: [PATCH] podnetwork: Support CNI plugins like PTP and GKE CNI plugins like PTP and GKE remove a route that is automatically added by kernel for eth0, and then add another route for the same destination. This patch changes the code to manipulates routes to support such CNI plugins. Fixes #1909 Signed-off-by: Yohei Ueda --- .../pkg/podnetwork/podnetwork_test.go | 73 +++++++++++-------- .../pkg/podnetwork/podnode.go | 23 +++++- .../pkg/podnetwork/tunneler/tunneler.go | 10 ++- .../pkg/podnetwork/workernode.go | 8 +- 4 files changed, 77 insertions(+), 37 deletions(-) diff --git a/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go b/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go index 3c3a628566..78dbf16186 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go @@ -99,10 +99,13 @@ func TestWorkerNode(t *testing.T) { require.Equal(t, expected.workerNodeIP, config.WorkerNodeIP.String(), "hostInterface=%q", hostInterface) require.Equal(t, mockTunnelType, config.TunnelType, "hostInterface=%q", hostInterface) - require.Equal(t, len(config.Routes), 1, "hostInterface=%q", hostInterface) + require.Equal(t, len(config.Routes), 2, "hostInterface=%q", hostInterface) require.Equal(t, config.Routes[0].Dst.String(), "0.0.0.0/0", "hostInterface=%q", hostInterface) require.Equal(t, config.Routes[0].GW.String(), "172.16.0.1", "hostInterface=%q", hostInterface) require.Equal(t, config.Routes[0].Dev, "eth0", "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[1].Dst.String(), "172.16.0.0/24", "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[1].GW.IsValid(), false, "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[1].Dev, "eth0", "hostInterface=%q", hostInterface) err = workerNode.Teardown(workerPodNS.Path(), config) require.Nil(t, err, "hostInterface=%q", hostInterface) @@ -128,9 +131,6 @@ func TestPodNode(t *testing.T) { tuntest.AddrAdd(t, podNodeNS, "ens1", "192.168.1.3/24") tuntest.RouteAdd(t, podNodeNS, "", "192.168.0.1", "ens0") - podNS := tuntest.NewNamedNS(t, "test-pod") - defer tuntest.DeleteNamedNS(t, podNS) - for hostInterface, expected := range map[string]struct { podNodeIP string workerNodeIP string @@ -149,35 +149,48 @@ func TestPodNode(t *testing.T) { }, } { - err := podNodeNS.Run(func() error { - - config := &tunneler.Config{ - PodIP: netip.MustParsePrefix("172.16.0.2/24"), - Routes: []*tunneler.Route{ - { - GW: netip.MustParseAddr("172.16.0.1"), - Dev: "eth0", + podNS := tuntest.NewNamedNS(t, "test-pod") + func() { + defer tuntest.DeleteNamedNS(t, podNS) + + tuntest.BridgeAdd(t, podNS, "eth0") + tuntest.AddrAdd(t, podNS, "eth0", "172.16.0.2/24") + + err := podNodeNS.Run(func() error { + + config := &tunneler.Config{ + PodIP: netip.MustParsePrefix("172.16.0.2/24"), + Routes: []*tunneler.Route{ + { + Dst: netip.MustParsePrefix("0.0.0.0/0"), + GW: netip.MustParseAddr("172.16.0.1"), + Dev: "eth0", + }, + { + Dst: netip.MustParsePrefix("172.16.0.0/24"), + Dev: "eth0", + }, }, - }, - InterfaceName: "eth0", - MTU: 1500, - WorkerNodeIP: netip.MustParsePrefix(expected.workerNodeIP), - TunnelType: mockTunnelType, - Dedicated: hostInterface == "ens1", - } - - podNode := NewPodNode(podNS.Path(), hostInterface, config) - require.NotNil(t, podNode, "hostInterface=%q", hostInterface) - - err := podNode.Setup() - require.Nil(t, err, "hostInterface=%q", hostInterface) + InterfaceName: "eth0", + MTU: 1500, + WorkerNodeIP: netip.MustParsePrefix(expected.workerNodeIP), + TunnelType: mockTunnelType, + Dedicated: hostInterface == "ens1", + } - err = podNode.Teardown() - require.Nil(t, err, "hostInterface=%q", hostInterface) + podNode := NewPodNode(podNS.Path(), hostInterface, config) + require.NotNil(t, podNode, "hostInterface=%q", hostInterface) - return nil - }) - require.Nil(t, err, "hostInterface=%q", hostInterface) + err := podNode.Setup() + require.Nil(t, err, "hostInterface=%q", hostInterface) + + err = podNode.Teardown() + require.Nil(t, err, "hostInterface=%q", hostInterface) + + return nil + }) + require.Nil(t, err, "hostInterface=%q", hostInterface) + }() } } diff --git a/src/cloud-api-adaptor/pkg/podnetwork/podnode.go b/src/cloud-api-adaptor/pkg/podnetwork/podnode.go index b832f832e0..0cd193d912 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/podnode.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/podnode.go @@ -95,6 +95,20 @@ func (n *podNode) Setup() error { return fmt.Errorf("failed to set up tunnel %q: %w", n.config.TunnelType, err) } + // Delete the nRoute that was automatically added by kernel for eth0 + // CNI plugins like PTP and GKE need this trick, otherwise adding a route will fail in a later step. + // The deleted route will be restored again in the cases of usual CNI plugins such as Flannel and Calico. + // https://github.com/containernetworking/plugins/blob/acf8ddc8e1128e6f68a34f7fe91122afeb1fa93d/plugins/main/ptp/ptp.go#L58-L61 + + nRoute := netops.Route{ + Destination: n.config.PodIP.Masked(), + Device: n.config.InterfaceName, + } + if err := podNS.RouteDel(&nRoute); err != nil { + return fmt.Errorf("failed to remove route %s dev %s: %v", nRoute.Destination, nRoute.Device, err) + } + logger.Printf("removed route %s dev %s", nRoute.Destination, nRoute.Device) + // We need to process routes without gateway address first. Processing routes with a gateway causes an error if the gateway is not reachable. // Calico sets up routes with this pattern. // https://github.com/projectcalico/cni-plugin/blob/7495c0279c34faac315b82c1838bca638e23dbbe/pkg/dataplane/linux/dataplane_linux.go#L158-L167 @@ -110,7 +124,14 @@ func (n *podNode) Setup() error { routes := append(first, second...) for _, route := range routes { - if err := podNS.RouteAdd(&netops.Route{Destination: route.Dst, Gateway: route.GW, Device: route.Dev}); err != nil { + nRoute := netops.Route{ + Destination: route.Dst, + Gateway: route.GW, + Device: route.Dev, + Protocol: route.Protocol, + Scope: route.Scope, + } + if err := podNS.RouteAdd(&nRoute); err != nil { return fmt.Errorf("failed to add a route to %s via %s on pod network namespace %s: %w", route.Dst, route.GW, podNS.Path(), err) } } diff --git a/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go b/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go index f66ce761f7..fa9f8f1c87 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go @@ -6,6 +6,8 @@ package tunneler import ( "fmt" "net/netip" + + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/netops" ) type Tunneler interface { @@ -28,9 +30,11 @@ type Config struct { } type Route struct { - Dst netip.Prefix - GW netip.Addr - Dev string + Dst netip.Prefix + GW netip.Addr + Dev string + Protocol netops.RouteProtocol + Scope netops.RouteScope } type driver struct { diff --git a/src/cloud-api-adaptor/pkg/podnetwork/workernode.go b/src/cloud-api-adaptor/pkg/podnetwork/workernode.go index dbaed59e44..0ff47224f8 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/workernode.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/workernode.go @@ -164,9 +164,11 @@ func (n *workerNode) Inspect(nsPath string) (*tunneler.Config, error) { for _, route := range routes { r := &tunneler.Route{ - Dst: route.Destination, - Dev: route.Device, - GW: route.Gateway, + Dst: route.Destination, + Dev: route.Device, + GW: route.Gateway, + Protocol: route.Protocol, + Scope: route.Scope, } config.Routes = append(config.Routes, r) }