diff --git a/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go b/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go index 3c3a62856..78dbf1618 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go @@ -99,10 +99,13 @@ func TestWorkerNode(t *testing.T) { require.Equal(t, expected.workerNodeIP, config.WorkerNodeIP.String(), "hostInterface=%q", hostInterface) require.Equal(t, mockTunnelType, config.TunnelType, "hostInterface=%q", hostInterface) - require.Equal(t, len(config.Routes), 1, "hostInterface=%q", hostInterface) + require.Equal(t, len(config.Routes), 2, "hostInterface=%q", hostInterface) require.Equal(t, config.Routes[0].Dst.String(), "0.0.0.0/0", "hostInterface=%q", hostInterface) require.Equal(t, config.Routes[0].GW.String(), "172.16.0.1", "hostInterface=%q", hostInterface) require.Equal(t, config.Routes[0].Dev, "eth0", "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[1].Dst.String(), "172.16.0.0/24", "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[1].GW.IsValid(), false, "hostInterface=%q", hostInterface) + require.Equal(t, config.Routes[1].Dev, "eth0", "hostInterface=%q", hostInterface) err = workerNode.Teardown(workerPodNS.Path(), config) require.Nil(t, err, "hostInterface=%q", hostInterface) @@ -128,9 +131,6 @@ func TestPodNode(t *testing.T) { tuntest.AddrAdd(t, podNodeNS, "ens1", "192.168.1.3/24") tuntest.RouteAdd(t, podNodeNS, "", "192.168.0.1", "ens0") - podNS := tuntest.NewNamedNS(t, "test-pod") - defer tuntest.DeleteNamedNS(t, podNS) - for hostInterface, expected := range map[string]struct { podNodeIP string workerNodeIP string @@ -149,35 +149,48 @@ func TestPodNode(t *testing.T) { }, } { - err := podNodeNS.Run(func() error { - - config := &tunneler.Config{ - PodIP: netip.MustParsePrefix("172.16.0.2/24"), - Routes: []*tunneler.Route{ - { - GW: netip.MustParseAddr("172.16.0.1"), - Dev: "eth0", + podNS := tuntest.NewNamedNS(t, "test-pod") + func() { + defer tuntest.DeleteNamedNS(t, podNS) + + tuntest.BridgeAdd(t, podNS, "eth0") + tuntest.AddrAdd(t, podNS, "eth0", "172.16.0.2/24") + + err := podNodeNS.Run(func() error { + + config := &tunneler.Config{ + PodIP: netip.MustParsePrefix("172.16.0.2/24"), + Routes: []*tunneler.Route{ + { + Dst: netip.MustParsePrefix("0.0.0.0/0"), + GW: netip.MustParseAddr("172.16.0.1"), + Dev: "eth0", + }, + { + Dst: netip.MustParsePrefix("172.16.0.0/24"), + Dev: "eth0", + }, }, - }, - InterfaceName: "eth0", - MTU: 1500, - WorkerNodeIP: netip.MustParsePrefix(expected.workerNodeIP), - TunnelType: mockTunnelType, - Dedicated: hostInterface == "ens1", - } - - podNode := NewPodNode(podNS.Path(), hostInterface, config) - require.NotNil(t, podNode, "hostInterface=%q", hostInterface) - - err := podNode.Setup() - require.Nil(t, err, "hostInterface=%q", hostInterface) + InterfaceName: "eth0", + MTU: 1500, + WorkerNodeIP: netip.MustParsePrefix(expected.workerNodeIP), + TunnelType: mockTunnelType, + Dedicated: hostInterface == "ens1", + } - err = podNode.Teardown() - require.Nil(t, err, "hostInterface=%q", hostInterface) + podNode := NewPodNode(podNS.Path(), hostInterface, config) + require.NotNil(t, podNode, "hostInterface=%q", hostInterface) - return nil - }) - require.Nil(t, err, "hostInterface=%q", hostInterface) + err := podNode.Setup() + require.Nil(t, err, "hostInterface=%q", hostInterface) + + err = podNode.Teardown() + require.Nil(t, err, "hostInterface=%q", hostInterface) + + return nil + }) + require.Nil(t, err, "hostInterface=%q", hostInterface) + }() } } diff --git a/src/cloud-api-adaptor/pkg/podnetwork/podnode.go b/src/cloud-api-adaptor/pkg/podnetwork/podnode.go index b832f832e..218b9afdd 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/podnode.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/podnode.go @@ -95,6 +95,22 @@ func (n *podNode) Setup() error { return fmt.Errorf("failed to set up tunnel %q: %w", n.config.TunnelType, err) } + if !n.config.PodIP.IsSingleIP() { + // Delete the nRoute that was automatically added by kernel for eth0 + // CNI plugins like PTP and GKE need this trick, otherwise adding a route will fail in a later step. + // The deleted route will be restored again in the cases of usual CNI plugins such as Flannel and Calico. + // https://github.com/containernetworking/plugins/blob/acf8ddc8e1128e6f68a34f7fe91122afeb1fa93d/plugins/main/ptp/ptp.go#L58-L61 + + nRoute := netops.Route{ + Destination: n.config.PodIP.Masked(), + Device: n.config.InterfaceName, + } + if err := podNS.RouteDel(&nRoute); err != nil { + return fmt.Errorf("failed to remove route %s dev %s: %v", nRoute.Destination, nRoute.Device, err) + } + logger.Printf("removed route %s dev %s", nRoute.Destination, nRoute.Device) + } + // We need to process routes without gateway address first. Processing routes with a gateway causes an error if the gateway is not reachable. // Calico sets up routes with this pattern. // https://github.com/projectcalico/cni-plugin/blob/7495c0279c34faac315b82c1838bca638e23dbbe/pkg/dataplane/linux/dataplane_linux.go#L158-L167 @@ -110,7 +126,14 @@ func (n *podNode) Setup() error { routes := append(first, second...) for _, route := range routes { - if err := podNS.RouteAdd(&netops.Route{Destination: route.Dst, Gateway: route.GW, Device: route.Dev}); err != nil { + nRoute := netops.Route{ + Destination: route.Dst, + Gateway: route.GW, + Device: route.Dev, + Protocol: route.Protocol, + Scope: route.Scope, + } + if err := podNS.RouteAdd(&nRoute); err != nil { return fmt.Errorf("failed to add a route to %s via %s on pod network namespace %s: %w", route.Dst, route.GW, podNS.Path(), err) } } diff --git a/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go b/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go index f66ce761f..fa9f8f1c8 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go @@ -6,6 +6,8 @@ package tunneler import ( "fmt" "net/netip" + + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/netops" ) type Tunneler interface { @@ -28,9 +30,11 @@ type Config struct { } type Route struct { - Dst netip.Prefix - GW netip.Addr - Dev string + Dst netip.Prefix + GW netip.Addr + Dev string + Protocol netops.RouteProtocol + Scope netops.RouteScope } type driver struct { diff --git a/src/cloud-api-adaptor/pkg/podnetwork/workernode.go b/src/cloud-api-adaptor/pkg/podnetwork/workernode.go index dbaed59e4..0ff47224f 100644 --- a/src/cloud-api-adaptor/pkg/podnetwork/workernode.go +++ b/src/cloud-api-adaptor/pkg/podnetwork/workernode.go @@ -164,9 +164,11 @@ func (n *workerNode) Inspect(nsPath string) (*tunneler.Config, error) { for _, route := range routes { r := &tunneler.Route{ - Dst: route.Destination, - Dev: route.Device, - GW: route.Gateway, + Dst: route.Destination, + Dev: route.Device, + GW: route.Gateway, + Protocol: route.Protocol, + Scope: route.Scope, } config.Routes = append(config.Routes, r) }