commit 88579443b8eade56814b4d9c13bcc0b16dc78761 Author: Travis Raines <571832+rainest@users.noreply.github.com> Date: Mon Mar 28 12:58:58 2022 -0700 feat(udproute) support parentRef port diff --git a/internal/dataplane/parser/translate_udproute.go b/internal/dataplane/parser/translate_udproute.go index c0b90c59..2246fe89 100644 --- a/internal/dataplane/parser/translate_udproute.go +++ b/internal/dataplane/parser/translate_udproute.go @@ -58,6 +58,16 @@ func ingressRulesFromUDPRoute(result *ingressRules, udproute *gatewayv1alpha2.UD return fmt.Errorf("no rules provided") } + // fetch inbound ports to match per https://gateway-api.sigs.k8s.io/geps/gep-957/ + // TODO unclear if we need to do anything special here to account for these being on separate Gateways + // For now we assume that all referenced Gateways are handled by our Kong instance + var ports []int + for _, parent := range udproute.Spec.ParentRefs { + if parent.Port != nil { + ports = append(ports, int(*parent.Port)) + } + } + // each rule may represent a different set of backend services that will be accepting // traffic, so we make separate routes and Kong services for every present rule. for ruleNumber, rule := range spec.Rules { @@ -74,7 +84,7 @@ func ingressRulesFromUDPRoute(result *ingressRules, udproute *gatewayv1alpha2.UD } // determine the routes needed to route traffic to services for this rule - routes, err := generateKongRoutesFromUDPRouteRule(udproute, ruleNumber, rule) + routes, err := generateKongRoutesFromUDPRouteRule(udproute, ruleNumber, rule, ports) if err != nil { return err } @@ -96,7 +106,8 @@ func ingressRulesFromUDPRoute(result *ingressRules, udproute *gatewayv1alpha2.UD // generateKongRoutesFromUDPRouteRule converts an UDPRoute rule to one or more // Kong Route objects to route traffic to services. -func generateKongRoutesFromUDPRouteRule(udproute *gatewayv1alpha2.UDPRoute, ruleNumber int, rule gatewayv1alpha2.UDPRouteRule) ([]kongstate.Route, error) { +func generateKongRoutesFromUDPRouteRule(udproute *gatewayv1alpha2.UDPRoute, ruleNumber int, + rule gatewayv1alpha2.UDPRouteRule, ports []int) ([]kongstate.Route, error) { // gather the k8s object information and hostnames from the udproute objectInfo := util.FromK8sObject(udproute) @@ -104,67 +115,41 @@ func generateKongRoutesFromUDPRouteRule(udproute *gatewayv1alpha2.UDPRoute, rule if len(rule.Matches) > 0 { // As of 2022-03-04, matches are supported only in experimental CRDs. if you apply a UDPRoute with matches against // the stable CRDs, the matches disappear into the ether (only if doing it via client-go, kubectl rejects them) - - // the UDPRoute specification upstream doesn't clearly indicate whether matches are ANDed or ORed, so we OR for - // now, same as HTTPRoute. TODO clarify upstream - for matchNumber, _ := range rule.Matches { - routeName := kong.String(fmt.Sprintf( - "udproute.%s.%s.%d.%d", - udproute.Namespace, - udproute.Name, - ruleNumber, - matchNumber, - )) - - // build the route object - r := kongstate.Route{ - Ingress: objectInfo, - Route: kong.Route{ - Name: routeName, - Protocols: kong.StringSlice("udp"), - //Sources: TODO extract from match.SourceAddresses - //Destinations: TODO extract from match.DestinationAddresses and backendRef - // TODO upstream does not appear to have many defined means of applying PNAT: matches only support - // IPAddress and NamedAddress address matches. although NamedAddress is documented as - // "vendor-specific behavior, you're on your own", its naming doesn't suggest we should use it for - // ports, though we could. However, there doesn't appear to be anything else we'd need to use it - // for at present, given that SNI has dedicated functionality in TLSRoute. For now, we hard-code - // this to our test port, to defer this question while testing the rest of the feature. - Destinations: []*kong.CIDRPort{ - { - Port: kong.Int(9999), - }, - }, - }, - } - - routes = append(routes, r) - } + // We do not intend to implement these until they are stable per https://github.com/Kong/kubernetes-ingress-controller/issues/2087#issuecomment-1079053290 + return routes, fmt.Errorf("UDPRoute Matches are not yet supported") + } + if len(rule.BackendRefs) > 1 { + // TODO refactor around the solution used for https://github.com/Kong/kubernetes-ingress-controller/issues/2173 + return routes, fmt.Errorf("Support for multiple backends in UDPRoute rules is not yet supported") + } + routeName := kong.String(fmt.Sprintf( + "udproute.%s.%s.%d.%d", + udproute.Namespace, + udproute.Name, + ruleNumber, + 0, + )) + + var destinations []*kong.CIDRPort + if len(ports) == 0 { + // TODO ditto handling multiple backendRefs + // if we don't have any ports from UDPRoute.ParentRefs, we just use the backend port as the proxy port + destinations = append(destinations, &kong.CIDRPort{Port: kong.Int(int(*rule.BackendRefs[0].Port))}) } else { - routeName := kong.String(fmt.Sprintf( - "udproute.%s.%s.%d.%d", - udproute.Namespace, - udproute.Name, - ruleNumber, - 0, - )) - - // build the route object - r := kongstate.Route{ - Ingress: objectInfo, - Route: kong.Route{ - Name: routeName, - Protocols: kong.StringSlice("udp"), - Destinations: []*kong.CIDRPort{ - { - Port: kong.Int(9999), - }, - }, - }, + for _, port := range ports { + destinations = append(destinations, &kong.CIDRPort{Port: kong.Int(port)}) } - - routes = append(routes, r) } + r := kongstate.Route{ + Ingress: objectInfo, + Route: kong.Route{ + Name: routeName, + Protocols: kong.StringSlice("udp"), + Destinations: destinations, + }, + } + + routes = append(routes, r) return routes, nil } diff --git a/test/integration/udpingress_test.go b/test/integration/udpingress_test.go index ba562e62..9c724ab6 100644 --- a/test/integration/udpingress_test.go +++ b/test/integration/udpingress_test.go @@ -398,4 +398,23 @@ const corefile = ` reload loadbalance } +.:9999 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance +} ` diff --git a/test/integration/udproute_test.go b/test/integration/udproute_test.go index f5677315..c8ca368e 100644 --- a/test/integration/udproute_test.go +++ b/test/integration/udproute_test.go @@ -101,6 +101,8 @@ func TestUDPRouteEssentials(t *testing.T) { t.Log("configuring a coredns deployent to deploy for UDP testing") container := generators.NewContainer("coredns", coreDNSImage, 53) container.Ports[0].Protocol = corev1.ProtocolUDP + container.Ports = append(container.Ports, container.Ports[0]) + container.Ports[1].ContainerPort = 9999 container.VolumeMounts = []corev1.VolumeMount{{Name: "config-volume", MountPath: "/etc/coredns"}} container.Args = []string{"-conf", "/etc/coredns/Corefile"} deployment := generators.NewDeploymentForContainer(container) @@ -124,6 +126,8 @@ func TestUDPRouteEssentials(t *testing.T) { t.Logf("exposing deployment %s via service", deployment.Name) service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeLoadBalancer) + service.Spec.Ports[0].Name = "alpha" + service.Spec.Ports[1].Name = "beta" service, err = env.Cluster().Client().CoreV1().Services(ns.Name).Create(ctx, service, metav1.CreateOptions{}) assert.NoError(t, err) @@ -133,7 +137,8 @@ func TestUDPRouteEssentials(t *testing.T) { }() t.Logf("creating a udproute to access deployment %s via kong", deployment.Name) - udpPort := gatewayv1alpha2.PortNumber(53) + udpPort9999 := gatewayv1alpha2.PortNumber(9999) + udpPort53 := gatewayv1alpha2.PortNumber(53) udproute := &gatewayv1alpha2.UDPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: uuid.NewString(), @@ -146,27 +151,68 @@ func TestUDPRouteEssentials(t *testing.T) { }}, }, Rules: []gatewayv1alpha2.UDPRouteRule{{ - Matches: []gatewayv1alpha2.AddressRouteMatches{ - { - // TODO we're just ignoring this but uh... idk what we do here in practice since we don't - // control the addresses in tests. empty and rely entirely on the backendRef port stuff? we - // want some way to test addresses - DestinationAddresses: []gatewayv1alpha2.AddressMatch{ - gatewayv1alpha2.AddressMatch{ - Value: "192.0.2.254", - }, - }, + BackendRefs: []gatewayv1alpha2.BackendRef{{ + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(service.Name), + Port: &udpPort9999, }, - }, + }}, + }}, + }, + } + udprouteWithRefPort := &gatewayv1alpha2.UDPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + Annotations: map[string]string{}, + }, + Spec: gatewayv1alpha2.UDPRouteSpec{ + CommonRouteSpec: gatewayv1alpha2.CommonRouteSpec{ + ParentRefs: []gatewayv1alpha2.ParentReference{{ + Name: gatewayv1alpha2.ObjectName(gw.Name), + Port: &udpPort9999, + }}, + }, + Rules: []gatewayv1alpha2.UDPRouteRule{{ BackendRefs: []gatewayv1alpha2.BackendRef{{ BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ Name: gatewayv1alpha2.ObjectName(service.Name), - Port: &udpPort, + Port: &udpPort53, }, }}, }}, }, } + + t.Log("configurating a net.Resolver to resolve DNS via the proxy") + resolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: time.Second * 5, + } + return d.DialContext(ctx, network, fmt.Sprintf("%s:%d", proxyUDPURL.Hostname(), ktfkong.DefaultUDPServicePort)) + }, + } + + // check routing when the Kong route destination port is set via the UDPRoute's ParentRef + udprouteWithRefPort, err = c.GatewayV1alpha2().UDPRoutes(ns.Name).Create(ctx, udprouteWithRefPort, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Log("verifying that the Gateway gets linked to the route via status") + udpeventuallyGatewayIsLinkedInStatus(t, c, ns.Name, udprouteWithRefPort.Name) + + t.Logf("checking DNS to resolve via UDPIngress %s", udprouteWithRefPort.Name) + assert.Eventually(t, func() bool { + _, err := resolver.LookupHost(ctx, "kernel.org") + return err == nil + }, ingressWait, waitTick) + + // delete the ref port UDPRoute and complete the remaining tests using the UDPRoute with only backendRef ports + // aside from routing, they behave identically, so we don't need to test both to validate gateway linking/unlinking + // behavior + err = c.GatewayV1alpha2().UDPRoutes(ns.Name).Delete(ctx, udprouteWithRefPort.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + udproute, err = c.GatewayV1alpha2().UDPRoutes(ns.Name).Create(ctx, udproute, metav1.CreateOptions{}) require.NoError(t, err) @@ -182,17 +228,6 @@ func TestUDPRouteEssentials(t *testing.T) { t.Log("verifying that the Gateway gets linked to the route via status") udpeventuallyGatewayIsLinkedInStatus(t, c, ns.Name, udproute.Name) - t.Log("configurating a net.Resolver to resolve DNS via the proxy") - resolver := &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - d := net.Dialer{ - Timeout: time.Second * 5, - } - return d.DialContext(ctx, network, fmt.Sprintf("%s:%d", proxyUDPURL.Hostname(), ktfkong.DefaultUDPServicePort)) - }, - } - t.Logf("checking DNS to resolve via UDPIngress %s", udproute.Name) assert.Eventually(t, func() bool { _, err := resolver.LookupHost(ctx, "kernel.org") @@ -301,7 +336,7 @@ func TestUDPRouteEssentials(t *testing.T) { Listeners: []gatewayv1alpha2.Listener{{ Name: "udp", Protocol: gatewayv1alpha2.UDPProtocolType, - Port: gatewayv1alpha2.PortNumber(80), + Port: gatewayv1alpha2.PortNumber(9999), }}, }, }