diff --git a/.changelog/156.txt b/.changelog/156.txt new file mode 100644 index 000000000..71e22459c --- /dev/null +++ b/.changelog/156.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +k8s/controllers: watch for ReferencePolicy changes to reconcile and revalidate affected HTTPRoutes +``` diff --git a/internal/commands/server/k8s_e2e_test.go b/internal/commands/server/k8s_e2e_test.go index 778e992b3..c2021b9bc 100644 --- a/internal/commands/server/k8s_e2e_test.go +++ b/internal/commands/server/k8s_e2e_test.go @@ -75,10 +75,10 @@ func TestGatewayWithClassConfigChange(t *testing.T) { // Create a Gateway and wait for it to be ready firstGatewayName := envconf.RandomName("gw", 16) - firstGateway := createGateway(ctx, t, cfg, firstGatewayName, gc, 443) + firstGateway := createGateway(ctx, t, cfg, firstGatewayName, gc, 443, nil) require.Eventually(t, func() bool { err := resources.Get(ctx, firstGatewayName, namespace, firstGateway) - return err == nil && conditionAccepted(firstGateway.Status.Conditions) + return err == nil && conditionReady(firstGateway.Status.Conditions) }, 60*time.Second, checkInterval, "no gateway found in the allotted time") require.Eventually(t, gatewayStatusCheck(ctx, resources, firstGatewayName, namespace, conditionReady), 30*time.Second, checkInterval, "no gateway found in the allotted time") checkGatewayConfigAnnotation(t, firstGateway, firstConfig) @@ -93,10 +93,10 @@ func TestGatewayWithClassConfigChange(t *testing.T) { // Create a second Gateway and wait for it to be ready secondGatewayName := envconf.RandomName("gw", 16) - secondGateway := createGateway(ctx, t, cfg, secondGatewayName, gc, 443) + secondGateway := createGateway(ctx, t, cfg, secondGatewayName, gc, 443, nil) require.Eventually(t, func() bool { err := resources.Get(ctx, secondGatewayName, namespace, secondGateway) - return err == nil && conditionAccepted(secondGateway.Status.Conditions) + return err == nil && conditionReady(secondGateway.Status.Conditions) }, 30*time.Second, checkInterval, "no gateway found in the allotted time") require.Eventually(t, gatewayStatusCheck(ctx, resources, secondGatewayName, namespace, conditionReady), 30*time.Second, checkInterval, "no gateway found in the allotted time") checkGatewayConfigAnnotation(t, secondGateway, secondConfig) @@ -129,7 +129,7 @@ func TestGatewayBasic(t *testing.T) { return err == nil && conditionAccepted(created.Status.Conditions) }, checkTimeout, checkInterval, "gatewayclass not accepted in the allotted time") - _ = createGateway(ctx, t, cfg, gatewayName, gc, 443) + _ = createGateway(ctx, t, cfg, gatewayName, gc, 443, nil) require.Eventually(t, func() bool { err := resources.Get(ctx, gatewayName, namespace, &apps.Deployment{}) @@ -139,7 +139,7 @@ func TestGatewayBasic(t *testing.T) { created := &gateway.Gateway{} require.Eventually(t, func() bool { err := resources.Get(ctx, gatewayName, namespace, created) - return err == nil && conditionAccepted(created.Status.Conditions) + return err == nil && conditionReady(created.Status.Conditions) }, checkTimeout, checkInterval, "no gateway found in the allotted time") checkGatewayConfigAnnotation(t, created, gcc) @@ -190,7 +190,7 @@ func TestServiceListeners(t *testing.T) { gatewayName := envconf.RandomName("gw", 16) _, gc := createGatewayClass(ctx, t, cfg) - gw := createGateway(ctx, t, cfg, gatewayName, gc, 443) + gw := createGateway(ctx, t, cfg, gatewayName, gc, 443, nil) require.Eventually(t, func() bool { service := &core.Service{} @@ -294,7 +294,7 @@ func TestHTTPRouteFlattening(t *testing.T) { require.NoError(t, err) checkPort := e2e.HTTPFlattenedPort(ctx) - gw := createGateway(ctx, t, cfg, gatewayName, gc, gateway.PortNumber(checkPort)) + gw := createGateway(ctx, t, cfg, gatewayName, gc, gateway.PortNumber(checkPort), nil) require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), checkTimeout, checkInterval, "no gateway found in the allotted time") port := gateway.PortNumber(serviceOne.Spec.Ports[0].Port) @@ -462,7 +462,7 @@ func TestHTTPMeshService(t *testing.T) { err = resources.Create(ctx, gc) require.NoError(t, err) - gw := createGateway(ctx, t, cfg, gatewayName, gc, gateway.PortNumber(e2e.HTTPPort(ctx))) + gw := createGateway(ctx, t, cfg, gatewayName, gc, gateway.PortNumber(e2e.HTTPPort(ctx)), nil) require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), checkTimeout, checkInterval, "no gateway found in the allotted time") // route 1 @@ -780,7 +780,16 @@ func TestTCPMeshService(t *testing.T) { err = resources.Create(ctx, routeOne) require.NoError(t, err) - require.Eventually(t, tcpRouteStatusCheck(ctx, resources, gatewayName, routeOneName, namespace, routeRefErrors), checkTimeout, checkInterval, "route status not set in allotted time") + require.Eventually(t, tcpRouteStatusCheck( + ctx, + resources, + gatewayName, + routeOneName, + namespace, + createConditionsCheck([]meta.Condition{ + {Type: "ResolvedRefs", Status: "False", Reason: "Errors"}, + }), + ), checkTimeout, checkInterval, "route status not set in allotted time") // route 2 meshServiceGroup := gateway.Group(apigwv1alpha1.Group) @@ -964,6 +973,192 @@ func TestTCPMeshService(t *testing.T) { testenv.Test(t, feature.Feature()) } +func TestHTTPRouteReferencePolicyLifecycle(t *testing.T) { + feature := features.New("http route reference policy"). + Assess("http route controller watches reference policy", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + serviceOne, err := e2e.DeployHTTPMeshService(ctx, cfg) + require.NoError(t, err) + + namespace := e2e.Namespace(ctx) + configName := envconf.RandomName("gcc", 16) + className := envconf.RandomName("gc", 16) + gatewayName := envconf.RandomName("gw", 16) + routeName := envconf.RandomName("route", 16) + routeNamespace := envconf.RandomName("ns", 16) + refPolicyName := envconf.RandomName("refpolicy", 16) + + resources := cfg.Client().Resources(namespace) + + gcc := &apigwv1alpha1.GatewayClassConfig{ + ObjectMeta: meta.ObjectMeta{ + Name: configName, + }, + Spec: apigwv1alpha1.GatewayClassConfigSpec{ + ImageSpec: apigwv1alpha1.ImageSpec{ + ConsulAPIGateway: e2e.DockerImage(ctx), + }, + UseHostPorts: true, + LogLevel: "trace", + ConsulSpec: apigwv1alpha1.ConsulSpec{ + Address: hostRoute, + Scheme: "https", + PortSpec: apigwv1alpha1.PortSpec{ + GRPC: e2e.ConsulGRPCPort(ctx), + HTTP: e2e.ConsulHTTPPort(ctx), + }, + AuthSpec: apigwv1alpha1.AuthSpec{ + Method: "consul-api-gateway", + Account: "consul-api-gateway", + }, + }, + }, + } + err = resources.Create(ctx, gcc) + require.NoError(t, err) + + gc := &gateway.GatewayClass{ + ObjectMeta: meta.ObjectMeta{ + Name: className, + }, + Spec: gateway.GatewayClassSpec{ + ControllerName: k8s.ControllerName, + ParametersRef: &gateway.ParametersReference{ + Group: apigwv1alpha1.Group, + Kind: apigwv1alpha1.GatewayClassConfigKind, + Name: configName, + }, + }, + } + err = resources.Create(ctx, gc) + require.NoError(t, err) + + checkPort := e2e.HTTPReferencePolicyPort(ctx) + + // Allow routes to bind from a different namespace for testing + // cross-namespace ReferencePolicy enforcement + all := gateway.NamespacesFromAll + allowedRoutes := &gateway.AllowedRoutes{ + Namespaces: &gateway.RouteNamespaces{ + From: &all, + }, + } + + gw := createGateway(ctx, t, cfg, gatewayName, gc, gateway.PortNumber(checkPort), allowedRoutes) + require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), checkTimeout, checkInterval, "no gateway found in the allotted time") + + // Create a different namespace for the route + ns := &core.Namespace{ + ObjectMeta: meta.ObjectMeta{ + Name: routeNamespace, + }, + } + err = resources.Create(ctx, ns) + require.NoError(t, err) + + port := gateway.PortNumber(serviceOne.Spec.Ports[0].Port) + gwNamespace := gateway.Namespace(namespace) + route := &gateway.HTTPRoute{ + ObjectMeta: meta.ObjectMeta{ + Name: routeName, + Namespace: routeNamespace, + }, + Spec: gateway.HTTPRouteSpec{ + CommonRouteSpec: gateway.CommonRouteSpec{ + ParentRefs: []gateway.ParentRef{{ + Name: gateway.ObjectName(gatewayName), + Namespace: &gwNamespace, + }}, + }, + Hostnames: []gateway.Hostname{"test.foo"}, + Rules: []gateway.HTTPRouteRule{{ + BackendRefs: []gateway.HTTPBackendRef{{ + BackendRef: gateway.BackendRef{ + BackendObjectReference: gateway.BackendObjectReference{ + Name: gateway.ObjectName(serviceOne.Name), + Namespace: &gwNamespace, + Port: &port, + }, + }, + }}, + }}, + }, + } + err = resources.Create(ctx, route) + require.NoError(t, err) + + // Expect that route sets + // ResolvedRefs{ status: False, reason: RefNotPermitted } + // due to missing ReferencePolicy for BackendRef in other namespace + httpRouteStatusCheckRefNotPermitted := httpRouteStatusCheck( + ctx, + resources, + gatewayName, + routeName, + routeNamespace, + createConditionsCheck([]meta.Condition{ + {Type: "Accepted", Status: "False"}, + {Type: "ResolvedRefs", Status: "False", Reason: "RefNotPermitted"}, + }), + ) + require.Eventually(t, httpRouteStatusCheckRefNotPermitted, checkTimeout, checkInterval, "route status not set in allotted time") + + // create ReferencePolicy allowing BackendRef + serviceOneObjectName := gateway.ObjectName(serviceOne.Name) + referencePolicy := &gateway.ReferencePolicy{ + ObjectMeta: meta.ObjectMeta{ + Name: refPolicyName, + Namespace: namespace, + }, + Spec: gateway.ReferencePolicySpec{ + From: []gateway.ReferencePolicyFrom{{ + Group: "gateway.networking.k8s.io", + Kind: "HTTPRoute", + Namespace: gateway.Namespace(routeNamespace), + }}, + To: []gateway.ReferencePolicyTo{{ + Group: "", + Kind: "Service", + Name: &serviceOneObjectName, + }}, + }, + } + err = resources.Create(ctx, referencePolicy) + require.NoError(t, err) + + // Expect that route sets + // ResolvedRefs{ status: True, reason: ResolvedRefs } + // now that ReferencePolicy allows BackendRef in other namespace + require.Eventually(t, httpRouteStatusCheck( + ctx, + resources, + gatewayName, + routeName, + routeNamespace, + createConditionsCheck([]meta.Condition{ + {Type: "Accepted", Status: "True"}, + {Type: "ResolvedRefs", Status: "True", Reason: "ResolvedRefs"}, + }), + ), checkTimeout, checkInterval, "route status not set in allotted time") + + // Check that route is successfully resolved and routing traffic + checkRoute(t, checkPort, "/", serviceOne.Name, map[string]string{ + "Host": "test.foo", + }, "service one not routable in allotted time") + + // Delete ReferencePolicy, check for RefNotPermitted again + err = resources.Delete(ctx, referencePolicy) + require.NoError(t, err) + require.Eventually(t, httpRouteStatusCheckRefNotPermitted, checkTimeout, checkInterval, "route status not set in allotted time") + + err = resources.Delete(ctx, gw) + require.NoError(t, err) + + return ctx + }) + + testenv.Test(t, feature.Feature()) +} + func gatewayStatusCheck(ctx context.Context, resources *resources.Resources, gatewayName, namespace string, checkFn func([]meta.Condition) bool) func() bool { return func() bool { updated := &gateway.Gateway{} @@ -1003,6 +1198,21 @@ func listenerStatusCheck(ctx context.Context, resources *resources.Resources, ga } } +func httpRouteStatusCheck(ctx context.Context, resources *resources.Resources, gatewayName, routeName, namespace string, checkFn func([]meta.Condition) bool) func() bool { + return func() bool { + updated := &gateway.HTTPRoute{} + if err := resources.Get(ctx, routeName, namespace, updated); err != nil { + return false + } + for _, status := range updated.Status.Parents { + if string(status.ParentRef.Name) == gatewayName { + return checkFn(status.Conditions) + } + } + return false + } +} + func tcpRouteStatusCheck(ctx context.Context, resources *resources.Resources, gatewayName, routeName, namespace string, checkFn func([]meta.Condition) bool) func() bool { return func() bool { updated := &gateway.TCPRoute{} @@ -1018,48 +1228,47 @@ func tcpRouteStatusCheck(ctx context.Context, resources *resources.Resources, ga } } -func routeRefErrors(conditions []meta.Condition) bool { - for _, condition := range conditions { - if condition.Type == "ResolvedRefs" && - condition.Status == "False" && - condition.Reason == "Errors" { - return true +func createConditionsCheck(expected []meta.Condition) func([]meta.Condition) bool { + return func(actual []meta.Condition) bool { + for _, eCondition := range expected { + matched := false + for _, aCondition := range actual { + if aCondition.Type == eCondition.Type && + aCondition.Status == eCondition.Status && + // Match if expected condition doesn't define an expected reason + (aCondition.Reason == eCondition.Reason || eCondition.Reason == "") { + matched = true + break + } + } + + if !matched { + return false + } } + return true } - return false } func conditionAccepted(conditions []meta.Condition) bool { - for _, condition := range conditions { - if condition.Type == "Accepted" || - condition.Status == "True" { - return true - } - } - return false + return createConditionsCheck([]meta.Condition{ + {Type: "Accepted", Status: "True"}, + })(conditions) } func conditionReady(conditions []meta.Condition) bool { - for _, condition := range conditions { - if condition.Type == "Ready" && - condition.Status == "True" { - return true - } - } - return false + return createConditionsCheck([]meta.Condition{ + {Type: "Ready", Status: "True"}, + })(conditions) } func conditionInSync(conditions []meta.Condition) bool { - for _, condition := range conditions { - if condition.Type == "InSync" && - condition.Status == "True" { - return true - } - } - return false + return createConditionsCheck([]meta.Condition{ + {Type: "InSync", Status: "True"}, + })(conditions) } -func createGateway(ctx context.Context, t *testing.T, cfg *envconf.Config, gatewayName string, gc *gateway.GatewayClass, listenerPort gateway.PortNumber) *gateway.Gateway { +func createGateway(ctx context.Context, t *testing.T, cfg *envconf.Config, gatewayName string, gc *gateway.GatewayClass, listenerPort gateway.PortNumber, listenerAllowedRoutes *gateway.AllowedRoutes) *gateway.Gateway { t.Helper() namespace := e2e.Namespace(ctx) @@ -1084,6 +1293,7 @@ func createGateway(ctx context.Context, t *testing.T, cfg *envconf.Config, gatew Namespace: &gatewayNamespace, }}, }, + AllowedRoutes: listenerAllowedRoutes, }}, }, } diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index bae429939..28a8016fc 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -189,6 +189,7 @@ func (k *Kubernetes) Start(ctx context.Context) error { } err = (&controllers.HTTPRouteReconciler{ + Context: ctx, Client: gwClient, Log: k.logger.Named("HTTPRoute"), Manager: reconcileManager, diff --git a/internal/k8s/controllers/http_route_controller.go b/internal/k8s/controllers/http_route_controller.go index a2cdc9bdd..f4c90dd0b 100644 --- a/internal/k8s/controllers/http_route_controller.go +++ b/internal/k8s/controllers/http_route_controller.go @@ -3,7 +3,12 @@ package controllers import ( "context" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" gateway "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient" @@ -13,6 +18,7 @@ import ( // HTTPRouteReconciler reconciles a HTTPRoute object type HTTPRouteReconciler struct { + Context context.Context Client gatewayclient.Client Log hclog.Logger ControllerName string @@ -53,5 +59,67 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&gateway.HTTPRoute{}). + Watches( + &source.Kind{Type: &gateway.ReferencePolicy{}}, + handler.EnqueueRequestsFromMapFunc(r.referencePolicyToRouteRequests), + ). Complete(gatewayclient.NewRequeueingMiddleware(r.Log, r)) } + +// For UpdateEvents which contain both a new and old object, this transformation +// function is run on both objects and both sets of Requests are enqueued. +// +// This is needed to reconcile any objects matched by both current and prior +// state in case a ReferencePolicy has been modified to revoke permission from a +// namespace or to a service +// +// It may be possible to improve performance here by filtering Routes by +// BackendRefs selectable by the To fields, but currently we just revalidate +// all Routes allowed in the From Namespaces +func (r *HTTPRouteReconciler) referencePolicyToRouteRequests(object client.Object) []reconcile.Request { + refPolicy := object.(*gateway.ReferencePolicy) + + routes := r.getRoutesAffectedByReferencePolicy(refPolicy) + requests := []reconcile.Request{} + + for _, route := range routes { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: route.Name, + Namespace: route.Namespace, + }, + }) + } + + return requests +} + +func (r *HTTPRouteReconciler) getRoutesAffectedByReferencePolicy(refPolicy *gateway.ReferencePolicy) []gateway.HTTPRoute { + matches := []gateway.HTTPRoute{} + + routes := r.getReferencePolicyObjectsFrom(refPolicy) + + // TODO: match only routes with BackendRefs selectable by a + // ReferencePolicyTo instead of appending all routes. This seems expensive, + // so not sure if it would actually improve performance or not. + matches = append(matches, routes...) + + return matches +} + +func (r *HTTPRouteReconciler) getReferencePolicyObjectsFrom(refPolicy *gateway.ReferencePolicy) []gateway.HTTPRoute { + matches := []gateway.HTTPRoute{} + + for _, from := range refPolicy.Spec.From { + // TODO: search by from.Group and from.Kind instead of assuming HTTPRoute + routes, err := r.Client.GetHTTPRoutesInNamespace(r.Context, string(from.Namespace)) + if err != nil { + r.Log.Error("error fetching routes", err) + return matches + } + + matches = append(matches, routes...) + } + + return matches +} diff --git a/internal/k8s/controllers/http_route_controller_test.go b/internal/k8s/controllers/http_route_controller_test.go index 32491f735..9e4763fcc 100644 --- a/internal/k8s/controllers/http_route_controller_test.go +++ b/internal/k8s/controllers/http_route_controller_test.go @@ -6,12 +6,22 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" + gw "sigs.k8s.io/gateway-api/apis/v1alpha2" + "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient" "github.com/hashicorp/consul-api-gateway/internal/k8s/gatewayclient/mocks" reconcilerMocks "github.com/hashicorp/consul-api-gateway/internal/k8s/reconciler/mocks" "github.com/hashicorp/go-hclog" + + apigwv1alpha1 "github.com/hashicorp/consul-api-gateway/pkg/apis/v1alpha1" ) var ( @@ -57,6 +67,7 @@ func TestHTTPRoute(t *testing.T) { } controller := &HTTPRouteReconciler{ + Context: context.Background(), Client: client, Log: hclog.NewNullLogger(), ControllerName: mockControllerName, @@ -75,3 +86,100 @@ func TestHTTPRoute(t *testing.T) { }) } } + +func TestHTTPRouteReferencePolicyToRouteRequests(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + serviceNamespace := gw.Namespace("namespace3") + + backendObjRef := gw.BackendObjectReference{ + Name: gw.ObjectName("service"), + Namespace: &serviceNamespace, + } + + httpRouteSpec := gw.HTTPRouteSpec{ + Rules: []gw.HTTPRouteRule{{ + BackendRefs: []gw.HTTPBackendRef{{ + BackendRef: gw.BackendRef{ + BackendObjectReference: backendObjRef, + }, + }}, + }}, + } + + refPolicy := gw.ReferencePolicy{ + TypeMeta: metav1.TypeMeta{Kind: "ReferencePolicy"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "namespace3"}, + Spec: gw.ReferencePolicySpec{ + From: []gw.ReferencePolicyFrom{{ + Group: "gateway.networking.k8s.io", + Kind: "HTTPRoute", + Namespace: "namespace1", + }}, + To: []gw.ReferencePolicyTo{{ + Kind: "Service", + }}, + }, + } + + gatewayclient := NewTestClient( + &gw.HTTPRouteList{ + Items: []gw.HTTPRoute{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "httproute", + Namespace: "namespace1", + }, + Spec: httpRouteSpec, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "httproute", + Namespace: "namespace2", + }, + Spec: httpRouteSpec, + }, + }, + }, + &refPolicy, + ) + + controller := &HTTPRouteReconciler{ + Client: gatewayclient, + Log: hclog.NewNullLogger(), + ControllerName: mockControllerName, + Manager: reconcilerMocks.NewMockReconcileManager(ctrl), + } + + requests := controller.referencePolicyToRouteRequests(&refPolicy) + + require.Equal(t, []reconcile.Request{{ + NamespacedName: types.NamespacedName{ + Name: "httproute", + Namespace: "namespace1", + }, + }}, requests) +} + +// FIXME: this should be refactored into a test utility package +func NewTestClient(list client.ObjectList, objects ...client.Object) gatewayclient.Client { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(gw.AddToScheme(scheme)) + apigwv1alpha1.RegisterTypes(scheme) + + builder := fake. + NewClientBuilder(). + WithScheme(scheme) + if list != nil { + builder = builder.WithLists(list) + } + if len(objects) > 0 { + builder = builder.WithObjects(objects...) + } + + return gatewayclient.New(builder.Build(), scheme, "") +} diff --git a/internal/k8s/gatewayclient/gatewayclient.go b/internal/k8s/gatewayclient/gatewayclient.go index a30750a68..1cff8d9b8 100644 --- a/internal/k8s/gatewayclient/gatewayclient.go +++ b/internal/k8s/gatewayclient/gatewayclient.go @@ -33,6 +33,7 @@ type Client interface { GetSecret(ctx context.Context, key types.NamespacedName) (*core.Secret, error) GetService(ctx context.Context, key types.NamespacedName) (*core.Service, error) GetHTTPRoute(ctx context.Context, key types.NamespacedName) (*gateway.HTTPRoute, error) + GetHTTPRoutesInNamespace(ctx context.Context, ns string) ([]gateway.HTTPRoute, error) GetTCPRoute(ctx context.Context, key types.NamespacedName) (*gateway.TCPRoute, error) GetMeshService(ctx context.Context, key types.NamespacedName) (*apigwv1alpha1.MeshService, error) GetNamespace(ctx context.Context, key types.NamespacedName) (*core.Namespace, error) @@ -250,6 +251,15 @@ func (g *gatewayClient) GetHTTPRoute(ctx context.Context, key types.NamespacedNa return route, nil } +// TODO: Make this generic over Group and Kind, returning []client.Object +func (g *gatewayClient) GetHTTPRoutesInNamespace(ctx context.Context, ns string) ([]gateway.HTTPRoute, error) { + routeList := &gateway.HTTPRouteList{} + if err := g.Client.List(ctx, routeList, client.InNamespace(ns)); err != nil { + return []gateway.HTTPRoute{}, NewK8sError(err) + } + return routeList.Items, nil +} + func (g *gatewayClient) GetTCPRoute(ctx context.Context, key types.NamespacedName) (*gateway.TCPRoute, error) { route := &gateway.TCPRoute{} if err := g.Client.Get(ctx, key, route); err != nil { diff --git a/internal/k8s/gatewayclient/gatewayclient_test.go b/internal/k8s/gatewayclient/gatewayclient_test.go index 35992f884..b60f81e57 100644 --- a/internal/k8s/gatewayclient/gatewayclient_test.go +++ b/internal/k8s/gatewayclient/gatewayclient_test.go @@ -100,6 +100,25 @@ func TestGetHTTPRoute(t *testing.T) { require.NotNil(t, route) } +func TestGetHTTPRoutesInNamespace(t *testing.T) { + t.Parallel() + + gatewayclient := newTestClient(nil, &gateway.HTTPRoute{ + ObjectMeta: meta.ObjectMeta{ + Name: "httproute", + Namespace: "namespace1", + }, + }) + + routes, err := gatewayclient.GetHTTPRoutesInNamespace(context.Background(), "namespace1") + require.NoError(t, err) + require.Equal(t, len(routes), 1) + + routes, err = gatewayclient.GetHTTPRoutesInNamespace(context.Background(), "namespace2") + require.NoError(t, err) + require.Equal(t, len(routes), 0) +} + func TestGetGatewayClass(t *testing.T) { t.Parallel() diff --git a/internal/k8s/gatewayclient/mocks/gatewayclient.go b/internal/k8s/gatewayclient/mocks/gatewayclient.go index a4598f866..7f601db97 100644 --- a/internal/k8s/gatewayclient/mocks/gatewayclient.go +++ b/internal/k8s/gatewayclient/mocks/gatewayclient.go @@ -259,6 +259,21 @@ func (mr *MockClientMockRecorder) GetHTTPRoute(ctx, key interface{}) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHTTPRoute", reflect.TypeOf((*MockClient)(nil).GetHTTPRoute), ctx, key) } +// GetHTTPRoutesInNamespace mocks base method. +func (m *MockClient) GetHTTPRoutesInNamespace(ctx context.Context, ns string) ([]v1alpha2.HTTPRoute, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHTTPRoutesInNamespace", ctx, ns) + ret0, _ := ret[0].([]v1alpha2.HTTPRoute) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetHTTPRoutesInNamespace indicates an expected call of GetHTTPRoutesInNamespace. +func (mr *MockClientMockRecorder) GetHTTPRoutesInNamespace(ctx, ns interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHTTPRoutesInNamespace", reflect.TypeOf((*MockClient)(nil).GetHTTPRoutesInNamespace), ctx, ns) +} + // GetMeshService mocks base method. func (m *MockClient) GetMeshService(ctx context.Context, key types.NamespacedName) (*v1alpha1.MeshService, error) { m.ctrl.T.Helper() diff --git a/internal/testing/e2e/consul.go b/internal/testing/e2e/consul.go index ed221082c..e810c8fe8 100644 --- a/internal/testing/e2e/consul.go +++ b/internal/testing/e2e/consul.go @@ -83,19 +83,20 @@ func init() { } type consulTestEnvironment struct { - ca []byte - consulClient *api.Client - token string - policy *api.ACLPolicy - httpPort int - httpFlattenedPort int - grpcPort int - extraHTTPPort int - extraTCPPort int - extraTCPTLSPort int - extraTCPTLSPortTwo int - namespace string - ip string + ca []byte + consulClient *api.Client + token string + policy *api.ACLPolicy + httpPort int + httpFlattenedPort int + httpReferencePolicyPort int + grpcPort int + extraHTTPPort int + extraTCPPort int + extraTCPTLSPort int + extraTCPTLSPortTwo int + namespace string + ip string } func CreateTestConsulContainer(name, namespace string) env.Func { @@ -109,6 +110,7 @@ func CreateTestConsulContainer(name, namespace string) env.Func { cluster := clusterVal.(*kindCluster) httpsPort := cluster.httpsPort httpFlattenedPort := cluster.httpsFlattenedPort + httpReferencePolicyPort := cluster.httpsReferencePolicyPort grpcPort := cluster.grpcPort extraTCPPort := cluster.extraTCPPort extraTCPTLSPort := cluster.extraTCPTLSPort @@ -197,16 +199,17 @@ func CreateTestConsulContainer(name, namespace string) env.Func { } env := &consulTestEnvironment{ - ca: rootCA.CertBytes, - consulClient: consulClient, - httpPort: httpsPort, - httpFlattenedPort: httpFlattenedPort, - grpcPort: grpcPort, - extraHTTPPort: extraHTTPPort, - extraTCPPort: extraTCPPort, - extraTCPTLSPort: extraTCPTLSPort, - extraTCPTLSPortTwo: extraTCPTLSPortTwo, - ip: ip, + ca: rootCA.CertBytes, + consulClient: consulClient, + httpPort: httpsPort, + httpFlattenedPort: httpFlattenedPort, + httpReferencePolicyPort: httpReferencePolicyPort, + grpcPort: grpcPort, + extraHTTPPort: extraHTTPPort, + extraTCPPort: extraTCPPort, + extraTCPTLSPort: extraTCPTLSPort, + extraTCPTLSPortTwo: extraTCPTLSPortTwo, + ip: ip, } return context.WithValue(ctx, consulTestContextKey, env), nil @@ -487,6 +490,14 @@ func HTTPFlattenedPort(ctx context.Context) int { return consulEnvironment.(*consulTestEnvironment).httpFlattenedPort } +func HTTPReferencePolicyPort(ctx context.Context) int { + consulEnvironment := ctx.Value(consulTestContextKey) + if consulEnvironment == nil { + panic("must run this with an integration test that has called CreateTestConsul") + } + return consulEnvironment.(*consulTestEnvironment).httpReferencePolicyPort +} + func ConsulHTTPPort(ctx context.Context) int { consulEnvironment := ctx.Value(consulTestContextKey) if consulEnvironment == nil { diff --git a/internal/testing/e2e/kind.go b/internal/testing/e2e/kind.go index 5010230ed..7b0e7a5fe 100644 --- a/internal/testing/e2e/kind.go +++ b/internal/testing/e2e/kind.go @@ -40,6 +40,9 @@ nodes: - containerPort: {{ .HTTPSFlattenedPort }} hostPort: {{ .HTTPSFlattenedPort }} protocol: TCP + - containerPort: {{ .HTTPSReferencePolicyPort }} + hostPort: {{ .HTTPSReferencePolicyPort }} + protocol: TCP - containerPort: {{ .GRPCPort }} hostPort: {{ .GRPCPort }} protocol: TCP @@ -66,31 +69,33 @@ func init() { // based off github.com/kubernetes-sigs/e2e-framework/support/kind type kindCluster struct { - name string - e *gexe.Echo - kubecfgFile string - config string - httpsPort int - httpsFlattenedPort int - grpcPort int - extraHTTPPort int - extraTCPPort int - extraTCPTLSPort int - extraTCPTLSPortTwo int + name string + e *gexe.Echo + kubecfgFile string + config string + httpsPort int + httpsFlattenedPort int + httpsReferencePolicyPort int + grpcPort int + extraHTTPPort int + extraTCPPort int + extraTCPTLSPort int + extraTCPTLSPortTwo int } func newKindCluster(name string) *kindCluster { - ports := freeport.MustTake(7) + ports := freeport.MustTake(8) return &kindCluster{ - name: name, - e: gexe.New(), - httpsPort: ports[0], - httpsFlattenedPort: ports[1], - grpcPort: ports[2], - extraHTTPPort: ports[3], - extraTCPPort: ports[4], - extraTCPTLSPort: ports[5], - extraTCPTLSPortTwo: ports[6], + name: name, + e: gexe.New(), + httpsPort: ports[0], + httpsFlattenedPort: ports[1], + httpsReferencePolicyPort: ports[2], + grpcPort: ports[3], + extraHTTPPort: ports[4], + extraTCPPort: ports[5], + extraTCPTLSPort: ports[6], + extraTCPTLSPortTwo: ports[7], } } @@ -99,21 +104,23 @@ func (k *kindCluster) Create() (string, error) { var kindConfig bytes.Buffer err := kindTemplate.Execute(&kindConfig, &struct { - HTTPSPort int - HTTPSFlattenedPort int - GRPCPort int - ExtraTCPPort int - ExtraTCPTLSPort int - ExtraTCPTLSPortTwo int - ExtraHTTPPort int + HTTPSPort int + HTTPSFlattenedPort int + HTTPSReferencePolicyPort int + GRPCPort int + ExtraTCPPort int + ExtraTCPTLSPort int + ExtraTCPTLSPortTwo int + ExtraHTTPPort int }{ - HTTPSPort: k.httpsPort, - HTTPSFlattenedPort: k.httpsFlattenedPort, - GRPCPort: k.grpcPort, - ExtraTCPPort: k.extraTCPPort, - ExtraTCPTLSPort: k.extraTCPTLSPort, - ExtraTCPTLSPortTwo: k.extraTCPTLSPortTwo, - ExtraHTTPPort: k.extraHTTPPort, + HTTPSPort: k.httpsPort, + HTTPSFlattenedPort: k.httpsFlattenedPort, + HTTPSReferencePolicyPort: k.httpsReferencePolicyPort, + GRPCPort: k.grpcPort, + ExtraTCPPort: k.extraTCPPort, + ExtraTCPTLSPort: k.extraTCPTLSPort, + ExtraTCPTLSPortTwo: k.extraTCPTLSPortTwo, + ExtraHTTPPort: k.extraHTTPPort, }) if err != nil { return "", err diff --git a/internal/testing/e2e/kubernetes.go b/internal/testing/e2e/kubernetes.go index 265192288..581abeeec 100644 --- a/internal/testing/e2e/kubernetes.go +++ b/internal/testing/e2e/kubernetes.go @@ -55,6 +55,7 @@ func InstallGatewayCRDs(ctx context.Context, cfg *envconf.Config) (context.Conte &gateway.HTTPRouteList{}, &gateway.TCPRoute{}, &gateway.TCPRouteList{}, + &gateway.ReferencePolicy{}, ) meta.AddToGroupVersion(scheme.Scheme, gateway.SchemeGroupVersion) diff --git a/internal/testing/e2e/stack.go b/internal/testing/e2e/stack.go index 46434cffa..6ae7bfa3e 100644 --- a/internal/testing/e2e/stack.go +++ b/internal/testing/e2e/stack.go @@ -50,9 +50,11 @@ func SetUpStack(hostRoute string) env.Func { func TearDownStack(ctx context.Context, cfg *envconf.Config) (context.Context, error) { var err error + namespace := Namespace(ctx) + for _, f := range []env.Func{ DestroyTestGatewayServer, - envfuncs.DeleteNamespace(Namespace(ctx)), + envfuncs.DeleteNamespace(namespace), DestroyKindCluster(ClusterName(ctx)), } { ctx, err = f(ctx, cfg)