From 6081f6591d954ab1c24387224cef99b8e6ee62a5 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Sat, 18 May 2024 06:36:55 +0000 Subject: [PATCH 01/10] set grpc as default protocol Signed-off-by: Jintao Zhang --- examples/gateway-grpcroute-via-http.yaml | 2 +- examples/gateway-grpcroute-via-https.yaml | 2 +- .../subtranslator/grpcroute_test.go | 18 +++--- .../translator/translate_grpcroute.go | 19 ++++-- test/conformance/gateway_conformance_test.go | 1 + test/consts.go | 3 +- test/integration/isolated/grpc_test.go | 58 +++++-------------- test/internal/helpers/gatewayapi.go | 4 +- 8 files changed, 42 insertions(+), 65 deletions(-) diff --git a/examples/gateway-grpcroute-via-http.yaml b/examples/gateway-grpcroute-via-http.yaml index bc7ebe3839..50b61c7999 100644 --- a/examples/gateway-grpcroute-via-http.yaml +++ b/examples/gateway-grpcroute-via-http.yaml @@ -58,7 +58,7 @@ spec: protocol: HTTP port: 80 --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-http diff --git a/examples/gateway-grpcroute-via-https.yaml b/examples/gateway-grpcroute-via-https.yaml index 3317bfc837..12e0871ec3 100644 --- a/examples/gateway-grpcroute-via-https.yaml +++ b/examples/gateway-grpcroute-via-https.yaml @@ -70,7 +70,7 @@ spec: certificateRefs: - name: grpcroute-example --- -apiVersion: gateway.networking.k8s.io/v1alpha2 +apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: grpcbin-via-https diff --git a/internal/dataplane/translator/subtranslator/grpcroute_test.go b/internal/dataplane/translator/subtranslator/grpcroute_test.go index ada1c15518..246aef4441 100644 --- a/internal/dataplane/translator/subtranslator/grpcroute_test.go +++ b/internal/dataplane/translator/subtranslator/grpcroute_test.go @@ -16,13 +16,13 @@ import ( var grpcRouteGVK = schema.GroupVersionKind{ Group: "gateway.networking.k8s.io", - Version: "v1alpha2", + Version: "v1", Kind: "GRPCRoute", } var grpcRouteTypeMeta = metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", } func makeTestGRPCRoute( @@ -33,7 +33,7 @@ func makeTestGRPCRoute( return &gatewayapi.GRPCRoute{ TypeMeta: metav1.TypeMeta{ Kind: "GRPCRoute", - APIVersion: "gateway.networking.k8s.io/v1alpha2", + APIVersion: "gateway.networking.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -98,7 +98,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -139,7 +139,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -194,7 +194,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -214,7 +214,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -243,7 +243,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), }, }, @@ -270,7 +270,7 @@ func TestGenerateKongRoutesFromGRPCRouteRule(t *testing.T) { "k8s-namespace:default", "k8s-kind:GRPCRoute", "k8s-group:gateway.networking.k8s.io", - "k8s-version:v1alpha2", + "k8s-version:v1", ), Paths: kong.StringSlice("/"), }, diff --git a/internal/dataplane/translator/translate_grpcroute.go b/internal/dataplane/translator/translate_grpcroute.go index 2487333df1..be99d0d143 100644 --- a/internal/dataplane/translator/translate_grpcroute.go +++ b/internal/dataplane/translator/translate_grpcroute.go @@ -53,10 +53,9 @@ func (t *Translator) ingressRulesFromGRPCRoute(result *ingressRules, grpcroute * // each rule may represent a different set of backend services that will be accepting // traffic, so we make separate routes and Kong services for every present rule. for ruleNumber, rule := range spec.Rules { - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. service, err := generateKongServiceFromBackendRefWithRuleNumber( - t.logger, t.storer, result, grpcroute, ruleNumber, "grpcs", grpcBackendRefsToBackendRefs(rule.BackendRefs)..., + t.logger, t.storer, result, grpcroute, ruleNumber, t.getProtocolForKongService(grpcroute), grpcBackendRefsToBackendRefs(rule.BackendRefs)..., ) if err != nil { return err @@ -116,15 +115,14 @@ func (t *Translator) ingressRulesFromGRPCRouteWithPriority( serviceName := subtranslator.KongServiceNameFromSplitGRPCRouteMatch(match) - // Create a service and attach the routes to it. Protocol for Service can be set via K8s object annotation - // "konghq.com/protocol", by default use "grpcs" to not break existing behavior when annotation is not specified. + // Create a service and attach the routes to it. kongService, _ := generateKongServiceFromBackendRefWithName( t.logger, t.storer, rules, serviceName, grpcRoute, - "grpcs", + t.getProtocolForKongService(grpcRoute), grpcBackendRefsToBackendRefs(grpcRouteRule.BackendRefs)..., ) kongService.Routes = append( @@ -144,3 +142,12 @@ func grpcBackendRefsToBackendRefs(grpcBackendRef []gatewayapi.GRPCBackendRef) [] } return backendRefs } + +// getProtocolForKongService returns the protocol for the Kong service based on the Gateway listening ports +func (t *Translator) getProtocolForKongService(grpcRoute *gatewayapi.GRPCRoute) string { + // When Gateway listens on HTTP use "grpc" protocol for the service. Otherwise for HTTPS use "grpcs". + if len(t.getGatewayListeningPorts(grpcRoute.Namespace, gatewayapi.HTTPProtocolType, grpcRoute.Spec.ParentRefs)) > 0 { + return "grpc" + } + return "grpcs" +} diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 014bf6b3a6..9567589e79 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -43,6 +43,7 @@ var expressionRoutesSupportedFeatures = []features.SupportedFeature{ // core features features.SupportGateway, features.SupportHTTPRoute, + features.SupportGRPCRoute, // extended features features.SupportHTTPRouteQueryParamMatching, features.SupportHTTPRouteMethodMatching, diff --git a/test/consts.go b/test/consts.go index 7bdb4e05fb..d3a9db6f5d 100644 --- a/test/consts.go +++ b/test/consts.go @@ -24,7 +24,8 @@ const ( // GRPCBinImage is the container image name we use for deploying the "grpcbin" GRPC testing tool. // See: https://github.com/Kong/grpcbin GRPCBinImage = "kong/grpcbin:latest" - GRPCBinPort = 9001 + GRPCBinPort = 9000 + GRPCSBinPort = 9001 // EnvironmentCleanupTimeout is the amount of time that will be given by the test suite to the // testing environment to perform its cleanup when the test suite is shutting down. diff --git a/test/integration/isolated/grpc_test.go b/test/integration/isolated/grpc_test.go index 0681cbd974..a1388ef328 100644 --- a/test/integration/isolated/grpc_test.go +++ b/test/integration/isolated/grpc_test.go @@ -22,7 +22,6 @@ import ( "google.golang.org/grpc/metadata" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" gatewayclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" @@ -30,7 +29,6 @@ import ( "github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi" "github.com/kong/kubernetes-ingress-controller/v3/internal/util/builder" "github.com/kong/kubernetes-ingress-controller/v3/test" - "github.com/kong/kubernetes-ingress-controller/v3/test/helpers/certificate" "github.com/kong/kubernetes-ingress-controller/v3/test/integration/consts" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/helpers" "github.com/kong/kubernetes-ingress-controller/v3/test/internal/testlabels" @@ -43,9 +41,12 @@ func TestGRPCRouteEssentials(t *testing.T) { New("essentials"). WithLabel(testlabels.NetworkingFamily, testlabels.NetworkingFamilyGatewayAPI). WithLabel(testlabels.Kind, testlabels.KindGRPCRoute). - WithSetup("deploy kong addon into cluster", featureSetup()). - Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTPS", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - // On purpose omit protocol annotation to test defaulting to "grpcs" that is preserved to not break users' configs. + WithSetup("deploy kong addon into cluster", featureSetup( + withKongProxyEnvVars(map[string]string{ + "PROXY_LISTEN": `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`, + }), + )). + Assess("deploying Gateway and example GRPC service (without konghq.com/protocol annotation) exposed via GRPCRoute over HTTP", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { cleaner := GetFromCtxForT[*clusters.Cleaner](ctx, t) cluster := GetClusterFromCtx(ctx) namespace := GetNamespaceForT(ctx, t) @@ -61,45 +62,12 @@ func TestGRPCRouteEssentials(t *testing.T) { assert.NoError(t, err) cleaner.Add(gwc) - t.Log("configuring secret") - const tlsRouteHostname = "tls-route.example" - tlsRouteExampleTLSCert, tlsRouteExampleTLSKey := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithCommonName(tlsRouteHostname)) - const tlsSecretName = "secret-test" - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - UID: k8stypes.UID("7428fb98-180b-4702-a91f-61351a33c6e8"), - Name: tlsSecretName, - Namespace: namespace, - }, - Data: map[string][]byte{ - "tls.crt": tlsRouteExampleTLSCert, - "tls.key": tlsRouteExampleTLSKey, - }, - } - - t.Log("deploying secret") - secret, err = cluster.Client().CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - assert.NoError(t, err) - cleaner.Add(secret) - t.Log("deploying a new gateway") gateway, err := helpers.DeployGateway(ctx, gatewayClient, namespace, gatewayClassName, func(gw *gatewayapi.Gateway) { - // Besides default HTTP listener, add a HTTPS listener. - gw.Spec.Listeners = append( - gw.Spec.Listeners, - builder.NewListener("https"). - HTTPS(). - WithPort(ktfkong.DefaultProxyTLSServicePort). - WithHostname(testHostname). - WithTLSConfig(&gatewayapi.GatewayTLSConfig{ - CertificateRefs: []gatewayapi.SecretObjectReference{ - { - Name: gatewayapi.ObjectName(secret.Name), - }, - }, - }). - Build(), - ) + gw.Spec.Listeners = builder.NewListener("grpc"). + HTTP(). + WithPort(ktfkong.DefaultProxyHTTPPort). + IntoSlice() }) assert.NoError(t, err) cleaner.Add(gateway) @@ -168,7 +136,7 @@ func TestGRPCRouteEssentials(t *testing.T) { return ctx }). Assess("checking if GRPCRoute is linked correctly and client can connect properly to the exposed service", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { - grpcAddr := GetHTTPSURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTPS, but without the scheme (https://). + grpcAddr := GetHTTPURLFromCtx(ctx).Host // For GRPC, we use the same address as for HTTP, but without the scheme (http://). namespace := GetNamespaceForT(ctx, t) gatewayClient := GetFromCtxForT[*gatewayclient.Clientset](ctx, t) grpcRoute := GetFromCtxForT[*gatewayapi.GRPCRoute](ctx, t) @@ -184,14 +152,14 @@ func TestGRPCRouteEssentials(t *testing.T) { t.Log("waiting for routes from GRPCRoute to become operational") assert.Eventually(t, func() bool { - err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", true) + err := grpcEchoResponds(ctx, grpcAddr, testHostname, "kong", false) if err != nil { t.Log(err) } return err == nil }, consts.IngressWait, consts.WaitTick) - client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, true) + client, closeGrpcConn, err := grpcBinClient(grpcAddr, testHostname, false) assert.NoError(t, err) t.Cleanup(func() { err := closeGrpcConn() diff --git a/test/internal/helpers/gatewayapi.go b/test/internal/helpers/gatewayapi.go index cfa42fea88..78bc86883f 100644 --- a/test/internal/helpers/gatewayapi.go +++ b/test/internal/helpers/gatewayapi.go @@ -116,7 +116,7 @@ func gatewayLinkStatusMatches( switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", err) @@ -211,7 +211,7 @@ func verifyProgrammedConditionStatus(t *testing.T, switch protocolType { case gatewayapi.HTTPProtocolType: route, err := c.GatewayV1().HTTPRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) - groute, gerr := c.GatewayV1alpha2().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) + groute, gerr := c.GatewayV1().GRPCRoutes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil && gerr != nil { t.Logf("error getting http route: %v", err) t.Logf("error getting grpc route: %v", err) From 1c6a1cb5a3adab7d60c2628ae413985e7af39259 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Sat, 18 May 2024 19:21:37 +0000 Subject: [PATCH 02/10] test(conformance): pass GRPCRoute conformance test Signed-off-by: Jintao Zhang --- CHANGELOG.md | 5 +++++ internal/controllers/gateway/grpcroute_controller.go | 2 +- internal/controllers/gateway/route_parent_status.go | 5 +++++ internal/dataplane/translator/translate_grpcroute.go | 2 +- test/conformance/gateway_conformance_test.go | 11 +++++++++++ test/conformance/suite_test.go | 4 ++++ 6 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98b6f2eb90..5b70f1f3e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,9 @@ Adding a new version? You'll need three changes: performance benefits, however, so labeling plugin configuration Secrets and enabling the filter is recommended as soon as is convenient. [#5856](https://github.com/Kong/kubernetes-ingress-controller/pull/5856) +- Dynamically set the proxy protocol of GRPCRoute to `grpc` or `grpcs` based on the port listened by Gateway. + If you don't set the protocol for Service via "konghq.com/protocol" annotation, Kong will use grpc instead of grpcs. + [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) ### Added @@ -158,6 +161,8 @@ Adding a new version? You'll need three changes: [#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965) - Fallback configuration no longer omits licenses and vaults. [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048) +- Add support for pass Kubernetes Gateway API GRPCRoute conformance test. + [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) ### Fixed diff --git a/internal/controllers/gateway/grpcroute_controller.go b/internal/controllers/gateway/grpcroute_controller.go index c033c57574..60264a2426 100644 --- a/internal/controllers/gateway/grpcroute_controller.go +++ b/internal/controllers/gateway/grpcroute_controller.go @@ -470,7 +470,7 @@ func (r *GRPCRouteReconciler) ensureGatewayReferenceStatusAdded(ctx context.Cont // if the reference already exists and doesn't require any changes // then just leave it alone. - parentRefKey := gateway.gateway.Namespace + "/" + gateway.gateway.Name + parentRefKey := fmt.Sprintf("%s/%s/%s", gateway.gateway.Namespace, gateway.gateway.Name, gateway.listenerName) if existingGatewayParentStatus, exists := parentStatuses[parentRefKey]; exists { // check if the parentRef and controllerName are equal, and whether the new condition is present in existing conditions if reflect.DeepEqual(existingGatewayParentStatus.ParentRef, gatewayParentStatus.ParentRef) && diff --git a/internal/controllers/gateway/route_parent_status.go b/internal/controllers/gateway/route_parent_status.go index 37b8065fe0..020ef6dec5 100644 --- a/internal/controllers/gateway/route_parent_status.go +++ b/internal/controllers/gateway/route_parent_status.go @@ -46,6 +46,11 @@ func routeParentStatusKey[routeT gatewayapi.RouteT]( namespace, parentRef.GetName(), parentRef.GetSectionName().OrEmpty()) + case *gatewayapi.GRPCRoute: + return fmt.Sprintf("%s/%s/%s", + namespace, + parentRef.GetName(), + parentRef.GetSectionName().OrEmpty()) default: return fmt.Sprintf("%s/%s", namespace, parentRef.GetName()) } diff --git a/internal/dataplane/translator/translate_grpcroute.go b/internal/dataplane/translator/translate_grpcroute.go index be99d0d143..c0269a56ad 100644 --- a/internal/dataplane/translator/translate_grpcroute.go +++ b/internal/dataplane/translator/translate_grpcroute.go @@ -143,7 +143,7 @@ func grpcBackendRefsToBackendRefs(grpcBackendRef []gatewayapi.GRPCBackendRef) [] return backendRefs } -// getProtocolForKongService returns the protocol for the Kong service based on the Gateway listening ports +// getProtocolForKongService returns the protocol for the Kong service based on the Gateway listening ports. func (t *Translator) getProtocolForKongService(grpcRoute *gatewayapi.GRPCRoute) string { // When Gateway listens on HTTP use "grpc" protocol for the service. Otherwise for HTTPS use "grpcs". if len(t.getGatewayListeningPorts(grpcRoute.Namespace, gatewayapi.HTTPProtocolType, grpcRoute.Spec.ParentRefs)) > 0 { diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 9567589e79..c51bdf0fe6 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -24,12 +24,22 @@ import ( var skippedTestsForTraditionalRoutes = []string{ // core conformance tests.HTTPRouteHeaderMatching.ShortName, + // Kong gateway does not support this feature + tests.GRPCRouteListenerHostnameMatching.ShortName, +} + +var skippedTestsForExpressionRoutes = []string{ + // core conformance + tests.HTTPRouteHeaderMatching.ShortName, + // Kong gateway does not support this feature + tests.GRPCRouteListenerHostnameMatching.ShortName, } var traditionalRoutesSupportedFeatures = []features.SupportedFeature{ // core features features.SupportGateway, features.SupportHTTPRoute, + features.SupportGRPCRoute, // extended features features.SupportHTTPRouteResponseHeaderModification, features.SupportHTTPRoutePathRewrite, @@ -71,6 +81,7 @@ func TestGatewayConformance(t *testing.T) { supportedFeatures = traditionalRoutesSupportedFeatures mode = string(dpconf.RouterFlavorTraditionalCompatible) case dpconf.RouterFlavorExpressions: + skippedTests = skippedTestsForExpressionRoutes supportedFeatures = expressionRoutesSupportedFeatures mode = string(dpconf.RouterFlavorExpressions) default: diff --git a/test/conformance/suite_test.go b/test/conformance/suite_test.go index 51c37929ed..1e0abe24b8 100644 --- a/test/conformance/suite_test.go +++ b/test/conformance/suite_test.go @@ -82,6 +82,10 @@ func TestMain(m *testing.M) { kongBuilder = kongBuilder.WithProxyEnvVar("router_flavor", string(dpconf.RouterFlavorExpressions)) } + // The test cases for GRPCRoute in the current GatewayAPI all use the h2c protocol. + // In order to pass conformance tests, the proxy must listen http2 and http on the same port. + kongBuilder.WithProxyEnvVar("PROXY_LISTEN", `0.0.0.0:8000 http2\, 0.0.0.0:8443 http2 ssl`) + // Pin the Helm chart version. kongBuilder.WithHelmChartVersion(testenv.KongHelmChartVersion()) From 6ce71db74b681b73d78a7e09e7c3661447c4f7dc Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Sat, 18 May 2024 22:57:23 +0000 Subject: [PATCH 03/10] skip unsupported case. Signed-off-by: Jintao Zhang --- test/conformance/gateway_conformance_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index c51bdf0fe6..c8a626ed2a 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -26,6 +26,7 @@ var skippedTestsForTraditionalRoutes = []string{ tests.HTTPRouteHeaderMatching.ShortName, // Kong gateway does not support this feature tests.GRPCRouteListenerHostnameMatching.ShortName, + tests.GRPCRouteHeaderMatching.ShortName, } var skippedTestsForExpressionRoutes = []string{ From 8cd6b9b8a51c261fdd62373bea0a9bf977538e35 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Sun, 19 May 2024 00:06:40 +0000 Subject: [PATCH 04/10] add GRPCRoute report profile Signed-off-by: Jintao Zhang --- test/conformance/gateway_conformance_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index c8a626ed2a..0197a60040 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -99,6 +99,7 @@ func TestGatewayConformance(t *testing.T) { opts.SkipTests = skippedTests opts.ConformanceProfiles = sets.New( suite.GatewayHTTPConformanceProfileName, + suite.GatewayGRPCConformanceProfileName, ) opts.Implementation = conformancev1.Implementation{ Organization: metadata.Organization, From 8e0286ff950edfa99ea06be692b6290e97aa1a18 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Tue, 21 May 2024 14:42:01 +0800 Subject: [PATCH 05/10] avoid hardcoding grpc ports. Signed-off-by: Jintao Zhang --- test/consts.go | 5 +++-- test/integration/isolated/ingress_test.go | 14 +++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/test/consts.go b/test/consts.go index d3a9db6f5d..65d47d268e 100644 --- a/test/consts.go +++ b/test/consts.go @@ -24,8 +24,9 @@ const ( // GRPCBinImage is the container image name we use for deploying the "grpcbin" GRPC testing tool. // See: https://github.com/Kong/grpcbin GRPCBinImage = "kong/grpcbin:latest" - GRPCBinPort = 9000 - GRPCSBinPort = 9001 + + GRPCBinPort int32 = 9000 + GRPCSBinPort int32 = 9001 // EnvironmentCleanupTimeout is the amount of time that will be given by the test suite to the // testing environment to perform its cleanup when the test suite is shutting down. diff --git a/test/integration/isolated/ingress_test.go b/test/integration/isolated/ingress_test.go index 6b14d4b851..ef531444d8 100644 --- a/test/integration/isolated/ingress_test.go +++ b/test/integration/isolated/ingress_test.go @@ -72,23 +72,19 @@ func TestIngressGRPC(t *testing.T) { gRPC kongProtocolAnnotation = "grpc" gRPCS kongProtocolAnnotation = "grpcs" ) - const ( - gRPCBinPort int32 = 9000 - gRPCSBinPort int32 = 9001 - ) t.Log("deploying a minimal gRPC container deployment to test Ingress routes") container := generators.NewContainer("grpcbin", test.GRPCBinImage, 0) // Overwrite ports to specify gRPC over HTTP (9000) and gRPC over HTTPS (9001). - container.Ports = []corev1.ContainerPort{{ContainerPort: gRPCBinPort, Name: string(gRPC)}, {ContainerPort: gRPCSBinPort, Name: string(gRPCS)}} + container.Ports = []corev1.ContainerPort{{ContainerPort: test.GRPCBinPort, Name: string(gRPC)}, {ContainerPort: test.GRPCSBinPort, Name: string(gRPCS)}} deployment := generators.NewDeploymentForContainer(container) deployment, err = cluster.Client().AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) assert.NoError(t, err) cleaner.Add(deployment) exposeWithService := func(p kongProtocolAnnotation) *corev1.Service { - grpcBinPort := gRPCBinPort + grpcBinPort := test.GRPCBinPort if p == gRPCS { - grpcBinPort = gRPCSBinPort + grpcBinPort = test.GRPCSBinPort } kongProtocol := string(p) t.Logf("exposing deployment gRPC (%s) port %s via service", kongProtocol, deployment.Name) @@ -122,7 +118,7 @@ func TestIngressGRPC(t *testing.T) { Service: &netv1.IngressServiceBackend{ Name: serviceGRPCS.Name, Port: netv1.ServiceBackendPort{ - Number: gRPCSBinPort, + Number: test.GRPCSBinPort, }, }, }, @@ -142,7 +138,7 @@ func TestIngressGRPC(t *testing.T) { Service: &netv1.IngressServiceBackend{ Name: serviceGRPC.Name, Port: netv1.ServiceBackendPort{ - Number: gRPCBinPort, + Number: test.GRPCBinPort, }, }, }, From 84d97f7d31ea2c11ae8f917fb0eeb6df9c9aa8eb Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Mon, 3 Jun 2024 16:40:26 +0800 Subject: [PATCH 06/10] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patryk Małek --- CHANGELOG.md | 4 ++-- internal/dataplane/translator/translate_grpcroute.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b70f1f3e2..26fc1c03d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,7 +103,7 @@ Adding a new version? You'll need three changes: enabling the filter is recommended as soon as is convenient. [#5856](https://github.com/Kong/kubernetes-ingress-controller/pull/5856) - Dynamically set the proxy protocol of GRPCRoute to `grpc` or `grpcs` based on the port listened by Gateway. - If you don't set the protocol for Service via "konghq.com/protocol" annotation, Kong will use grpc instead of grpcs. + If you don't set the protocol for Service via "konghq.com/protocol" annotation, Kong will use `grpc` instead of `grpcs`. [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) ### Added @@ -161,7 +161,7 @@ Adding a new version? You'll need three changes: [#5965](https://github.com/Kong/kubernetes-ingress-controller/pull/5965) - Fallback configuration no longer omits licenses and vaults. [#6048](https://github.com/Kong/kubernetes-ingress-controller/pull/6048) -- Add support for pass Kubernetes Gateway API GRPCRoute conformance test. +- Add support for Gateway API GRPCRoute and pass related Gateway API conformance test. [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) ### Fixed diff --git a/internal/dataplane/translator/translate_grpcroute.go b/internal/dataplane/translator/translate_grpcroute.go index c0269a56ad..f88407d3be 100644 --- a/internal/dataplane/translator/translate_grpcroute.go +++ b/internal/dataplane/translator/translate_grpcroute.go @@ -143,7 +143,8 @@ func grpcBackendRefsToBackendRefs(grpcBackendRef []gatewayapi.GRPCBackendRef) [] return backendRefs } -// getProtocolForKongService returns the protocol for the Kong service based on the Gateway listening ports. +// getProtocolForKongService returns the protocol for the Kong service configuration. +// In order to get the protocol, provided route's parentRefs are searched for a Gateway that has the matching listening ports. func (t *Translator) getProtocolForKongService(grpcRoute *gatewayapi.GRPCRoute) string { // When Gateway listens on HTTP use "grpc" protocol for the service. Otherwise for HTTPS use "grpcs". if len(t.getGatewayListeningPorts(grpcRoute.Namespace, gatewayapi.HTTPProtocolType, grpcRoute.Spec.ParentRefs)) > 0 { From 75bd0d70d7ffa8cfd60f46bf38b60b3f93ad001c Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Wed, 5 Jun 2024 12:49:01 +0000 Subject: [PATCH 07/10] clean up comments Signed-off-by: Jintao Zhang --- test/conformance/gateway_conformance_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 0197a60040..4c48fe6463 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -24,15 +24,17 @@ import ( var skippedTestsForTraditionalRoutes = []string{ // core conformance tests.HTTPRouteHeaderMatching.ShortName, - // Kong gateway does not support this feature + // There is an issue with KIC when processing this scenario. + // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6136 tests.GRPCRouteListenerHostnameMatching.ShortName, - tests.GRPCRouteHeaderMatching.ShortName, + // tests.GRPCRouteHeaderMatching.ShortName and tests.GRPCExactMethodMatching.ShortName may + // have some conflicts, skipping either one will still pass normally. + tests.GRPCExactMethodMatching.ShortName, } var skippedTestsForExpressionRoutes = []string{ - // core conformance - tests.HTTPRouteHeaderMatching.ShortName, - // Kong gateway does not support this feature + // When processingvthis scenario, + // the expression router of Kong currently needs to add `priority` to the route. tests.GRPCRouteListenerHostnameMatching.ShortName, } From 945546426fda47915060e6aeda4b24296fc4eff1 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Thu, 6 Jun 2024 05:56:50 +0800 Subject: [PATCH 08/10] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patryk Małek --- CHANGELOG.md | 2 +- test/conformance/gateway_conformance_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07d34ea047..2f5cf38f27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,7 +103,7 @@ Adding a new version? You'll need three changes: enabling the filter is recommended as soon as is convenient. [#5856](https://github.com/Kong/kubernetes-ingress-controller/pull/5856) - Dynamically set the proxy protocol of GRPCRoute to `grpc` or `grpcs` based on the port listened by Gateway. - If you don't set the protocol for Service via "konghq.com/protocol" annotation, Kong will use `grpc` instead of `grpcs`. + If you don't set the protocol for Service via `konghq.com/protocol` annotation, Kong will use `grpc` instead of `grpcs`. [#5776](https://github.com/Kong/kubernetes-ingress-controller/pull/5776) - The `/debug/config/failed` and `/debug/config/successful` diagnostic endpoints now nest configuration dumps under a `config` key. These endpoints diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index 4c48fe6463..bf0f3fd41a 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -33,8 +33,8 @@ var skippedTestsForTraditionalRoutes = []string{ } var skippedTestsForExpressionRoutes = []string{ - // When processingvthis scenario, - // the expression router of Kong currently needs to add `priority` to the route. + // When processing this scenario, the Kong's expression router currently needs to + // add `priority` to the route. tests.GRPCRouteListenerHostnameMatching.ShortName, } From 76af203f6126225730f723255721e8478edfdbbc Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Thu, 6 Jun 2024 05:48:51 +0000 Subject: [PATCH 09/10] add TODO about test case conflicts Signed-off-by: Jintao Zhang --- test/conformance/gateway_conformance_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index bf0f3fd41a..c781129c4e 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -29,6 +29,7 @@ var skippedTestsForTraditionalRoutes = []string{ tests.GRPCRouteListenerHostnameMatching.ShortName, // tests.GRPCRouteHeaderMatching.ShortName and tests.GRPCExactMethodMatching.ShortName may // have some conflicts, skipping either one will still pass normally. + // TODO: https://github.com/Kong/kubernetes-ingress-controller/issues/6144 tests.GRPCExactMethodMatching.ShortName, } From f9e0ed5ae2df3309e9b1c893287d752488ac7910 Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Thu, 6 Jun 2024 09:37:27 +0000 Subject: [PATCH 10/10] Update comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jintao Zhang Co-authored-by: Patryk Małek --- test/conformance/gateway_conformance_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/conformance/gateway_conformance_test.go b/test/conformance/gateway_conformance_test.go index c781129c4e..c7f89fc338 100644 --- a/test/conformance/gateway_conformance_test.go +++ b/test/conformance/gateway_conformance_test.go @@ -34,8 +34,9 @@ var skippedTestsForTraditionalRoutes = []string{ } var skippedTestsForExpressionRoutes = []string{ - // When processing this scenario, the Kong's expression router currently needs to - // add `priority` to the route. + // When processing this scenario, the Kong's expressions router requires `priority` + // to be specified for routes. + // We cannot provide that for routes that are part of the conformance suite. tests.GRPCRouteListenerHostnameMatching.ShortName, }