helm: support webhook.enablePolicyApi

changelog/v1.18.0-beta1/helm-policy-validation-api.yaml

@@ -0,0 +1,8 @@
+changelog:
+  - type: HELM
+    issueLink: https://github.com/solo-io/solo-projects/issues/6352
+    resolvesIssue: false
+    description: >-
+      Introduce `gateway.validation.webhook.enablePolicyApi` which controls whether or not RouteOptions and
+      VirtualHostOptions CRs are subject to validation. By default, this value is true.
+      The validation of these Policy APIs only runs if the Kubernetes Gateway integration is enabled (`kubeGateway.enabled`).

docs/content/reference/values.txt

@@ -454,6 +454,7 @@
 |gateway.validation.webhook.timeoutSeconds|int||the timeout for the webhook, defaults to 10|
 |gateway.validation.webhook.extraAnnotations.NAME|string||extra annotations to add to the webhook|
 |gateway.validation.webhook.skipDeleteValidationResources[]|string||resource types in this list will not use webhook valdaition for DELETEs. Use '*' to skip validation for all resources. Valid values are 'virtualservices', 'routetables','upstreams', 'secrets', 'ratelimitconfigs', and '*'. Invalid values will be accepted but will not be used.|
+|gateway.validation.webhook.enablePolicyApi|bool|true|enable validation of Policy Api resources (RouteOptions, VirtualHostOptions) (default: true). NOTE: This only applies if the Kubernetes Gateway Integration is also enabled (kubeGateway.enabled).|
 |gateway.validation.webhook.kubeResourceOverride.NAME|interface||override fields in the generated resource by specifying the yaml structure to override under the top-level key.|
 |gateway.validation.validationServerGrpcMaxSizeBytes|int|104857600|gRPC max message size in bytes for the gloo validation server|
 |gateway.validation.livenessProbeEnabled|bool||Set to true to enable a liveness probe for the gateway (default is false). You must also set the 'Probes' value to true.|

install/helm/gloo/generate/values.go

@@ -450,6 +450,12 @@ type Webhook struct {
 	TimeoutSeconds                *int              `json:"timeoutSeconds,omitempty" desc:"the timeout for the webhook, defaults to 10"`
 	ExtraAnnotations              map[string]string `json:"extraAnnotations,omitempty" desc:"extra annotations to add to the webhook"`
 	SkipDeleteValidationResources []string          `json:"skipDeleteValidationResources,omitempty" desc:"resource types in this list will not use webhook valdaition for DELETEs. Use '*' to skip validation for all resources. Valid values are 'virtualservices', 'routetables','upstreams', 'secrets', 'ratelimitconfigs', and '*'. Invalid values will be accepted but will not be used."`
+	// EnablePolicyApi provides granular access to users to opt-out of Policy validation
+	// There are some known race conditions in our Gloo Gateway processes resource references,
+	// even when allowWarnings=true: https://github.com/solo-io/solo-projects/issues/6321
+	// As a result, this is intended as a short-term solution to provide users a way to opt-out of Policy API validation.
+	// The desired long-term strategy is that our validation logic is stable, and users can leverage it
+	EnablePolicyApi *bool `json:"enablePolicyApi,omitempty" desc:"enable validation of Policy Api resources (RouteOptions, VirtualHostOptions) (default: true). NOTE: This only applies if the Kubernetes Gateway Integration is also enabled (kubeGateway.enabled)."`
 	*KubeResourceOverride
 }
 

install/helm/gloo/templates/5-gateway-validation-webhook-configuration.yaml

@@ -24,7 +24,7 @@ webhooks:
       path: "/validation"
     caBundle: "" # update manually or use certgen job or cert-manager's ca-injector
   rules:
-{{- if .Values.kubeGateway.enabled }}
+{{- if and .Values.kubeGateway.enabled .Values.gateway.validation.webhook.enablePolicyApi }}
   - operations: [ "CREATE", "UPDATE" ]
     # RouteOption and VirtualHostOption DELETEs are not supported.
     # Their validation is currently limited to usage as Kube Gateway API Policies
@@ -33,7 +33,7 @@ webhooks:
     apiGroups: ["gateway.solo.io"]
     apiVersions: ["v1"]
     resources: ["routeoptions", "virtualhostoptions"]
-{{- end }}{{/* if .Values.kubeGateway.enabled */}}
+{{- end }}{{/* if and .Values.kubeGateway.enabled .Values.gateway.validation.webhook.enablePolicyApi */}}
   - operations: {{ include "gloo.webhookvalidation.operationsForResource" (list "virtualservices" .Values.gateway.validation.webhook.skipDeleteValidationResources) }}
     apiGroups: ["gateway.solo.io"]
     apiVersions: ["v1"]

install/helm/gloo/values-template.yaml

@@ -113,6 +113,10 @@ gateway:
       enabled: true
       disableHelmHook: false
       extraAnnotations: {}
+      # We have learned that defaulting validation behavior leads to unintentional usage of it
+      # https://github.com/solo-io/gloo/issues/9309
+      # As a result, for our Policy API, we default it to on, and provide users the way to opt-out of it
+      enablePolicyApi: true
   certGenJob:
     enabled: true
     image:

install/test/5-gateway-validation-webhook-configuration_test.go

@@ -1,11 +1,16 @@
 package test
 
 import (
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 
+	"github.com/golang/protobuf/ptypes/wrappers"
+	"github.com/onsi/gomega/types"
+	v1 "k8s.io/api/admissionregistration/v1"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	gloostringutils "github.com/solo-io/gloo/pkg/utils/stringutils"
@@ -15,6 +20,10 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+const (
+	validatingWebhookConfigurationKind = "ValidatingWebhookConfiguration"
+)
+
 var _ = Describe("WebhookValidationConfiguration helm test", func() {
 	var allTests = func(rendererTestCase renderTestCase) {
 
@@ -55,6 +64,115 @@ var _ = Describe("WebhookValidationConfiguration helm test", func() {
 			Entry("all", []string{"*"}, 5),
 			Entry("empty", []string{}, 0),
 		)
+
+		Context("enablePolicyApi", func() {
+
+			// containPolicyApiOperation returns a GomegaMatcher which will assert that a provided ValidatingWebhookConfiguration
+			// contains the Rule that includes the Policy APIs
+			containPolicyApiOperation := func() types.GomegaMatcher {
+				policyApiOperation := v1.RuleWithOperations{
+					Operations: []v1.OperationType{
+						v1.Create,
+						v1.Update,
+					},
+					Rule: v1.Rule{
+						APIGroups:   []string{"gateway.solo.io"},
+						APIVersions: []string{"v1"},
+						Resources:   []string{"routeoptions", "virtualhostoptions"},
+					},
+				}
+				return WithTransform(func(config *v1.ValidatingWebhookConfiguration) []v1.RuleWithOperations {
+					if config == nil {
+						return nil
+					}
+					return config.Webhooks[0].Rules
+				}, ContainElement(policyApiOperation))
+			}
+
+			type enablePolicyApiCase struct {
+				enableKubeGatewayApi         *wrappers.BoolValue
+				enablePolicyApi              *wrappers.BoolValue
+				expectedWebhookConfiguration types.GomegaMatcher
+			}
+
+			DescribeTable("respects helm values",
+				func(testCase enablePolicyApiCase) {
+					var valuesArgs []string
+					if testCase.enableKubeGatewayApi != nil {
+						valuesArgs = append(valuesArgs, fmt.Sprintf(`kubeGateway.enabled=%t`, testCase.enableKubeGatewayApi.GetValue()))
+					}
+					if testCase.enablePolicyApi != nil {
+						valuesArgs = append(valuesArgs, fmt.Sprintf(`gateway.validation.webhook.enablePolicyApi=%t`, testCase.enablePolicyApi.GetValue()))
+					}
+
+					prepareMakefile(namespace, helmValues{
+						valuesArgs: valuesArgs,
+					})
+
+					testManifest.ExpectUnstructured(
+						validatingWebhookConfigurationKind,
+						"", // ValidatingWebhookConfiguration is cluster-scoped
+						fmt.Sprintf("gloo-gateway-validation-webhook-%s", namespace),
+					).To(WithTransform(func(unstructuredVwc *unstructured.Unstructured) *v1.ValidatingWebhookConfiguration {
+						// convert the unstructured validating webhook configuration to a structured object
+						if unstructuredVwc == nil {
+							return nil
+						}
+
+						rawJson, err := json.Marshal(unstructuredVwc.Object)
+						if err != nil {
+							return nil
+						}
+
+						var structuredVwc *v1.ValidatingWebhookConfiguration
+						err = json.Unmarshal(rawJson, &structuredVwc)
+						if err != nil {
+							return nil
+						}
+
+						return structuredVwc
+					}, testCase.expectedWebhookConfiguration))
+				},
+				Entry("unset", enablePolicyApiCase{
+					enableKubeGatewayApi:         nil,
+					enablePolicyApi:              nil,
+					expectedWebhookConfiguration: Not(containPolicyApiOperation()),
+				}),
+				Entry("enableKubeGatewayApi=false,", enablePolicyApiCase{
+					enableKubeGatewayApi:         &wrappers.BoolValue{Value: false},
+					enablePolicyApi:              nil,
+					expectedWebhookConfiguration: Not(containPolicyApiOperation()),
+				}),
+				Entry("enableKubeGatewayApi=true,", enablePolicyApiCase{
+					enableKubeGatewayApi:         &wrappers.BoolValue{Value: true},
+					enablePolicyApi:              nil, // default is true
+					expectedWebhookConfiguration: containPolicyApiOperation(),
+				}),
+				Entry("enableKubeGatewayApi=true, enablePolicyApi=true", enablePolicyApiCase{
+					enableKubeGatewayApi:         &wrappers.BoolValue{Value: true},
+					enablePolicyApi:              &wrappers.BoolValue{Value: true},
+					expectedWebhookConfiguration: containPolicyApiOperation(),
+				}),
+				// This is the critical test case, which demonstrates that a user can enabled the K8s Gateway Integration,
+				// but disable validation for the Policy APIs
+				Entry("enableKubeGatewayApi=false, enablePolicyApi=true", enablePolicyApiCase{
+					enableKubeGatewayApi:         &wrappers.BoolValue{Value: false},
+					enablePolicyApi:              &wrappers.BoolValue{Value: true},
+					expectedWebhookConfiguration: Not(containPolicyApiOperation()),
+				}),
+				Entry("enableKubeGatewayApi=false, enablePolicyApi=true", enablePolicyApiCase{
+					enableKubeGatewayApi:         &wrappers.BoolValue{Value: false},
+					enablePolicyApi:              &wrappers.BoolValue{Value: true},
+					expectedWebhookConfiguration: Not(containPolicyApiOperation()),
+				}),
+				Entry("enableKubeGatewayApi=false, enablePolicyApi=false", enablePolicyApiCase{
+					enableKubeGatewayApi:         &wrappers.BoolValue{Value: false},
+					enablePolicyApi:              &wrappers.BoolValue{Value: false},
+					expectedWebhookConfiguration: Not(containPolicyApiOperation()),
+				}),
+			)
+		})
+
 	}
 	runTests(allTests)
 })