[1.17] Warn on missing TLS secret

.github/workflows/nightly-tests.yaml

@@ -50,7 +50,7 @@ jobs:
     name: End-to-End (branch=${{ github.ref_name }}, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
     if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'workflow_initiating_branch' }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 150
+    timeout-minutes: 180
     strategy:
       # Since we are running these on a schedule, there is no value in failing fast
       # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking
@@ -60,7 +60,7 @@ jobs:
           # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel.
           # As a result, we increase the threshold for the tests, since they all run serially on a single cluster
           - cluster-name: 'cluster-one'
-            go-test-args: '-v -timeout=120m'
+            go-test-args: '-v -timeout=150m'
             go-test-run-regex: ${{ inputs.kubernetes-end-to-end-run-regex }}
         # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support
         # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
@@ -110,7 +110,7 @@ jobs:
     name: End-to-End (branch=main, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
     if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'main') || github.event.schedule == '0 5 * * 1-5' }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 150
+    timeout-minutes: 180
     strategy:
       # Since we are running these on a schedule, there is no value in failing fast
       # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking
@@ -120,7 +120,7 @@ jobs:
           # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel.
           # As a result, we increase the threshold for the tests, since they all run serially on a single cluster
           - cluster-name: 'cluster-one'
-            go-test-args: '-v -timeout=120m'
+            go-test-args: '-v -timeout=150m'
             go-test-run-regex: ""
         # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support
         # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/

changelog/v1.17.5/missing-tls-secret.yaml

@@ -0,0 +1,20 @@
+changelog:
+  - type: FIX
+    issueLink: https://github.com/solo-io/gloo/issues/6957
+    resolvesIssue: false
+    description: >-
+      Fix for issue where a missing TLS secret was treated by validation as an error,
+      potentially bringing down the entire HTTPS gateway if the gloo pod restarts while 
+      in this bad state. This is a breaking change in the default behavior of validation.
+
+      To enable this behavior, use the helm setting `gateway.validation.warnMissingTlsSecret=true`
+      or the same field on the Settings CR. This field has no effect if allowWarnings is false or 
+      acceptAllResources is true.
+  - type: HELM
+    issueLink: https://github.com/solo-io/gloo/issues/6957
+    resolvesIssue: false
+    description: >-
+      New field gateway.validation.warnMissingTlsSecret controls whether missing TLS secrets referenced
+      in SslConfig and UpstreamSslConfig will be treated as a warning instead of an error during validation.
+      Defaults to false. This field has no effect if allowWarnings is false or acceptAllResources is true.
+

docs/content/reference/values.txt

@@ -482,6 +482,7 @@
 |gateway.validation.enabled|bool|true|enable Gloo Edge API Gateway validation hook (default true)|
 |gateway.validation.alwaysAcceptResources|bool|true|unless this is set this to false in order to ensure validation webhook rejects invalid resources. by default, validation webhook will only log and report metrics for invalid resource admission without rejecting them outright.|
 |gateway.validation.allowWarnings|bool|true|set this to false in order to ensure validation webhook rejects resources that would have warning status or rejected status, rather than just rejected.|
+|gateway.validation.warnMissingTlsSecret|bool|false|set this to true in order to treat missing tls secret references as warnings, causing validation to allow this state. This supports eventually consistent workflows where TLS secrets may not yet be present when VirtualServices that reference them are created. This field has no effect if allowWarnings is false or acceptAllResources is true.|
 |gateway.validation.serverEnabled|bool|true|By providing the validation field (parent of this object) the user is implicitly opting into validation. This field allows the user to opt out of the validation server, while still configuring pre-existing fields such as warn_route_short_circuiting and disable_transformation_validation.|
 |gateway.validation.disableTransformationValidation|bool|false|set this to true to disable transformation validation. This may bring signifigant performance benefits if using many transformations, at the cost of possibly incorrect transformations being sent to Envoy. When using this value make sure to pre-validate transformations.|
 |gateway.validation.warnRouteShortCircuiting|bool|false|Write a warning to route resources if validation produced a route ordering warning (defaults to false). By setting to true, this means that Gloo Edge will start assigning warnings to resources that would result in route short-circuiting within a virtual host.|

install/helm/gloo/crds/gloo.solo.io_v1_Settings.yaml

@@ -546,6 +546,9 @@ spec:
                         type: string
                       validationWebhookTlsKey:
                         type: string
+                      warnMissingTlsSecret:
+                        nullable: true
+                        type: boolean
                       warnRouteShortCircuiting:
                         nullable: true
                         type: boolean

install/helm/gloo/generate/values.go

@@ -460,6 +460,7 @@ type GatewayValidation struct {
 	Enabled                          *bool    `json:"enabled,omitempty" desc:"enable Gloo Edge API Gateway validation hook (default true)"`
 	AlwaysAcceptResources            *bool    `json:"alwaysAcceptResources,omitempty" desc:"unless this is set this to false in order to ensure validation webhook rejects invalid resources. by default, validation webhook will only log and report metrics for invalid resource admission without rejecting them outright."`
 	AllowWarnings                    *bool    `json:"allowWarnings,omitempty" desc:"set this to false in order to ensure validation webhook rejects resources that would have warning status or rejected status, rather than just rejected."`
+	WarnMissingTlsSecret             *bool    `json:"warnMissingTlsSecret,omitempty" desc:"set this to true in order to treat missing tls secret references as warnings, causing validation to allow this state. This supports eventually consistent workflows where TLS secrets may not yet be present when VirtualServices that reference them are created. This field has no effect if allowWarnings is false or acceptAllResources is true."`
 	ServerEnabled                    *bool    `json:"serverEnabled,omitempty" desc:"By providing the validation field (parent of this object) the user is implicitly opting into validation. This field allows the user to opt out of the validation server, while still configuring pre-existing fields such as warn_route_short_circuiting and disable_transformation_validation."`
 	DisableTransformationValidation  *bool    `json:"disableTransformationValidation,omitempty" desc:"set this to true to disable transformation validation. This may bring signifigant performance benefits if using many transformations, at the cost of possibly incorrect transformations being sent to Envoy. When using this value make sure to pre-validate transformations."`
 	WarnRouteShortCircuiting         *bool    `json:"warnRouteShortCircuiting,omitempty" desc:"Write a warning to route resources if validation produced a route ordering warning (defaults to false). By setting to true, this means that Gloo Edge will start assigning warnings to resources that would result in route short-circuiting within a virtual host."`

install/helm/gloo/templates/18-settings.yaml

@@ -130,9 +130,9 @@ spec:
 {{- if .Values.gateway.validation.enabled }}
     validation:
       proxyValidationServerAddr: "gloo:{{ .Values.gloo.deployment.validationPort }}"
-{{- /* need to do this weird if/else because Helm cannot differentiate between 'false' and 'unset' */}}
       alwaysAccept: {{ .Values.gateway.validation.alwaysAcceptResources }}
       allowWarnings: {{ .Values.gateway.validation.allowWarnings }}
+      warnMissingTlsSecret: {{ .Values.gateway.validation.warnMissingTlsSecret }}
       serverEnabled: {{ .Values.gateway.validation.serverEnabled }}
       disableTransformationValidation: {{ .Values.gateway.validation.disableTransformationValidation }}
       warnRouteShortCircuiting: {{ .Values.gateway.validation.warnRouteShortCircuiting }}

install/helm/gloo/values-template.yaml

@@ -124,6 +124,10 @@ gateway:
     secretName: gateway-validation-certs
     alwaysAcceptResources: true
     allowWarnings: true
+    # Explicitly defaulting this setting to false in order to emphasize the opt-in 
+    # nature of this behavior in this version. The default value for this field 
+    # is changed to true in 1.18, making the behavior opt-out.
+    warnMissingTlsSecret: false
     serverEnabled: true
     disableTransformationValidation: false
     warnRouteShortCircuiting: false
@@ -310,4 +314,4 @@ global:
   # additionalLabels adds a label to all object metadata
   additionalLabels: {}
   # securitySettings defines global security settings such as `floatingUserId`
-  securitySettings: {}
+  securitySettings: {}

install/test/fixtures/settings/compressed_proxy_spec.yaml

@@ -7,31 +7,31 @@ metadata:
   name: default
   namespace: {{ . }}
 spec:
- discovery:
-   fdsMode: WHITELIST
- gateway:
-   readGatewaysFromAllNamespaces: false
-   compressedProxySpec: true
-   enableGatewayController: true
-   isolateVirtualHostsBySslConfig: false
- gloo:
-   regexMaxProgramSize: 1024
-   enableRestEds: false
-   xdsBindAddr: 0.0.0.0:9977
-   restXdsBindAddr: 0.0.0.0:9976
-   proxyDebugBindAddr: 0.0.0.0:9966
-   disableKubernetesDestinations: false
-   disableProxyGarbageCollection: false
-   invalidConfigPolicy:
-     invalidRouteResponseBody: Gloo Gateway has invalid configuration. Administrators should run `glooctl check` to find and fix config errors.
-     invalidRouteResponseCode: 404
-     replaceInvalidRoutes: false
-   istioOptions:
-     appendXForwardedHost: true
-     enableAutoMtls: false
-     enableIntegration: false
- kubernetesArtifactSource: {}
- kubernetesConfigSource: {}
- kubernetesSecretSource: {}
- refreshRate: 60s
- discoveryNamespace: {{ . }}
+  discovery:
+    fdsMode: WHITELIST
+  gateway:
+    readGatewaysFromAllNamespaces: false
+    compressedProxySpec: true
+    enableGatewayController: true
+    isolateVirtualHostsBySslConfig: false
+  gloo:
+    regexMaxProgramSize: 1024
+    enableRestEds: false
+    xdsBindAddr: 0.0.0.0:9977
+    restXdsBindAddr: 0.0.0.0:9976
+    proxyDebugBindAddr: 0.0.0.0:9966
+    disableKubernetesDestinations: false
+    disableProxyGarbageCollection: false
+    invalidConfigPolicy:
+      invalidRouteResponseBody: Gloo Gateway has invalid configuration. Administrators should run `glooctl check` to find and fix config errors.
+      invalidRouteResponseCode: 404
+      replaceInvalidRoutes: false
+    istioOptions:
+      appendXForwardedHost: true
+      enableAutoMtls: false
+      enableIntegration: false
+  kubernetesArtifactSource: {}
+  kubernetesConfigSource: {}
+  kubernetesSecretSource: {}
+  refreshRate: 60s
+  discoveryNamespace: {{ . }}

install/test/fixtures/settings/consul_config_upstream_discovery.yaml

@@ -7,45 +7,46 @@ metadata:
   name: default
   namespace: {{ . }}
 spec:
- discovery:
-   fdsMode: WHITELIST
- gateway:
-   enableGatewayController: true
-   readGatewaysFromAllNamespaces: false
-   isolateVirtualHostsBySslConfig: false
-   validation:
-     alwaysAccept: true
-     allowWarnings: true
-     serverEnabled: true
-     disableTransformationValidation: false
-     warnRouteShortCircuiting: false
-     proxyValidationServerAddr: gloo:9988
-     validationServerGrpcMaxSizeBytes: 104857600
- gloo:
-   regexMaxProgramSize: 1024
-   enableRestEds: false
-   xdsBindAddr: 0.0.0.0:9977
-   restXdsBindAddr: 0.0.0.0:9976
-   proxyDebugBindAddr: 0.0.0.0:9966
-   disableKubernetesDestinations: false
-   disableProxyGarbageCollection: false
-   invalidConfigPolicy:
-     invalidRouteResponseBody: Gloo Gateway has invalid configuration. Administrators should run `glooctl check` to find and fix config errors.
-     invalidRouteResponseCode: 404
-     replaceInvalidRoutes: false
-   istioOptions:
-     appendXForwardedHost: true
-     enableAutoMtls: false
-     enableIntegration: false
- consulDiscovery:
-   useTlsTagging: true
-   tlsTagName: tag
-   splitTlsServices: true
-   rootCa:
-     name: testName
-     namespace: testNamespace
- kubernetesArtifactSource: {}
- kubernetesConfigSource: {}
- kubernetesSecretSource: {}
- refreshRate: 60s
- discoveryNamespace: {{ . }}
+  discovery:
+    fdsMode: WHITELIST
+  gateway:
+    enableGatewayController: true
+    readGatewaysFromAllNamespaces: false
+    isolateVirtualHostsBySslConfig: false
+    validation:
+      alwaysAccept: true
+      allowWarnings: true
+      warnMissingTlsSecret: false
+      serverEnabled: true
+      disableTransformationValidation: false
+      warnRouteShortCircuiting: false
+      proxyValidationServerAddr: gloo:9988
+      validationServerGrpcMaxSizeBytes: 104857600
+  gloo:
+    regexMaxProgramSize: 1024
+    enableRestEds: false
+    xdsBindAddr: 0.0.0.0:9977
+    restXdsBindAddr: 0.0.0.0:9976
+    proxyDebugBindAddr: 0.0.0.0:9966
+    disableKubernetesDestinations: false
+    disableProxyGarbageCollection: false
+    invalidConfigPolicy:
+      invalidRouteResponseBody: Gloo Gateway has invalid configuration. Administrators should run `glooctl check` to find and fix config errors.
+      invalidRouteResponseCode: 404
+      replaceInvalidRoutes: false
+    istioOptions:
+      appendXForwardedHost: true
+      enableAutoMtls: false
+      enableIntegration: false
+  consulDiscovery:
+    useTlsTagging: true
+    tlsTagName: tag
+    splitTlsServices: true
+    rootCa:
+      name: testName
+      namespace: testNamespace
+  kubernetesArtifactSource: {}
+  kubernetesConfigSource: {}
+  kubernetesSecretSource: {}
+  refreshRate: 60s
+  discoveryNamespace: {{ . }}