Merge refs/heads/main into argocd

.github/workflows/pr-kubernetes-tests.yaml

@@ -75,10 +75,10 @@ jobs:
           go-test-args: '-v -timeout=30m'
           go-test-run-regex: '(^TestK8sGatewayIstio$$|^TestGlooGatewayEdgeGateway$$|^TestGlooctlIstioInjectEdgeApiGateway$$|^TestK8sGatewayNoValidation$$)'
 
-        # September 24, 2024: 22 minutes
+        # October 1, 2024: 27 minutes
         - cluster-name: 'cluster-five'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '^TestValidationStrict$$|^TestValidationAlwaysAccept$$|^TestTransformationValidationDisabled$$|^TestGloomtlsGatewayEdgeGateway$$|^TestHelm$$|^TestHelmSettings$$|^TestWatchNamespaceSelector$$'
+          go-test-run-regex: '^TestFullEnvoyValidation$$|^TestValidationStrict$$|^TestValidationAlwaysAccept$$|^TestTransformationValidationDisabled$$|^TestGloomtlsGatewayEdgeGateway$$|^TestHelm$$|^TestHelmSettings$$|^TestWatchNamespaceSelector$$'
 
         # In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support
         # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/

changelog/v1.18.0-beta25/expand-envoy-validation-functionality.yaml

@@ -0,0 +1,14 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/gloo/issues/5720
+    resolvesIssue: false
+    description: >-
+      Adds feature to utilize Envoy's validate mode to validate all config during translation. 
+      This will be helpful in catching issues which Gloo translation cannot or otherwise does 
+      not view as errors before the config gets served to Envoy.
+  - type: HELM
+    issueLink: https://github.com/solo-io/gloo/issues/5720
+    resolvesIssue: false
+    description: >-
+      Add value to enable full Envoy validation after translation. This functionality
+      is disabled by default but can be enabled with gateway.validation.fullEnvoyValidation=true.

pkg/utils/envoyutils/bootstrap/bootstrap.go

@@ -2,17 +2,30 @@ package bootstrap
 
 import (
 	"bytes"
+	"context"
+	"errors"
 
 	envoy_config_bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
+	envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	envoy_config_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	envoy_extensions_filters_network_http_connection_manager_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
 	anypb "github.com/golang/protobuf/ptypes/any"
+	"github.com/rotisserie/eris"
 	"github.com/solo-io/gloo/projects/gloo/pkg/utils"
+	envoycache "github.com/solo-io/solo-kit/pkg/api/v1/control-plane/cache"
+)
+
+var (
+	// errNoHcm represents a situation where a filter chain does not have an HttpConnectionManager.
+	// Because this could occur for valid reasons, such as a TCP proxy, we use
+	// this as a sentinel error to inform us it's ok to ignore it and continue.
+	errNoHcm = eris.New("no HttpConnectionManager found")
 )
 
 func FromEnvoyResources(resources *EnvoyResources) (string, error) {
@@ -102,3 +115,226 @@ func FromFilter(filterName string, msg proto.Message) (string, error) {
 
 	return FromEnvoyResources(&EnvoyResources{Listeners: []*envoy_config_listener_v3.Listener{listener}})
 }
+
+// FromSnapshot accepts an xds Snapshot and converts it into valid bootstrap json.
+func FromSnapshot(
+	ctx context.Context,
+	snap envoycache.Snapshot,
+) (string, error) {
+
+	// Get the resources we're going to need as concrete types.
+	resources, err := resourcesFromSnapshot(snap)
+	if err != nil {
+		return "", err
+	}
+
+	// This map will hold the aggregate of all cluster names that are routed to
+	// by a FilterChain.
+	routedCluster := map[string]struct{}{}
+
+	// Gather up all of the clusters that we target with RouteConfigs that are associated with a FilterChain.
+	if err := extractRoutedClustersFromListeners(routedCluster, resources.Listeners, resources.routes); err != nil {
+		return "", err
+	}
+
+	// Next, we will look through our Snapshot's clusters and delete the ones which are
+	// already routed to.
+	convertToStaticClusters(routedCluster, resources.Clusters, resources.endpoints)
+
+	// We now need to find clusters which do not exist, even though they are targeted by
+	// a route. In static mode, envoy won't start without these. At this point in the
+	// processing, routedClusters holds this list, so we use it to create blackhole
+	// clusters for these routes to target. It is important to have unique clusters
+	// for the targets since some envoy functionality relies on such setup, like
+	// weighted destinations.
+	resources.Clusters = addBlackholeClusters(routedCluster, resources.Clusters)
+
+	return FromEnvoyResources(resources)
+}
+
+// extractRoutedClustersFromListeners accepts a hash set of strings containing the names of clusters
+// to which routes point, a slice of pointers to Listener structs,
+// and a slice of pointers to RouteConfiguration structs from the snapshot. It looks
+// through the FilterChains on each Listener for an HttpConnectionManager, gets the
+// routes on that hcm, and gets all of the clusters targeted by those routes. It then
+// converts the hcm config to use static RouteConfiguration. routedCluster and elements
+// of listeners are mutated in this function.
+func extractRoutedClustersFromListeners(
+	routedCluster map[string]struct{},
+	listeners []*envoy_config_listener_v3.Listener,
+	routes []*envoy_config_route_v3.RouteConfiguration,
+) error {
+	for _, l := range listeners {
+		for _, fc := range l.GetFilterChains() {
+			// Get the HttpConnectionManager for this FilterChain if it exists.
+			hcm, f, err := getHcmForFilterChain(fc)
+			if err != nil {
+				// If we just don't have an hcm on this filter chain, skip to the next one.
+				if errors.Is(err, errNoHcm) {
+					continue
+				}
+				// If we encountered any other error, fail loudly.
+				return err
+			}
+
+			// We use Route Discovery Service (RDS) in lieu of static route table config, so we
+			// need to get the RouteConfiguration name to lookup in our Snapshot-provided routes,
+			// which contain what we serve over RDS.
+			routeConfigName := hcm.GetRds().GetRouteConfigName()
+			if routeConfigName == "" {
+				continue
+			}
+			// Find matching route config from snapshot.
+			for _, r := range routes {
+				if r.GetName() != routeConfigName {
+					// These aren't the routes you're looking for.
+					continue
+				}
+
+				// Add clusters targeted by routes on this config to our aggregate list of all targeted clusters
+				findTargetedClusters(r, routedCluster)
+
+				// We need to add our route table as a static config to this hcm instead of
+				// relying on RDS, the we pack it back up and set it back on the filter chain.
+				if err = setStaticRouteConfig(f, hcm, r); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// convertToStaticClusters accepts a hash set of strings containing the names of clusters
+// to which routes point, a slice of pointers to Cluster structs,
+// and a slice of pointers to ClusterLoadAssignment structs from the snapshot. It
+// deletes all clusters that exist from the routedCluster hash set, then converts
+// the cluster's EDS config to static config using the endpoints from the snapshot.
+// clusters is mutated in this function.
+func convertToStaticClusters(
+	routedCluster map[string]struct{},
+	clusters []*envoy_config_cluster_v3.Cluster,
+	endpoints []*envoy_config_endpoint_v3.ClusterLoadAssignment,
+) {
+	for _, c := range clusters {
+		delete(routedCluster, c.GetName())
+
+		// We use Endpoint Discovery Service (EDS) in lieu of static endpoint config, so we
+		// need to get the EDS ServiceName name to lookup in our Snapshot-provided endpoints,
+		// which contain what we serve over EDS.
+		if c.GetEdsClusterConfig() != nil {
+			clusterName := c.GetName()
+			if edsServiceName := c.GetEdsClusterConfig().GetServiceName(); edsServiceName != "" {
+				clusterName = edsServiceName
+			}
+
+			// Find endpoints matching our EDS config and convert the cluster to use
+			// static endpoint config matching that which would have been served over EDS.
+			for _, e := range endpoints {
+				if e.GetClusterName() == clusterName {
+					c.LoadAssignment = e
+					c.EdsClusterConfig = nil
+					c.ClusterDiscoveryType = &envoy_config_cluster_v3.Cluster_Type{
+						Type: envoy_config_cluster_v3.Cluster_STRICT_DNS,
+					}
+				}
+			}
+		}
+	}
+}
+
+// addBlackholeClusters accepts a hash set of strings containing the names of clusters
+// to which routes point and a slice of pointers to Cluster structs from the snapshot. It
+// adds an cluster to clusters for each entry in the routedCluster set. clusters is mutated
+// by this function.
+func addBlackholeClusters(
+	routedCluster map[string]struct{},
+	clusters []*envoy_config_cluster_v3.Cluster,
+) []*envoy_config_cluster_v3.Cluster {
+	for c := range routedCluster {
+		clusters = append(clusters, &envoy_config_cluster_v3.Cluster{
+			Name: c,
+			ClusterDiscoveryType: &envoy_config_cluster_v3.Cluster_Type{
+				Type: envoy_config_cluster_v3.Cluster_STATIC,
+			},
+			LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
+				ClusterName: c,
+				Endpoints:   []*envoy_config_endpoint_v3.LocalityLbEndpoints{},
+			},
+		})
+	}
+	return clusters
+}
+
+// getHcmForFilterChain accepts a pointer to a FilterChain and looks for the HttpConnectionManager
+// network filter if one exists. It returns a pointer to the HttpConnectionManager struct and
+// a pointer to the filter that actually contained it. This function has no side effects.
+func getHcmForFilterChain(fc *envoy_config_listener_v3.FilterChain) (
+	*envoy_extensions_filters_network_http_connection_manager_v3.HttpConnectionManager,
+	*envoy_config_listener_v3.Filter,
+	error,
+) {
+	for _, f := range fc.GetFilters() {
+		if f.GetTypedConfig().GetTypeUrl() == "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" {
+			hcmAny, err := utils.AnyToMessage(f.GetTypedConfig())
+			if err != nil {
+				return nil, nil, err
+			}
+			// This check can be unreliable if the proto *Any format can be successfully unmarshalled to this concrete type,
+			// which is surprisingly easy to do. This codepath is not tested as I was unable to force a failure, but we're
+			// leaving the check in to guard against NPE from the concrete cast.
+			if hcm, ok := hcmAny.(*envoy_extensions_filters_network_http_connection_manager_v3.HttpConnectionManager); ok {
+				return hcm, f, nil
+			} else {
+				return nil, nil, eris.Errorf("filter %v has hcm type url but casting to concrete failed", f.GetName())
+			}
+		}
+	}
+	return nil, nil, errNoHcm
+}
+
+// findTargetedClusters accepts a pointer to a RouteConfiguration and a hash set of strings. It
+// finds all clusters and weighted clusters targeted by routes on the virtual hosts in the RouteConfiguration
+// and adds their names to the routedCluster hash set. routedCluster is mutated in this function.
+func findTargetedClusters(r *envoy_config_route_v3.RouteConfiguration, routedCluster map[string]struct{}) {
+	for _, v := range r.GetVirtualHosts() {
+		for _, r := range v.GetRoutes() {
+			if r.GetRoute() == nil {
+				continue
+			}
+
+			if c := r.GetRoute().GetCluster(); c != "" {
+				routedCluster[c] = struct{}{}
+			}
+			if wc := r.GetRoute().GetWeightedClusters().GetClusters(); len(wc) != 0 {
+				for _, c := range wc {
+					routedCluster[c.GetName()] = struct{}{}
+				}
+			}
+		}
+	}
+}
+
+// setStaticRouteConfig accepts pointers to each of a Filter, HttpConnectionManager, and RouteConfiguration.
+// It adds the RouteConfiguration to the HttpConnectionManager as static, marshals the hcm, and sets the filter's
+// TypedConfig. f and hcm are mutated in this function.
+func setStaticRouteConfig(
+	f *envoy_config_listener_v3.Filter,
+	hcm *envoy_extensions_filters_network_http_connection_manager_v3.HttpConnectionManager,
+	r *envoy_config_route_v3.RouteConfiguration,
+) error {
+	hcm.RouteSpecifier = &envoy_extensions_filters_network_http_connection_manager_v3.HttpConnectionManager_RouteConfig{
+		RouteConfig: r,
+	}
+
+	hcmAny, err := utils.MessageToAny(hcm)
+	if err != nil {
+		return err
+	}
+
+	f.ConfigType = &envoy_config_listener_v3.Filter_TypedConfig{
+		TypedConfig: hcmAny,
+	}
+	return nil
+}

pkg/utils/envoyutils/bootstrap/bootstrap_suite_test.go

@@ -1,4 +1,4 @@
-package bootstrap_test
+package bootstrap
 
 import (
 	"testing"