enable running e2e tests against envoy deployment style manifests

Signed-off-by: Steve Sloka <slokas@vmware.com>

test/e2e/deployment.go

@@ -45,6 +45,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
 	apimachinery_util_yaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -83,6 +84,7 @@ type Deployment struct {
 	EnvoyService              *v1.Service
 	ContourDeployment         *apps_v1.Deployment
 	EnvoyDaemonSet            *apps_v1.DaemonSet
+	EnvoyDeployment           *apps_v1.Deployment
 
 	// Ratelimit deployment.
 	RateLimitDeployment       *apps_v1.Deployment
@@ -99,19 +101,33 @@ func (d *Deployment) UnmarshalResources() error {
 	if !ok {
 		return errors.New("could not get path to this source file (test/e2e/deployment.go)")
 	}
-	renderedManifestPath := filepath.Join(filepath.Dir(thisFile), "..", "..", "examples", "render", "contour.yaml")
-	file, err := os.Open(renderedManifestPath)
+	renderedManifestPathDaemonset := filepath.Join(filepath.Dir(thisFile), "..", "..", "examples", "render", "contour.yaml")
+	file, err := os.Open(renderedManifestPathDaemonset)
 	if err != nil {
 		return err
 	}
 	defer file.Close()
+
+	renderedManifestPathDeployment := filepath.Join(filepath.Dir(thisFile), "..", "..", "examples", "render", "contour-deployment.yaml")
+	fileDeployment, err := os.Open(renderedManifestPathDeployment)
+	if err != nil {
+		return err
+	}
+	defer fileDeployment.Close()
+
 	decoder := apimachinery_util_yaml.NewYAMLToJSONDecoder(file)
+	decoderDeployment := apimachinery_util_yaml.NewYAMLToJSONDecoder(fileDeployment)
 
 	// Discard empty document.
 	if err := decoder.Decode(new(struct{})); err != nil {
 		return err
 	}
 
+	// Discard empty document.
+	if err := decoderDeployment.Decode(new(struct{})); err != nil {
+		return err
+	}
+
 	d.Namespace = new(v1.Namespace)
 	d.ContourServiceAccount = new(v1.ServiceAccount)
 	d.EnvoyServiceAccount = new(v1.ServiceAccount)
@@ -131,6 +147,7 @@ func (d *Deployment) UnmarshalResources() error {
 	d.EnvoyService = new(v1.Service)
 	d.ContourDeployment = new(apps_v1.Deployment)
 	d.EnvoyDaemonSet = new(apps_v1.DaemonSet)
+	d.EnvoyDeployment = new(apps_v1.Deployment)
 	objects := []interface{}{
 		d.Namespace,
 		d.ContourServiceAccount,
@@ -152,9 +169,42 @@ func (d *Deployment) UnmarshalResources() error {
 		d.ContourDeployment,
 		d.EnvoyDaemonSet,
 	}
-	for _, o := range objects {
-		if err := decoder.Decode(o); err != nil {
-			return err
+
+	objectsDeployment := []interface{}{
+		d.Namespace,
+		d.ContourServiceAccount,
+		d.EnvoyServiceAccount,
+		d.ContourConfigMap,
+		d.ExtensionServiceCRD,
+		d.HTTPProxyCRD,
+		d.TLSCertDelegationCRD,
+		d.ContourConfigurationCRD,
+		d.ContourDeploymentCRD,
+		d.CertgenServiceAccount,
+		d.ContourRoleBinding,
+		d.CertgenRole,
+		d.CertgenJob,
+		d.ContourClusterRoleBinding,
+		d.ContourClusterRole,
+		d.ContourService,
+		d.EnvoyService,
+		d.ContourDeployment,
+		d.EnvoyDeployment,
+	}
+
+	if UseEnvoyDaemonsetDeploymentModel() {
+
+		for _, o := range objects {
+			if err := decoder.Decode(o); err != nil {
+				return err
+			}
+		}
+	} else {
+
+		for _, o := range objectsDeployment {
+			if err := decoderDeployment.Decode(o); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -339,17 +389,36 @@ func (d *Deployment) EnsureEnvoyDaemonSet() error {
 	return d.ensureResource(d.EnvoyDaemonSet, new(apps_v1.DaemonSet))
 }
 
-func (d *Deployment) WaitForEnvoyDaemonSetUpdated() error {
-	daemonSetUpdated := func() (bool, error) {
-		tempDS := new(apps_v1.DaemonSet)
-		if err := d.client.Get(context.TODO(), client.ObjectKeyFromObject(d.EnvoyDaemonSet), tempDS); err != nil {
+func (d *Deployment) EnsureEnvoyDeployment() error {
+	return d.ensureResource(d.EnvoyDeployment, new(apps_v1.Deployment))
+}
+
+func (d *Deployment) WaitForEnvoyUpdated() error {
+
+	if UseEnvoyDaemonsetDeploymentModel() {
+
+		daemonSetUpdated := func() (bool, error) {
+			tempDS := new(apps_v1.DaemonSet)
+			if err := d.client.Get(context.TODO(), client.ObjectKeyFromObject(d.EnvoyDaemonSet), tempDS); err != nil {
+				return false, err
+			}
+			// This might work for now while we have only one worker node, but
+			// if we expand to more, we will have to rethink this.
+			return tempDS.Status.NumberReady > 0, nil
+		}
+		return wait.PollImmediate(time.Millisecond*50, time.Minute*3, daemonSetUpdated)
+	}
+
+	deploymentSetUpdated := func() (bool, error) {
+		tempDeployment := new(apps_v1.Deployment)
+		if err := d.client.Get(context.TODO(), client.ObjectKeyFromObject(d.EnvoyDeployment), tempDeployment); err != nil {
 			return false, err
 		}
 		// This might work for now while we have only one worker node, but
 		// if we expand to more, we will have to rethink this.
-		return tempDS.Status.NumberReady > 0, nil
+		return tempDeployment.Status.ReadyReplicas > 0, nil
 	}
-	return wait.PollImmediate(time.Millisecond*50, time.Minute*3, daemonSetUpdated)
+	return wait.PollImmediate(time.Millisecond*50, time.Minute*3, deploymentSetUpdated)
 }
 
 func (d *Deployment) EnsureRateLimitResources(namespace string, configContents string) error {
@@ -396,8 +465,8 @@ func (d *Deployment) EnsureRateLimitResources(namespace string, configContents s
 // - CRDs
 // - Envoy service
 // - ConfigMap with Envoy bootstrap config
-// - Envoy DaemonSet modified for local Contour xDS server
-func (d *Deployment) EnsureResourcesForLocalContour() error {
+// - Envoy DaemonSet/Deployment modified for local Contour xDS server
+func (d *Deployment) EnsureResourcesForLocalContour(useDaemonset bool) error {
 	if err := d.EnsureNamespace(); err != nil {
 		return err
 	}
@@ -419,6 +488,10 @@ func (d *Deployment) EnsureResourcesForLocalContour() error {
 	if err := d.EnsureContourDeploymentCRD(); err != nil {
 		return err
 	}
+
+	// Set the Envoy type to clusterIP so that external load balancers
+	// aren't spun up accidentally.
+	d.EnvoyService.Spec.Type = v1.ServiceTypeNodePort
 	if err := d.EnsureEnvoyService(); err != nil {
 		return err
 	}
@@ -468,6 +541,50 @@ func (d *Deployment) EnsureResourcesForLocalContour() error {
 		return err
 	}
 
+	if !useDaemonset {
+		// Set the replica count to 1 since this is on a single node cluster.
+		d.EnvoyDeployment.Spec.Replicas = pointer.Int32Ptr(1)
+
+		// Add bootstrap ConfigMap as volume and add envoy admin volume on Envoy pods (also removes cert volume).
+		d.EnvoyDeployment.Spec.Template.Spec.Volumes = []v1.Volume{{
+			Name: "envoy-config",
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: "envoy-bootstrap",
+					},
+				},
+			},
+		}, {
+			Name: "envoy-admin",
+			VolumeSource: v1.VolumeSource{
+				EmptyDir: &v1.EmptyDirVolumeSource{},
+			},
+		}}
+
+		// Remove cert volume mount.
+		d.EnvoyDeployment.Spec.Template.Spec.Containers[1].VolumeMounts = []v1.VolumeMount{
+			d.EnvoyDeployment.Spec.Template.Spec.Containers[1].VolumeMounts[0], // Config mount
+			d.EnvoyDeployment.Spec.Template.Spec.Containers[1].VolumeMounts[2], // Admin mount
+		}
+
+		// Remove init container.
+		d.EnvoyDeployment.Spec.Template.Spec.InitContainers = nil
+
+		// Remove shutdown-manager container.
+		d.EnvoyDeployment.Spec.Template.Spec.Containers = d.EnvoyDeployment.Spec.Template.Spec.Containers[1:]
+
+		// Expose the metrics & admin interfaces via host port to test from outside the kind cluster.
+		d.EnvoyDeployment.Spec.Template.Spec.Containers[0].Ports = append(d.EnvoyDeployment.Spec.Template.Spec.Containers[0].Ports,
+			v1.ContainerPort{
+				Name:          "metrics",
+				ContainerPort: 8002,
+				HostPort:      8002,
+				Protocol:      v1.ProtocolTCP,
+			})
+		return d.EnsureEnvoyDeployment()
+	}
+
 	// Add bootstrap ConfigMap as volume and add envoy admin volume on Envoy pods (also removes cert volume).
 	d.EnvoyDaemonSet.Spec.Template.Spec.Volumes = []v1.Volume{{
 		Name: "envoy-config",
@@ -539,8 +656,17 @@ func (d *Deployment) DeleteResourcesForLocalContour() error {
 		return nil
 	}
 
+	if UseEnvoyDaemonsetDeploymentModel() {
+		if err := ensureDeleted(d.EnvoyDaemonSet); err != nil {
+			return err
+		}
+	} else {
+		if err := ensureDeleted(d.EnvoyDeployment); err != nil {
+			return err
+		}
+	}
+
 	for _, r := range []client.Object{
-		d.EnvoyDaemonSet,
 		d.ContourConfigMap,
 		d.EnvoyService,
 		d.TLSCertDelegationCRD,

test/e2e/framework.go

@@ -19,6 +19,7 @@ package e2e
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -447,3 +448,19 @@ func UsingContourConfigCRD() bool {
 	useContourConfiguration, found := os.LookupEnv("USE_CONTOUR_CONFIGURATION_CRD")
 	return found && useContourConfiguration == "true"
 }
+
+// UseEnvoyDaemonsetDeploymentModel determines if Envoy should be deployed as a Daemonset or a Deployment
+// when running E2E tests. It looks for the env var "USE_ENVOY_DEPLOYMENT". If it exists and is valid, then
+// that value is returned, otherwise true is returned to indicate that the default daemonset model is to be used.
+func UseEnvoyDaemonsetDeploymentModel() bool {
+
+	if useEnvoyDeployment, found := os.LookupEnv("USE_ENVOY_DEPLOYMENT"); found {
+		got, err := strconv.ParseBool(useEnvoyDeployment)
+		if err != nil {
+			fmt.Printf("error trying to parse \"USE_ENVOY_DEPLOYMENT\" env var, defaulting to use Daemonset: %v\n", err)
+			return true
+		}
+		return !got
+	}
+	return true
+}

test/e2e/gateway/gateway_test.go

@@ -43,7 +43,7 @@ func TestGatewayAPI(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour())
+	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour(e2e.UseEnvoyDaemonsetDeploymentModel()))
 })
 
 var _ = AfterSuite(func() {
@@ -118,7 +118,7 @@ var _ = Describe("Gateway API", func() {
 		require.NoError(f.T(), err)
 
 		// Wait for Envoy to be healthy.
-		require.NoError(f.T(), f.Deployment.WaitForEnvoyDaemonSetUpdated())
+		require.NoError(f.T(), f.Deployment.WaitForEnvoyUpdated())
 
 		f.CreateGatewayClassAndWaitFor(contourGatewayClass, gatewayClassValid)
 		f.CreateGatewayAndWaitFor(contourGateway, gatewayValid)

test/e2e/gateway/multiple_gateways_and_classes_test.go

@@ -76,7 +76,7 @@ var _ = Describe("GatewayClass/Gateway admission tests", func() {
 		require.NoError(f.T(), err)
 
 		// Wait for Envoy to be healthy.
-		require.NoError(f.T(), f.Deployment.WaitForEnvoyDaemonSetUpdated())
+		require.NoError(f.T(), f.Deployment.WaitForEnvoyUpdated())
 	})
 
 	AfterEach(func() {

test/e2e/httpproxy/httpproxy_test.go

@@ -44,7 +44,7 @@ func TestHTTPProxy(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour())
+	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour(e2e.UseEnvoyDaemonsetDeploymentModel()))
 })
 
 var _ = AfterSuite(func() {
@@ -88,7 +88,7 @@ var _ = Describe("HTTPProxy", func() {
 		require.NoError(f.T(), err)
 
 		// Wait for Envoy to be healthy.
-		require.NoError(f.T(), f.Deployment.WaitForEnvoyDaemonSetUpdated())
+		require.NoError(f.T(), f.Deployment.WaitForEnvoyUpdated())
 	})
 
 	AfterEach(func() {

test/e2e/infra/infra_test.go

@@ -37,7 +37,7 @@ func TestInfra(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour())
+	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour(e2e.UseEnvoyDaemonsetDeploymentModel()))
 })
 
 var _ = AfterSuite(func() {
@@ -82,9 +82,14 @@ var _ = Describe("Infra", func() {
 		require.NoError(f.T(), err)
 
 		// Wait for Envoy to be healthy.
-		require.NoError(f.T(), f.Deployment.WaitForEnvoyDaemonSetUpdated())
+		require.NoError(f.T(), f.Deployment.WaitForEnvoyUpdated())
 
-		kubectlCmd, err = f.Kubectl.StartKubectlPortForward(19001, 9001, "projectcontour", "daemonset/envoy", additionalContourArgs...)
+		envoyObjectName := "daemonset/envoy"
+		if !e2e.UseEnvoyDaemonsetDeploymentModel() {
+			envoyObjectName = "deployment/envoy"
+		}
+
+		kubectlCmd, err = f.Kubectl.StartKubectlPortForward(19001, 9001, "projectcontour", envoyObjectName, additionalContourArgs...)
 		require.NoError(f.T(), err)
 	})
 

test/e2e/ingress/ingress_test.go

@@ -42,7 +42,7 @@ func TestIngress(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour())
+	require.NoError(f.T(), f.Deployment.EnsureResourcesForLocalContour(e2e.UseEnvoyDaemonsetDeploymentModel()))
 })
 
 var _ = AfterSuite(func() {
@@ -84,7 +84,7 @@ var _ = Describe("Ingress", func() {
 		require.NoError(f.T(), err)
 
 		// Wait for Envoy to be healthy.
-		require.NoError(f.T(), f.Deployment.WaitForEnvoyDaemonSetUpdated())
+		require.NoError(f.T(), f.Deployment.WaitForEnvoyUpdated())
 	})
 
 	AfterEach(func() {

test/e2e/upgrade/upgrade_test.go

@@ -103,7 +103,7 @@ var _ = Describe("upgrading Contour", func() {
 			require.NoError(f.T(), f.Deployment.WaitForContourDeploymentUpdated())
 
 			By("waiting for envoy daemonset to be updated")
-			require.NoError(f.T(), f.Deployment.WaitForEnvoyDaemonSetUpdated())
+			require.NoError(f.T(), f.Deployment.WaitForEnvoyUpdated())
 
 			By("ensuring app is still routable")
 			checkRoutability(appHost)