Merge refs/heads/main into cors_on_direct_response

Makefile

@@ -145,7 +145,7 @@ check-spelling:
 
 GINKGO_VERSION ?= $(shell echo $(shell go list -m github.com/onsi/ginkgo/v2) | cut -d' ' -f2)
 GINKGO_ENV ?= GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore ACK_GINKGO_RC=true ACK_GINKGO_DEPRECATIONS=$(GINKGO_VERSION)
-GINKGO_FLAGS ?= -tags=purego --trace -progress -race --fail-fast -fail-on-pending --randomize-all --compilers=4
+GINKGO_FLAGS ?= -tags=purego --trace -progress -race --fail-fast -fail-on-pending --randomize-all --compilers=5
 GINKGO_REPORT_FLAGS ?= --json-report=test-report.json --junit-report=junit.xml -output-dir=$(OUTPUT_DIR)
 GINKGO_COVERAGE_FLAGS ?= --cover --covermode=count --coverprofile=coverage.cov
 TEST_PKG ?= ./... # Default to run all tests

changelog/v1.15.0-beta16/parallel-tests.yaml

@@ -0,0 +1,9 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/5143
+    resolvesIssue: true
+    description: >-
+      Enable tests to be run in parallel. Turn this OFF by default in CI.
+
+      skipCI-kube-tests:true
+      skipCI-docs-build:true

ci/cloudbuild/run-tests.yaml

@@ -134,6 +134,9 @@ options:
   - 'RUN_VAULT_TESTS=1'
   - 'DOCKER_CONFIG=/workspace/.docker/'
   - 'SKIP_TEMP_DISABLED=1' # https://github.com/solo-io/solo-projects/issues/4515
+  # Gloo supports running tests in parallel (https://github.com/solo-io/gloo/pull/8423) but occasionally
+  # Ginkgo reports a failure, even when all test suites pass. As a result, we do not run tests in parallel in CI.
+  - 'GINKGO_USER_FLAGS=-procs=1'
   volumes:
   - name: 'gopath'
     path: '/go'

projects/gloo/cli/pkg/cmd/add/add_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"log"
 
+	"github.com/solo-io/gloo/test/ginkgo/decorators"
+
 	"github.com/hashicorp/consul/api"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -12,7 +14,7 @@ import (
 	"github.com/solo-io/gloo/test/testutils"
 )
 
-var _ = Describe("Add", func() {
+var _ = Describe("Add", decorators.Consul, func() {
 	if !testutils.IsEnvTruthy(testutils.RunConsulTests) {
 		log.Print("This test downloads and runs consul and is disabled by default. To enable, set RUN_CONSUL_TESTS=1 in your env.")
 		return

projects/gloo/cli/pkg/cmd/create/create_test.go

@@ -4,14 +4,16 @@ import (
 	"context"
 	"log"
 
+	"github.com/solo-io/gloo/test/ginkgo/decorators"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers"
 	cliutils "github.com/solo-io/gloo/projects/gloo/cli/pkg/testutils"
 	"github.com/solo-io/gloo/test/testutils"
 )
 
-var _ = Describe("Create", func() {
+var _ = Describe("Create", decorators.Consul, func() {
 	if !testutils.IsEnvTruthy(testutils.RunConsulTests) {
 		log.Print("This test downloads and runs consul and is disabled by default. To enable, set RUN_CONSUL_TESTS=1 in your env.")
 		return

projects/gloo/cli/pkg/cmd/create/secret/create_test.go

@@ -5,14 +5,16 @@ import (
 	"fmt"
 	"log"
 
+	"github.com/solo-io/gloo/test/ginkgo/decorators"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/testutils"
 	glootestutils "github.com/solo-io/gloo/test/testutils"
 )
 
-var _ = Describe("Create", func() {
+var _ = Describe("Create", decorators.Vault, func() {
 
 	if !glootestutils.IsEnvTruthy(glootestutils.RunVaultTests) {
 		log.Print("This test downloads and runs vault and is disabled by default. To enable, set RUN_VAULT_TESTS=1 in your env.")

projects/gloo/cli/pkg/cmd/install/install_suite_test.go

@@ -35,32 +35,36 @@ const (
 	expectedHelmFilenameFmt = "gloo-%s.tgz"
 )
 
-// NOTE: This needs to be run from the root of the repo as the working directory
-var _ = BeforeSuite(func() {
-
-	// Make sure we don't hit a real cluster during any of the tests in this suite
-	helpers.UseMemoryClients()
-
+func setupVariables() {
 	cwd, err := os.Getwd()
 	Expect(err).NotTo(HaveOccurred())
 	RootDir = filepath.Join(cwd, "../../../../../..")
 
 	// the regression test depend on having only one chart in _test.
 	// so run these in a different location.
 	dir = filepath.Join(RootDir, "_unit_test/")
+	file = filepath.Join(dir, fmt.Sprintf(expectedHelmFilenameFmt, version.Version))
+
+	values1 = filepath.Join(dir, "values-namespace1.yaml")
+	values2 = filepath.Join(dir, "values-namespace2.yaml")
+}
+
+// NOTE: This needs to be run from the root of the repo as the working directory
+var beforeSuiteOnce = func() {
+
+	// Make sure we don't hit a real cluster during any of the tests in this suite
+	helpers.UseMemoryClients()
+
+	setupVariables()
 	os.Mkdir(dir, 0755)
 
-	err = testutils.Make(RootDir, "build-test-chart TEST_ASSET_DIR=\""+dir+"\"")
+	err := testutils.Make(RootDir, "build-test-chart TEST_ASSET_DIR=\""+dir+"\"")
 	Expect(err).NotTo(HaveOccurred())
 
 	// Some tests need the Gloo/GlooE version that gets linked into the glooctl binary at build time
 	err = testutils.Make(RootDir, "glooctl")
 	Expect(err).NotTo(HaveOccurred())
 
-	file = filepath.Join(dir, fmt.Sprintf(expectedHelmFilenameFmt, version.Version))
-
-	values1 = filepath.Join(dir, "values-namespace1.yaml")
-	values2 = filepath.Join(dir, "values-namespace2.yaml")
 	f, err := os.Create(values1)
 	Expect(err).NotTo(HaveOccurred())
 	_, err = f.WriteString(`
@@ -100,9 +104,13 @@ settings:
 		}
 	}
 	Expect(install.GlooCrdNames).To(ContainElements(crdNames))
-})
+}
+
+var beforeSuiteAll = func() {
+	setupVariables()
+}
 
-var _ = AfterSuite(func() {
+var afterSuiteOnce = func() {
 	err := os.Remove(file)
 	Expect(err).NotTo(HaveOccurred())
 	err = os.Remove(values1)
@@ -111,4 +119,9 @@ var _ = AfterSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	err = os.RemoveAll(dir)
 	Expect(err).NotTo(HaveOccurred())
-})
+}
+
+var (
+	_ = SynchronizedBeforeSuite(beforeSuiteOnce, beforeSuiteAll)
+	_ = SynchronizedAfterSuite(func() {}, afterSuiteOnce)
+)

projects/gloo/pkg/plugins/consul/consul_suite_test.go

@@ -9,13 +9,11 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var T *testing.T
-
 func TestConsul(t *testing.T) {
+	// This has caused issues when tests are run in parallel (not enabled in CI)
 	leakDetector := helpers.DeferredGoroutineLeakDetector(t)
 	defer leakDetector()
 
 	RegisterFailHandler(Fail)
-	T = t
 	RunSpecs(t, "Consul Plugin Suite")
 }

projects/gloo/pkg/plugins/consul/eds_test.go

@@ -46,7 +46,7 @@ var _ = Describe("Consul EDS", func() {
 	const writeNamespace = defaults.GlooSystem
 
 	BeforeEach(func() {
-		ctrl = gomock.NewController(T)
+		ctrl = gomock.NewController(GinkgoT())
 	})
 
 	AfterEach(func() {

projects/gloo/pkg/plugins/consul/plugin_test.go

@@ -24,10 +24,9 @@ var _ = Describe("Resolve", func() {
 	)
 
 	BeforeEach(func() {
-		ctrl = gomock.NewController(T)
+		ctrl = gomock.NewController(GinkgoT())
 
 		consulWatcherMock = mock_consul.NewMockConsulWatcher(ctrl)
-
 	})
 
 	AfterEach(func() {

projects/gloo/pkg/translator/hashing_benchmark_test.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"time"
 
+	"github.com/solo-io/gloo/test/ginkgo/decorators"
+
 	"github.com/solo-io/gloo/test/ginkgo/labels"
 
 	"github.com/onsi/gomega/gmeasure"
@@ -42,7 +44,7 @@ import (
 )
 
 // These tests are meant to demonstrate that FNV hashing is more efficient than hashstructure hashing
-var _ = Describe("Hashing Benchmarks", Serial, Label(labels.Performance), func() {
+var _ = Describe("Hashing Benchmarks", decorators.Performance, Label(labels.Performance), func() {
 	var allUpstreams v1.UpstreamList
 	BeforeEach(func() {
 		var upstreams []interface{}

projects/gloo/pkg/translator/performance_test.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/solo-io/gloo/test/ginkgo/decorators"
+
 	"github.com/solo-io/go-utils/contextutils"
 
 	validationutils "github.com/solo-io/gloo/projects/gloo/pkg/utils/validation"
@@ -49,7 +51,7 @@ type benchmarkConfig struct {
 // More info on that machine can be found here: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
 // When developing new tests, users should manually run that action in order to test performance under the same parameters
 // Results can then be found in the logs for that instance of the action
-var _ = Describe("Translation - Benchmarking Tests", Serial, Label(labels.Performance), func() {
+var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Label(labels.Performance), func() {
 	var (
 		ctrl       *gomock.Controller
 		settings   *v1.Settings

projects/sds/pkg/run/run_e2e_test.go

@@ -18,7 +18,9 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("SDS Server E2E Test", func() {
+var _ = Describe("SDS Server E2E Test", Serial, func() {
+
+	// These tests use the Serial decorator because they rely on a hard-coded port for the SDS server (8236)
 
 	var (
 		ctx                                                             context.Context

test/consulvaulte2e/consul_vault_test.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/solo-io/gloo/test/ginkgo/decorators"
+
 	"github.com/solo-io/gloo/test/services/envoy"
 
 	"google.golang.org/protobuf/types/known/wrapperspb"
@@ -40,7 +42,7 @@ import (
 	"github.com/solo-io/solo-kit/pkg/api/v1/resources/core"
 )
 
-var _ = Describe("Consul + Vault Configuration Happy Path e2e", func() {
+var _ = Describe("Consul + Vault Configuration Happy Path e2e", decorators.Vault, decorators.Consul, func() {
 
 	var (
 		ctx            context.Context

test/consulvaulte2e/e2e_suite_test.go

@@ -24,6 +24,7 @@ func TestE2e(t *testing.T) {
 
 	helpers.RegisterCommonFailHandlers()
 	helpers.SetupLog()
+
 	RunSpecs(t, "Consul+Vault E2e Suite", Label(labels.E2E))
 }
 

test/e2e/README.md

@@ -95,6 +95,15 @@ If you have made changes to the component, you will have had to rebuild the imag
 ENVOY_IMAGE_TAG=0.0.1-local TEST_PKG=./test/e2e/... make test
 ```
 
+#### Running Tests in Parallel
+It is possible to run tests in parallel locally, however it is not recommended, because debugging failures becomes more difficult. If you do want to run tests in parallel, you can do so by passing in the relevant `GINKGO_USER_FLAGS` values:
+```bash
+GINKGO_USER_FLAGS=-procs=4 TEST_PKG=./test/e2e/... make test
+```
+
+*Note: When using Docker to run Envoy, we have seen occasional failures: `Error response from daemon: dial unix docker.raw.sock: connect: connection refused`*
+
+
 ### Debugging Tests
 #### Use WAIT_ON_FAIL
 When Ginkgo encounters a [test failure](https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure) it will attempt to clean up relevant resources, which includes stopping the running instance of Envoy, preventing the developer from inspecting the state of the Envoy instance for useful clues.

test/e2e/aws_test.go

@@ -64,14 +64,11 @@ var _ = Describe("AWS Lambda", func() {
 		envoyInstance *envoy.Instance
 		secret        *gloov1.Secret
 		upstream      *gloov1.Upstream
-		httpClient    *http.Client
 		runOptions    *services.RunOptions
 	)
 
 	BeforeEach(func() {
 		testutils.ValidateRequirementsAndNotifyGinkgo(testutils.AwsCredentials())
-		httpClient = http.DefaultClient
-		httpClient.Timeout = 10 * time.Second
 		runOptions = &services.RunOptions{
 			NsToWrite: writeNamespace,
 			NsToWatch: []string{"default", writeNamespace},
@@ -486,6 +483,8 @@ var _ = Describe("AWS Lambda", func() {
 		var body []byte
 		path := "transforms-req-test"
 		waitForLambdaAndGetBody := func() error {
+			httpClient := testutils.DefaultClientBuilder().WithTimeout(time.Second * 10).Build()
+
 			req, err := http.NewRequest("POST", fmt.Sprintf("http://%s:%d/%s?foo=bar", "localhost", envoyInstance.HttpPort, path), bytes.NewBufferString(`"test"`))
 			Expect(err).NotTo(HaveOccurred())
 			req.Header.Set("Content-Type", "application/octet-stream")

test/e2e/consul_test.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"time"
 
+	"github.com/solo-io/gloo/test/ginkgo/decorators"
+
 	"github.com/golang/protobuf/ptypes/duration"
 	v1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1"
 	"github.com/solo-io/gloo/test/e2e"
@@ -28,7 +30,7 @@ import (
 	"github.com/solo-io/gloo/test/v1helpers"
 )
 
-var _ = Describe("Consul e2e", func() {
+var _ = Describe("Consul e2e", decorators.Consul, func() {
 
 	Context("Consul Service Registry", func() {
 

test/e2e/ratelimit_test.go

@@ -125,7 +125,9 @@ func (s *metadataCheckingRateLimitServer) getActionsForServer() []*rltypes.RateL
 	}
 }
 
-var _ = Describe("Rate Limit", func() {
+var _ = Describe("Rate Limit", Serial, func() {
+
+	// These tests use the Serial decorator because they rely on a hard-coded port for the RateLimit server (18081)
 
 	var (
 		ctx         context.Context

test/e2e/staged_transformation_test.go

@@ -3,8 +3,6 @@ package e2e_test
 import (
 	"encoding/base64"
 
-	"github.com/solo-io/gloo/test/services"
-	"github.com/solo-io/gloo/test/services/envoy"
 	"github.com/solo-io/gloo/test/testutils"
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/protobuf/types/known/wrapperspb"
@@ -33,26 +31,17 @@ var _ = Describe("Staged Transformation", func() {
 
 	var (
 		testContext *e2e.TestContext
-
-		// This test relies on running the gateway-proxy with debug logging enabled
-		// This variable allows us to reset the original log level after the test
-		resetLogLevel func()
 	)
 
 	BeforeEach(func() {
-		originalProxyLogLevel := services.GetLogLevel(envoy.ServiceName)
-		services.SetLogLevel(envoy.ServiceName, zapcore.DebugLevel)
-		resetLogLevel = func() {
-			services.SetLogLevel(envoy.ServiceName, originalProxyLogLevel)
-		}
-
 		testContext = testContextFactory.NewTestContext()
 		testContext.BeforeEach()
+
+		// This test relies on running the gateway-proxy with debug logging enabled
+		testContext.EnvoyInstance().LogLevel = zapcore.DebugLevel.String()
 	})
 
 	AfterEach(func() {
-		resetLogLevel()
-
 		testContext.AfterEach()
 	})