From f9ee93daaa8534ea278f4e209ab3bce3245616e4 Mon Sep 17 00:00:00 2001 From: Sam Heilbron Date: Fri, 30 Jun 2023 13:44:19 -0600 Subject: [PATCH] Stability: Support Running Tests in Parallel (#8423) * add docker/unique container name in envoy, e2e tests running in parallel htiting docker daemon issues after ~15 successful tests * include multi-process in ci, for e2e tests only * fix zipkin_test port re-use * fix consul_test port re-use * run tests in parallel, not just e2e tests * consul/vault serial * move serial decorator * apply serial decorator to vaut tests * serial ratelimit tests * rotate port for vault * advnace by 2 * serial vault test * handle sychronized managmenet of install assets * handle sychronized managmenet of install assets * increase parallel factor,serial consul tests' * manage log level in ordered test * set log level per test * groutine leak detector doesn't play with parallel test * use shared advance port * serial sds tests * update concurrency patterns in ci * staged transofmration tests no longer need to be ordered * add changelog * update readme * Retry Consul/Vault run * Revert "Retry Consul/Vault run" This reverts commit dad47834b85c41df7623df01e19f861822c8ebf5. * use 1000 not 100 for offset, try to avoid dupes * conslidate runConfig for envoy instance * include denyList of ports taht are hard-coded * safe port allocation, adjustments * portInUseListen support * update docker.go * cleanup testutils.env * fix race in aws-test * more cleanup for local parallel tests * disable parallel tests in CI * update changelog * fix typo * shared decorators * Apply suggestions from code review Co-authored-by: Bernie Birnbaum * move unused function into test * add back leak detection * remove duplicate comment * goimports -w . * remove ginkgo parallel flag in makefile --------- Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> Co-authored-by: Bernie Birnbaum --- Makefile | 2 +- changelog/v1.15.0-beta16/parallel-tests.yaml | 9 ++ ci/cloudbuild/run-tests.yaml | 3 + projects/gloo/cli/pkg/cmd/add/add_test.go | 4 +- .../gloo/cli/pkg/cmd/create/create_test.go | 4 +- .../cli/pkg/cmd/create/secret/create_test.go | 4 +- .../cli/pkg/cmd/install/install_suite_test.go | 41 ++++-- .../pkg/plugins/consul/consul_suite_test.go | 4 +- projects/gloo/pkg/plugins/consul/eds_test.go | 2 +- .../gloo/pkg/plugins/consul/plugin_test.go | 3 +- .../pkg/translator/hashing_benchmark_test.go | 4 +- .../gloo/pkg/translator/performance_test.go | 4 +- projects/sds/pkg/run/run_e2e_test.go | 4 +- test/consulvaulte2e/consul_vault_test.go | 4 +- test/consulvaulte2e/e2e_suite_test.go | 1 + test/e2e/README.md | 9 ++ test/e2e/aws_test.go | 5 +- test/e2e/consul_test.go | 4 +- test/e2e/ratelimit_test.go | 4 +- test/e2e/staged_transformation_test.go | 17 +-- test/e2e/test_context.go | 13 +- test/e2e/vault_aws_test.go | 3 +- test/e2e/vault_test.go | 3 +- test/e2e/zipkin_test.go | 4 +- test/ginkgo/decorators/decorators.go | 19 +++ test/ginkgo/parallel/parallel_suite_test.go | 14 ++ test/ginkgo/parallel/ports.go | 60 +++++++- test/ginkgo/parallel/ports_test.go | 38 +++++ test/helpers/debug.go | 6 +- test/services/docker.go | 138 ++++++++++++++++++ test/services/envoy/factory.go | 5 +- test/services/envoy/instance.go | 138 +++++++----------- test/services/envoy/ports.go | 6 +- test/services/gloo.go | 5 +- test/services/vault.go | 4 +- test/testutils/env.go | 15 ++ 36 files changed, 442 insertions(+), 161 deletions(-) create mode 100644 changelog/v1.15.0-beta16/parallel-tests.yaml create mode 100644 test/ginkgo/decorators/decorators.go create mode 100644 test/ginkgo/parallel/parallel_suite_test.go create mode 100644 test/ginkgo/parallel/ports_test.go create mode 100644 test/services/docker.go diff --git a/Makefile b/Makefile index abff954674d..7be6a928a6d 100644 --- a/Makefile +++ b/Makefile @@ -145,7 +145,7 @@ check-spelling: GINKGO_VERSION ?= $(shell echo $(shell go list -m github.com/onsi/ginkgo/v2) | cut -d' ' -f2) GINKGO_ENV ?= GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore ACK_GINKGO_RC=true ACK_GINKGO_DEPRECATIONS=$(GINKGO_VERSION) -GINKGO_FLAGS ?= -tags=purego --trace -progress -race --fail-fast -fail-on-pending --randomize-all --compilers=4 +GINKGO_FLAGS ?= -tags=purego --trace -progress -race --fail-fast -fail-on-pending --randomize-all --compilers=5 GINKGO_REPORT_FLAGS ?= --json-report=test-report.json --junit-report=junit.xml -output-dir=$(OUTPUT_DIR) GINKGO_COVERAGE_FLAGS ?= --cover --covermode=count --coverprofile=coverage.cov TEST_PKG ?= ./... # Default to run all tests diff --git a/changelog/v1.15.0-beta16/parallel-tests.yaml b/changelog/v1.15.0-beta16/parallel-tests.yaml new file mode 100644 index 00000000000..f81f89c7420 --- /dev/null +++ b/changelog/v1.15.0-beta16/parallel-tests.yaml @@ -0,0 +1,9 @@ +changelog: + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/5143 + resolvesIssue: true + description: >- + Enable tests to be run in parallel. Turn this OFF by default in CI. + + skipCI-kube-tests:true + skipCI-docs-build:true \ No newline at end of file diff --git a/ci/cloudbuild/run-tests.yaml b/ci/cloudbuild/run-tests.yaml index 53a458d7193..929083d43cf 100644 --- a/ci/cloudbuild/run-tests.yaml +++ b/ci/cloudbuild/run-tests.yaml @@ -134,6 +134,9 @@ options: - 'RUN_VAULT_TESTS=1' - 'DOCKER_CONFIG=/workspace/.docker/' - 'SKIP_TEMP_DISABLED=1' # https://github.com/solo-io/solo-projects/issues/4515 + # Gloo supports running tests in parallel (https://github.com/solo-io/gloo/pull/8423) but occasionally + # Ginkgo reports a failure, even when all test suites pass. As a result, we do not run tests in parallel in CI. + - 'GINKGO_USER_FLAGS=-procs=1' volumes: - name: 'gopath' path: '/go' diff --git a/projects/gloo/cli/pkg/cmd/add/add_test.go b/projects/gloo/cli/pkg/cmd/add/add_test.go index 7b3c4537cdd..c95fcc88d3d 100644 --- a/projects/gloo/cli/pkg/cmd/add/add_test.go +++ b/projects/gloo/cli/pkg/cmd/add/add_test.go @@ -4,6 +4,8 @@ import ( "context" "log" + "github.com/solo-io/gloo/test/ginkgo/decorators" + "github.com/hashicorp/consul/api" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -12,7 +14,7 @@ import ( "github.com/solo-io/gloo/test/testutils" ) -var _ = Describe("Add", func() { +var _ = Describe("Add", decorators.Consul, func() { if !testutils.IsEnvTruthy(testutils.RunConsulTests) { log.Print("This test downloads and runs consul and is disabled by default. To enable, set RUN_CONSUL_TESTS=1 in your env.") return diff --git a/projects/gloo/cli/pkg/cmd/create/create_test.go b/projects/gloo/cli/pkg/cmd/create/create_test.go index 0bb5a0e395c..a2659868c1f 100644 --- a/projects/gloo/cli/pkg/cmd/create/create_test.go +++ b/projects/gloo/cli/pkg/cmd/create/create_test.go @@ -4,6 +4,8 @@ import ( "context" "log" + "github.com/solo-io/gloo/test/ginkgo/decorators" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers" @@ -11,7 +13,7 @@ import ( "github.com/solo-io/gloo/test/testutils" ) -var _ = Describe("Create", func() { +var _ = Describe("Create", decorators.Consul, func() { if !testutils.IsEnvTruthy(testutils.RunConsulTests) { log.Print("This test downloads and runs consul and is disabled by default. To enable, set RUN_CONSUL_TESTS=1 in your env.") return diff --git a/projects/gloo/cli/pkg/cmd/create/secret/create_test.go b/projects/gloo/cli/pkg/cmd/create/secret/create_test.go index 98647b8a98b..1a306f9b4c4 100644 --- a/projects/gloo/cli/pkg/cmd/create/secret/create_test.go +++ b/projects/gloo/cli/pkg/cmd/create/secret/create_test.go @@ -5,6 +5,8 @@ import ( "fmt" "log" + "github.com/solo-io/gloo/test/ginkgo/decorators" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers" @@ -12,7 +14,7 @@ import ( glootestutils "github.com/solo-io/gloo/test/testutils" ) -var _ = Describe("Create", func() { +var _ = Describe("Create", decorators.Vault, func() { if !glootestutils.IsEnvTruthy(glootestutils.RunVaultTests) { log.Print("This test downloads and runs vault and is disabled by default. To enable, set RUN_VAULT_TESTS=1 in your env.") diff --git a/projects/gloo/cli/pkg/cmd/install/install_suite_test.go b/projects/gloo/cli/pkg/cmd/install/install_suite_test.go index aaa84f888fb..d631fdd8e6e 100644 --- a/projects/gloo/cli/pkg/cmd/install/install_suite_test.go +++ b/projects/gloo/cli/pkg/cmd/install/install_suite_test.go @@ -35,12 +35,7 @@ const ( expectedHelmFilenameFmt = "gloo-%s.tgz" ) -// NOTE: This needs to be run from the root of the repo as the working directory -var _ = BeforeSuite(func() { - - // Make sure we don't hit a real cluster during any of the tests in this suite - helpers.UseMemoryClients() - +func setupVariables() { cwd, err := os.Getwd() Expect(err).NotTo(HaveOccurred()) RootDir = filepath.Join(cwd, "../../../../../..") @@ -48,19 +43,28 @@ var _ = BeforeSuite(func() { // the regression test depend on having only one chart in _test. // so run these in a different location. dir = filepath.Join(RootDir, "_unit_test/") + file = filepath.Join(dir, fmt.Sprintf(expectedHelmFilenameFmt, version.Version)) + + values1 = filepath.Join(dir, "values-namespace1.yaml") + values2 = filepath.Join(dir, "values-namespace2.yaml") +} + +// NOTE: This needs to be run from the root of the repo as the working directory +var beforeSuiteOnce = func() { + + // Make sure we don't hit a real cluster during any of the tests in this suite + helpers.UseMemoryClients() + + setupVariables() os.Mkdir(dir, 0755) - err = testutils.Make(RootDir, "build-test-chart TEST_ASSET_DIR=\""+dir+"\"") + err := testutils.Make(RootDir, "build-test-chart TEST_ASSET_DIR=\""+dir+"\"") Expect(err).NotTo(HaveOccurred()) // Some tests need the Gloo/GlooE version that gets linked into the glooctl binary at build time err = testutils.Make(RootDir, "glooctl") Expect(err).NotTo(HaveOccurred()) - file = filepath.Join(dir, fmt.Sprintf(expectedHelmFilenameFmt, version.Version)) - - values1 = filepath.Join(dir, "values-namespace1.yaml") - values2 = filepath.Join(dir, "values-namespace2.yaml") f, err := os.Create(values1) Expect(err).NotTo(HaveOccurred()) _, err = f.WriteString(` @@ -100,9 +104,13 @@ settings: } } Expect(install.GlooCrdNames).To(ContainElements(crdNames)) -}) +} + +var beforeSuiteAll = func() { + setupVariables() +} -var _ = AfterSuite(func() { +var afterSuiteOnce = func() { err := os.Remove(file) Expect(err).NotTo(HaveOccurred()) err = os.Remove(values1) @@ -111,4 +119,9 @@ var _ = AfterSuite(func() { Expect(err).NotTo(HaveOccurred()) err = os.RemoveAll(dir) Expect(err).NotTo(HaveOccurred()) -}) +} + +var ( + _ = SynchronizedBeforeSuite(beforeSuiteOnce, beforeSuiteAll) + _ = SynchronizedAfterSuite(func() {}, afterSuiteOnce) +) diff --git a/projects/gloo/pkg/plugins/consul/consul_suite_test.go b/projects/gloo/pkg/plugins/consul/consul_suite_test.go index bfb81c964fc..982cb010197 100644 --- a/projects/gloo/pkg/plugins/consul/consul_suite_test.go +++ b/projects/gloo/pkg/plugins/consul/consul_suite_test.go @@ -9,13 +9,11 @@ import ( . "github.com/onsi/gomega" ) -var T *testing.T - func TestConsul(t *testing.T) { + // This has caused issues when tests are run in parallel (not enabled in CI) leakDetector := helpers.DeferredGoroutineLeakDetector(t) defer leakDetector() RegisterFailHandler(Fail) - T = t RunSpecs(t, "Consul Plugin Suite") } diff --git a/projects/gloo/pkg/plugins/consul/eds_test.go b/projects/gloo/pkg/plugins/consul/eds_test.go index 5e3f2b30d16..d922865d239 100644 --- a/projects/gloo/pkg/plugins/consul/eds_test.go +++ b/projects/gloo/pkg/plugins/consul/eds_test.go @@ -46,7 +46,7 @@ var _ = Describe("Consul EDS", func() { const writeNamespace = defaults.GlooSystem BeforeEach(func() { - ctrl = gomock.NewController(T) + ctrl = gomock.NewController(GinkgoT()) }) AfterEach(func() { diff --git a/projects/gloo/pkg/plugins/consul/plugin_test.go b/projects/gloo/pkg/plugins/consul/plugin_test.go index e58a37aa87b..3004be2b48a 100644 --- a/projects/gloo/pkg/plugins/consul/plugin_test.go +++ b/projects/gloo/pkg/plugins/consul/plugin_test.go @@ -24,10 +24,9 @@ var _ = Describe("Resolve", func() { ) BeforeEach(func() { - ctrl = gomock.NewController(T) + ctrl = gomock.NewController(GinkgoT()) consulWatcherMock = mock_consul.NewMockConsulWatcher(ctrl) - }) AfterEach(func() { diff --git a/projects/gloo/pkg/translator/hashing_benchmark_test.go b/projects/gloo/pkg/translator/hashing_benchmark_test.go index 92221863b6e..18b560b3040 100644 --- a/projects/gloo/pkg/translator/hashing_benchmark_test.go +++ b/projects/gloo/pkg/translator/hashing_benchmark_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "time" + "github.com/solo-io/gloo/test/ginkgo/decorators" + "github.com/solo-io/gloo/test/ginkgo/labels" "github.com/onsi/gomega/gmeasure" @@ -42,7 +44,7 @@ import ( ) // These tests are meant to demonstrate that FNV hashing is more efficient than hashstructure hashing -var _ = Describe("Hashing Benchmarks", Serial, Label(labels.Performance), func() { +var _ = Describe("Hashing Benchmarks", decorators.Performance, Label(labels.Performance), func() { var allUpstreams v1.UpstreamList BeforeEach(func() { var upstreams []interface{} diff --git a/projects/gloo/pkg/translator/performance_test.go b/projects/gloo/pkg/translator/performance_test.go index 7db470affff..ad36dade077 100644 --- a/projects/gloo/pkg/translator/performance_test.go +++ b/projects/gloo/pkg/translator/performance_test.go @@ -5,6 +5,8 @@ import ( "fmt" "strings" + "github.com/solo-io/gloo/test/ginkgo/decorators" + "github.com/solo-io/go-utils/contextutils" validationutils "github.com/solo-io/gloo/projects/gloo/pkg/utils/validation" @@ -49,7 +51,7 @@ type benchmarkConfig struct { // More info on that machine can be found here: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources // When developing new tests, users should manually run that action in order to test performance under the same parameters // Results can then be found in the logs for that instance of the action -var _ = Describe("Translation - Benchmarking Tests", Serial, Label(labels.Performance), func() { +var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Label(labels.Performance), func() { var ( ctrl *gomock.Controller settings *v1.Settings diff --git a/projects/sds/pkg/run/run_e2e_test.go b/projects/sds/pkg/run/run_e2e_test.go index 4aeba755a76..50c75a6c7f0 100644 --- a/projects/sds/pkg/run/run_e2e_test.go +++ b/projects/sds/pkg/run/run_e2e_test.go @@ -18,7 +18,9 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("SDS Server E2E Test", func() { +var _ = Describe("SDS Server E2E Test", Serial, func() { + + // These tests use the Serial decorator because they rely on a hard-coded port for the SDS server (8236) var ( ctx context.Context diff --git a/test/consulvaulte2e/consul_vault_test.go b/test/consulvaulte2e/consul_vault_test.go index 36c275f5044..e3b5ea29964 100644 --- a/test/consulvaulte2e/consul_vault_test.go +++ b/test/consulvaulte2e/consul_vault_test.go @@ -8,6 +8,8 @@ import ( "path/filepath" "time" + "github.com/solo-io/gloo/test/ginkgo/decorators" + "github.com/solo-io/gloo/test/services/envoy" "google.golang.org/protobuf/types/known/wrapperspb" @@ -40,7 +42,7 @@ import ( "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" ) -var _ = Describe("Consul + Vault Configuration Happy Path e2e", func() { +var _ = Describe("Consul + Vault Configuration Happy Path e2e", decorators.Vault, decorators.Consul, func() { var ( ctx context.Context diff --git a/test/consulvaulte2e/e2e_suite_test.go b/test/consulvaulte2e/e2e_suite_test.go index 269964e01d6..36fd796593a 100644 --- a/test/consulvaulte2e/e2e_suite_test.go +++ b/test/consulvaulte2e/e2e_suite_test.go @@ -24,6 +24,7 @@ func TestE2e(t *testing.T) { helpers.RegisterCommonFailHandlers() helpers.SetupLog() + RunSpecs(t, "Consul+Vault E2e Suite", Label(labels.E2E)) } diff --git a/test/e2e/README.md b/test/e2e/README.md index 404e76debea..28882c28b80 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -95,6 +95,15 @@ If you have made changes to the component, you will have had to rebuild the imag ENVOY_IMAGE_TAG=0.0.1-local TEST_PKG=./test/e2e/... make test ``` +#### Running Tests in Parallel +It is possible to run tests in parallel locally, however it is not recommended, because debugging failures becomes more difficult. If you do want to run tests in parallel, you can do so by passing in the relevant `GINKGO_USER_FLAGS` values: +```bash +GINKGO_USER_FLAGS=-procs=4 TEST_PKG=./test/e2e/... make test +``` + +*Note: When using Docker to run Envoy, we have seen occasional failures: `Error response from daemon: dial unix docker.raw.sock: connect: connection refused`* + + ### Debugging Tests #### Use WAIT_ON_FAIL When Ginkgo encounters a [test failure](https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure) it will attempt to clean up relevant resources, which includes stopping the running instance of Envoy, preventing the developer from inspecting the state of the Envoy instance for useful clues. diff --git a/test/e2e/aws_test.go b/test/e2e/aws_test.go index d924d0a5e23..4d2bb6beffa 100644 --- a/test/e2e/aws_test.go +++ b/test/e2e/aws_test.go @@ -64,14 +64,11 @@ var _ = Describe("AWS Lambda", func() { envoyInstance *envoy.Instance secret *gloov1.Secret upstream *gloov1.Upstream - httpClient *http.Client runOptions *services.RunOptions ) BeforeEach(func() { testutils.ValidateRequirementsAndNotifyGinkgo(testutils.AwsCredentials()) - httpClient = http.DefaultClient - httpClient.Timeout = 10 * time.Second runOptions = &services.RunOptions{ NsToWrite: writeNamespace, NsToWatch: []string{"default", writeNamespace}, @@ -486,6 +483,8 @@ var _ = Describe("AWS Lambda", func() { var body []byte path := "transforms-req-test" waitForLambdaAndGetBody := func() error { + httpClient := testutils.DefaultClientBuilder().WithTimeout(time.Second * 10).Build() + req, err := http.NewRequest("POST", fmt.Sprintf("http://%s:%d/%s?foo=bar", "localhost", envoyInstance.HttpPort, path), bytes.NewBufferString(`"test"`)) Expect(err).NotTo(HaveOccurred()) req.Header.Set("Content-Type", "application/octet-stream") diff --git a/test/e2e/consul_test.go b/test/e2e/consul_test.go index e1fadf4ff07..d31942d14cd 100644 --- a/test/e2e/consul_test.go +++ b/test/e2e/consul_test.go @@ -5,6 +5,8 @@ import ( "errors" "time" + "github.com/solo-io/gloo/test/ginkgo/decorators" + "github.com/golang/protobuf/ptypes/duration" v1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1" "github.com/solo-io/gloo/test/e2e" @@ -28,7 +30,7 @@ import ( "github.com/solo-io/gloo/test/v1helpers" ) -var _ = Describe("Consul e2e", func() { +var _ = Describe("Consul e2e", decorators.Consul, func() { Context("Consul Service Registry", func() { diff --git a/test/e2e/ratelimit_test.go b/test/e2e/ratelimit_test.go index f062e30f6ee..432a447fb1d 100644 --- a/test/e2e/ratelimit_test.go +++ b/test/e2e/ratelimit_test.go @@ -125,7 +125,9 @@ func (s *metadataCheckingRateLimitServer) getActionsForServer() []*rltypes.RateL } } -var _ = Describe("Rate Limit", func() { +var _ = Describe("Rate Limit", Serial, func() { + + // These tests use the Serial decorator because they rely on a hard-coded port for the RateLimit server (18081) var ( ctx context.Context diff --git a/test/e2e/staged_transformation_test.go b/test/e2e/staged_transformation_test.go index fe7642e1c91..276a6d20b70 100644 --- a/test/e2e/staged_transformation_test.go +++ b/test/e2e/staged_transformation_test.go @@ -3,8 +3,6 @@ package e2e_test import ( "encoding/base64" - "github.com/solo-io/gloo/test/services" - "github.com/solo-io/gloo/test/services/envoy" "github.com/solo-io/gloo/test/testutils" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/wrapperspb" @@ -33,26 +31,17 @@ var _ = Describe("Staged Transformation", func() { var ( testContext *e2e.TestContext - - // This test relies on running the gateway-proxy with debug logging enabled - // This variable allows us to reset the original log level after the test - resetLogLevel func() ) BeforeEach(func() { - originalProxyLogLevel := services.GetLogLevel(envoy.ServiceName) - services.SetLogLevel(envoy.ServiceName, zapcore.DebugLevel) - resetLogLevel = func() { - services.SetLogLevel(envoy.ServiceName, originalProxyLogLevel) - } - testContext = testContextFactory.NewTestContext() testContext.BeforeEach() + + // This test relies on running the gateway-proxy with debug logging enabled + testContext.EnvoyInstance().LogLevel = zapcore.DebugLevel.String() }) AfterEach(func() { - resetLogLevel() - testContext.AfterEach() }) diff --git a/test/e2e/test_context.go b/test/e2e/test_context.go index 9f40a325673..1d10c096d38 100644 --- a/test/e2e/test_context.go +++ b/test/e2e/test_context.go @@ -138,10 +138,8 @@ func (c *TestContext) BeforeEach() { } func (c *TestContext) AfterEach() { - ginkgo.By("TestContext.AfterEach: Stopping Envoy and cancelling test context") - // Stop Envoy - c.envoyInstance.Clean() - + ginkgo.By("TestContext.AfterEach: Cancelling test context") + // All services connected to the TestContext are tied to the context, so cancelling it will clean those up c.cancel() } @@ -152,7 +150,12 @@ func (c *TestContext) JustBeforeEach() { c.testClients = services.RunGlooGatewayUdsFds(c.ctx, c.runOptions) // Run Envoy - err := c.envoyInstance.RunWithRole(envoyRole, c.testClients.GlooPort) + err := c.envoyInstance.RunWith(envoy.RunConfig{ + Context: c.ctx, + Role: envoyRole, + Port: uint32(c.testClients.GlooPort), + RestXdsPort: uint32(c.testClients.RestXdsPort), + }) ExpectWithOffset(1, err).NotTo(HaveOccurred()) // Create Resources diff --git a/test/e2e/vault_aws_test.go b/test/e2e/vault_aws_test.go index 000a9a7db59..3f36decada5 100644 --- a/test/e2e/vault_aws_test.go +++ b/test/e2e/vault_aws_test.go @@ -5,6 +5,7 @@ import ( v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1" bootstrap "github.com/solo-io/gloo/projects/gloo/pkg/bootstrap/clients" "github.com/solo-io/gloo/test/e2e" + "github.com/solo-io/gloo/test/ginkgo/decorators" "github.com/solo-io/gloo/test/testutils" "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" @@ -25,7 +26,7 @@ const ( vaultRole = "vault-role" ) -var _ = Describe("Vault Secret Store (AWS Auth)", func() { +var _ = Describe("Vault Secret Store (AWS Auth)", decorators.Vault, func() { var ( testContext *e2e.TestContextWithVault diff --git a/test/e2e/vault_test.go b/test/e2e/vault_test.go index 06d8c55bf12..a7aaa616952 100644 --- a/test/e2e/vault_test.go +++ b/test/e2e/vault_test.go @@ -5,6 +5,7 @@ import ( v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/extauth/v1" bootstrap "github.com/solo-io/gloo/projects/gloo/pkg/bootstrap/clients" "github.com/solo-io/gloo/test/e2e" + "github.com/solo-io/gloo/test/ginkgo/decorators" "github.com/solo-io/gloo/test/services" "github.com/solo-io/solo-kit/pkg/api/v1/clients" "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" @@ -13,7 +14,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Vault Secret Store (Token Auth)", func() { +var _ = Describe("Vault Secret Store (Token Auth)", decorators.Vault, func() { var ( testContext *e2e.TestContextWithVault diff --git a/test/e2e/zipkin_test.go b/test/e2e/zipkin_test.go index 3bb64017e6e..2895fa04bcc 100644 --- a/test/e2e/zipkin_test.go +++ b/test/e2e/zipkin_test.go @@ -42,7 +42,9 @@ const ( zipkinCollectionPath = "/api/v2/spans" ) -var _ = Describe("Tracing config loading", func() { +var _ = Describe("Tracing config loading", Serial, func() { + + // These tests use the Serial decorator because they rely on a hard-coded port for the tracing collector (9411) var ( ctx context.Context diff --git a/test/ginkgo/decorators/decorators.go b/test/ginkgo/decorators/decorators.go new file mode 100644 index 00000000000..e61eba65d75 --- /dev/null +++ b/test/ginkgo/decorators/decorators.go @@ -0,0 +1,19 @@ +package decorators + +import "github.com/onsi/ginkgo/v2" + +// Ginkgo Decorators (https://onsi.github.io/ginkgo/#decorator-reference) + +const ( + // Vault is a decorator that allows you to mark a spec or container as requiring our Vault service (test/services/vault.go) + // The Vault service uses a hard-coded port, so only one test can use it at a time. + Vault = ginkgo.Serial + + // Consul is a decorator that allows you to mark a spec or container as requiring our Consul service (test/services/consul.go) + // The Consul service uses on a hard-coded port for the Consul agent (8300) + Consul = ginkgo.Serial + + // Performance is a decorator that allows you to mark a spec or container as running performance tests + // These often require more resources/time to complete, and for consistency, should not be run in parallel with other tests + Performance = ginkgo.Serial +) diff --git a/test/ginkgo/parallel/parallel_suite_test.go b/test/ginkgo/parallel/parallel_suite_test.go new file mode 100644 index 00000000000..f569744375f --- /dev/null +++ b/test/ginkgo/parallel/parallel_suite_test.go @@ -0,0 +1,14 @@ +package parallel_test + +import ( + "testing" + + "github.com/solo-io/go-utils/testutils" + + . "github.com/onsi/ginkgo/v2" +) + +func TestParallel(t *testing.T) { + testutils.RegisterCommonFailHandlers() + RunSpecs(t, "Parallel Suite") +} diff --git a/test/ginkgo/parallel/ports.go b/test/ginkgo/parallel/ports.go index 4bcf02a376b..cbada6ba207 100644 --- a/test/ginkgo/parallel/ports.go +++ b/test/ginkgo/parallel/ports.go @@ -1,23 +1,67 @@ package parallel -import "github.com/onsi/ginkgo/v2" +import ( + "fmt" + "net" + "sync/atomic" + "time" -func init() { - parallelGinkgoProcesses = ginkgo.GinkgoParallelProcess() -} + "github.com/rotisserie/eris" + + "github.com/avast/retry-go" -var ( - parallelGinkgoProcesses int + "github.com/onsi/ginkgo/v2" ) // GetParallelProcessCount returns the parallel process number for the current ginkgo process func GetParallelProcessCount() int { - return parallelGinkgoProcesses + return ginkgo.GinkgoParallelProcess() } // GetPortOffset returns the number of parallel Ginkgo processes * 1000 // This is intended to be used by tests which need to produce unique ports so that they can be run // in parallel without port conflict func GetPortOffset() int { - return parallelGinkgoProcesses * 1000 + return GetParallelProcessCount() * 1000 +} + +// AdvancePortSafe advances the provided port by 1 until it returns a port that is safe to use +// The availability of the port is determined by the errIfPortInUse function +func AdvancePortSafe(p *uint32, errIfPortInUse func(proposedPort uint32) error) uint32 { + var newPort uint32 + + _ = retry.Do(func() error { + newPort = AdvancePort(p) + return errIfPortInUse(newPort) + }, + retry.RetryIf(func(err error) bool { + return err != nil + }), + retry.Attempts(3), + retry.Delay(time.Millisecond*0)) + + return newPort +} + +// AdvancePort advances the provided port by 1, and adds an offset to support running tests in parallel +func AdvancePort(p *uint32) uint32 { + return atomic.AddUint32(p, 1) + uint32(GetPortOffset()) +} + +// AdvancePortSafeListen returns a port that is safe to use in parallel tests +// It relies on pinging the port to see if it is in use +func AdvancePortSafeListen(p *uint32) uint32 { + return AdvancePortSafe(p, portInUseListen) +} + +func portInUseListen(proposedPort uint32) error { + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", proposedPort)) + + if err != nil { + return eris.Wrapf(err, "port %d is in use", proposedPort) + } + + _ = ln.Close() + // Port is available + return nil } diff --git a/test/ginkgo/parallel/ports_test.go b/test/ginkgo/parallel/ports_test.go new file mode 100644 index 00000000000..6832404c132 --- /dev/null +++ b/test/ginkgo/parallel/ports_test.go @@ -0,0 +1,38 @@ +package parallel_test + +import ( + "github.com/rotisserie/eris" + "github.com/solo-io/gloo/test/ginkgo/parallel" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Ports", func() { + + Context("AdvancePortSafe", func() { + + portInUseDenylist := func(proposedPort uint32) error { + var denyList = map[uint32]struct{}{ + 10010: {}, // used by Gloo, when devMode is enabled + } + + if _, ok := denyList[proposedPort]; ok { + return eris.Errorf("port %d is in use", proposedPort) + } + return nil + } + + It("skips ports based on errIfPortInUse", func() { + portInDenylist := uint32(10010) + advanceAmount := uint32(1 + parallel.GetPortOffset()) + portInDenylistMinusOffset := portInDenylist - advanceAmount + + selectedPort := parallel.AdvancePortSafe(&portInDenylistMinusOffset, portInUseDenylist) + Expect(selectedPort).NotTo(Equal(portInDenylist), "should have skipped the port in the denylist") + Expect(selectedPort).To(Equal(portInDenylist+1), "should have selected the next port") + }) + + }) + +}) diff --git a/test/helpers/debug.go b/test/helpers/debug.go index d13d593a1e2..da18daebfcd 100644 --- a/test/helpers/debug.go +++ b/test/helpers/debug.go @@ -9,9 +9,9 @@ import ( // DeferredGoroutineLeakDetector returns a function that can be used in tests to identify goroutine leaks // Example usage: // -// leakDetector := DeferredGoroutineLeakDetector(t) -// defer leakDetector() -// ... +// leakDetector := DeferredGoroutineLeakDetector(t) +// defer leakDetector() +// ... // // NOTE TO DEVS: We would like to extend the usage of this across more test suites: https://github.com/solo-io/gloo/issues/7147 func DeferredGoroutineLeakDetector(t *testing.T) func() { diff --git a/test/services/docker.go b/test/services/docker.go new file mode 100644 index 00000000000..bf49b3b2fae --- /dev/null +++ b/test/services/docker.go @@ -0,0 +1,138 @@ +package services + +import ( + "fmt" + "os" + "os/exec" + "time" + + "github.com/solo-io/gloo/test/testutils" + + "github.com/avast/retry-go" + + . "github.com/onsi/gomega" + + "github.com/onsi/ginkgo/v2" + "github.com/pkg/errors" +) + +const ( + // defaultNetwork is the default docker network driver + // https://docs.docker.com/network/drivers/bridge/ + defaultNetwork = "bridge" + + // cloudbuildNetwork is the docker network driver used by Google Cloudbuild + // https://cloud.google.com/build/docs/build-config-file-schema#network + cloudbuildNetwork = "cloudbuild" +) + +func RunContainer(containerName string, args []string) error { + runArgs := append([]string{ + "run", + "--rm", + "--name", + containerName, + }, args...) + cmd := exec.Command("docker", runArgs...) + cmd.Stdout = ginkgo.GinkgoWriter + cmd.Stderr = ginkgo.GinkgoWriter + err := cmd.Run() + if err != nil { + return errors.Wrap(err, "Unable to start "+containerName+" container") + } + return nil +} + +// ContainerExistsWithName returns an empty string if the container does not exist +func ContainerExistsWithName(containerName string) string { + cmd := exec.Command("docker", "ps", "-aq", "-f", "name=^/"+containerName+"$") + out, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("cmd.Run() [%s %s] failed with %s\n", cmd.Path, cmd.Args, err) + } + return string(out) +} + +func ExecOnContainer(containerName string, args []string) ([]byte, error) { + arguments := []string{"exec", containerName} + arguments = append(arguments, args...) + cmd := exec.Command("docker", arguments...) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, errors.Wrapf(err, "Unable to execute command %v on [%s] container [%s]", arguments, containerName, out) + } + return out, nil +} + +func MustStopAndRemoveContainer(containerName string) { + StopContainer(containerName) + + // We assume that the container was run with auto-remove, and thus stopping the container will cause it to be removed + err := WaitUntilContainerRemoved(containerName) + Expect(err).ToNot(HaveOccurred()) + + // CI host may be extremely CPU-bound as it's often building test assets in tandem with other tests, + // as well as other CI builds running in parallel. When that happens, the tests can run much slower, + // thus they need a longer timeout. see https://github.com/solo-io/solo-projects/issues/1701#issuecomment-620873754 + Eventually(ContainerExistsWithName(containerName), "30s", "2s").Should(BeEmpty()) +} + +func StopContainer(containerName string) { + cmd := exec.Command("docker", "stop", containerName) + cmd.Stdout = ginkgo.GinkgoWriter + cmd.Stderr = ginkgo.GinkgoWriter + err := cmd.Run() + if err != nil { + // We have seen this trip, even when the container is successfully stopped + // log.Printf("Error stopping container %s: %v", containerName, err) + } +} + +// WaitUntilContainerRemoved polls docker for removal of the container named containerName - block until +// successful or fail after a small number of retries +func WaitUntilContainerRemoved(containerName string) error { + return retry.Do(func() error { + inspectErr := exec.Command("docker", "inspect", containerName).Run() + if inspectErr == nil { + // If there is no error, it means the container still exists, so we want to retry + return errors.Errorf("container %s still exists", containerName) + } + return nil + }, + retry.RetryIf(func(err error) bool { + return err != nil + }), + retry.Attempts(10), + retry.Delay(time.Millisecond*500), + retry.DelayType(retry.BackOffDelay), + retry.LastErrorOnly(true), + ) +} + +func RunningInDocker() bool { + if _, err := os.Stat("/.dockerenv"); os.IsNotExist(err) { + // magic docker env file doesn't exist. not running in docker + return false + } + return true +} + +func GetDockerHost(containerName string) string { + if RunningInDocker() { + return containerName + } else { + return "127.0.0.1" + } +} + +func GetContainerNetwork() string { + network := defaultNetwork + if RunningInDocker() { + if !testutils.IsRunningInCloudbuild() { + // We error loudly here so that if/when we move off of Google Cloudbuild, we can clean up this logic + ginkgo.Fail("Running in docker but not in cloudbuild. Could not determine docker network") + } + network = cloudbuildNetwork + } + return network +} diff --git a/test/services/envoy/factory.go b/test/services/envoy/factory.go index 15aadb36577..c9aa27aefe1 100644 --- a/test/services/envoy/factory.go +++ b/test/services/envoy/factory.go @@ -208,11 +208,12 @@ func (f *factoryImpl) newInstanceOrError() (*Instance, error) { envoypath: f.envoypath, UseDocker: f.useDocker, DockerImage: f.dockerImage, + DockerContainerName: fmt.Sprintf("e2e_envoy-%d", defaults.HttpPort), GlooAddr: gloo, AccessLogPort: NextAccessLogPort(), AccessLogAddr: gloo, ApiVersion: "V3", - logLevel: getInstanceLogLevel(), + LogLevel: getInstanceLogLevel(), RequestPorts: &RequestPorts{ HttpPort: defaults.HttpPort, HttpsPort: defaults.HttpsPort, @@ -221,6 +222,8 @@ func (f *factoryImpl) newInstanceOrError() (*Instance, error) { AdminPort: defaults.EnvoyAdminPort, }, } + + log.Printf("Envoy Instance (%s) Ports: %+v", ei.DockerContainerName, ei.RequestPorts) f.instances = append(f.instances, ei) return ei, nil diff --git a/test/services/envoy/instance.go b/test/services/envoy/instance.go index faffb485d81..ae14319412d 100644 --- a/test/services/envoy/instance.go +++ b/test/services/envoy/instance.go @@ -9,6 +9,8 @@ import ( "net/http" "os/exec" + "github.com/solo-io/gloo/test/services" + adminv3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" "github.com/golang/protobuf/jsonpb" @@ -22,7 +24,6 @@ import ( ) const ( - containerName = "e2e_envoy" DefaultProxyName = "default~proxy" ) @@ -41,7 +42,7 @@ type Instance struct { envoypath string envoycfg string logs *SafeBuffer - logLevel string + LogLevel string cmd *exec.Cmd GlooAddr string // address for gloo and services Port uint32 @@ -51,8 +52,9 @@ type Instance struct { ApiVersion string DockerOptions - UseDocker bool - DockerImage string + UseDocker bool + DockerImage string + DockerContainerName string *RequestPorts } @@ -75,106 +77,81 @@ type DockerOptions struct { Env []string } +// Deprecated: use RunWith instead func (ei *Instance) Run(port int) error { - return ei.RunWithRole(DefaultProxyName, port) -} - -func (ei *Instance) RunWith(eic InstanceConfig) error { - return ei.runWithAll(eic, &templateBootstrapBuilder{ - template: ei.defaultBootstrapTemplate, + return ei.RunWith(RunConfig{ + Role: DefaultProxyName, + Port: uint32(port), + Context: context.TODO(), }) } +// Deprecated: use RunWith instead func (ei *Instance) RunWithRole(role string, port int) error { - eic := &envoyInstanceConfig{ - role: role, - port: uint32(port), - context: context.TODO(), - } - boostrapBuilder := &templateBootstrapBuilder{ - template: ei.defaultBootstrapTemplate, - } - return ei.runWithAll(eic, boostrapBuilder) + return ei.RunWith(RunConfig{ + Role: role, + Port: uint32(port), + Context: context.TODO(), + }) } +// Deprecated: use RunWith instead func (ei *Instance) RunWithRoleAndRestXds(role string, glooPort, restXdsPort int) error { - eic := &envoyInstanceConfig{ - role: role, - port: uint32(glooPort), - restXdsPort: uint32(restXdsPort), - context: context.TODO(), - } - boostrapBuilder := &templateBootstrapBuilder{ + return ei.RunWith(RunConfig{ + Role: role, + Port: uint32(glooPort), + RestXdsPort: uint32(restXdsPort), + Context: context.TODO(), + }) +} + +func (ei *Instance) RunWith(runConfig RunConfig) error { + return ei.runWithAll(runConfig, &templateBootstrapBuilder{ template: ei.defaultBootstrapTemplate, - } - return ei.runWithAll(eic, boostrapBuilder) + }) } func (ei *Instance) RunWithConfigFile(port int, configFile string) error { - eic := &envoyInstanceConfig{ - role: "gloo-system~gateway-proxy", - port: uint32(port), - context: context.TODO(), + runConfig := RunConfig{ + Role: "gloo-system~gateway-proxy", + Port: uint32(port), + Context: context.TODO(), } boostrapBuilder := &fileBootstrapBuilder{ file: configFile, } - return ei.runWithAll(eic, boostrapBuilder) + return ei.runWithAll(runConfig, boostrapBuilder) } -type InstanceConfig interface { - Role() string - Port() uint32 - RestXdsPort() uint32 +type RunConfig struct { + Context context.Context - Context() context.Context + Role string + Port uint32 + RestXdsPort uint32 } -type envoyInstanceConfig struct { - role string - port uint32 - restXdsPort uint32 - - context context.Context -} - -func (eic *envoyInstanceConfig) Role() string { - return eic.role -} - -func (eic *envoyInstanceConfig) Port() uint32 { - return eic.port -} - -func (eic *envoyInstanceConfig) RestXdsPort() uint32 { - return eic.restXdsPort -} - -func (eic *envoyInstanceConfig) Context() context.Context { - return eic.context -} - -func (ei *Instance) runWithAll(eic InstanceConfig, bootstrapBuilder bootstrapBuilder) error { +func (ei *Instance) runWithAll(runConfig RunConfig, bootstrapBuilder bootstrapBuilder) error { go func() { - <-eic.Context().Done() + <-runConfig.Context.Done() ei.Clean() }() if ei.ID == "" { ei.ID = "ingress~for-testing" } - ei.Role = eic.Role() - ei.Port = eic.Port() - ei.RestXdsPort = eic.RestXdsPort() + ei.Role = runConfig.Role + ei.Port = runConfig.Port + ei.RestXdsPort = runConfig.RestXdsPort ei.envoycfg = bootstrapBuilder.Build(ei) if ei.UseDocker { - return ei.runContainer(eic.Context()) + return ei.runContainer(runConfig.Context) } - args := []string{"--config-yaml", ei.envoycfg, "--disable-hot-restart", "--log-level", ei.logLevel} + args := []string{"--config-yaml", ei.envoycfg, "--disable-hot-restart", "--log-level", ei.LogLevel} // run directly - cmd := exec.CommandContext(eic.Context(), ei.envoypath, args...) + cmd := exec.CommandContext(runConfig.Context, ei.envoypath, args...) safeBuffer := &SafeBuffer{ buffer: &bytes.Buffer{}, @@ -225,14 +202,14 @@ func (ei *Instance) Clean() { } if ei.UseDocker { - // No need to handle the error here as the call to quitquitquit above should kill and exit the container + // An earlier call to quitquitquit should kill and exit the container // This is just a backup to make sure it really gets deleted - _ = stopContainer() + services.MustStopAndRemoveContainer(ei.DockerContainerName) } } func (ei *Instance) runContainer(ctx context.Context) error { - args := []string{"run", "--rm", "--name", containerName, + args := []string{"run", "--rm", "--name", ei.DockerContainerName, "-p", fmt.Sprintf("%d:%d", ei.HttpPort, ei.HttpPort), "-p", fmt.Sprintf("%d:%d", ei.HttpsPort, ei.HttpsPort), "-p", fmt.Sprintf("%d:%d", ei.TcpPort, ei.TcpPort), @@ -252,7 +229,7 @@ func (ei *Instance) runContainer(ctx context.Context) error { "--entrypoint=envoy", ei.DockerImage, "--disable-hot-restart", - "--log-level", ei.logLevel, + "--log-level", ei.LogLevel, "--config-yaml", ei.envoycfg, ) @@ -293,20 +270,9 @@ func (ei *Instance) waitForEnvoyToBeRunning() error { } } -func stopContainer() error { - cmd := exec.Command("docker", "stop", containerName) - cmd.Stdout = ginkgo.GinkgoWriter - cmd.Stderr = ginkgo.GinkgoWriter - err := cmd.Run() - if err != nil { - return errors.Wrap(err, "Error stopping container "+containerName) - } - return nil -} - func (ei *Instance) Logs() (string, error) { if ei.UseDocker { - logsArgs := []string{"logs", containerName} + logsArgs := []string{"logs", ei.DockerContainerName} cmd := exec.Command("docker", logsArgs...) byt, err := cmd.CombinedOutput() if err != nil { diff --git a/test/services/envoy/ports.go b/test/services/envoy/ports.go index c4d644e5b41..5c42de2d0f8 100644 --- a/test/services/envoy/ports.go +++ b/test/services/envoy/ports.go @@ -1,8 +1,6 @@ package envoy import ( - "sync/atomic" - "github.com/solo-io/gloo/projects/gloo/pkg/defaults" "github.com/solo-io/gloo/test/ginkgo/parallel" @@ -29,6 +27,6 @@ func advanceRequestPorts() { defaults.EnvoyAdminPort = advancePort(&baseAdminPort) } -func advancePort(p *uint32) uint32 { - return atomic.AddUint32(p, 1) + uint32(parallel.GetPortOffset()) +func advancePort(port *uint32) uint32 { + return parallel.AdvancePortSafeListen(port) } diff --git a/test/services/gloo.go b/test/services/gloo.go index 0a3070df99f..53abb375636 100644 --- a/test/services/gloo.go +++ b/test/services/gloo.go @@ -11,7 +11,6 @@ import ( "net" "net/http" "reflect" - "sync/atomic" "time" "github.com/hashicorp/consul/api" @@ -73,7 +72,7 @@ import ( "k8s.io/client-go/kubernetes" ) -var glooPortBase = int32(30400) +var glooPortBase = uint32(30400) const ( GlooServiceName = "gloo" @@ -82,7 +81,7 @@ const ( ) func AllocateGlooPort() int32 { - return atomic.AddInt32(&glooPortBase, 1) + int32(parallel.GetPortOffset()) + return int32(parallel.AdvancePortSafeListen(&glooPortBase)) } // RunOptions are options for running an in-memory e2e test diff --git a/test/services/vault.go b/test/services/vault.go index 5769f628365..a8806d972be 100644 --- a/test/services/vault.go +++ b/test/services/vault.go @@ -37,6 +37,7 @@ type VaultFactory struct { vaultPath string tmpdir string useTls bool + basePort uint32 } // NewVaultFactory returns a VaultFactory @@ -59,6 +60,7 @@ func NewVaultFactory() (*VaultFactory, error) { return &VaultFactory{ vaultPath: binaryPath, tmpdir: tmpdir, + basePort: DefaultPort, }, nil } @@ -104,7 +106,7 @@ func (vf *VaultFactory) NewVaultInstance() (*VaultInstance, error) { useTls: false, // this is not used currently but we know we will need to support it soon token: DefaultVaultToken, hostname: DefaultHost, - port: DefaultPort, + port: vf.basePort, }, nil } diff --git a/test/testutils/env.go b/test/testutils/env.go index cc84464fb4f..42320cf3b90 100644 --- a/test/testutils/env.go +++ b/test/testutils/env.go @@ -54,6 +54,10 @@ const ( // ServiceLogLevel is used to set the log level for the test services. See services/logging.go for more details ServiceLogLevel = "SERVICE_LOG_LEVEL" + // GcloudBuildId is used by Cloudbuild to identify the build id + // This is set when running tests in Cloudbuild + GcloudBuildId = "GCLOUD_BUILD_ID" + // ReleasedVersion can be used when running KubeE2E tests to have the test suite use a previously released version of Gloo Edge // If set to 'LATEST', the most recently released version will be used // If set to another value, the test suite will use that version (ie '1.15.0-beta1') @@ -79,9 +83,20 @@ func ShouldSkipTempDisabledTests() bool { return IsEnvTruthy(SkipTempDisabledTests) } +// IsRunningInCloudbuild returns true if tests are running in Cloudbuild +func IsRunningInCloudbuild() bool { + return IsEnvDefined(GcloudBuildId) +} + // IsEnvTruthy returns true if a given environment variable has a truthy value // Examples of truthy values are: "1", "t", "T", "true", "TRUE", "True". Anything else is considered false. func IsEnvTruthy(envVarName string) bool { envValue, _ := strconv.ParseBool(os.Getenv(envVarName)) return envValue } + +// IsEnvDefined returns true if a given environment variable has any value +func IsEnvDefined(envVarName string) bool { + envValue := os.Getenv(envVarName) + return len(envValue) > 0 +}