Increase timeout for all Pods to be eventually ready (elastic#2348)

Some of our E2E tests fail because the CheckExpectedPodsEventuallyReady
test reaches its 5min timeout.
On good conditions, it takes more than 3 minutes for some rolling
upgrades to be completely applied (for eg.
TestMutationNodeSetReplacementWithChangeBudget).

Depending on external factors (slow Pod scheduling, slow
PersistentVolume binding, etc.), we can easily reach the fixed 5min
timeout.

I propose we increase the timeout to 15min for this particular check.
This is an arbitrary value (unfortunately), but I think we're OK with
the eventual consistency nature of k8s Pods scheduling.

We could make the test smarter (continue waiting if we see there's some
small progress), but we'd still have to pick up some arbitrary timeout
values anyway, so let's keep things simple.

test/e2e/test/elasticsearch/checks_k8s.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"time"
 
 	esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates"
@@ -227,11 +228,16 @@ func CheckESPassword(b Builder, k *test.K8sClient) test.Step {
 }
 
 func CheckExpectedPodsEventuallyReady(b Builder, k *test.K8sClient) test.Step {
+	// Most tests require less than 5 minutes for all Pods to be running and ready,
+	// but it occasionally takes longer for various reasons (long pod Pod creation time, long volume binding, etc.).
+	// We use a longer timeout here to not be impacted too much by those external factors, and only fail
+	// if things seem to be stuck.
+	timeout := 15 * time.Minute
 	return test.Step{
 		Name: "All expected Pods should eventually be ready",
-		Test: test.Eventually(func() error {
+		Test: test.UntilSuccess(func() error {
 			return checkExpectedPodsReady(b, k)
-		}),
+		}, timeout),
 	}
 }
 

test/e2e/test/utils.go

@@ -66,16 +66,19 @@ func ExitOnErr(err error) {
 	}
 }
 
-// Eventually runs the given function until success,
-// with a default timeout
+// Eventually runs the given function until success with a default timeout.
 func Eventually(f func() error) func(*testing.T) {
+	return UntilSuccess(f, ctx.TestTimeout)
+}
+
+// UntilSuccess executes f until it succeeds, or the timeout is reached.
+func UntilSuccess(f func() error, timeout time.Duration) func(*testing.T) {
 	return func(t *testing.T) {
-		defaultTimeout := Ctx().TestTimeout
-		fmt.Printf("Retries (%s timeout): ", defaultTimeout)
+		fmt.Printf("Retries (%s timeout): ", timeout)
 		err := retry.UntilSuccess(func() error {
 			fmt.Print(".") // super modern progress bar 2.0!
 			return f()
-		}, defaultTimeout, DefaultRetryDelay)
+		}, timeout, DefaultRetryDelay)
 		fmt.Println()
 		require.NoError(t, err)
 	}