diff --git a/test/e2e/test/elasticsearch/checks_k8s.go b/test/e2e/test/elasticsearch/checks_k8s.go index 2bb329d97e..6cb9e37273 100644 --- a/test/e2e/test/elasticsearch/checks_k8s.go +++ b/test/e2e/test/elasticsearch/checks_k8s.go @@ -9,6 +9,7 @@ import ( "fmt" "reflect" "sort" + "time" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" @@ -22,6 +23,15 @@ import ( "k8s.io/apimachinery/pkg/types" ) +const ( + // RollingUpgradeTimeout is used for checking a rolling upgrade is complete. + // Most tests require less than 5 minutes for all Pods to be running and ready, + // but it occasionally takes longer for various reasons (long Pod creation time, long volume binding, etc.). + // We use a longer timeout here to not be impacted too much by those external factors, and only fail + // if things seem to be stuck. + RollingUpgradeTimeout = 15 * time.Minute +) + func (b Builder) CheckK8sTestSteps(k *test.K8sClient) test.StepList { return test.StepList{ CheckCertificateAuthority(b, k), @@ -229,9 +239,9 @@ func CheckESPassword(b Builder, k *test.K8sClient) test.Step { func CheckExpectedPodsEventuallyReady(b Builder, k *test.K8sClient) test.Step { return test.Step{ Name: "All expected Pods should eventually be ready", - Test: test.Eventually(func() error { + Test: test.UntilSuccess(func() error { return checkExpectedPodsReady(b, k) - }), + }, RollingUpgradeTimeout), } } diff --git a/test/e2e/test/elasticsearch/checks_keystore.go b/test/e2e/test/elasticsearch/checks_keystore.go index 5c3091c77a..22fec214e4 100644 --- a/test/e2e/test/elasticsearch/checks_keystore.go +++ b/test/e2e/test/elasticsearch/checks_keystore.go @@ -19,7 +19,7 @@ import ( func CheckESKeystoreEntries(k *test.K8sClient, b Builder, expectedKeys []string) test.Step { return test.Step{ Name: "Elasticsearch secure settings should eventually be set in all nodes keystore", - Test: test.Eventually(func() error { + Test: test.UntilSuccess(func() error { pods, err := k.GetPods(test.ESPodListOptions(b.Elasticsearch.Namespace, b.Elasticsearch.Name)...) if err != nil { return err @@ -59,6 +59,6 @@ func CheckESKeystoreEntries(k *test.K8sClient, b Builder, expectedKeys []string) } return nil - }), + }, RollingUpgradeTimeout), } } diff --git a/test/e2e/test/utils.go b/test/e2e/test/utils.go index 5f7cd90b9c..d2d54321b9 100644 --- a/test/e2e/test/utils.go +++ b/test/e2e/test/utils.go @@ -66,16 +66,19 @@ func ExitOnErr(err error) { } } -// Eventually runs the given function until success, -// with a default timeout +// Eventually runs the given function until success with a default timeout. func Eventually(f func() error) func(*testing.T) { + return UntilSuccess(f, ctx.TestTimeout) +} + +// UntilSuccess executes f until it succeeds, or the timeout is reached. +func UntilSuccess(f func() error, timeout time.Duration) func(*testing.T) { return func(t *testing.T) { - defaultTimeout := Ctx().TestTimeout - fmt.Printf("Retries (%s timeout): ", defaultTimeout) + fmt.Printf("Retries (%s timeout): ", timeout) err := retry.UntilSuccess(func() error { fmt.Print(".") // super modern progress bar 2.0! return f() - }, defaultTimeout, DefaultRetryDelay) + }, timeout, DefaultRetryDelay) fmt.Println() require.NoError(t, err) }