From e04a85bd84c7fa890017e08a811bb62ce1e90eb2 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Mon, 24 Jul 2023 15:52:32 -0500 Subject: [PATCH] Retry only specified tests (#8825) This list was compiled by finding every test failure that succeeded on a retry in the past month (approx. June 20-July 20, 2023). The [flaky-test-finder.rb][1] script was used to collect this list. ``` ruby flaky-test-finder.rb -s 18000 -e 20681 | sed 's/[0-9]*[[:space:]]\(.*\)\..*/includeClasses\.add(\"\1\")/' | sort | uniq ``` [1]: https://gist.github.com/andrross/ee07a8a05beb63f1173bcb98523918b9 Signed-off-by: Andrew Ross Signed-off-by: Ivan Brusic --- TESTING.md | 5 ++++ build.gradle | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/TESTING.md b/TESTING.md index 93b4615da6f0b..a76ee07e1faac 100644 --- a/TESTING.md +++ b/TESTING.md @@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Test groups](#test-groups) - [Load balancing and caches](#load-balancing-and-caches) - [Test compatibility](#test-compatibility) + - [Retries](#retries) - [Miscellaneous](#miscellaneous) - [Running verification tasks](#running-verification-tasks) - [Testing the REST layer](#testing-the-rest-layer) @@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to ./gradlew test -Dtests.compatibility=1.0.0 +## Retries + +The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label. + ## Miscellaneous Run all tests without stopping on errors (inspect log files). diff --git a/build.gradle b/build.gradle index 88c485dae9350..c7b6987b1103f 100644 --- a/build.gradle +++ b/build.gradle @@ -470,6 +470,82 @@ subprojects { maxFailures = 10 } failOnPassedAfterRetry = false + classRetry { + includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests") + includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT") + includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT") + includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT") + includeClasses.add("org.opensearch.aliases.IndexAliasesIT") + includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.blocks.SimpleBlocksIT") + includeClasses.add("org.opensearch.client.PitIT") + includeClasses.add("org.opensearch.client.ReindexIT") + includeClasses.add("org.opensearch.cluster.ClusterHealthIT") + includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT") + includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT") + includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT") + includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests") + includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests") + includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT") + includeClasses.add("org.opensearch.cluster.service.MasterServiceTests") + includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests") + includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT") + includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT") + includeClasses.add("org.opensearch.http.SearchRestCancellationIT") + includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests") + includeClasses.add("org.opensearch.index.IndexServiceTests") + includeClasses.add("org.opensearch.index.IndexSettingsTests") + includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT") + includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests") + includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests") + includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests") + includeClasses.add("org.opensearch.index.shard.IndexShardIT") + includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests") + includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests") + includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT") + includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests") + includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT") + includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests") + includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT") + includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests") + includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT") + includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT") + includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT") + includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT") + includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT") + includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT") + includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT") + includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT") + includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT") + includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests") + } } } }