From 6434a28b615058fd95dffbbc8c7da3c61510afdf Mon Sep 17 00:00:00 2001
From: Anton Malinskiy <anton@malinskiy.com>
Date: Thu, 16 Feb 2023 16:34:10 +1000
Subject: [PATCH 1/7] feat(core): rework test FSM and support execution
 strategies

fixes test count expectation
---
 .idea/runConfigurations/android_app.xml       |   4 +-
 .idea/runConfigurations/android_library.xml   |  14 +-
 .../marathon/config/Configuration.kt          |  75 +-
 .../ExecutionStrategyConfiguration.kt         |  31 +
 .../serialization/ConfigurationFactoryTest.kt |   1 -
 .../kotlin/com/malinskiy/marathon/Marathon.kt |   9 +-
 .../com/malinskiy/marathon/device/Device.kt   |   2 -
 .../com/malinskiy/marathon/di/Modules.kt      |   4 +-
 .../marathon/execution/DevicePoolActor.kt     |   8 +-
 .../malinskiy/marathon/execution/Scheduler.kt |  17 +-
 .../marathon/execution/device/DeviceActor.kt  |   4 +-
 .../progress/PoolProgressAccumulator.kt       | 409 +++++++++
 .../execution/progress/ProgressReporter.kt    |  77 +-
 .../progress/tracker/PoolProgressTracker.kt   | 149 ----
 .../progress/tracker/ProgressEvent.kt         |   7 -
 .../progress/tracker/ProgressTestState.kt     |   8 -
 .../marathon/execution/queue/QueueActor.kt    |  34 +-
 .../marathon/execution/queue/TestAction.kt    |   5 +-
 .../marathon/execution/queue/TestEvent.kt     |  11 +-
 .../execution/queue/TestResultReporter.kt     | 168 ----
 .../marathon/execution/queue/TestState.kt     |  34 +-
 .../marathon/execution/queue/test_state.md    |  41 +
 .../com/malinskiy/marathon/TestGenerator.kt   |   6 +-
 .../malinskiy/marathon/device/DeviceStub.kt   |   5 +-
 .../progress/PoolProgressAccumulatorTest.kt   | 809 ++++++++++++++++++
 .../progress/ProgressReporterTest.kt          | 102 +--
 .../tracker/PoolProgressTrackerTest.kt        |  99 ---
 ...kt => TestResultReporterAllSuccessTest.kt} |  13 +-
 .../marathon/report/JUnitReporterTest.kt      |   4 +-
 .../marathon/gradle/MarathonPlugin.kt         |   1 -
 .../marathon/android/BaseAndroidDevice.kt     |   8 +-
 .../android/adam/AdamAndroidDevice.kt         |   6 +-
 .../android/adam/AndroidDeviceTestRunner.kt   |   1 -
 .../listeners/ProgressTestRunListener.kt      |  44 -
 ...tRunListener.kt => TestResultsListener.kt} |   6 +-
 .../marathon/android/ApkParserTest.kt         |   3 +-
 .../android/adam/TestConfigurationFactory.kt  |   1 -
 .../listeners/CompositeTestRunListenerTest.kt |   2 +-
 ...enerTest.kt => TestResultsListenerTest.kt} |  32 +-
 .../marathon/ios/AppleSimulatorDevice.kt      |  16 +-
 .../listener/ProgressReportingListener.kt     |  24 -
 .../executor/listener/TestResultsListener.kt  |  16 +
 .../com/malinskiy/marathon/test/StubDevice.kt |   4 +-
 .../marathon/test/factory/MarathonFactory.kt  |   3 +-
 44 files changed, 1462 insertions(+), 855 deletions(-)
 create mode 100644 configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
 create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
 delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTracker.kt
 delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressEvent.kt
 delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressTestState.kt
 delete mode 100644 core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporter.kt
 create mode 100644 core/src/main/kotlin/com/malinskiy/marathon/execution/queue/test_state.md
 create mode 100644 core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt
 delete mode 100644 core/src/test/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTrackerTest.kt
 rename core/src/test/kotlin/com/malinskiy/marathon/execution/queue/{TestResultReporterTemporalTest.kt => TestResultReporterAllSuccessTest.kt} (90%)
 delete mode 100644 vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListener.kt
 rename vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/{TestRunResultsTestRunListener.kt => TestResultsListener.kt} (95%)
 rename vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/{TestRunResultsListenerTest.kt => TestResultsListenerTest.kt} (91%)
 delete mode 100644 vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/ProgressReportingListener.kt

diff --git a/.idea/runConfigurations/android_app.xml b/.idea/runConfigurations/android_app.xml
index 367f2db76..523e7dfb8 100644
--- a/.idea/runConfigurations/android_app.xml
+++ b/.idea/runConfigurations/android_app.xml
@@ -5,9 +5,9 @@
     <shortenClasspath name="NONE" />
     <option name="WORKING_DIRECTORY" value="sample/android-app" />
     <method v="2">
-      <option name="Gradle.BeforeRunTask" enabled="false" tasks="publishAllPublicationsToLocalRepository" externalProjectPath="$PROJECT_DIR$" vmOptions="" scriptParameters="" />
+      <option name="Gradle.BeforeRunTask" enabled="false" tasks="publishAllPublicationsToLocalRepository publishPluginMavenPublicationToMavenLocal" externalProjectPath="$PROJECT_DIR$" vmOptions="" scriptParameters="" />
       <option name="Gradle.BeforeRunTask" enabled="false" tasks=":app:assembleDebug :app:assembleDebugAndroidTest" externalProjectPath="$PROJECT_DIR$/sample/android-app" vmOptions="" scriptParameters="" />
       <option name="Make" enabled="true" />
     </method>
   </configuration>
-</component>
+</component>
\ No newline at end of file
diff --git a/.idea/runConfigurations/android_library.xml b/.idea/runConfigurations/android_library.xml
index 466470bdd..d95e2935a 100644
--- a/.idea/runConfigurations/android_library.xml
+++ b/.idea/runConfigurations/android_library.xml
@@ -1,15 +1,11 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="android-library" type="JetRunConfigurationType" factoryName="Kotlin">
-    <module name="marathon.cli.main" />
-    <option name="VM_PARAMETERS" value="" />
-    <option name="PROGRAM_PARAMETERS" value="" />
-    <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
-    <option name="ALTERNATIVE_JRE_PATH" />
-    <option name="PASS_PARENT_ENVS" value="true" />
+  <configuration default="false" name="android-library" type="JetRunConfigurationType">
     <option name="MAIN_CLASS_NAME" value="com.malinskiy.marathon.cli.ApplicationViewKt" />
-    <option name="WORKING_DIRECTORY" value="file://sample/android-library" />
+    <module name="marathon.cli.main" />
+    <shortenClasspath name="NONE" />
+    <option name="WORKING_DIRECTORY" value="sample/android-library" />
     <method v="2">
-      <option name="Gradle.BeforeRunTask" enabled="true" tasks="publishAllPublicationsToLocalRepository" externalProjectPath="$PROJECT_DIR$" vmOptions="" scriptParameters="" />
+      <option name="Gradle.BeforeRunTask" enabled="false" tasks="publishAllPublicationsToLocalRepository publishPluginMavenPublicationToMavenLocal" externalProjectPath="$PROJECT_DIR$" vmOptions="" scriptParameters="" />
       <option name="Gradle.BeforeRunTask" enabled="false" tasks=":library:assembleDebug :library:assembleDebugAndroidTest" externalProjectPath="$PROJECT_DIR$/sample/android-library" vmOptions="" scriptParameters="" />
       <option name="Make" enabled="true" />
     </method>
diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
index 4e619a185..956f0629f 100644
--- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
@@ -2,6 +2,7 @@ package com.malinskiy.marathon.config
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.malinskiy.marathon.config.strategy.BatchingStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.PoolingStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
@@ -18,7 +19,7 @@ private const val DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS = 180_000L
 data class Configuration private constructor(
     val name: String,
     val outputDir: File,
-    
+
     val outputConfiguration: OutputConfiguration,
 
     val analyticsConfiguration: AnalyticsConfiguration,
@@ -31,8 +32,8 @@ data class Configuration private constructor(
     val filteringConfiguration: FilteringConfiguration,
 
     val ignoreFailures: Boolean,
+    val executionStrategy: ExecutionStrategyConfiguration,
     val isCodeCoverageEnabled: Boolean,
-    val strictMode: Boolean,
     val uncompletedTestRetryQuota: Int,
 
     val testClassRegexes: Collection<Regex>,
@@ -65,7 +66,7 @@ data class Configuration private constructor(
             "filtering" to filteringConfiguration.toString(),
             "ignoreFailures" to ignoreFailures.toString(),
             "isCodeCoverageEnabled" to isCodeCoverageEnabled.toString(),
-            "strictMode" to strictMode.toString(),
+            "executionStrategy" to executionStrategy.toString(),
             "testClassRegexes" to testClassRegexes.toString(),
             "includeSerialRegexes" to includeSerialRegexes.toString(),
             "excludeSerialRegexes" to excludeSerialRegexes.toString(),
@@ -96,7 +97,7 @@ data class Configuration private constructor(
         if (filteringConfiguration != other.filteringConfiguration) return false
         if (ignoreFailures != other.ignoreFailures) return false
         if (isCodeCoverageEnabled != other.isCodeCoverageEnabled) return false
-        if (strictMode != other.strictMode) return false
+        if (executionStrategy != other.executionStrategy) return false
         if (uncompletedTestRetryQuota != other.uncompletedTestRetryQuota) return false
         //For testing we need to compare configuration instances. Unfortunately Regex equality is broken so need to map it to String
         if (testClassRegexes.map { it.pattern } != other.testClassRegexes.map { it.pattern }) return false
@@ -127,7 +128,7 @@ data class Configuration private constructor(
         result = 31 * result + filteringConfiguration.hashCode()
         result = 31 * result + ignoreFailures.hashCode()
         result = 31 * result + isCodeCoverageEnabled.hashCode()
-        result = 31 * result + strictMode.hashCode()
+        result = 31 * result + executionStrategy.hashCode()
         result = 31 * result + uncompletedTestRetryQuota
         result = 31 * result + testClassRegexes.hashCode()
         result = 31 * result + includeSerialRegexes.hashCode()
@@ -142,37 +143,37 @@ data class Configuration private constructor(
         return result
     }
      data class Builder(
-        val name: String,
-        val outputDir: File,
-        var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
-        var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
-        var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
-        var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
-        var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
-        var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
-        var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
-        var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
-
-        var ignoreFailures: Boolean = false,
-        var isCodeCoverageEnabled: Boolean = false,
-        var strictMode: Boolean = false,
-        var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
-
-        var testClassRegexes: Collection<Regex> = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
-        var includeSerialRegexes: Collection<Regex> = emptyList(),
-        var excludeSerialRegexes: Collection<Regex> = emptyList(),
-
-        var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
-        var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
-        var debug: Boolean = true,
-
-        var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
-
-        var analyticsTracking: Boolean = false,
-        var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
-
-        var outputConfiguration: OutputConfiguration = OutputConfiguration(),
-        var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
+         val name: String,
+         val outputDir: File,
+         var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
+         var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
+         var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
+         var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
+         var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
+         var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
+         var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
+         var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
+
+         var ignoreFailures: Boolean = false,
+         var isCodeCoverageEnabled: Boolean = false,
+         var executionStrategy: ExecutionStrategyConfiguration = ExecutionStrategyConfiguration(),
+         var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
+
+         var testClassRegexes: Collection<Regex> = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
+         var includeSerialRegexes: Collection<Regex> = emptyList(),
+         var excludeSerialRegexes: Collection<Regex> = emptyList(),
+
+         var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
+         var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
+         var debug: Boolean = true,
+
+         var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
+
+         var analyticsTracking: Boolean = false,
+         var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
+
+         var outputConfiguration: OutputConfiguration = OutputConfiguration(),
+         var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
     ) {
         fun build(): Configuration {
             return Configuration(
@@ -189,7 +190,7 @@ data class Configuration private constructor(
                 filteringConfiguration = filteringConfiguration,
                 ignoreFailures = ignoreFailures,
                 isCodeCoverageEnabled = isCodeCoverageEnabled,
-                strictMode = strictMode,
+                executionStrategy = executionStrategy,
                 uncompletedTestRetryQuota = uncompletedTestRetryQuota,
                 testClassRegexes = testClassRegexes,
                 includeSerialRegexes = includeSerialRegexes,
diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
new file mode 100644
index 000000000..d2d22530e
--- /dev/null
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
@@ -0,0 +1,31 @@
+package com.malinskiy.marathon.config.strategy
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+/**
+ * @property fast fail-fast or success-fast depending on the value of [mode]. This doesn't affect the result
+ *  of the run, it only saves compute time
+ */
+data class ExecutionStrategyConfiguration(
+    @JsonProperty("mode") val mode: ExecutionMode = ExecutionMode.ANY_SUCCESS,
+    @JsonProperty("fast") val fast: Boolean = true,
+)
+
+/**
+ * @property ANY_SUCCESS test passes if any of executions is passing
+ *  this mode works only if there is no complex sharding strategy applied
+ *  
+ *  Why: it doesn't make sense when user asks for N number of tests
+ *  to run explicitly, and we pass on the first one. Sharding used to verify probability of passing with an explicit boundary for precision
+ * @property ALL_SUCCESS test passes if and only if all the executions are passing
+ *  this mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy
+ *  
+ *  Why: when adding retries to tests with retry+flakiness strategies users want to trade-off cost for realiability, i.e. add more retries
+ *  and pass if one of them passes, so retries only make sense for the [ANY_SUCCESS] mode. When we use [ALL_SUCCESS] mode it means user
+ *  wants to verify each test with a number of tries (they are not retries per se) and pass only if all of them succeed. This is the case
+ *  when fixing a flaky test or adding a new test, and we want to have a signal that the test is fixed/not flaky.
+ */
+enum class ExecutionMode {
+    @JsonProperty("ANY_SUCCESS") ANY_SUCCESS,
+    @JsonProperty("ALL_SUCCESS") ALL_SUCCESS,
+}
diff --git a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt
index bec6847c7..28e52d838 100644
--- a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt
+++ b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt
@@ -163,7 +163,6 @@ class ConfigurationFactoryTest {
         configuration.excludeSerialRegexes.joinToString(separator = "") { it.pattern } shouldBeEqualTo """emulator-5002""".toRegex().pattern
         configuration.ignoreFailures shouldBeEqualTo false
         configuration.isCodeCoverageEnabled shouldBeEqualTo false
-        configuration.strictMode shouldBeEqualTo true
         configuration.testBatchTimeoutMillis shouldBeEqualTo 20_000
         configuration.testOutputTimeoutMillis shouldBeEqualTo 30_000
         configuration.debug shouldBeEqualTo true
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt
index f31d1cb3f..102e25211 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt
@@ -19,7 +19,6 @@ import com.malinskiy.marathon.execution.TestParser
 import com.malinskiy.marathon.execution.TestShard
 import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
 import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.execution.withRetry
 import com.malinskiy.marathon.extension.toFlakinessStrategy
 import com.malinskiy.marathon.extension.toShardingStrategy
@@ -34,7 +33,6 @@ import com.malinskiy.marathon.usageanalytics.UsageAnalytics
 import com.malinskiy.marathon.usageanalytics.tracker.Event
 import kotlinx.coroutines.NonCancellable
 import kotlinx.coroutines.async
-import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import kotlinx.coroutines.withTimeoutOrNull
@@ -52,7 +50,6 @@ class Marathon(
     private val logConfigurator: MarathonLogConfigurator,
     private val tracker: TrackerInternal,
     private val analytics: Analytics,
-    private val progressReporter: ProgressReporter,
     private val track: Track,
     private val timer: Timer,
     private val marathonTestParseCommand: MarathonTestParseCommand
@@ -142,7 +139,6 @@ class Marathon(
             analytics,
             configuration,
             shard,
-            progressReporter,
             track,
             timer,
             testBundleIdentifier,
@@ -155,12 +151,13 @@ class Marathon(
         }
         configuration.outputDir.mkdirs()
 
-        if (parsedFilteredTests.isNotEmpty()) {
+        val result = if (parsedFilteredTests.isNotEmpty()) {
             scheduler.execute()
+        } else {
+            true
         }
 
         onFinish(analytics, deviceProvider)
-        val result = progressReporter.aggregateResult()
 
         stopKoin()
         return result
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt b/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt
index d06267498..92d3fe326 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt
@@ -2,7 +2,6 @@ package com.malinskiy.marathon.device
 
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.test.TestBatch
 import kotlinx.coroutines.CompletableDeferred
 import mu.KLogger
@@ -45,7 +44,6 @@ interface Device {
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
         deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter
     )
 
     /**
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt b/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt
index 3628c430f..f59596aa6 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt
@@ -7,7 +7,6 @@ import com.malinskiy.marathon.analytics.external.AnalyticsFactory
 import com.malinskiy.marathon.analytics.internal.pub.Track
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.json.FileSerializer
 import com.malinskiy.marathon.time.SystemTimer
@@ -37,12 +36,11 @@ val coreModule = module {
     }
     single<Clock> { Clock.systemDefaultZone() }
     single<Timer> { SystemTimer(get()) }
-    single { ProgressReporter(get()) }
     single {
         val configuration = get<Configuration>()
         MarathonTestParseCommand(configuration.outputDir)
     }
-    single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
+    single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
 }
 
 fun marathonStartKoin(configuration: Configuration, modules: List<Module>): KoinApplication {
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
index 92c223193..84db4841e 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
@@ -12,7 +12,7 @@ import com.malinskiy.marathon.device.toDeviceInfo
 import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
 import com.malinskiy.marathon.execution.device.DeviceActor
 import com.malinskiy.marathon.execution.device.DeviceEvent
-import com.malinskiy.marathon.execution.progress.ProgressReporter
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
 import com.malinskiy.marathon.execution.queue.QueueActor
 import com.malinskiy.marathon.execution.queue.QueueMessage
 import com.malinskiy.marathon.log.MarathonLogging
@@ -26,9 +26,9 @@ import kotlin.coroutines.CoroutineContext
 class DevicePoolActor(
     private val poolId: DevicePoolId,
     private val configuration: Configuration,
+    private val poolProgressAccumulator: PoolProgressAccumulator,
     analytics: Analytics,
     shard: TestShard,
-    private val progressReporter: ProgressReporter,
     track: Track,
     timer: Timer,
     parent: Job,
@@ -64,10 +64,10 @@ class DevicePoolActor(
         analytics,
         this,
         poolId,
-        progressReporter,
         track,
         timer,
         testBundleIdentifier,
+        poolProgressAccumulator,
         poolJob,
         context
     )
@@ -153,7 +153,7 @@ class DevicePoolActor(
         }
 
         logger.debug { "add device ${device.serialNumber}" }
-        val actor = DeviceActor(poolId, this, configuration, device, progressReporter, poolJob, coroutineContext)
+        val actor = DeviceActor(poolId, this, configuration, device, poolJob, coroutineContext)
         devices[device.serialNumber] = actor
         actor.safeSend(DeviceEvent.Initialize)
     }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
index 0031819cf..1eb6adf5e 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
@@ -12,7 +12,7 @@ import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler
 import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler.AddDevice
 import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler.RemoveDevice
 import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
-import com.malinskiy.marathon.execution.progress.ProgressReporter
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
 import com.malinskiy.marathon.extension.toPoolingStrategy
 import com.malinskiy.marathon.log.MarathonLogging
 import com.malinskiy.marathon.time.Timer
@@ -32,13 +32,11 @@ import kotlin.coroutines.CoroutineContext
  * 1) Subscribe on DeviceProvider
  * 2) Create device pools using PoolingStrategy
  */
-
 class Scheduler(
     private val deviceProvider: DeviceProvider,
     private val analytics: Analytics,
     private val configuration: Configuration,
     private val shard: TestShard,
-    private val progressReporter: ProgressReporter,
     private val track: Track,
     private val timer: Timer,
     private val testBundleIdentifier: TestBundleIdentifier?,
@@ -46,12 +44,13 @@ class Scheduler(
 ) : CoroutineScope {
 
     private val job = Job()
-    private val pools = ConcurrentHashMap<DevicePoolId, SendChannel<FromScheduler>>()
+    private val pools = ConcurrentHashMap<DevicePoolId, DevicePoolActor>()
+    private val results = ConcurrentHashMap<DevicePoolId, PoolProgressAccumulator>()
     private val poolingStrategy = configuration.poolingStrategy.toPoolingStrategy()
 
     private val logger = MarathonLogging.logger("Scheduler")
 
-    suspend fun execute() {
+    suspend fun execute() : Boolean {
         subscribeOnDevices(job)
         try {
             withTimeout(deviceProvider.deviceInitializationTimeoutMillis) {
@@ -66,6 +65,8 @@ class Scheduler(
         for (child in job.children) {
             child.join()
         }
+
+        return results.all { it.value.aggregateResult() }
     }
 
     private fun subscribeOnDevices(job: Job): Job {
@@ -109,9 +110,13 @@ class Scheduler(
 
         val poolId = poolingStrategy.associate(device)
         logger.debug { "device ${device.serialNumber} associated with poolId ${poolId.name}" }
+        val accumulator = results.computeIfAbsent(poolId) { id ->
+            PoolProgressAccumulator(id, shard, configuration, track)
+        }
+
         pools.computeIfAbsent(poolId) { id ->
             logger.debug { "pool actor ${id.name} is being created" }
-            DevicePoolActor(id, configuration, analytics, shard, progressReporter, track, timer, parent, context, testBundleIdentifier)
+            DevicePoolActor(id, configuration, accumulator, analytics, shard, track, timer, parent, context, testBundleIdentifier)
         }
         pools[poolId]?.send(AddDevice(device)) ?: logger.debug {
             "not sending the AddDevice event " +
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt
index eec934399..f90415551 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt
@@ -10,7 +10,6 @@ import com.malinskiy.marathon.exceptions.TestBatchExecutionException
 import com.malinskiy.marathon.execution.DevicePoolMessage
 import com.malinskiy.marathon.execution.DevicePoolMessage.FromDevice.IsReady
 import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.execution.withRetry
 import com.malinskiy.marathon.log.MarathonLogging
 import com.malinskiy.marathon.test.TestBatch
@@ -29,7 +28,6 @@ class DeviceActor(
     private val pool: SendChannel<DevicePoolMessage>,
     private val configuration: Configuration,
     val device: Device,
-    private val progressReporter: ProgressReporter,
     parent: Job,
     context: CoroutineContext
 ) :
@@ -184,7 +182,7 @@ class DeviceActor(
         logger.debug { "executeBatch ${device.serialNumber}" }
         job = async {
             try {
-                device.execute(configuration, devicePoolId, batch, result, progressReporter)
+                device.execute(configuration, devicePoolId, batch, result)
                 state.transition(DeviceEvent.Complete)
             } catch (e: CancellationException) {
                 logger.warn(e) { "Device execution has been cancelled" }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
new file mode 100644
index 000000000..07cc9a8a6
--- /dev/null
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
@@ -0,0 +1,409 @@
+package com.malinskiy.marathon.execution.progress
+
+import com.malinskiy.marathon.actor.StateMachine
+import com.malinskiy.marathon.analytics.internal.pub.Track
+import com.malinskiy.marathon.config.Configuration
+import com.malinskiy.marathon.config.strategy.ExecutionMode
+import com.malinskiy.marathon.device.DeviceInfo
+import com.malinskiy.marathon.device.DevicePoolId
+import com.malinskiy.marathon.execution.TestResult
+import com.malinskiy.marathon.execution.TestShard
+import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.execution.queue.TestAction
+import com.malinskiy.marathon.execution.queue.TestEvent
+import com.malinskiy.marathon.execution.queue.TestState
+import com.malinskiy.marathon.log.MarathonLogging
+import com.malinskiy.marathon.test.Test
+import com.malinskiy.marathon.test.toTestName
+import kotlin.math.roundToInt
+
+class PoolProgressAccumulator(
+    private val poolId: DevicePoolId,
+    shard: TestShard,
+    private val configuration: Configuration,
+    private val track: Track
+) {
+    private val tests: HashMap<String, StateMachine<TestState, TestEvent, TestAction>> = HashMap()
+    private val logger = MarathonLogging.logger {}
+    private val executionStrategy = configuration.executionStrategy
+
+    private fun createState(initialCount: Int) = StateMachine.create<TestState, TestEvent, TestAction> {
+        initialState(TestState.Added(initialCount))
+        state<TestState.Added> {
+            on<TestEvent.Started> {
+                transitionTo(TestState.Added(total, running + 1))
+            }
+            on<TestEvent.Passed> {
+                when (executionStrategy.mode) {
+                    ExecutionMode.ANY_SUCCESS -> {
+                        if (executionStrategy.fast || total <= 1) {
+                            transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Passing(total = total, done = 1, running = running - 1))
+                        }
+                    }
+
+                    ExecutionMode.ALL_SUCCESS -> {
+                        if (total <= 1) {
+                            transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Passing(total = total, done = 1, running = running - 1))
+                        }
+                    }
+                }
+            }
+            on<TestEvent.Failed> {
+                when (executionStrategy.mode) {
+                    ExecutionMode.ANY_SUCCESS -> {
+                        if (total <= 1) {
+                            transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Failing(total = total, done = 1, running = running - 1))
+                        }
+                    }
+
+                    ExecutionMode.ALL_SUCCESS -> {
+                        if (executionStrategy.fast || total <= 1) {
+                            transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Failing(total = total, done = 1, running = running - 1))
+                        }
+                    }
+                }
+            }
+            on<TestEvent.Incomplete> {
+                if (it.final) {
+                    transitionTo(TestState.Failed(total = total, done = 0, running = running - 1), TestAction.Conclude)
+                } else {
+                    transitionTo(TestState.Added(total = total, running = running - 1))
+                }
+            }
+            on<TestEvent.AddRetry> {
+                transitionTo(TestState.Added(total = total + 1, running = running))
+            }
+            on<TestEvent.RemoveAttempts> {
+                transitionTo(TestState.Added(total = total - it.count, running = running))
+            }
+        }
+        state<TestState.Passing> {
+            on<TestEvent.Started> {
+                transitionTo(TestState.Passing(total = total, running = running + 1, done = done))
+            }
+            on<TestEvent.Passed> {
+                when (executionStrategy.mode) {
+                    ExecutionMode.ANY_SUCCESS -> {
+                        if (executionStrategy.fast || done + 1 >= total) {
+                            transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Passing(total = total, running = running - 1, done = done + 1))
+                        }
+                    }
+
+                    ExecutionMode.ALL_SUCCESS -> {
+                        if (total <= done + 1) {
+                            transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Passing(total = total, running = running - 1, done = done + 1))
+                        }
+                    }
+                }
+            }
+            on<TestEvent.Failed> {
+                when (executionStrategy.mode) {
+                    ExecutionMode.ANY_SUCCESS -> {
+                        if (executionStrategy.fast || done + 1 >= total) {
+                            transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Passing(total = total, done = done + 1, running = running - 1))
+                        }
+                    }
+
+                    ExecutionMode.ALL_SUCCESS -> {
+                        if (executionStrategy.fast || done + 1 >= total) {
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+                        }
+                    }
+                }
+            }
+            on<TestEvent.Incomplete> {
+                if (it.final) {
+                    when (executionStrategy.mode) {
+                        ExecutionMode.ANY_SUCCESS -> {
+                            transitionTo(TestState.Passed(total = total, done = done, running = running - 1), TestAction.Conclude)
+                        }
+
+                        ExecutionMode.ALL_SUCCESS -> {
+                            transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Conclude)
+                        }
+                    }
+                } else {
+                    transitionTo(TestState.Passing(total = total, done = done, running = running - 1))
+                }
+            }
+            on<TestEvent.RemoveAttempts> {
+                transitionTo(TestState.Passing(total = total - it.count, running = running, done = done))
+            }
+            on<TestEvent.AddRetry> {
+                transitionTo(TestState.Passing(total = total + 1, running = running, done = done))
+            }
+        }
+        state<TestState.Failing> {
+            on<TestEvent.Started> {
+                transitionTo(TestState.Failing(total = total, running = running + 1, done = done))
+            }
+            on<TestEvent.Passed> {
+                when (executionStrategy.mode) {
+                    ExecutionMode.ANY_SUCCESS -> {
+                        if (executionStrategy.fast || done + 1 >= total) {
+                            transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Passing(total = total, done = done + 1, running = running - 1))
+                        }
+                    }
+
+                    ExecutionMode.ALL_SUCCESS -> {
+                        if (executionStrategy.fast || done + 1 >= total) {
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+                        }
+                    }
+                }
+            }
+            on<TestEvent.Failed> {
+                when (executionStrategy.mode) {
+                    ExecutionMode.ANY_SUCCESS -> {
+                        if (done + 1 >= total) {
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+                        }
+                    }
+
+                    ExecutionMode.ALL_SUCCESS -> {
+                        if (executionStrategy.fast || done + 1 >= total) {
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                        } else {
+                            transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+                        }
+                    }
+                }
+            }
+            on<TestEvent.Incomplete> {
+                if (it.final) {
+                    transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Conclude)
+                } else {
+                    transitionTo(TestState.Failing(total = total, done = done, running = running - 1))
+                }
+            }
+            on<TestEvent.RemoveAttempts> {
+                transitionTo(TestState.Failing(total = total - it.count, done = done, running = running))
+            }
+            on<TestEvent.AddRetry> {
+                transitionTo(TestState.Failing(total = total + 1, done = done, running = running))
+            }
+        }
+        state<TestState.Failed> {
+            on<TestEvent.Started> {
+                transitionTo(TestState.Failed(total = total, running = running + 1, done = done))
+            }
+            on<TestEvent.Passed> {
+                transitionTo(TestState.Failed(total = total, running = running - 1, done = done + 1))
+            }
+            on<TestEvent.Failed> {
+                transitionTo(TestState.Failed(total = total, running = running - 1, done = done + 1))
+            }
+            on<TestEvent.Incomplete> {
+                transitionTo(TestState.Failed(total = total, running = running - 1, done = done))
+            }
+            on<TestEvent.AddRetry> {
+                transitionTo(TestState.Failed(total = total + 1, running = running, done = done))
+            }
+            on<TestEvent.RemoveAttempts> {
+                transitionTo(TestState.Failed(total = total - it.count, running = running, done = done))
+            }
+        }
+        state<TestState.Passed> {
+            on<TestEvent.Started> {
+                transitionTo(TestState.Passed(total = total, running = running + 1, done = done))
+            }
+            on<TestEvent.Passed> {
+                transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1))
+            }
+            on<TestEvent.Failed> {
+                transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1))
+            }
+            on<TestEvent.Incomplete> {
+                transitionTo(TestState.Passed(total = total, running = running - 1, done = done))
+            }
+            on<TestEvent.AddRetry> {
+                transitionTo(TestState.Passed(total = total + 1, running = running, done = done))
+            }
+            on<TestEvent.RemoveAttempts> {
+                transitionTo(TestState.Passed(total = total - it.count, running = running, done = done))
+            }
+        }
+        onTransition {
+            if (it as? StateMachine.Transition.Valid !is StateMachine.Transition.Valid) {
+                logger.error { "from ${it.fromState} event ${it.event}" }
+            }
+            trackTestTransition(poolId, it)
+        }
+    }
+
+    init {
+        val allTests = shard.tests + shard.flakyTests
+        allTests.groupBy { it }.map {
+            val count = it.value.size
+            it.key.toTestName() to createState(count)
+        }.also {
+            tests.putAll(it)
+        }
+    }
+
+    fun testStarted(device: DeviceInfo, test: Test) {
+        tests[test.toTestName()]?.transition(TestEvent.Started)
+        println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started")
+    }
+
+    /**
+     * @param final used for incomplete tests to signal no more retries left, hence a decision on the status has to be made
+     */
+    fun testEnded(device: DeviceInfo, testResult: TestResult, final: Boolean = false) {
+        when(testResult.status) {
+            TestStatus.FAILURE -> {
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Failed(device, testResult))
+                println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} failed")
+            }
+            TestStatus.PASSED -> {
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult))
+                println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ended")
+            }
+            TestStatus.IGNORED, TestStatus.ASSUMPTION_FAILURE -> {
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult))
+                println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ignored")
+            }
+            TestStatus.INCOMPLETE -> {
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Incomplete(device, testResult, final))
+                println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} incomplete")
+            }
+        }
+    }
+
+    /**
+     * Should always be called before testEnded, otherwise the FSM might transition into a terminal state prematurely
+     */
+    fun retryTest(test: Test) {
+        tests[test.toTestName()]?.transition(TestEvent.AddRetry)
+    }
+
+    fun removeTest(test: Test, diff: Int) {
+        tests[test.toTestName()]?.transition(TestEvent.RemoveAttempts(diff))
+    }
+
+    private fun trackTestTransition(poolId: DevicePoolId, transition: StateMachine.Transition<TestState, TestEvent, TestAction>) {
+        val validTransition = transition as? StateMachine.Transition.Valid
+        val final = if (validTransition is StateMachine.Transition.Valid) {
+            when (validTransition.sideEffect) {
+                is TestAction.Conclude -> true
+                else -> false
+            }
+        } else false
+
+        val (testResult: TestResult?, device: DeviceInfo?) = extractEventAndDevice(transition)
+        if (testResult == null || device == null) return
+
+        track.test(poolId, device, testResult, final)
+    }
+
+    private fun extractEventAndDevice(transition: StateMachine.Transition<TestState, TestEvent, TestAction>): Pair<TestResult?, DeviceInfo?> {
+        val event = transition.event
+        val testResult: TestResult? = when (event) {
+            is TestEvent.Passed -> event.testResult
+            is TestEvent.Failed -> event.testResult
+            is TestEvent.Incomplete -> event.testResult
+            else -> null
+        }
+        val device: DeviceInfo? = when (event) {
+            is TestEvent.Passed -> event.device
+            is TestEvent.Failed -> event.device
+            is TestEvent.Incomplete -> event.device
+            else -> null
+        }
+        return Pair(testResult, device)
+    }
+
+    fun aggregateResult(): Boolean {
+        synchronized(tests) {
+            return tests.map {
+                when (it.value.state) {
+                    is TestState.Added -> {
+                        logger.error { "Expected to run ${it.key} but no events received" }
+                        false
+                    }
+
+                    is TestState.Failed -> false
+                    is TestState.Failing -> {
+                        logger.error { "Expected to run ${it.key} more, but no terminal events received" }
+                        false
+                    }
+
+                    is TestState.Passed -> true
+                    is TestState.Passing -> {
+                        logger.error { "Expected to run ${it.key} more, but no terminal events received" }
+                        //The test is passing but the execution mode might require all the 
+                        //runs to pass before considering this an actual pass 
+                        executionStrategy.mode == ExecutionMode.ANY_SUCCESS
+                    }
+                }
+            }.reduce { a, b -> a && b }
+        }
+    }
+
+    fun progress(): Float {
+        var done = 0
+        var total = 0
+
+        tests.values.forEach {
+            val state = it.state
+            when (state) {
+                is TestState.Added -> {
+                    total += state.total
+                }
+
+                is TestState.Failed -> {
+                    total += state.total
+                    done += state.done
+                }
+
+                is TestState.Failing -> {
+                    total += state.total
+                    done += state.done
+                }
+
+                is TestState.Passed -> {
+                    total += state.total
+                    done += state.done
+                }
+
+                is TestState.Passing -> {
+                    total += state.total
+                    done += state.done
+                }
+            }
+        }
+        return done.toFloat() / total.toFloat()
+    }
+
+    private fun toPercent(float: Float): String {
+        val percent = (float * HUNDRED_PERCENT_IN_FLOAT).roundToInt()
+        val format = "%02d%%"
+        return String.format(format, percent)
+    }
+
+    companion object {
+        const val HUNDRED_PERCENT_IN_FLOAT: Float = 100.0f
+    }
+}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
index 93df275bf..9a846f340 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
@@ -1,82 +1,25 @@
 package com.malinskiy.marathon.execution.progress
 
-import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.device.DeviceInfo
 import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.execution.progress.tracker.PoolProgressTracker
 import com.malinskiy.marathon.test.Test
+import com.malinskiy.marathon.test.TestBatch
 import com.malinskiy.marathon.test.toTestName
-import java.util.concurrent.ConcurrentHashMap
-import kotlin.math.roundToInt
 
-const val HUNDRED_PERCENT_IN_FLOAT: Float = 100.0f
-
-class ProgressReporter(private val configuration: Configuration) {
-    private val reporters = ConcurrentHashMap<DevicePoolId, PoolProgressTracker>()
-
-    private inline fun <T> execute(poolId: DevicePoolId, f: (PoolProgressTracker) -> T): T {
-        val reporter = reporters[poolId] ?: PoolProgressTracker(configuration)
-        val result = f(reporter)
-        reporters[poolId] = reporter
-        return result
-    }
-
-    private fun toPercent(float: Float): String {
-        val percent = (float * HUNDRED_PERCENT_IN_FLOAT).roundToInt()
-        val format = "%02d%%"
-        return String.format(format, percent)
-    }
-
-    fun testStarted(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
-        execute(poolId) { it.testStarted(test) }
-        println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started")
-    }
-
-    fun testFailed(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
-        execute(poolId) { it.testFailed(test) }
-        println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} failed")
-    }
-
-    fun testPassed(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
-        execute(poolId) { it.testPassed(test) }
-        println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ended")
-    }
-
-    fun testIgnored(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
-        execute(poolId) { it.testIgnored(test) }
-        println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ignored")
-    }
-
-    fun aggregateResult(): Boolean {
-        return reporters.isNotEmpty() && reporters.values.all {
-            it.aggregateResult()
-        }
-    }
-
-    fun testCountExpectation(poolId: DevicePoolId, size: Int) {
-        execute(poolId) { it.testCountExpectation(size) }
-    }
-
-    fun removeTests(poolId: DevicePoolId, count: Int) {
-        execute(poolId) { it.removeTests(count) }
-    }
-
-    fun addTestDiscoveredDuringRuntime(poolId: DevicePoolId, test: Test) {
-        execute(poolId) { it.addTestDiscoveredDuringRuntime(test) }
+class ProgressReporter(private val batch: TestBatch, private val poolId: DevicePoolId, private val device: DeviceInfo) {
+    fun testStarted(test: Test) {
+        println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started")
     }
 
-    fun addRetries(poolId: DevicePoolId, count: Int) {
-        execute(poolId) { it.addTestRetries(count) }
+    fun testFailed(test: Test) {
+        println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} failed")
     }
 
-    fun progress(): Float {
-        val size = reporters.size
-        return reporters.values.sumOf {
-            it.progress().toDouble()
-        }.toFloat() / size
+    fun testPassed(test: Test) {
+        println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ended")
     }
 
-    fun progress(poolId: DevicePoolId): Float {
-        return execute(poolId) { it.progress() }
+    fun testIgnored(test: Test) {
+        println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ignored")
     }
 }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTracker.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTracker.kt
deleted file mode 100644
index e0db4b2f1..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTracker.kt
+++ /dev/null
@@ -1,149 +0,0 @@
-package com.malinskiy.marathon.execution.progress.tracker
-
-import com.malinskiy.marathon.actor.StateMachine
-import com.malinskiy.marathon.config.Configuration
-import com.malinskiy.marathon.log.MarathonLogging
-import com.malinskiy.marathon.test.Test
-import java.util.concurrent.atomic.AtomicInteger
-
-class PoolProgressTracker(private val configuration: Configuration) {
-
-    private val tests = mutableMapOf<Test, StateMachine<ProgressTestState, ProgressEvent, Any>>()
-    private val runtimeDiscoveredTests = mutableSetOf<Test>()
-    private val logger = MarathonLogging.logger {}
-
-    private fun createState() = StateMachine.create<ProgressTestState, ProgressEvent, Any> {
-        initialState(ProgressTestState.Started)
-        state<ProgressTestState.Started> {
-            on<ProgressEvent.Failed> {
-                transitionTo(ProgressTestState.Failed)
-            }
-            on<ProgressEvent.Passed> {
-                transitionTo(ProgressTestState.Passed)
-            }
-            on<ProgressEvent.Ignored> {
-                transitionTo(ProgressTestState.Ignored)
-            }
-        }
-        state<ProgressTestState.Passed> {
-            on<ProgressEvent.Failed> {
-                if (configuration.strictMode) {
-                    transitionTo(ProgressTestState.Failed)
-                } else {
-                    dontTransition()
-                }
-            }
-            on<ProgressEvent.Ignored> {
-                dontTransition()
-            }
-        }
-        state<ProgressTestState.Failed> {
-            on<ProgressEvent.Passed> {
-                if (configuration.strictMode) {
-                    dontTransition()
-                } else {
-                    transitionTo(ProgressTestState.Passed)
-                }
-            }
-
-        }
-        state<ProgressTestState.Ignored> {
-            on<ProgressEvent.Passed> {
-                transitionTo(ProgressTestState.Passed)
-            }
-        }
-    }
-
-    private fun updateStatus(test: Test, newStatus: ProgressEvent) {
-        synchronized(tests) {
-            tests[test]?.transition(newStatus)
-        }
-    }
-
-    private val expectedTestCount = AtomicInteger(0)
-    private val completed = AtomicInteger(0)
-    private val failed = AtomicInteger(0)
-    private val ignored = AtomicInteger(0)
-    private val retries = AtomicInteger(0)
-
-    fun testStarted(test: Test) {
-        synchronized(tests) {
-            tests.computeIfAbsent(test) { _ -> createState() }
-        }
-    }
-
-    fun testFailed(test: Test) {
-        failed.updateAndGet {
-            it + 1
-        }
-        updateStatus(test, ProgressEvent.Failed)
-    }
-
-    fun testPassed(test: Test) {
-        completed.updateAndGet {
-            it + 1
-        }
-        updateStatus(test, ProgressEvent.Passed)
-    }
-
-    fun testIgnored(test: Test) {
-        ignored.updateAndGet {
-            it + 1
-        }
-        updateStatus(test, ProgressEvent.Ignored)
-    }
-
-    fun aggregateResult(): Boolean {
-        synchronized(tests) {
-            val expected = expectedTestCount.get()
-            val actual = tests.size
-            return if (actual == expected) {
-                tests.all {
-                    when (it.value.state) {
-                        is ProgressTestState.Passed -> true
-                        is ProgressTestState.Ignored -> true
-                        else -> false
-                    }
-                }
-            } else {
-                logger.error { "Expected to run $expected tests but received results for only $actual" }
-                false
-            }
-        }
-    }
-
-    fun testCountExpectation(size: Int) {
-        expectedTestCount.set(size)
-    }
-
-    fun removeTests(count: Int) {
-        expectedTestCount.updateAndGet {
-            it - count
-        }
-    }
-
-    fun progress(): Float {
-        return (completed.toFloat() + failed.toFloat() + ignored.toFloat()) / (expectedTestCount.toFloat() + retries.toFloat())
-    }
-
-    /**
-     * This is for parameterized test discovery that can happen at runtime
-     * Unfortunately a runtime discovered tests retries it will go through discovery process again, so we have to collect these
-     */
-    fun addTestDiscoveredDuringRuntime(test: Test) {
-        synchronized(runtimeDiscoveredTests) {
-            val before = runtimeDiscoveredTests.size
-            runtimeDiscoveredTests.add(test)
-            val after = runtimeDiscoveredTests.size
-            expectedTestCount.updateAndGet {
-                it + (after - before)
-            }
-        }
-    }
-
-    fun addTestRetries(count: Int) {
-        retries.updateAndGet {
-            it + count
-        }
-    }
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressEvent.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressEvent.kt
deleted file mode 100644
index a41c5ff3f..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressEvent.kt
+++ /dev/null
@@ -1,7 +0,0 @@
-package com.malinskiy.marathon.execution.progress.tracker
-
-sealed class ProgressEvent {
-    object Passed : ProgressEvent()
-    object Failed : ProgressEvent()
-    object Ignored : ProgressEvent()
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressTestState.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressTestState.kt
deleted file mode 100644
index f8619c366..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressTestState.kt
+++ /dev/null
@@ -1,8 +0,0 @@
-package com.malinskiy.marathon.execution.progress.tracker
-
-sealed class ProgressTestState {
-    object Started : ProgressTestState()
-    object Passed : ProgressTestState()
-    object Failed : ProgressTestState()
-    object Ignored : ProgressTestState()
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
index a96cf1e50..1c255ec5a 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
@@ -6,14 +6,13 @@ import com.malinskiy.marathon.analytics.internal.pub.Track
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.device.DeviceInfo
 import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.execution.DevicePoolMessage
 import com.malinskiy.marathon.execution.DevicePoolMessage.FromQueue
 import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestShard
 import com.malinskiy.marathon.execution.TestStatus
 import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
-import com.malinskiy.marathon.execution.progress.ProgressReporter
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
 import com.malinskiy.marathon.extension.toBatchingStrategy
 import com.malinskiy.marathon.extension.toRetryStrategy
 import com.malinskiy.marathon.extension.toSortingStrategy
@@ -36,10 +35,10 @@ class QueueActor(
     private val analytics: Analytics,
     private val pool: SendChannel<FromQueue>,
     private val poolId: DevicePoolId,
-    private val progressReporter: ProgressReporter,
     private val track: Track,
     private val timer: Timer,
     private val testBundleIdentifier: TestBundleIdentifier?,
+    private val poolProgressAccumulator: PoolProgressAccumulator,
     poolJob: Job,
     coroutineContext: CoroutineContext
 ) :
@@ -56,11 +55,8 @@ class QueueActor(
     private val activeBatches = mutableMapOf<String, TestBatch>()
     private val uncompletedTestsRetryCount = mutableMapOf<Test, Int>()
 
-    private val testResultReporter = TestResultReporter(poolId, analytics, testShard, configuration, track)
-
     init {
         queue.addAll(testShard.tests + testShard.flakyTests)
-        progressReporter.testCountExpectation(poolId, queue.size)
     }
 
     override suspend fun receive(msg: QueueMessage) {
@@ -118,16 +114,15 @@ class QueueActor(
                 it.copy(status = TestStatus.FAILURE)
             }
             for (test in uncompletedToFailed) {
-                testResultReporter.testIncomplete(device, test, final = true)
+                poolProgressAccumulator.testEnded(device, test, final = true)
             }
         }
 
         if (uncompleted.isNotEmpty()) {
             for (test in uncompleted) {
-                testResultReporter.testIncomplete(device, test, final = false)
+                poolProgressAccumulator.testEnded(device, test, final = false)
             }
             returnTests(uncompleted.map { it.test })
-            progressReporter.addRetries(poolId, uncompleted.size)
         }
     }
 
@@ -163,8 +158,8 @@ class QueueActor(
     }
 
     private fun handleFinishedTests(finished: Collection<TestResult>, device: DeviceInfo) {
-        finished.filter { testShard.flakyTests.contains(it.test) }.let {
-            it.forEach {
+        finished.filter { testShard.flakyTests.contains(it.test) }.let { testResults ->
+            testResults.forEach {
                 val oldSize = queue.size
                 queue.removeAll(listOf(it.test))
                 /**
@@ -174,12 +169,11 @@ class QueueActor(
                  * 3. Another parallel run of test X finishes and should be counted towards reducing the expected tests
                  */
                 val diff = max(oldSize - queue.size, 1)
-                testResultReporter.removeTest(it.test, diff)
-                progressReporter.removeTests(poolId, diff)
+                poolProgressAccumulator.removeTest(it.test, diff)
             }
         }
         finished.forEach {
-            testResultReporter.testFinished(device, it)
+            poolProgressAccumulator.testEnded(device, it)
         }
     }
 
@@ -190,18 +184,17 @@ class QueueActor(
         logger.debug { "handle failed tests ${device.serialNumber}" }
         val retryList = retryStrategy.process(poolId, failed, testShard)
 
-        progressReporter.addRetries(poolId, retryList.size)
-        queue.addAll(retryList.map { it.test })
         retryList.forEach {
-            testResultReporter.retryTest(device, it)
+            poolProgressAccumulator.retryTest(it.test)
         }
+        queue.addAll(retryList.map { it.test })
 
         val (_, noRetries) = failed.partition { testResult ->
             retryList.map { retry -> retry.test }.contains(testResult.test)
         }
 
         noRetries.forEach {
-            testResultReporter.testFailed(device, it)
+            poolProgressAccumulator.testEnded(device, it)
         }
     }
 
@@ -218,12 +211,11 @@ class QueueActor(
             return
         }
         if (queueIsEmpty && activeBatches.isEmpty()) {
-            pool.send(DevicePoolMessage.FromQueue.Terminated)
+            pool.send(FromQueue.Terminated)
             onTerminate()
         } else if (queueIsEmpty) {
             logger.debug {
-                "queue is empty but there are active batches present for " +
-                    "${activeBatches.keys.joinToString { it }}"
+                "queue is empty but there are active batches present for " + activeBatches.keys.joinToString { it }
             }
         }
     }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
index f864fdce4..210b47df4 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
@@ -1,8 +1,5 @@
 package com.malinskiy.marathon.execution.queue
 
-import com.malinskiy.marathon.device.DeviceInfo
-import com.malinskiy.marathon.execution.TestResult
-
 sealed class TestAction {
-    data class SaveReport(val deviceInfo: DeviceInfo, val testResult: TestResult) : TestAction()
+    object Conclude : TestAction()
 }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt
index e9a69a0ef..520147d76 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt
@@ -4,6 +4,8 @@ import com.malinskiy.marathon.device.DeviceInfo
 import com.malinskiy.marathon.execution.TestResult
 
 sealed class TestEvent {
+    object Started : TestEvent()
+    
     data class Failed(
         val device: DeviceInfo,
         val testResult: TestResult
@@ -13,13 +15,10 @@ sealed class TestEvent {
         val device: DeviceInfo,
         val testResult: TestResult
     ) : TestEvent()
+    
+    data class RemoveAttempts(val count: Int) : TestEvent()
 
-    data class Remove(val diff: Int) : TestEvent()
-
-    data class Retry(
-        val device: DeviceInfo,
-        val testResult: TestResult
-    ) : TestEvent()
+    object AddRetry : TestEvent()
 
     data class Incomplete(
         val device: DeviceInfo,
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporter.kt
deleted file mode 100644
index b1c09949c..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporter.kt
+++ /dev/null
@@ -1,168 +0,0 @@
-package com.malinskiy.marathon.execution.queue
-
-import com.malinskiy.marathon.actor.StateMachine
-import com.malinskiy.marathon.analytics.external.Analytics
-import com.malinskiy.marathon.analytics.internal.pub.Track
-import com.malinskiy.marathon.config.Configuration
-import com.malinskiy.marathon.device.DeviceInfo
-import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.execution.TestResult
-import com.malinskiy.marathon.execution.TestShard
-import com.malinskiy.marathon.log.MarathonLogging
-import com.malinskiy.marathon.test.Test
-import com.malinskiy.marathon.test.toTestName
-
-class TestResultReporter(
-    private val poolId: DevicePoolId,
-    private val analytics: Analytics,
-    shard: TestShard,
-    private val configuration: Configuration,
-    private val track: Track
-) {
-
-    private val tests: HashMap<String, StateMachine<TestState, TestEvent, TestAction>> = HashMap()
-
-    private val logger = MarathonLogging.logger("TestResultReporter")
-
-    private fun createState(initialCount: Int) = StateMachine.create<TestState, TestEvent, TestAction> {
-        initialState(TestState.Added(initialCount))
-        state<TestState.Added> {
-            on<TestEvent.Passed> {
-                if (!configuration.strictMode || count <= 1) {
-                    transitionTo(TestState.Passed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
-                } else {
-                    transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
-                }
-            }
-            on<TestEvent.Failed> {
-                if (configuration.strictMode || count <= 1) {
-                    transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
-                } else {
-                    transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
-                }
-            }
-            on<TestEvent.Incomplete> {
-                if (it.final) {
-                    transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
-                } else {
-                    transitionTo(TestState.Executed(it.device, it.testResult, count))
-                }
-            }
-            on<TestEvent.Retry> {
-                dontTransition()
-            }
-            on<TestEvent.Remove> {
-                transitionTo(this.copy(count = this.count - it.diff))
-            }
-        }
-        state<TestState.Executed> {
-            on<TestEvent.Failed> {
-                if (configuration.strictMode || count <= 1) {
-                    transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
-                } else {
-                    transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
-                }
-            }
-            on<TestEvent.Incomplete> {
-                if (it.final) {
-                    transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
-                } else {
-                    transitionTo(TestState.Executed(it.device, it.testResult, count))
-                }
-            }
-            on<TestEvent.Remove> {
-                transitionTo(this.copy(count = this.count - it.diff))
-            }
-            on<TestEvent.Passed> {
-                if (!configuration.strictMode || count <= 1) {
-                    transitionTo(TestState.Passed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
-                } else {
-                    transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
-                }
-            }
-            on<TestEvent.Retry> {
-                transitionTo(this.copy(count = this.count + 1))
-            }
-        }
-        state<TestState.Failed> {
-        }
-        state<TestState.Passed> {
-            on<TestEvent.Failed> {
-                dontTransition()
-            }
-            on<TestEvent.Incomplete> {
-                dontTransition()
-            }
-        }
-        onTransition {
-            if (it as? StateMachine.Transition.Valid !is StateMachine.Transition.Valid) {
-                logger.error { "from ${it.fromState} event ${it.event}" }
-            }
-            trackTestTransition(poolId, it)
-        }
-    }
-
-    init {
-        val allTests = shard.tests + shard.flakyTests
-        allTests.groupBy { it }.map {
-            val count = it.value.size
-            it.key.toTestName() to createState(count)
-        }.also {
-            tests.putAll(it)
-        }
-    }
-
-    fun testFinished(device: DeviceInfo, testResult: TestResult) {
-        tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult))
-    }
-
-    fun testFailed(device: DeviceInfo, testResult: TestResult) {
-        tests[testResult.test.toTestName()]?.transition(TestEvent.Failed(device, testResult))
-    }
-
-    fun testIncomplete(device: DeviceInfo, testResult: TestResult, final: Boolean) {
-        tests[testResult.test.toTestName()]?.transition(TestEvent.Incomplete(device, testResult, final))
-    }
-
-    fun retryTest(device: DeviceInfo, testResult: TestResult) {
-        tests[testResult.test.toTestName()]?.transition(TestEvent.Retry(device, testResult))
-    }
-
-    fun removeTest(test: Test, diff: Int) {
-        tests[test.toTestName()]?.transition(TestEvent.Remove(diff))
-    }
-
-    private fun trackTestTransition(poolId: DevicePoolId, transition: StateMachine.Transition<TestState, TestEvent, TestAction>) {
-        val validTransition = transition as? StateMachine.Transition.Valid
-        val final = if (validTransition is StateMachine.Transition.Valid) {
-            when (validTransition.sideEffect) {
-                is TestAction.SaveReport -> true
-                else -> false
-            }
-        } else false
-
-        val (testResult: TestResult?, device: DeviceInfo?) = extractEventAndDevice(transition)
-        if (testResult == null || device == null) return
-
-        track.test(poolId, device, testResult, final)
-    }
-
-    private fun extractEventAndDevice(transition: StateMachine.Transition<TestState, TestEvent, TestAction>): Pair<TestResult?, DeviceInfo?> {
-        val event = transition.event
-        val testResult: TestResult? = when (event) {
-            is TestEvent.Passed -> event.testResult
-            is TestEvent.Failed -> event.testResult
-            is TestEvent.Retry -> event.testResult
-            is TestEvent.Incomplete -> event.testResult
-            else -> null
-        }
-        val device: DeviceInfo? = when (event) {
-            is TestEvent.Passed -> event.device
-            is TestEvent.Failed -> event.device
-            is TestEvent.Retry -> event.device
-            is TestEvent.Incomplete -> event.device
-            else -> null
-        }
-        return Pair(testResult, device)
-    }
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt
index b70c83690..98cad7590 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt
@@ -1,24 +1,36 @@
 package com.malinskiy.marathon.execution.queue
 
-import com.malinskiy.marathon.device.DeviceInfo
-import com.malinskiy.marathon.execution.TestResult
 
+/**
+ * TestState accumulates all the information about a single test case for one pool
+ */
 sealed class TestState {
-    data class Added(val count: Int) : TestState()
+    data class Added(
+        val total: Int, 
+        val running: Int = 0,
+    ) : TestState()
+    
+    data class Passing(
+        val total: Int,
+        val running: Int,
+        val done: Int,
+    ) : TestState()
 
-    data class Executed(
-        val device: DeviceInfo,
-        val testResult: TestResult,
-        val count: Int
+    data class Failing(
+        val total: Int,
+        val running: Int,
+        val done: Int,
     ) : TestState()
 
     data class Failed(
-        val device: DeviceInfo,
-        val testResult: TestResult
+        val total: Int,
+        val running: Int,
+        val done: Int,
     ) : TestState()
 
     data class Passed(
-        val device: DeviceInfo,
-        val testResult: TestResult
+        val total: Int,
+        val running: Int,
+        val done: Int,
     ) : TestState()
 }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/test_state.md b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/test_state.md
new file mode 100644
index 000000000..d1c820188
--- /dev/null
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/test_state.md
@@ -0,0 +1,41 @@
+```mermaid
+stateDiagram-v2
+    [*] --> Started
+
+    Started --> Failed : Failed
+    Started --> Passed : Passed
+    Started --> Ignored : Ignored
+    
+    Passed --> Failed : Failed
+    
+    Failed --> Passed : Passed
+    
+    Ignored --> Passed : Passed
+
+    Failed --> [*]
+    Passed --> [*]
+    Ignored --> [*]
+```
+
+```mermaid
+stateDiagram-v2
+    [*] --> Added
+
+    Added --> Passed : Passed
+    Added --> Passing : Passed
+    Added --> Failed : Failed
+    Added --> Passing : Failed
+    Added --> Failed : Incomplete
+    Added --> Passing : Incomplete
+    
+    
+    Passed --> Failed : Failed
+    
+    Failed --> Passed : Passed
+    
+    Ignored --> Passed : Passed
+
+    Failed --> [*]
+    Passed --> [*]
+    Ignored --> [*]
+```
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt b/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt
index 2d44122ca..4df9d690f 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt
@@ -7,8 +7,8 @@ fun generateTest(
     pkg: String = "pkg",
     clazz: String = "clazz",
     method: String = "method",
-    annotations: List<MetaProperty> = emptyList()
-) = Test(pkg, clazz, method, annotations)
+    metaProperties: List<MetaProperty> = emptyList()
+) = Test(pkg, clazz, method, metaProperties)
 
 fun generateTests(
     count: Int,
@@ -20,4 +20,4 @@ fun generateTests(
     return (0 until count).map {
         Test("$pkg$it", "$clazz$it", "$method$it", annotations)
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt b/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt
index 47e9fbf2f..85c4b44e8 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt
@@ -2,11 +2,9 @@ package com.malinskiy.marathon.device
 
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.log.MarathonLogging
 import com.malinskiy.marathon.test.TestBatch
 import kotlinx.coroutines.CompletableDeferred
-import mu.KLogger
 
 class DeviceStub(
     override var operatingSystem: OperatingSystem = OperatingSystem("25"),
@@ -24,8 +22,7 @@ class DeviceStub(
         configuration: Configuration,
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
-        deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter
+        deferred: CompletableDeferred<TestBatchResults>
     ) {
     }
 
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt
new file mode 100644
index 000000000..9d38c8810
--- /dev/null
+++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt
@@ -0,0 +1,809 @@
+package com.malinskiy.marathon.execution.progress
+
+import com.malinskiy.marathon.analytics.external.Analytics
+import com.malinskiy.marathon.analytics.internal.pub.Track
+import com.malinskiy.marathon.config.Configuration
+import com.malinskiy.marathon.config.strategy.ExecutionMode
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
+import com.malinskiy.marathon.config.vendor.VendorConfiguration
+import com.malinskiy.marathon.device.DevicePoolId
+import com.malinskiy.marathon.execution.TestResult
+import com.malinskiy.marathon.execution.TestShard
+import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.generateTest
+import com.malinskiy.marathon.report.getDevice
+import com.nhaarman.mockitokotlin2.mock
+import com.nhaarman.mockitokotlin2.reset
+import org.amshove.kluent.shouldBeEqualTo
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import java.io.File
+
+class PoolProgressAccumulatorTest {
+    private val track = mock<Track>()
+    private val analytics = mock<Analytics>()
+
+    @BeforeEach
+    fun `setup mocks`() {
+        reset(track, analytics)
+    }
+
+    private val anySuccessConfig = Configuration.Builder(
+        name = "",
+        outputDir = File("")
+    ).apply {
+        vendorConfiguration = VendorConfiguration.StubVendorConfiguration
+        debug = false
+        analyticsTracking = false
+        executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ANY_SUCCESS, fast = false)
+
+    }.build()
+    private val allSuccessConfig =
+        anySuccessConfig.copy(executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS, fast = false))
+    private val successFastConfig =
+        anySuccessConfig.copy(executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ANY_SUCCESS, fast = true))
+    private val failFastConfig =
+        anySuccessConfig.copy(executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS, fast = true))
+    val test = generateTest()
+    private val poolId = DevicePoolId("test")
+    private val device = getDevice()
+
+    @Test
+    fun `ANY_SUCCESS should pass on pass, failure, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should pass on pass, pass, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should pass on pass, fail, fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.FAILURE, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should fail on no events`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should pass on retry, fail, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.retryTest(test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should pass with runtime discovery`() {
+        val testParameterized = generateTest(
+            pkg = "com.malinskiy.marathon",
+            clazz = "ParameterizedTest",
+            method = "test",
+            metaProperties = emptyList()
+        )
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(testParameterized)),
+            anySuccessConfig,
+            track
+        )
+        val test0 = generateTest(
+            pkg = "com.malinskiy.marathon",
+            clazz = "ParameterizedTest",
+            method = "test[0]",
+            metaProperties = emptyList()
+        )
+        val test1 = generateTest(
+            pkg = "com.malinskiy.marathon",
+            clazz = "ParameterizedTest",
+            method = "test[1]",
+            metaProperties = emptyList()
+        )
+
+        reporter.testStarted(device, test0)
+        reporter.testEnded(device, TestResult(test0, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test1)
+        reporter.testEnded(device, TestResult(test1, device, "1", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testEnded(device, TestResult(testParameterized, device, "1", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS fast should pass on pass, fail, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            successFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS fast should fail on fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            successFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ANY_SUCCESS fast should fail on fail with many tries`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            successFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should pass on pass, incomplete, incomplete`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.INCOMPLETE, 1, 2), false)
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.INCOMPLETE, 2, 3), true)
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should pass on pass, retry, fail, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.testStarted(device, test)
+        reporter.retryTest(test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should fail on fail, fail, fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.FAILURE, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should pass on fail, pass, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS fast should pass on fail, pass, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            successFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ANY_SUCCESS should fail on add retry, fail, add retry, fail, remove retry, fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.retryTest(test)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+
+        reporter.testStarted(device, test)
+        reporter.retryTest(test)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.removeTest(test, 1)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.FAILURE, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `should fail on incomplete for a bunch of test tries without a single pass or fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            successFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.INCOMPLETE, 0, 1), false)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.INCOMPLETE, 1, 2), false)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.INCOMPLETE, 2, 3), true)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `should fail on fail, incomplete, incomplete`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.INCOMPLETE, 1, 2), false)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.INCOMPLETE, 2, 3), true)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `should not change result after terminating to failed`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            anySuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.INCOMPLETE, 1, 2), false)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.retryTest(test)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.removeTest(test, 1)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `should not change result after terminating to pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.INCOMPLETE, 1, 2), false)
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.retryTest(test)
+        reporter.aggregateResult().shouldBeEqualTo(true)
+
+        reporter.removeTest(test, 1)
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ALL_SUCCESS passing with multiple tries passing`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            failFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on pass, pass, fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.FAILURE, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS fast should fail on pass, fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            failFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on pass, fail, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should pass on pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on fail with many test tries`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on first failure for a single test`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS fast should fail on failure for a many test tries`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            failFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on retry, pass, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test)),
+            failFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.retryTest(test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should pass on remove retry, pass, pass with 3 test tries`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test, test)),
+            failFastConfig,
+            track
+        )
+        //Just removes one attempt. Now we have 3 tests to execute with all_success mode
+        reporter.removeTest(test, 1)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+        //2 tests left to execute
+
+        val device2 = getDevice("2")
+        //Concurrent device executing one of the tries
+        reporter.testStarted(device, test)
+
+        //Removes one that is not yet processed by the queue
+        reporter.removeTest(test, 1)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testEnded(device2, TestResult(test, device2, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(true)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on pass, incomplete, incomplete`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.INCOMPLETE, 1, 2), false)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.INCOMPLETE, 2, 3), true)
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on fail, pass, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS fast should fail on fail, pass, pass`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            failFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.PASSED, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.PASSED, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS should fail on fail, fail, fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            allSuccessConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.FAILURE, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `ALL_SUCCESS fast should fail on fail, fail, fail`() {
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test, test, test)),
+            failFastConfig,
+            track
+        )
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "1", TestStatus.FAILURE, 0, 1))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+
+        reporter.testStarted(device, test)
+        reporter.testEnded(device, TestResult(test, device, "3", TestStatus.FAILURE, 2, 3))
+        reporter.aggregateResult().shouldBeEqualTo(false)
+    }
+
+    @Test
+    fun `should report progress`() {
+        val poolId = DevicePoolId("testpool")
+
+        val test1 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method1", emptyList())
+        val test2 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method2", emptyList())
+        val test3 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method3", emptyList())
+
+        val reporter = PoolProgressAccumulator(
+            poolId,
+            TestShard(listOf(test1, test2, test3)),
+            anySuccessConfig,
+            track
+        )
+        
+        reporter.progress().shouldBeEqualTo(.0f)
+
+        /**
+         * test 1 passed
+         */
+        reporter.testStarted(device, test1)
+        reporter.testEnded(device, TestResult(test1, device, "1", TestStatus.PASSED, 0, 1))
+        reporter.progress().shouldBeEqualTo(1 / 3f)
+
+        /**
+         * test 2 failed
+         */
+        reporter.testStarted(device, test2)
+        reporter.testEnded(device, TestResult(test2, device, "2", TestStatus.FAILURE, 1, 2))
+        reporter.progress().shouldBeEqualTo(2 / 3f)
+
+        /**
+         * adding 4 retries for test2 and then test 2 passes once
+         */
+        reporter.retryTest(test2)
+        reporter.retryTest(test2)
+        reporter.retryTest(test2)
+        reporter.retryTest(test2)
+        reporter.progress().shouldBeEqualTo(2 / 7f)
+        reporter.testStarted(device, test2)
+        reporter.testEnded(device, TestResult(test2, device, "2", TestStatus.PASSED, 2, 3))
+        reporter.progress().shouldBeEqualTo(3 / 7f)
+
+        /**
+         * 1 retry of test 2 fails
+         */
+        reporter.testStarted(device, test2)
+        reporter.testEnded(device, TestResult(test2, device, "3", TestStatus.FAILURE, 3, 4))
+        reporter.progress().shouldBeEqualTo(4 / 7f)
+
+        /**
+         * 1 retry of test 2 is ignored
+         */
+        reporter.testStarted(device, test2)
+        reporter.testEnded(device, TestResult(test2, device, "4", TestStatus.IGNORED, 4, 5))
+        reporter.progress().shouldBeEqualTo(5 / 7f)
+
+        /**
+         * removing one retry of test 2
+         */
+        reporter.removeTest(test2, 1)
+        reporter.progress().shouldBeEqualTo(5 / 6f)
+
+        /**
+         * test 3 is ignored (assumption failure or just ignore)
+         */
+        reporter.testStarted(device, test3)
+        reporter.testEnded(device, TestResult(test3, device, "5", TestStatus.IGNORED, 5, 6))
+        val progress = reporter.progress()
+        progress.shouldBeEqualTo(6 / 6f)
+    }
+
+    @Test
+    fun shouldReportProgressForOnePoolWithRuntimeDiscovery() {
+//        val poolId = DevicePoolId("testpool")
+//
+//        val test0 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method[0]", emptyList())
+//        val test1 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method[1]", emptyList())
+//        val test2 = com.malinskiy.marathon.test.Test("com.example", "SimpleTest", "method[2]", emptyList())
+//
+//        reporter.testCountExpectation(poolId, 1)
+//        reporter.progress().shouldBeEqualTo(.0f)
+//
+//        /**
+//         * [0] passed
+//         */
+//        reporter.testStarted(poolId, deviceInfo, test0)
+//        reporter.testPassed(poolId, deviceInfo, test0)
+//        reporter.progress().shouldBeEqualTo(1 / 1f)
+//
+//        /**
+//         * [1] passed
+//         */
+//        reporter.testStarted(poolId, deviceInfo, test1)
+//        reporter.testPassed(poolId, deviceInfo, test1)
+//        reporter.progress().shouldBeEqualTo(2 / 1f)
+//
+//        /**
+//         * [2] passed
+//         */
+//        reporter.testStarted(poolId, deviceInfo, test2)
+//        reporter.testPassed(poolId, deviceInfo, test2)
+//        reporter.progress().shouldBeEqualTo(3 / 1f)
+//
+//        val progress = reporter.progress()
+//        progress.shouldBeEqualTo(6 / 6f)
+    }
+}
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt
index e8cb45325..699a58848 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt
@@ -23,105 +23,5 @@ class ProgressReporterTest {
     )
     private val deviceInfo = StubDevice().toDeviceInfo()
 
-    @Test
-    fun shouldReportProgressForOnePool() {
-        val poolId = DevicePoolId("testpool")
-
-        val test1 = MarathonTest("com.example", "SimpleTest", "method1", emptyList())
-        val test2 = MarathonTest("com.example", "SimpleTest", "method2", emptyList())
-        val test3 = MarathonTest("com.example", "SimpleTest", "method3", emptyList())
-
-        reporter.testCountExpectation(poolId, 3)
-        reporter.progress().shouldBeEqualTo(.0f)
-
-        /**
-         * test 1 passed
-         */
-        reporter.testStarted(poolId, deviceInfo, test1)
-        reporter.testPassed(poolId, deviceInfo, test1)
-        reporter.progress().shouldBeEqualTo(1 / 3f)
-
-        /**
-         * test 2 failed
-         */
-        reporter.testStarted(poolId, deviceInfo, test2)
-        reporter.testFailed(poolId, deviceInfo, test2)
-        reporter.progress().shouldBeEqualTo(2 / 3f)
-
-        /**
-         * adding 4 retries for test2 and then test 2 passes once
-         */
-        reporter.addRetries(poolId, 4)
-        reporter.progress().shouldBeEqualTo(2 / 7f)
-        reporter.testStarted(poolId, deviceInfo, test2)
-        reporter.testPassed(poolId, deviceInfo, test2)
-        reporter.progress().shouldBeEqualTo(3 / 7f)
-
-        /**
-         * 1 retry of test 2 fails
-         */
-        reporter.testStarted(poolId, deviceInfo, test2)
-        reporter.testFailed(poolId, deviceInfo, test2)
-        reporter.progress().shouldBeEqualTo(4 / 7f)
-
-        /**
-         * 1 retry of test 2 is ignored
-         */
-        reporter.testStarted(poolId, deviceInfo, test2)
-        reporter.testIgnored(poolId, deviceInfo, test2)
-        reporter.progress().shouldBeEqualTo(5 / 7f)
-
-        /**
-         * removing one retry of test 2
-         */
-        reporter.removeTests(poolId, 1)
-        reporter.progress().shouldBeEqualTo(5 / 6f)
-
-        /**
-         * test 3 is ignored (assumption failure or just ignore)
-         */
-        reporter.testStarted(poolId, deviceInfo, test3)
-        reporter.testIgnored(poolId, deviceInfo, test3)
-        val progress = reporter.progress()
-        progress.shouldBeEqualTo(6 / 6f)
-    }
-
-    @Test
-    fun shouldReportProgressForOnePoolWithRuntimeDiscovery() {
-        val poolId = DevicePoolId("testpool")
-
-        val test0 = MarathonTest("com.example", "SimpleTest", "method[0]", emptyList())
-        val test1 = MarathonTest("com.example", "SimpleTest", "method[1]", emptyList())
-        val test2 = MarathonTest("com.example", "SimpleTest", "method[2]", emptyList())
-
-        reporter.testCountExpectation(poolId, 1)
-        reporter.progress().shouldBeEqualTo(.0f)
-
-        /**
-         * [0] passed
-         */
-        reporter.testStarted(poolId, deviceInfo, test0)
-        reporter.testPassed(poolId, deviceInfo, test0)
-        reporter.progress().shouldBeEqualTo(1 / 1f)
-
-        /**
-         * [1] passed
-         */
-        reporter.testStarted(poolId, deviceInfo, test1)
-        reporter.testPassed(poolId, deviceInfo, test1)
-        reporter.progress().shouldBeEqualTo(2 / 1f)
-
-        /**
-         * [2] passed
-         */
-        reporter.testStarted(poolId, deviceInfo, test2)
-        reporter.testPassed(poolId, deviceInfo, test2)
-        reporter.progress().shouldBeEqualTo(3 / 1f)
-
-        reporter.addTestDiscoveredDuringRuntime(poolId, test1)
-        reporter.addTestDiscoveredDuringRuntime(poolId, test2)
-
-        val progress = reporter.progress()
-        progress.shouldBeEqualTo(6 / 6f)
-    }
+    
 }
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTrackerTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTrackerTest.kt
deleted file mode 100644
index b74cee5d5..000000000
--- a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTrackerTest.kt
+++ /dev/null
@@ -1,99 +0,0 @@
-package com.malinskiy.marathon.execution.progress.tracker
-
-import com.malinskiy.marathon.config.Configuration
-import com.malinskiy.marathon.config.vendor.VendorConfiguration
-import org.amshove.kluent.shouldBeEqualTo
-import org.junit.jupiter.api.Test
-import java.io.File
-import com.malinskiy.marathon.test.Test as MarathonTest
-
-class PoolProgressTrackerTest {
-    val test = MarathonTest(
-        pkg = "com.malinskiy.marathon",
-        clazz = "SomeTest",
-        method = "someMethod",
-        metaProperties = emptyList()
-    )
-
-    private fun createConfiguration(strictMode: Boolean): Configuration {
-        return Configuration.Builder(
-            name = "",
-            outputDir = File("")
-        ).apply {
-            vendorConfiguration = VendorConfiguration.StubVendorConfiguration
-            this.strictMode = strictMode
-            debug = false
-            analyticsTracking = false
-        }.build()
-    }
-
-    @Test
-    fun nonStrictMode_case1() {
-        val tracker = PoolProgressTracker(createConfiguration(strictMode = false))
-        tracker.testCountExpectation(1)
-        tracker.testStarted(test)
-        tracker.testPassed(test)
-        tracker.testFailed(test)
-        tracker.aggregateResult().shouldBeEqualTo(true)
-        tracker.testPassed(test)
-        tracker.aggregateResult().shouldBeEqualTo(true)
-    }
-
-    @Test
-    fun strictMode_case1() {
-        val tracker = PoolProgressTracker(createConfiguration(strictMode = true))
-        tracker.testCountExpectation(1)
-        tracker.testStarted(test)
-        tracker.testPassed(test)
-        tracker.testFailed(test)
-        tracker.aggregateResult().shouldBeEqualTo(false)
-        tracker.testPassed(test)
-        tracker.aggregateResult().shouldBeEqualTo(false)
-    }
-
-    @Test
-    fun all_incomplete() {
-        val tracker = PoolProgressTracker(createConfiguration(strictMode = false)).apply {
-            testCountExpectation(1)
-        }
-        tracker.aggregateResult() shouldBeEqualTo false
-    }
-
-    @Test
-    fun withRetries() {
-        val tracker = PoolProgressTracker(createConfiguration(strictMode = false))
-
-        tracker.testCountExpectation(1)
-        tracker.testStarted(test)
-        tracker.testFailed(test)
-        tracker.addTestRetries(1)
-        tracker.testStarted(test)
-        tracker.testPassed(test)
-        tracker.aggregateResult().shouldBeEqualTo(true)
-    }
-
-    @Test
-    fun withRuntimeDiscovery() {
-        val tracker = PoolProgressTracker(createConfiguration(strictMode = false))
-        val test0 = MarathonTest(
-            pkg = "com.malinskiy.marathon",
-            clazz = "ParameterizedTest",
-            method = "test[0]",
-            metaProperties = emptyList()
-        )
-        val test1 = MarathonTest(
-            pkg = "com.malinskiy.marathon",
-            clazz = "ParameterizedTest",
-            method = "test[1]",
-            metaProperties = emptyList()
-        )
-
-        tracker.testCountExpectation(1)
-        tracker.testStarted(test0)
-        tracker.testPassed(test0)
-        tracker.testStarted(test1)
-        tracker.testPassed(test1)
-        tracker.addTestDiscoveredDuringRuntime(test1)
-        tracker.aggregateResult().shouldBeEqualTo(true)
-    }
-}
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterTemporalTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt
similarity index 90%
rename from core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterTemporalTest.kt
rename to core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt
index 7b454ec8c..dfe500bfe 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterTemporalTest.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt
@@ -3,12 +3,15 @@ package com.malinskiy.marathon.execution.queue
 import com.malinskiy.marathon.analytics.external.Analytics
 import com.malinskiy.marathon.analytics.internal.pub.Track
 import com.malinskiy.marathon.config.Configuration
+import com.malinskiy.marathon.config.strategy.ExecutionMode
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
 import com.malinskiy.marathon.config.vendor.VendorConfiguration
 import com.malinskiy.marathon.createDeviceInfo
 import com.malinskiy.marathon.device.DevicePoolId
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestShard
 import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
 import com.malinskiy.marathon.generateTest
 import com.nhaarman.mockitokotlin2.inOrder
 import com.nhaarman.mockitokotlin2.mock
@@ -18,7 +21,7 @@ import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import java.io.File
 
-class TestResultReporterTemporalTest {
+class TestResultReporterAllSuccessTest {
     private val track = mock<Track>()
     private val analytics = mock<Analytics>()
 
@@ -35,21 +38,19 @@ class TestResultReporterTemporalTest {
         debug = false
         analyticsTracking = false
     }.build()
-    private val strictConfig = defaultConfig.copy(strictMode = true)
+    private val strictConfig = defaultConfig.copy(executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS, fast = false))
     val test = generateTest()
     private val poolId = DevicePoolId("test")
 
-    private fun filterDefault() = TestResultReporter(
+    private fun filterDefault() = PoolProgressAccumulator(
         poolId,
-        analytics,
         TestShard(listOf(test, test, test)),
         defaultConfig,
         track
     )
 
-    private fun filterStrict() = TestResultReporter(
+    private fun filterStrict() = PoolProgressAccumulator(
         poolId,
-        analytics,
         TestShard(listOf(test, test, test)),
         strictConfig,
         track
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt
index 82eac2b67..503955eaf 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/report/JUnitReporterTest.kt
@@ -221,10 +221,10 @@ fun createTestEvent(
     )
 }
 
-fun getDevice() =
+fun getDevice(serialNumber: String = "xxyyzz") =
     DeviceInfo(
         operatingSystem = OperatingSystem("23"),
-        serialNumber = "xxyyzz",
+        serialNumber = serialNumber,
         model = "Android SDK built for x86",
         manufacturer = "unknown",
         networkState = NetworkState.CONNECTED,
diff --git a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt
index b8a8e7eaf..c79560a6d 100644
--- a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt
+++ b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt
@@ -175,7 +175,6 @@ class MarathonPlugin : Plugin<Project> {
                 config.filteringConfiguration?.toFilteringConfiguration()?.let { filteringConfiguration = it }
                 config.ignoreFailures?.let { ignoreFailures = it }
                 config.isCodeCoverageEnabled?.let { isCodeCoverageEnabled = it }
-                config.strictMode?.let { strictMode = it }
                 config.uncompletedTestRetryQuota?.let { uncompletedTestRetryQuota = it }
                 config.testClassRegexes?.map { it.toRegex() }?.let { testClassRegexes = it }
                 config.includeSerialRegexes?.map { it.toRegex() }?.let { includeSerialRegexes = it }
diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/BaseAndroidDevice.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/BaseAndroidDevice.kt
index 7916cab7e..dfddd79b6 100644
--- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/BaseAndroidDevice.kt
+++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/BaseAndroidDevice.kt
@@ -7,9 +7,8 @@ import com.malinskiy.marathon.android.executor.listeners.CompositeTestRunListene
 import com.malinskiy.marathon.android.executor.listeners.DebugTestRunListener
 import com.malinskiy.marathon.execution.listener.LogListener
 import com.malinskiy.marathon.android.executor.listeners.NoOpTestRunListener
-import com.malinskiy.marathon.android.executor.listeners.ProgressTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.TestRunListenerAdapter
-import com.malinskiy.marathon.android.executor.listeners.TestRunResultsTestRunListener
+import com.malinskiy.marathon.android.executor.listeners.TestResultsListener
 import com.malinskiy.marathon.android.executor.listeners.filesync.FileSyncTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.screenshot.AdamScreenCaptureTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.screenshot.ScreenCapturerTestRunListener
@@ -28,7 +27,6 @@ import com.malinskiy.marathon.device.OperatingSystem
 import com.malinskiy.marathon.device.toDeviceInfo
 import com.malinskiy.marathon.exceptions.DeviceSetupException
 import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.extension.withTimeoutOrNull
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.log.MarathonLogging
@@ -253,7 +251,6 @@ abstract class BaseAndroidDevice(
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
         deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter
     ): CompositeTestRunListener {
         val fileManager = FileManager(configuration.outputConfiguration.maxPath, configuration.outputDir)
         val attachmentProviders = mutableListOf<AttachmentProvider>()
@@ -281,9 +278,8 @@ abstract class BaseAndroidDevice(
             listOf(
                 recorderListener,
                 logListener,
-                TestRunResultsTestRunListener(testBatch, this, deferred, timer, progressReporter, devicePoolId, attachmentProviders),
+                TestResultsListener(testBatch, this, deferred, timer, devicePoolId, attachmentProviders),
                 DebugTestRunListener(this),
-                ProgressTestRunListener(this, devicePoolId, progressReporter),
                 adamScreenCaptureTestRunListener,
                 fileSyncTestRunListener
             )
diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamAndroidDevice.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamAndroidDevice.kt
index c5a4cc783..a515281f9 100644
--- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamAndroidDevice.kt
+++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamAndroidDevice.kt
@@ -52,7 +52,6 @@ import com.malinskiy.marathon.device.NetworkState
 import com.malinskiy.marathon.device.file.measureFileTransfer
 import com.malinskiy.marathon.exceptions.DeviceLostException
 import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.extension.escape
 import com.malinskiy.marathon.extension.withTimeout
 import com.malinskiy.marathon.extension.withTimeoutOrNull
@@ -383,13 +382,12 @@ class AdamAndroidDevice(
         configuration: Configuration,
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
-        deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter
+        deferred: CompletableDeferred<TestBatchResults>
     ) {
         try {
             async(coroutineContext) {
                 supervisorScope {
-                    val listener = createExecutionListeners(configuration, devicePoolId, testBatch, deferred, progressReporter)
+                    val listener = createExecutionListeners(configuration, devicePoolId, testBatch, deferred)
                     AndroidDeviceTestRunner(this@AdamAndroidDevice, testBundleIdentifier).execute(configuration, testBatch, listener)
                 }
             }.await()
diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AndroidDeviceTestRunner.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AndroidDeviceTestRunner.kt
index 448f98ae1..1a9074480 100644
--- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AndroidDeviceTestRunner.kt
+++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/adam/AndroidDeviceTestRunner.kt
@@ -37,7 +37,6 @@ class AndroidDeviceTestRunner(private val device: AdamAndroidDevice, private val
     private val logger = MarathonLogging.logger("AndroidDeviceTestRunner")
     private val argumentsFactory = ArgumentsFactory(device)
 
-    @OptIn(ExperimentalStdlibApi::class)
     suspend fun execute(
         configuration: Configuration,
         rawTestBatch: TestBatch,
diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListener.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListener.kt
deleted file mode 100644
index 7f9bd130b..000000000
--- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListener.kt
+++ /dev/null
@@ -1,44 +0,0 @@
-package com.malinskiy.marathon.android.executor.listeners
-
-import com.malinskiy.marathon.android.model.TestIdentifier
-import com.malinskiy.marathon.device.Device
-import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.device.toDeviceInfo
-import com.malinskiy.marathon.execution.progress.ProgressReporter
-
-class ProgressTestRunListener(
-    private val device: Device,
-    private val poolId: DevicePoolId,
-    private val progressTracker: ProgressReporter
-) : NoOpTestRunListener() {
-
-    private val failed = mutableMapOf<TestIdentifier, Boolean>()
-    private val ignored = mutableMapOf<TestIdentifier, Boolean>()
-
-    override suspend fun testStarted(test: TestIdentifier) {
-        failed[test] = false
-        ignored[test] = false
-        progressTracker.testStarted(poolId, device.toDeviceInfo(), test.toTest())
-    }
-
-    override suspend fun testFailed(test: TestIdentifier, trace: String) {
-        failed[test] = true
-    }
-
-    override suspend fun testAssumptionFailure(test: TestIdentifier, trace: String) {
-        testIgnored(test)
-    }
-
-    override suspend fun testEnded(test: TestIdentifier, testMetrics: Map<String, String>) {
-        if (failed[test] == true) {
-            progressTracker.testFailed(poolId, device.toDeviceInfo(), test.toTest())
-        } else if (ignored[test] == false) {
-            progressTracker.testPassed(poolId, device.toDeviceInfo(), test.toTest())
-        }
-    }
-
-    override suspend fun testIgnored(test: TestIdentifier) {
-        ignored[test] = true
-        progressTracker.testIgnored(poolId, device.toDeviceInfo(), test.toTest())
-    }
-}
diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsTestRunListener.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt
similarity index 95%
rename from vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsTestRunListener.kt
rename to vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt
index db67d37fa..4b1faacf4 100644
--- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsTestRunListener.kt
+++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt
@@ -18,18 +18,18 @@ import com.malinskiy.marathon.test.toTestName
 import com.malinskiy.marathon.time.Timer
 import kotlinx.coroutines.CompletableDeferred
 
-class TestRunResultsTestRunListener(
+class TestResultsListener(
     private val testBatch: TestBatch,
     private val device: Device,
     private val deferred: CompletableDeferred<TestBatchResults>,
     private val timer: Timer,
-    private val progressReporter: ProgressReporter,
     private val poolId: DevicePoolId,
     attachmentProviders: List<AttachmentProvider>,
     private val attachmentCollector: AttachmentCollector = AttachmentCollector(attachmentProviders),
 ) : AccumulatingResultTestRunListener(timer), AttachmentListener by attachmentCollector {
 
     private val logger = MarathonLogging.logger("TestRunResultsListener")
+    private val progressReporter = ProgressReporter(testBatch, poolId, device.toDeviceInfo())
 
     override suspend fun afterTestRun() {
         val results = mergeParameterisedResults(runResult.temporalTestResults)
@@ -98,8 +98,6 @@ class TestRunResultsTestRunListener(
                     result[realIdentifier] = e.value
                 } else {
                     result[realIdentifier]?.status = maybeExistingParameterizedResult.status + e.value.status
-                    //Needed for proper result aggregation
-                    progressReporter.addTestDiscoveredDuringRuntime(poolId, test)
                 }
             } else {
                 result[test] = e.value
diff --git a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/ApkParserTest.kt b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/ApkParserTest.kt
index 8bc629e8f..204463ddd 100644
--- a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/ApkParserTest.kt
+++ b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/ApkParserTest.kt
@@ -1,5 +1,6 @@
 package com.malinskiy.marathon.android
 
+import org.amshove.kluent.shouldBeEqualTo
 import org.amshove.kluent.shouldEqual
 import org.junit.jupiter.api.Test
 import java.io.File
@@ -11,7 +12,7 @@ class ApkParserTest {
     fun `should parser AndroidManifest and return InstrumentationInfo`() {
         val parser = ApkParser()
         val apkFile = File(javaClass.classLoader.getResource("android_test_1.apk").file)
-        parser.parseInstrumentationInfo(apkFile) shouldEqual InstrumentationInfo(
+        parser.parseInstrumentationInfo(apkFile) shouldBeEqualTo InstrumentationInfo(
             "com.example",
             "com.example.test",
             "android.support.test.runner.AndroidJUnitRunner"
diff --git a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/adam/TestConfigurationFactory.kt b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/adam/TestConfigurationFactory.kt
index 296dcfb04..f7b194f8c 100644
--- a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/adam/TestConfigurationFactory.kt
+++ b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/adam/TestConfigurationFactory.kt
@@ -34,7 +34,6 @@ object TestConfigurationFactory {
                 allureConfiguration = allureConfiguration
             )
             this.isCodeCoverageEnabled = isCodeCoverageEnabled
-            strictMode = false
             debug = false
             analyticsTracking = false
         }.build()
diff --git a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/CompositeTestRunListenerTest.kt b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/CompositeTestRunListenerTest.kt
index 6544d0f4b..c9ecb8084 100644
--- a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/CompositeTestRunListenerTest.kt
+++ b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/CompositeTestRunListenerTest.kt
@@ -10,7 +10,7 @@ import org.junit.jupiter.api.Test
 class CompositeTestRunListenerTest {
     @Test
     fun testDefault() {
-        val mock = mock<TestRunResultsTestRunListener>()
+        val mock = mock<TestResultsListener>()
         val listener = CompositeTestRunListener(listOf(mock))
 
         runBlocking {
diff --git a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsListenerTest.kt b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListenerTest.kt
similarity index 91%
rename from vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsListenerTest.kt
rename to vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListenerTest.kt
index 20fa17fa7..e158d7586 100644
--- a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsListenerTest.kt
+++ b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListenerTest.kt
@@ -23,12 +23,10 @@ import com.malinskiy.marathon.execution.AttachmentType
 import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestStatus
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.report.attachment.AttachmentProvider
 import com.malinskiy.marathon.test.TestBatch
 import com.nhaarman.mockitokotlin2.any
 import com.nhaarman.mockitokotlin2.mock
-import com.nhaarman.mockitokotlin2.times
 import com.nhaarman.mockitokotlin2.verify
 import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.runBlocking
@@ -38,7 +36,7 @@ import java.io.File
 import java.util.UUID
 
 @AdbTest
-class TestRunResultsListenerTest {
+class TestResultsListenerTest {
     @AdbClient
     lateinit var client: AndroidDebugBridgeClient
 
@@ -65,7 +63,7 @@ class TestRunResultsListenerTest {
         val deferred = CompletableDeferred<TestBatchResults>()
 
         val attachmentProvider = mock<AttachmentProvider>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), listOf(attachmentProvider))
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), listOf(attachmentProvider))
 
         runBlocking {
             server.multipleSessions {
@@ -134,7 +132,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -175,7 +173,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -226,7 +224,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -266,7 +264,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -312,7 +310,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -363,7 +361,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -414,7 +412,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -463,7 +461,7 @@ class TestRunResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -517,7 +515,7 @@ class TestRunResultsListenerTest {
         val deferred = CompletableDeferred<TestBatchResults>()
         val progressReporter = mock<ProgressReporter>()
         val poolId = DevicePoolId("omni")
-        val listener = TestRunResultsTestRunListener(batch, device, deferred, mock(), progressReporter, poolId, emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), progressReporter, poolId, emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -552,14 +550,6 @@ class TestRunResultsListenerTest {
             )
             assertThat(result.failed).isEmpty()
             assertThat(result.uncompleted).isEmpty()
-
-            /**
-             * Due to the need for consistency first parameterized test does not end up being reported to remove the need to remove the
-             * parent test
-             */
-            verify(progressReporter, times(0)).addTestDiscoveredDuringRuntime(poolId, id0.toTest())
-            verify(progressReporter, times(1)).addTestDiscoveredDuringRuntime(poolId, id1.toTest())
-            verify(progressReporter, times(1)).addTestDiscoveredDuringRuntime(poolId, id2.toTest())
         }
     }
 
diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleSimulatorDevice.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleSimulatorDevice.kt
index 888d91196..08bbcab70 100644
--- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleSimulatorDevice.kt
+++ b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/AppleSimulatorDevice.kt
@@ -20,7 +20,6 @@ import com.malinskiy.marathon.exceptions.DeviceSetupException
 import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.listener.LineListener
 import com.malinskiy.marathon.execution.listener.LogListener
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.ios.bin.AppleBinaryEnvironment
 import com.malinskiy.marathon.ios.cmd.CommandExecutor
@@ -29,7 +28,6 @@ import com.malinskiy.marathon.ios.cmd.FileBridge
 import com.malinskiy.marathon.ios.executor.listener.AppleTestRunListener
 import com.malinskiy.marathon.ios.executor.listener.CompositeTestRunListener
 import com.malinskiy.marathon.ios.executor.listener.DebugTestRunListener
-import com.malinskiy.marathon.ios.executor.listener.ProgressReportingListener
 import com.malinskiy.marathon.ios.executor.listener.ResultBundleRunListener
 import com.malinskiy.marathon.ios.executor.listener.video.ScreenRecordingListener
 import com.malinskiy.marathon.ios.executor.listener.TestResultsListener
@@ -237,15 +235,14 @@ class AppleSimulatorDevice(
         configuration: Configuration,
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
-        deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter
+        deferred: CompletableDeferred<TestBatchResults>
     ) {
         try {
             async(CoroutineName("execute $serialNumber")) {
                 supervisorScope {
                     var executionLineListeners = setOf<LineListener>()
                     try {
-                        val (listener, lineListeners) = createExecutionListeners(devicePoolId, testBatch, deferred, progressReporter)
+                        val (listener, lineListeners) = createExecutionListeners(devicePoolId, testBatch, deferred)
                         executionLineListeners = lineListeners.onEach { addLineListener(it) }
                         AppleDeviceTestRunner(this@AppleSimulatorDevice).execute(configuration, vendorConfiguration, testBatch, listener)
                     } finally {
@@ -285,16 +282,9 @@ class AppleSimulatorDevice(
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
         deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter,
     ): Pair<CompositeTestRunListener, Set<LineListener>> {
         val logWriter = LogWriter(fileManager)
 
-        val progressReportingListener = ProgressReportingListener(
-            deviceInfo = toDeviceInfo(),
-            poolId = devicePoolId,
-            progressReporter = progressReporter,
-        )
-
         val attachmentProviders = mutableListOf<AttachmentProvider>()
         val recorderListener =
             RecorderTypeSelector.selectRecorderType(deviceFeatures, vendorConfiguration.screenRecordConfiguration)?.let { feature ->
@@ -329,6 +319,7 @@ class AppleSimulatorDevice(
                     TestResultsListener(
                         testBatch,
                         this,
+                        devicePoolId,
                         deferred,
                         timer,
                         remoteFileManager,
@@ -336,7 +327,6 @@ class AppleSimulatorDevice(
                         attachmentProviders
                     ),
                     logListener,
-                    progressReportingListener,
                     DebugTestRunListener(this),
                     diagnosticLogsPathFinder,
                     sessionResultsPathFinder,
diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/ProgressReportingListener.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/ProgressReportingListener.kt
deleted file mode 100644
index b58c2486d..000000000
--- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/ProgressReportingListener.kt
+++ /dev/null
@@ -1,24 +0,0 @@
-package com.malinskiy.marathon.ios.executor.listener
-
-import com.malinskiy.marathon.device.DeviceInfo
-import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.execution.progress.ProgressReporter
-import com.malinskiy.marathon.test.Test
-
-class ProgressReportingListener(
-    private val deviceInfo: DeviceInfo,
-    private val poolId: DevicePoolId,
-    private val progressReporter: ProgressReporter,
-) : AppleTestRunListener {
-    override suspend fun testStarted(test: Test) {
-        progressReporter.testStarted(poolId, deviceInfo, test)
-    }
-
-    override suspend fun testFailed(test: Test, startTime: Long, endTime: Long, trace: String?) {
-        progressReporter.testFailed(poolId, deviceInfo, test)
-    }
-
-    override suspend fun testPassed(test: Test, startTime: Long, endTime: Long) {
-        progressReporter.testPassed(poolId, deviceInfo, test)
-    }
-}
diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt
index 4d96c7756..a7ede1a9c 100644
--- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt
+++ b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt
@@ -1,10 +1,12 @@
 package com.malinskiy.marathon.ios.executor.listener
 
 import com.malinskiy.marathon.device.Device
+import com.malinskiy.marathon.device.DevicePoolId
 import com.malinskiy.marathon.device.toDeviceInfo
 import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.execution.result.TemporalTestResult
 import com.malinskiy.marathon.ios.RemoteFileManager
 import com.malinskiy.marathon.ios.bin.xcrun.xcresulttool.ResultBundleFormat
@@ -25,6 +27,7 @@ import kotlin.system.measureTimeMillis
 class TestResultsListener(
     private val testBatch: TestBatch,
     private val device: Device,
+    private val devicePoolId: DevicePoolId,
     private val deferred: CompletableDeferred<TestBatchResults>,
     private val timer: Timer,
     private val remoteFileManager: RemoteFileManager,
@@ -35,6 +38,19 @@ class TestResultsListener(
 
     private val logger = MarathonLogging.logger {}
     private val enhanceUsingXcresult = true
+    private val progressReporter = ProgressReporter(testBatch, devicePoolId, device.toDeviceInfo())
+
+    override suspend fun testStarted(test: Test) {
+        progressReporter.testStarted(test)
+    }
+
+    override suspend fun testFailed(test: Test, startTime: Long, endTime: Long, trace: String?) {
+        progressReporter.testFailed(test)
+    }
+
+    override suspend fun testPassed(test: Test, startTime: Long, endTime: Long) {
+        progressReporter.testPassed(test)
+    }
 
     override suspend fun afterTestRun() {
         if(enhanceUsingXcresult) enhanceResults()
diff --git a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/StubDevice.kt b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/StubDevice.kt
index a0b2ec119..a6e589633 100644
--- a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/StubDevice.kt
+++ b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/StubDevice.kt
@@ -11,7 +11,6 @@ import com.malinskiy.marathon.exceptions.TestBatchExecutionException
 import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestStatus
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.log.MarathonLogging
 import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.delay
@@ -40,8 +39,7 @@ class StubDevice(
         configuration: Configuration,
         devicePoolId: DevicePoolId,
         testBatch: TestBatch,
-        deferred: CompletableDeferred<TestBatchResults>,
-        progressReporter: ProgressReporter
+        deferred: CompletableDeferred<TestBatchResults>
     ) {
         delay(testTimeMillis)
 
diff --git a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt
index 171581d8e..8d18a8589 100644
--- a/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt
+++ b/vendor/vendor-test/src/main/kotlin/com/malinskiy/marathon/test/factory/MarathonFactory.kt
@@ -42,12 +42,11 @@ class MarathonFactory {
             single { Gson() }
             single<Clock> { Clock.systemDefaultZone() }
             single { timer ?: SystemTimer(get()) }
-            single { ProgressReporter(get()) }
             single {
                 val configuration = get<Configuration>()
                 MarathonTestParseCommand(configuration.outputDir)
             }
-            single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
+            single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
         }
 
         val configurationModule = module {

From a309fed68aaacd846a226834e5aa538448d1ced0 Mon Sep 17 00:00:00 2001
From: Anton Malinskiy <anton@malinskiy.com>
Date: Sat, 4 Mar 2023 20:59:02 +1000
Subject: [PATCH 2/7] feat(core): conclusion on test result should remove
 retries

---
 .../progress/PoolProgressAccumulator.kt       | 30 +++++----
 .../execution/progress/ProgressReporter.kt    |  2 +-
 .../marathon/execution/queue/QueueActor.kt    | 61 +++++++++++--------
 3 files changed, 54 insertions(+), 39 deletions(-)

diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
index 07cc9a8a6..f2e581a33 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
@@ -271,23 +271,23 @@ class PoolProgressAccumulator(
     /**
      * @param final used for incomplete tests to signal no more retries left, hence a decision on the status has to be made
      */
-    fun testEnded(device: DeviceInfo, testResult: TestResult, final: Boolean = false) {
-        when(testResult.status) {
+    fun testEnded(device: DeviceInfo, testResult: TestResult, final: Boolean = false): TestAction? {
+        return when(testResult.status) {
             TestStatus.FAILURE -> {
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Failed(device, testResult))
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} failed")
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Failed(device, testResult)).sideffect()
             }
             TestStatus.PASSED -> {
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult))
-                println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ended")
+                println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} passed")
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult)).sideffect()
             }
             TestStatus.IGNORED, TestStatus.ASSUMPTION_FAILURE -> {
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult))
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ignored")
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult)).sideffect()
             }
             TestStatus.INCOMPLETE -> {
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Incomplete(device, testResult, final))
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} incomplete")
+                tests[testResult.test.toTestName()]?.transition(TestEvent.Incomplete(device, testResult, final)).sideffect()
             }
         }
     }
@@ -295,12 +295,12 @@ class PoolProgressAccumulator(
     /**
      * Should always be called before testEnded, otherwise the FSM might transition into a terminal state prematurely
      */
-    fun retryTest(test: Test) {
-        tests[test.toTestName()]?.transition(TestEvent.AddRetry)
+    fun retryTest(test: Test): TestAction? {
+        return tests[test.toTestName()]?.transition(TestEvent.AddRetry).sideffect()
     }
 
-    fun removeTest(test: Test, diff: Int) {
-        tests[test.toTestName()]?.transition(TestEvent.RemoveAttempts(diff))
+    fun removeTest(test: Test, diff: Int): TestAction? {
+        return tests[test.toTestName()]?.transition(TestEvent.RemoveAttempts(diff)).sideffect()
     }
 
     private fun trackTestTransition(poolId: DevicePoolId, transition: StateMachine.Transition<TestState, TestEvent, TestAction>) {
@@ -407,3 +407,11 @@ class PoolProgressAccumulator(
         const val HUNDRED_PERCENT_IN_FLOAT: Float = 100.0f
     }
 }
+
+private fun <STATE: Any, EVENT: Any, SIDE_EFFECT: Any> StateMachine.Transition<STATE, EVENT, SIDE_EFFECT>?.sideffect(): SIDE_EFFECT? {
+    return when(this) {
+        is StateMachine.Transition.Invalid -> null
+        is StateMachine.Transition.Valid -> this.sideEffect
+        null -> null
+    }
+}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
index 9a846f340..19be2590d 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
@@ -16,7 +16,7 @@ class ProgressReporter(private val batch: TestBatch, private val poolId: DeviceP
     }
 
     fun testPassed(test: Test) {
-        println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ended")
+        println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} passed")
     }
 
     fun testIgnored(test: Test) {
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
index 1c255ec5a..7d6fb40e5 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
@@ -64,15 +64,19 @@ class QueueActor(
             is QueueMessage.RequestBatch -> {
                 onRequestBatch(msg.device)
             }
+
             is QueueMessage.IsEmpty -> {
                 msg.deferred.complete(queue.isEmpty() && activeBatches.isEmpty())
             }
+
             is QueueMessage.Terminate -> {
                 onTerminate()
             }
+
             is QueueMessage.Completed -> {
                 onBatchCompleted(msg.device, msg.results)
             }
+
             is QueueMessage.ReturnBatch -> {
                 onReturnBatch(msg.device, msg.batch)
             }
@@ -108,21 +112,17 @@ class QueueActor(
             uncompletedTestsRetryCount[it.test] = (uncompletedTestsRetryCount[it.test] ?: 0) + 1
         }
 
-        if (uncompletedRetryQuotaExceeded.isNotEmpty()) {
-            logger.debug { "uncompletedRetryQuotaExceeded for ${uncompletedRetryQuotaExceeded.joinToString(separator = ", ") { it.test.toTestName() }}" }
-            val uncompletedToFailed = uncompletedRetryQuotaExceeded.map {
-                it.copy(status = TestStatus.FAILURE)
-            }
-            for (test in uncompletedToFailed) {
-                poolProgressAccumulator.testEnded(device, test, final = true)
-            }
+        for (test in uncompletedRetryQuotaExceeded) {
+            logger.debug { "uncompletedTestRetryQuota exceeded for ${test.test.toTestName()}}" }
+            val testAction = poolProgressAccumulator.testEnded(device, test, final = true)
+            processTestAction(testAction, test)
         }
 
         if (uncompleted.isNotEmpty()) {
             for (test in uncompleted) {
-                poolProgressAccumulator.testEnded(device, test, final = false)
+                val testAction = poolProgressAccumulator.testEnded(device, test, final = false)
+                processTestAction(testAction, test)
             }
-            returnTests(uncompleted.map { it.test })
         }
     }
 
@@ -149,31 +149,35 @@ class QueueActor(
         }
     }
 
-    private fun returnTests(tests: Collection<Test>) {
-        queue.addAll(tests)
+    private fun rerunTest(test: Test) {
+        queue.add(test)
     }
 
     private fun onTerminate() {
         close()
     }
 
-    private fun handleFinishedTests(finished: Collection<TestResult>, device: DeviceInfo) {
-        finished.filter { testShard.flakyTests.contains(it.test) }.let { testResults ->
-            testResults.forEach {
+    private fun processTestAction(testAction: TestAction?, testResult: TestResult) {
+        when (testAction) {
+            TestAction.Conclude -> {
+                //Test has reached final state. No need to run any of the other retries
+                //This doesn't do anything with retries currently in progress
                 val oldSize = queue.size
-                queue.removeAll(listOf(it.test))
-                /**
-                 * Important edge case:
-                 * 1. Multiple runs of test X scheduled via flaky tests
-                 * 2. One run of test X finishes and removes other non started flaky retries
-                 * 3. Another parallel run of test X finishes and should be counted towards reducing the expected tests
-                 */
-                val diff = max(oldSize - queue.size, 1)
-                poolProgressAccumulator.removeTest(it.test, diff)
+                queue.removeAll(setOf(testResult.test))
+                val diff = oldSize - queue.size
+                if (diff >= 0) {
+                    poolProgressAccumulator.removeTest(testResult.test, diff)
+                }
             }
+
+            null -> Unit
         }
+    }
+
+    private fun handleFinishedTests(finished: Collection<TestResult>, device: DeviceInfo) {
         finished.forEach {
-            poolProgressAccumulator.testEnded(device, it)
+            val testAction = poolProgressAccumulator.testEnded(device, it)
+            processTestAction(testAction, it)
         }
     }
 
@@ -186,15 +190,18 @@ class QueueActor(
 
         retryList.forEach {
             poolProgressAccumulator.retryTest(it.test)
+            val testAction = poolProgressAccumulator.testEnded(device, it)
+            processTestAction(testAction, it)
+            rerunTest(it.test)
         }
-        queue.addAll(retryList.map { it.test })
 
         val (_, noRetries) = failed.partition { testResult ->
             retryList.map { retry -> retry.test }.contains(testResult.test)
         }
 
         noRetries.forEach {
-            poolProgressAccumulator.testEnded(device, it)
+            val testAction = poolProgressAccumulator.testEnded(device, it)
+            processTestAction(testAction, it)
         }
     }
 

From 243f1c06a3a666fc6cb7208d4fd5fa423cf560d6 Mon Sep 17 00:00:00 2001
From: Anton Malinskiy <anton@malinskiy.com>
Date: Sat, 4 Mar 2023 21:15:15 +1000
Subject: [PATCH 3/7] feat(core): uncompleted tests should rerun unless
 concluded

---
 .../config/strategy/ExecutionStrategyConfiguration.kt    | 2 +-
 .../com/malinskiy/marathon/execution/queue/QueueActor.kt | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
index d2d22530e..baf7f3b3b 100644
--- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
@@ -20,7 +20,7 @@ data class ExecutionStrategyConfiguration(
  * @property ALL_SUCCESS test passes if and only if all the executions are passing
  *  this mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy
  *  
- *  Why: when adding retries to tests with retry+flakiness strategies users want to trade-off cost for realiability, i.e. add more retries
+ *  Why: when adding retries to tests with retry+flakiness strategies users want to trade-off cost for reliability, i.e. add more retries
  *  and pass if one of them passes, so retries only make sense for the [ANY_SUCCESS] mode. When we use [ALL_SUCCESS] mode it means user
  *  wants to verify each test with a number of tries (they are not retries per se) and pass only if all of them succeed. This is the case
  *  when fixing a flaky test or adding a new test, and we want to have a signal that the test is fixed/not flaky.
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
index 7d6fb40e5..9af4d6e4b 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
@@ -119,9 +119,12 @@ class QueueActor(
         }
 
         if (uncompleted.isNotEmpty()) {
-            for (test in uncompleted) {
-                val testAction = poolProgressAccumulator.testEnded(device, test, final = false)
-                processTestAction(testAction, test)
+            for (testResult in uncompleted) {
+                val testAction = poolProgressAccumulator.testEnded(device, testResult, final = false)
+                when (testAction) {
+                    TestAction.Conclude -> processTestAction(testAction, testResult)
+                    null -> rerunTest(testResult.test)
+                }
             }
         }
     }

From 3d074545f358da6e3eb3989fc4f56af1f65e517a Mon Sep 17 00:00:00 2001
From: Anton Malinskiy <anton@malinskiy.com>
Date: Sat, 4 Mar 2023 21:54:30 +1000
Subject: [PATCH 4/7] fix(core): fix tests

---
 .../resources/fixture/config/sample_1.yaml    |  4 +-
 .../resources/fixture/config/sample_1_rp.yaml |  4 +-
 .../marathon/execution/DevicePoolActor.kt     |  2 -
 .../malinskiy/marathon/execution/Scheduler.kt |  2 +-
 .../progress/PoolProgressAccumulator.kt       | 17 +++-
 .../marathon/execution/queue/QueueActor.kt    |  1 -
 .../progress/ProgressReporterTest.kt          | 27 ------
 .../execution/queue/QueueActorTest.kt         | 42 +++++----
 .../queue/TestResultReporterAllSuccessTest.kt | 26 +++---
 .../listeners/ProgressTestRunListenerTest.kt  | 91 -------------------
 .../listeners/TestResultsListenerTest.kt      | 22 ++---
 11 files changed, 67 insertions(+), 171 deletions(-)
 delete mode 100644 core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt
 delete mode 100644 vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListenerTest.kt

diff --git a/configuration/src/test/resources/fixture/config/sample_1.yaml b/configuration/src/test/resources/fixture/config/sample_1.yaml
index 100d84b45..754869033 100644
--- a/configuration/src/test/resources/fixture/config/sample_1.yaml
+++ b/configuration/src/test/resources/fixture/config/sample_1.yaml
@@ -68,7 +68,9 @@ excludeSerialRegexes:
   - "emulator-5002"
 ignoreFailures: false
 isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+  mode: ANY_SUCCESS
+  fast: true
 testBatchTimeoutMillis: 20000
 testOutputTimeoutMillis: 30000
 debug: true
diff --git a/configuration/src/test/resources/fixture/config/sample_1_rp.yaml b/configuration/src/test/resources/fixture/config/sample_1_rp.yaml
index 185b37872..7a1465277 100644
--- a/configuration/src/test/resources/fixture/config/sample_1_rp.yaml
+++ b/configuration/src/test/resources/fixture/config/sample_1_rp.yaml
@@ -66,7 +66,9 @@ excludeSerialRegexes:
   - "emulator-5002"
 ignoreFailures: false
 isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+  mode: ANY_SUCCESS
+  fast: true
 testBatchTimeoutMillis: 20000
 testOutputTimeoutMillis: 30000
 debug: true
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
index 84db4841e..715d5ce35 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
@@ -29,7 +29,6 @@ class DevicePoolActor(
     private val poolProgressAccumulator: PoolProgressAccumulator,
     analytics: Analytics,
     shard: TestShard,
-    track: Track,
     timer: Timer,
     parent: Job,
     context: CoroutineContext,
@@ -64,7 +63,6 @@ class DevicePoolActor(
         analytics,
         this,
         poolId,
-        track,
         timer,
         testBundleIdentifier,
         poolProgressAccumulator,
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
index 1eb6adf5e..037bd88f2 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
@@ -116,7 +116,7 @@ class Scheduler(
 
         pools.computeIfAbsent(poolId) { id ->
             logger.debug { "pool actor ${id.name} is being created" }
-            DevicePoolActor(id, configuration, accumulator, analytics, shard, track, timer, parent, context, testBundleIdentifier)
+            DevicePoolActor(id, configuration, accumulator, analytics, shard, timer, parent, context, testBundleIdentifier)
         }
         pools[poolId]?.send(AddDevice(device)) ?: logger.debug {
             "not sending the AddDevice event " +
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
index f2e581a33..dd7f19955 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
@@ -272,19 +272,22 @@ class PoolProgressAccumulator(
      * @param final used for incomplete tests to signal no more retries left, hence a decision on the status has to be made
      */
     fun testEnded(device: DeviceInfo, testResult: TestResult, final: Boolean = false): TestAction? {
-        return when(testResult.status) {
+        return when (testResult.status) {
             TestStatus.FAILURE -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} failed")
                 tests[testResult.test.toTestName()]?.transition(TestEvent.Failed(device, testResult)).sideffect()
             }
+
             TestStatus.PASSED -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} passed")
                 tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult)).sideffect()
             }
+
             TestStatus.IGNORED, TestStatus.ASSUMPTION_FAILURE -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ignored")
                 tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult)).sideffect()
             }
+
             TestStatus.INCOMPLETE -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} incomplete")
                 tests[testResult.test.toTestName()]?.transition(TestEvent.Incomplete(device, testResult, final)).sideffect()
@@ -323,7 +326,13 @@ class PoolProgressAccumulator(
         val testResult: TestResult? = when (event) {
             is TestEvent.Passed -> event.testResult
             is TestEvent.Failed -> event.testResult
-            is TestEvent.Incomplete -> event.testResult
+            is TestEvent.Incomplete -> {
+                if (event.final) {
+                    event.testResult.copy(status = TestStatus.FAILURE)
+                } else {
+                    event.testResult
+                }
+            }
             else -> null
         }
         val device: DeviceInfo? = when (event) {
@@ -408,8 +417,8 @@ class PoolProgressAccumulator(
     }
 }
 
-private fun <STATE: Any, EVENT: Any, SIDE_EFFECT: Any> StateMachine.Transition<STATE, EVENT, SIDE_EFFECT>?.sideffect(): SIDE_EFFECT? {
-    return when(this) {
+private fun <STATE : Any, EVENT : Any, SIDE_EFFECT : Any> StateMachine.Transition<STATE, EVENT, SIDE_EFFECT>?.sideffect(): SIDE_EFFECT? {
+    return when (this) {
         is StateMachine.Transition.Invalid -> null
         is StateMachine.Transition.Valid -> this.sideEffect
         null -> null
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
index 9af4d6e4b..a0bb72cae 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
@@ -35,7 +35,6 @@ class QueueActor(
     private val analytics: Analytics,
     private val pool: SendChannel<FromQueue>,
     private val poolId: DevicePoolId,
-    private val track: Track,
     private val timer: Timer,
     private val testBundleIdentifier: TestBundleIdentifier?,
     private val poolProgressAccumulator: PoolProgressAccumulator,
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt
deleted file mode 100644
index 699a58848..000000000
--- a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporterTest.kt
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.malinskiy.marathon.execution.progress
-
-import com.malinskiy.marathon.config.Configuration
-import com.malinskiy.marathon.config.vendor.VendorConfiguration
-import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.device.toDeviceInfo
-import com.malinskiy.marathon.test.StubDevice
-import org.amshove.kluent.shouldBeEqualTo
-import org.junit.jupiter.api.Test
-import java.io.File
-import com.malinskiy.marathon.test.Test as MarathonTest
-
-class ProgressReporterTest {
-    private val reporter = ProgressReporter(
-        Configuration.Builder(
-            name = "",
-            outputDir = File(""),
-        ).apply {
-            vendorConfiguration = VendorConfiguration.StubVendorConfiguration
-            debug = false
-            analyticsTracking = false
-        }.build()
-    )
-    private val deviceInfo = StubDevice().toDeviceInfo()
-
-    
-}
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt
index 405c28bf6..3c78df5d7 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/QueueActorTest.kt
@@ -13,6 +13,7 @@ import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestShard
 import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
 import com.nhaarman.mockitokotlin2.KArgumentCaptor
 import com.nhaarman.mockitokotlin2.any
 import com.nhaarman.mockitokotlin2.argumentCaptor
@@ -115,7 +116,7 @@ class QueueActorTest {
 
             verify(track, times(1)).test(any(), any(), testResultCaptor.capture(), any())
             testResultCaptor.firstValue.test shouldBe TEST_1
-            testResultCaptor.firstValue.status shouldBe TestStatus.FAILURE
+            testResultCaptor.firstValue.status shouldBe TestStatus.INCOMPLETE
         }
     }
 
@@ -165,7 +166,7 @@ class QueueActorTest {
 
             verify(track).test(any(), any(), testResultCaptor.capture(), any())
             testResultCaptor.firstValue.test shouldBe TEST_1
-            testResultCaptor.firstValue.status shouldBe TestStatus.FAILURE
+            testResultCaptor.firstValue.status shouldBe TestStatus.INCOMPLETE
         }
     }
 
@@ -212,7 +213,7 @@ class QueueActorTest {
         testResultCaptor = argumentCaptor<TestResult>()
         testBatchResults = createBatchResult(
             uncompleted = listOf(
-                createTestResult(TEST_1, TestStatus.FAILURE)
+                createTestResult(TEST_1, TestStatus.INCOMPLETE)
             )
         )
     }
@@ -247,24 +248,27 @@ private fun createTestResult(test: MarathonTest, status: TestStatus) = TestResul
 
 private fun createQueueActor(
     configuration: Configuration,
-    tests: List<MarathonTest>,
+    tests: List<com.malinskiy.marathon.test.Test>,
     poolChannel: SendChannel<FromQueue>,
     analytics: Analytics,
-    track: Track,
-    job: Job
-) = QueueActor(
-    configuration,
-    TestShard(tests, emptyList()),
-    analytics,
-    poolChannel,
-    DevicePoolId("test"),
-    mock(),
-    track,
-    mock(),
-    null,
-    job,
-    Dispatchers.Unconfined
-)
+    job: Job,
+    track: Track
+): QueueActor {
+    val devicePoolId = DevicePoolId("test")
+    val testShard = TestShard(tests, emptyList())
+    return QueueActor(
+        configuration,
+        testShard,
+        analytics,
+        poolChannel,
+        devicePoolId,
+        mock(),
+        null,
+        PoolProgressAccumulator(devicePoolId, testShard, configuration, track),
+        job,
+        Dispatchers.Unconfined
+  )
+}
 
 private val DEFAULT_CONFIGURATION = Configuration.Builder(
     name = "",
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt
index dfe500bfe..8a0f661cd 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporterAllSuccessTest.kt
@@ -38,7 +38,7 @@ class TestResultReporterAllSuccessTest {
         debug = false
         analyticsTracking = false
     }.build()
-    private val strictConfig = defaultConfig.copy(executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS, fast = false))
+    private val strictConfig = defaultConfig.copy(executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS, fast = true))
     val test = generateTest()
     private val poolId = DevicePoolId("test")
 
@@ -66,9 +66,9 @@ class TestResultReporterAllSuccessTest {
         val r2 = TestResult(test, deviceInfo, "stub-batch", TestStatus.FAILURE, 2, 3)
         val r3 = TestResult(test, deviceInfo, "stub-batch", TestStatus.FAILURE, 4, 5)
 
-        filter.testFinished(deviceInfo, r1)
-        filter.testFailed(deviceInfo, r2)
-        filter.testFailed(deviceInfo, r3)
+        filter.testEnded(deviceInfo, r1)
+        filter.testEnded(deviceInfo, r2)
+        filter.testEnded(deviceInfo, r3)
 
         inOrder(track) {
             verify(track).test(poolId, deviceInfo, r1, true)
@@ -86,9 +86,9 @@ class TestResultReporterAllSuccessTest {
         val r2 = TestResult(test, deviceInfo, "stub-batch", TestStatus.FAILURE, 2, 3)
         val r3 = TestResult(test, deviceInfo, "stub-batch", TestStatus.PASSED, 4, 5)
 
-        filter.testFailed(deviceInfo, r1)
-        filter.testFailed(deviceInfo, r2)
-        filter.testFinished(deviceInfo, r3)
+        filter.testEnded(deviceInfo, r1)
+        filter.testEnded(deviceInfo, r2)
+        filter.testEnded(deviceInfo, r3)
 
         inOrder(track) {
             verify(track).test(poolId, deviceInfo, r1, false)
@@ -106,9 +106,9 @@ class TestResultReporterAllSuccessTest {
         val r2 = TestResult(test, deviceInfo, "stub-batch", TestStatus.FAILURE, 2, 3)
         val r3 = TestResult(test, deviceInfo, "stub-batch", TestStatus.FAILURE, 4, 5)
 
-        filter.testFinished(deviceInfo, r1)
-        filter.testFailed(deviceInfo, r2)
-        filter.testFailed(deviceInfo, r3)
+        filter.testEnded(deviceInfo, r1)
+        filter.testEnded(deviceInfo, r2)
+        filter.testEnded(deviceInfo, r3)
 
         inOrder(track) {
             verify(track).test(poolId, deviceInfo, r1, false)
@@ -126,9 +126,9 @@ class TestResultReporterAllSuccessTest {
         val r2 = TestResult(test, deviceInfo, "stub-batch", TestStatus.PASSED, 2, 3)
         val r3 = TestResult(test, deviceInfo, "stub-batch", TestStatus.PASSED, 4, 5)
 
-        filter.testFailed(deviceInfo, r1)
-        filter.testFinished(deviceInfo, r2)
-        filter.testFinished(deviceInfo, r3)
+        filter.testEnded(deviceInfo, r1)
+        filter.testEnded(deviceInfo, r2)
+        filter.testEnded(deviceInfo, r3)
 
         inOrder(track) {
             verify(track).test(poolId, deviceInfo, r1, true)
diff --git a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListenerTest.kt b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListenerTest.kt
deleted file mode 100644
index 05f84eaa5..000000000
--- a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/ProgressTestRunListenerTest.kt
+++ /dev/null
@@ -1,91 +0,0 @@
-package com.malinskiy.marathon.android.executor.listeners
-
-import com.malinskiy.adam.AndroidDebugBridgeClient
-import com.malinskiy.adam.server.junit5.AdbClient
-import com.malinskiy.adam.server.junit5.AdbServer
-import com.malinskiy.adam.server.junit5.AdbTest
-import com.malinskiy.adam.server.stub.AndroidDebugBridgeServer
-import com.malinskiy.marathon.android.adam.TestConfigurationFactory
-import com.malinskiy.marathon.android.adam.TestDeviceFactory
-import com.malinskiy.marathon.android.adam.boot
-import com.malinskiy.marathon.android.adam.features
-import com.malinskiy.marathon.android.model.TestIdentifier
-import com.malinskiy.marathon.config.vendor.android.FileSyncConfiguration
-import com.malinskiy.marathon.config.vendor.android.FileSyncEntry
-import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.device.toDeviceInfo
-import com.malinskiy.marathon.execution.progress.ProgressReporter
-import com.nhaarman.mockitokotlin2.mock
-import com.nhaarman.mockitokotlin2.reset
-import com.nhaarman.mockitokotlin2.times
-import com.nhaarman.mockitokotlin2.verify
-import kotlinx.coroutines.runBlocking
-import org.junit.jupiter.api.Test
-
-@AdbTest
-class ProgressTestRunListenerTest {
-    @AdbClient
-    lateinit var client: AndroidDebugBridgeClient
-
-    @AdbServer
-    lateinit var server: AndroidDebugBridgeServer
-
-    @Test
-    fun testDefault() {
-        val configuration = TestConfigurationFactory.create(
-            fileSyncConfiguration = FileSyncConfiguration(
-                mutableSetOf(
-                    FileSyncEntry("screenshots")
-                )
-            )
-        )
-        val device = TestDeviceFactory.create(client, configuration, mock())
-        val progressReporter = mock<ProgressReporter>()
-        val poolId = DevicePoolId("testpool")
-        val listener = ProgressTestRunListener(device, poolId, progressReporter)
-
-        runBlocking {
-            server.multipleSessions {
-                serial("emulator-5554") {
-                    boot()
-                }
-                features("emulator-5554")
-            }
-
-            device.setup()
-
-            listener.testStarted(test0)
-            listener.testEnded(test0, mapOf("metric" to "value"))
-            verify(progressReporter, times(1)).testStarted(poolId, device.toDeviceInfo(), test0.toTest())
-            verify(progressReporter, times(1)).testPassed(poolId, device.toDeviceInfo(), test0.toTest())
-            reset(progressReporter)
-
-            listener.testStarted(test1)
-            listener.testFailed(test1, "trace")
-            listener.testEnded(test1, emptyMap())
-            verify(progressReporter, times(1)).testStarted(poolId, device.toDeviceInfo(), test1.toTest())
-            verify(progressReporter, times(1)).testFailed(poolId, device.toDeviceInfo(), test1.toTest())
-            reset(progressReporter)
-
-            listener.testStarted(test2)
-            listener.testAssumptionFailure(test2, "trace")
-            verify(progressReporter, times(1)).testStarted(poolId, device.toDeviceInfo(), test2.toTest())
-            verify(progressReporter, times(1)).testIgnored(poolId, device.toDeviceInfo(), test2.toTest())
-            reset(progressReporter)
-
-            listener.testStarted(test3)
-            listener.testIgnored(test3)
-            verify(progressReporter, times(1)).testStarted(poolId, device.toDeviceInfo(), test3.toTest())
-            verify(progressReporter, times(1)).testIgnored(poolId, device.toDeviceInfo(), test3.toTest())
-            reset(progressReporter)
-        }
-    }
-
-    companion object {
-        val test0 = TestIdentifier("com.example.Class", "method0")
-        val test1 = TestIdentifier("com.example.Class", "method1")
-        val test2 = TestIdentifier("com.example.Class", "method2")
-        val test3 = TestIdentifier("com.example.Class", "method3")
-        val test4 = TestIdentifier("com.example.Class", "method4")
-    }
-}
diff --git a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListenerTest.kt b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListenerTest.kt
index e158d7586..2e0348da4 100644
--- a/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListenerTest.kt
+++ b/vendor/vendor-android/src/test/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListenerTest.kt
@@ -23,6 +23,7 @@ import com.malinskiy.marathon.execution.AttachmentType
 import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.report.attachment.AttachmentProvider
 import com.malinskiy.marathon.test.TestBatch
 import com.nhaarman.mockitokotlin2.any
@@ -63,7 +64,7 @@ class TestResultsListenerTest {
         val deferred = CompletableDeferred<TestBatchResults>()
 
         val attachmentProvider = mock<AttachmentProvider>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), listOf(attachmentProvider))
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), listOf(attachmentProvider))
 
         runBlocking {
             server.multipleSessions {
@@ -132,7 +133,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -173,7 +174,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -224,7 +225,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -264,7 +265,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -310,7 +311,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -361,7 +362,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -412,7 +413,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -461,7 +462,7 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val listener = TestResultsListener(batch, device, deferred, mock(), mock(), DevicePoolId("omni"), emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), DevicePoolId("omni"), emptyList())
 
         runBlocking {
             server.multipleSessions {
@@ -513,9 +514,8 @@ class TestResultsListenerTest {
         )
         val device = TestDeviceFactory.create(client, configuration, mock())
         val deferred = CompletableDeferred<TestBatchResults>()
-        val progressReporter = mock<ProgressReporter>()
         val poolId = DevicePoolId("omni")
-        val listener = TestResultsListener(batch, device, deferred, mock(), progressReporter, poolId, emptyList())
+        val listener = TestResultsListener(batch, device, deferred, mock(), poolId, emptyList())
 
         runBlocking {
             server.multipleSessions {

From 162926da7aa14bbe2b161c152fa43eb10b3f20e1 Mon Sep 17 00:00:00 2001
From: Anton Malinskiy <anton@malinskiy.com>
Date: Sun, 5 Mar 2023 23:05:42 +1000
Subject: [PATCH 5/7] docs(core): update strictMode -> executionStrategy

---
 docs/docs/android/install.md                  |   1 -
 docs/docs/intro/configure.md                  | 111 ++++++++++++++++--
 .../marathon/gradle/MarathonExtension.kt      |  11 +-
 .../marathon/gradle/MarathonPlugin.kt         |   3 +-
 4 files changed, 110 insertions(+), 16 deletions(-)

diff --git a/docs/docs/android/install.md b/docs/docs/android/install.md
index f08e6b594..290103b17 100644
--- a/docs/docs/android/install.md
+++ b/docs/docs/android/install.md
@@ -121,7 +121,6 @@ marathon {
   isCodeCoverageEnabled = false
   fallbackToScreenshots = false
   testOutputTimeoutMillis = 30_000
-  strictMode = false
   debug = true
   autoGrantPermission = true
 }
diff --git a/docs/docs/intro/configure.md b/docs/docs/intro/configure.md
index d8389b40e..7233c0771 100644
--- a/docs/docs/intro/configure.md
+++ b/docs/docs/intro/configure.md
@@ -375,16 +375,113 @@ marathon {
 </TabItem>
 </Tabs>
 
-### Strict mode
-By default, if one of the test retries succeeds then the test is considered successfully executed. If you require success status only when
-all retries were executed successfully you can enable the strict mode. This may be useful to verify that flakiness of tests was fixed for
-example.
+### Execution strategy
+When executing tests with retries there are multiple trade-offs to be made. Two execution strategies are supported: any success or all success.
+By default, any success strategy is used with fast execution i.e. if one of the test retries succeeds then the test is considered successfully
+executed and all non-started retries are removed.
+
+#### Any success
+Test passes if any of executions is passing. This mode works only if there is no complex sharding strategy applied. This is the default.
+
+<Tabs>
+<TabItem value="YAML" label="Marathonfile">
+
+```yaml
+executionStrategy:
+  mode: ANY_SUCCESS
+```
+
+</TabItem>
+<TabItem value="kts" label="Kotlin DSL">
+
+```kotlin
+marathon {
+  executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ANY_SUCCESS)
+}
+```
+
+</TabItem>
+<TabItem value="groovy" label="Groovy DSL">
+
+```groovy
+marathon {
+  executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ANY_SUCCESS)
+}
+```
+
+</TabItem>
+</Tabs>
+
+:::info
+
+Complex sharding with any success mode doesn't make sense when user asks for N number of tests to run explicitly, and we pass on the first one.
+
+:::
+
+#### All success
+Test passes if and only if all the executions are passing. This mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy.
+
+<Tabs>
+<TabItem value="YAML" label="Marathonfile">
+
+```yaml
+executionStrategy:
+  mode: ALL_SUCCESS
+```
+
+</TabItem>
+<TabItem value="kts" label="Kotlin DSL">
+
+```kotlin
+marathon {
+  executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS)
+}
+```
+
+</TabItem>
+<TabItem value="groovy" label="Groovy DSL">
+
+```groovy
+marathon {
+  executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS)
+}
+```
+
+</TabItem>
+</Tabs>
+
+:::info
+
+Adding retries with retry/flakiness strategies means users wants to trade-off cost for reliability, i.e. add more retries and pass if one 
+of test retries passes, so retries only make sense for any success mode. 
+
+When we use all success mode it means user wants to verify each test with a number of tries (they are not retries per se) and pass only if 
+all of them succeed. This is the case when fixing a flaky test or adding a new test, and we want to have a signal that the test is fixed/not flaky.
+
+:::
+
+#### Fast execution mode
+When test reaches a state when a decision about it's state can be made marathon can remove additional in-progress retries.
+This decision point is different depending on the execution mode used. Let's walk through two examples.
+
+Assume any success strategy is used and 100 retries are scheduled for a test A via flakiness strategy. Let's say first 3 failed and the 4th attempt succeeded. At 
+this point the test should already be considered passed since any success out of all retries leads to the result by definition of any successs
+execution strategy. To save cost one can remove additional non-started retries by using fast execution mode (this is the default behavior for
+any success strategy). On the other hand one could disable fast execution and get much more accurate statistics about this test by executing
+more retries and calculating the probability of passing as a measure of flakiness for test A.
+
+Assume all success strategy is used and 100 retries are scheduled using sharding strategy. Let's say first 3 passed and the 4th attempt failed.
+At this point the test should already be considered failed since any failure out of all retries leads to the result by definition of all success
+execution strategy. You can save cost by removing additional non-started retries by using fast execution mode (this is the default behaviour for
+all success strategy). On the other hand one could disable fast execution and verify the flakiness rate with a defined precision, in this case
+there are 100 retries, so you would get precision up to 1% for test A.
 
 <Tabs>
 <TabItem value="YAML" label="Marathonfile">
 
 ```yaml
-strictMode: true
+executionStrategy:
+  fast: true
 ```
 
 </TabItem>
@@ -392,7 +489,7 @@ strictMode: true
 
 ```kotlin
 marathon {
-  strictMode = true
+  executionStrategy = ExecutionStrategyConfiguration(fast = true)
 }
 ```
 
@@ -401,7 +498,7 @@ marathon {
 
 ```groovy
 marathon {
-  strictMode = true
+  executionStrategy = ExecutionStrategyConfiguration(ExecutionMode.ALL_SUCCESS, true)
 }
 ```
 
diff --git a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt
index 6bf9bedd3..9c21542a7 100644
--- a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt
+++ b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt
@@ -1,7 +1,7 @@
 package com.malinskiy.marathon.gradle
 
 import com.malinskiy.marathon.config.ScreenRecordingPolicy
-import com.malinskiy.marathon.config.vendor.VendorConfiguration
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
 import com.malinskiy.marathon.config.vendor.android.AdbEndpoint
 import com.malinskiy.marathon.config.vendor.android.AllureConfiguration
 import com.malinskiy.marathon.config.vendor.android.FileSyncConfiguration
@@ -169,13 +169,12 @@ open class MarathonExtension {
      */
     var isCodeCoverageEnabled: Boolean? = null
 
-
     /**
-     * By default, if one of the test retries succeeds then the test is considered successfully executed. If you require success status only when
-     * all retries were executed successfully you can enable the strict mode. This may be useful to verify that flakiness of tests was fixed for
-     * example.
+     * When executing tests with retries there are multiple trade-offs to be made. Two execution strategies are supported: any success or all success.
+     * By default, any success strategy is used with fast execution i.e. if one of the test retries succeeds then the test is considered successfully
+     * executed and all non-started retries are removed.
      */
-    var strictMode: Boolean? = null
+    var executionStrategy: ExecutionStrategyConfiguration? = null
 
     /**
      * By default, tests that don't have any status reported after execution (for example a device disconnected during the execution) retry
diff --git a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt
index c79560a6d..c3c950a98 100644
--- a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt
+++ b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt
@@ -5,8 +5,6 @@ import com.android.build.api.variant.AndroidTest
 import com.android.build.api.variant.ApplicationAndroidComponentsExtension
 import com.android.build.api.variant.BuiltArtifactsLoader
 import com.android.build.api.variant.LibraryAndroidComponentsExtension
-import com.android.build.gradle.AppPlugin
-import com.android.build.gradle.LibraryPlugin
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.config.vendor.VendorConfiguration
 import com.malinskiy.marathon.gradle.configuration.toStrategy
@@ -175,6 +173,7 @@ class MarathonPlugin : Plugin<Project> {
                 config.filteringConfiguration?.toFilteringConfiguration()?.let { filteringConfiguration = it }
                 config.ignoreFailures?.let { ignoreFailures = it }
                 config.isCodeCoverageEnabled?.let { isCodeCoverageEnabled = it }
+                config.executionStrategy?.let { executionStrategy = it }
                 config.uncompletedTestRetryQuota?.let { uncompletedTestRetryQuota = it }
                 config.testClassRegexes?.map { it.toRegex() }?.let { testClassRegexes = it }
                 config.includeSerialRegexes?.map { it.toRegex() }?.let { includeSerialRegexes = it }

From 8abe591ca7e9668de1694fc66685b6f36575069f Mon Sep 17 00:00:00 2001
From: Anton Malinskiy <anton@malinskiy.com>
Date: Sun, 5 Mar 2023 23:38:17 +1000
Subject: [PATCH 6/7] fix(core): address review comments

---
 .../marathon/config/Configuration.kt          |  1 +
 .../config/LogicalConfigurationValidator.kt   | 24 ++++++++
 .../malinskiy/marathon/execution/Scheduler.kt |  3 +
 .../progress/PoolProgressAccumulator.kt       | 57 ++++++++++---------
 .../marathon/execution/queue/QueueActor.kt    |  6 +-
 .../marathon/execution/queue/TestAction.kt    |  6 +-
 .../executor/listeners/TestResultsListener.kt |  1 -
 .../executor/listener/TestResultsListener.kt  | 14 -----
 8 files changed, 66 insertions(+), 46 deletions(-)

diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
index 956f0629f..5dc0f7939 100644
--- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
@@ -142,6 +142,7 @@ data class Configuration private constructor(
         result = 31 * result + deviceInitializationTimeoutMillis.hashCode()
         return result
     }
+
      data class Builder(
          val name: String,
          val outputDir: File,
diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt
index 559b062e8..f734f7c6e 100644
--- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt
@@ -1,7 +1,9 @@
 package com.malinskiy.marathon.config
 
 import com.malinskiy.marathon.config.exceptions.ConfigurationException
+import com.malinskiy.marathon.config.strategy.ExecutionMode
 import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.ShardingStrategyConfiguration
 import com.malinskiy.marathon.config.vendor.VendorConfiguration
 
@@ -28,5 +30,27 @@ class LogicalConfigurationValidator : ConfigurationValidator {
 
             else -> Unit
         }
+
+        when(configuration.executionStrategy.mode) {
+            ExecutionMode.ANY_SUCCESS -> {
+                if (configuration.shardingStrategy !is ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration) {
+                    throw ConfigurationException(
+                        "Configuration is invalid: can't use complex sharding and any success execution strategy at the same time. Consult documentation for the any success execution logic"
+                    )
+                }
+            }
+            ExecutionMode.ALL_SUCCESS -> {
+                if (configuration.flakinessStrategy !is FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration) {
+                    throw ConfigurationException(
+                        "Configuration is invalid: can't use complex flakiness strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+                    )
+                }
+                if (configuration.retryStrategy !is RetryStrategyConfiguration.NoRetryStrategyConfiguration) {
+                    throw ConfigurationException(
+                        "Configuration is invalid: can't use complex retry strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+                    )
+                }
+            }
+        }
     }
 }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
index 037bd88f2..4e4cf89d8 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
@@ -50,6 +50,9 @@ class Scheduler(
 
     private val logger = MarathonLogging.logger("Scheduler")
 
+    /**
+     * @return true if scheduled tests passed successfully, false otherwise
+     */
     suspend fun execute() : Boolean {
         subscribeOnDevices(job)
         try {
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
index dd7f19955..0516c8e3c 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
@@ -37,7 +37,7 @@ class PoolProgressAccumulator(
                 when (executionStrategy.mode) {
                     ExecutionMode.ANY_SUCCESS -> {
                         if (executionStrategy.fast || total <= 1) {
-                            transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Passing(total = total, done = 1, running = running - 1))
                         }
@@ -45,7 +45,7 @@ class PoolProgressAccumulator(
 
                     ExecutionMode.ALL_SUCCESS -> {
                         if (total <= 1) {
-                            transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Passing(total = total, done = 1, running = running - 1))
                         }
@@ -56,7 +56,7 @@ class PoolProgressAccumulator(
                 when (executionStrategy.mode) {
                     ExecutionMode.ANY_SUCCESS -> {
                         if (total <= 1) {
-                            transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Failing(total = total, done = 1, running = running - 1))
                         }
@@ -64,7 +64,7 @@ class PoolProgressAccumulator(
 
                     ExecutionMode.ALL_SUCCESS -> {
                         if (executionStrategy.fast || total <= 1) {
-                            transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Failing(total = total, done = 1, running = running - 1))
                         }
@@ -73,7 +73,7 @@ class PoolProgressAccumulator(
             }
             on<TestEvent.Incomplete> {
                 if (it.final) {
-                    transitionTo(TestState.Failed(total = total, done = 0, running = running - 1), TestAction.Conclude)
+                    transitionTo(TestState.Failed(total = total, done = 0, running = running - 1), TestAction.Complete)
                 } else {
                     transitionTo(TestState.Added(total = total, running = running - 1))
                 }
@@ -93,7 +93,7 @@ class PoolProgressAccumulator(
                 when (executionStrategy.mode) {
                     ExecutionMode.ANY_SUCCESS -> {
                         if (executionStrategy.fast || done + 1 >= total) {
-                            transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Conclude)
+                            transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Passing(total = total, running = running - 1, done = done + 1))
                         }
@@ -101,7 +101,7 @@ class PoolProgressAccumulator(
 
                     ExecutionMode.ALL_SUCCESS -> {
                         if (total <= done + 1) {
-                            transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Conclude)
+                            transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Passing(total = total, running = running - 1, done = done + 1))
                         }
@@ -112,7 +112,7 @@ class PoolProgressAccumulator(
                 when (executionStrategy.mode) {
                     ExecutionMode.ANY_SUCCESS -> {
                         if (executionStrategy.fast || done + 1 >= total) {
-                            transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Passing(total = total, done = done + 1, running = running - 1))
                         }
@@ -120,7 +120,7 @@ class PoolProgressAccumulator(
 
                     ExecutionMode.ALL_SUCCESS -> {
                         if (executionStrategy.fast || done + 1 >= total) {
-                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
                         }
@@ -131,11 +131,11 @@ class PoolProgressAccumulator(
                 if (it.final) {
                     when (executionStrategy.mode) {
                         ExecutionMode.ANY_SUCCESS -> {
-                            transitionTo(TestState.Passed(total = total, done = done, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Passed(total = total, done = done, running = running - 1), TestAction.Complete)
                         }
 
                         ExecutionMode.ALL_SUCCESS -> {
-                            transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Complete)
                         }
                     }
                 } else {
@@ -157,7 +157,7 @@ class PoolProgressAccumulator(
                 when (executionStrategy.mode) {
                     ExecutionMode.ANY_SUCCESS -> {
                         if (executionStrategy.fast || done + 1 >= total) {
-                            transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Passing(total = total, done = done + 1, running = running - 1))
                         }
@@ -165,7 +165,7 @@ class PoolProgressAccumulator(
 
                     ExecutionMode.ALL_SUCCESS -> {
                         if (executionStrategy.fast || done + 1 >= total) {
-                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
                         }
@@ -176,7 +176,7 @@ class PoolProgressAccumulator(
                 when (executionStrategy.mode) {
                     ExecutionMode.ANY_SUCCESS -> {
                         if (done + 1 >= total) {
-                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
                         }
@@ -184,7 +184,7 @@ class PoolProgressAccumulator(
 
                     ExecutionMode.ALL_SUCCESS -> {
                         if (executionStrategy.fast || done + 1 >= total) {
-                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Conclude)
+                            transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
                         } else {
                             transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
                         }
@@ -193,7 +193,7 @@ class PoolProgressAccumulator(
             }
             on<TestEvent.Incomplete> {
                 if (it.final) {
-                    transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Conclude)
+                    transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Complete)
                 } else {
                     transitionTo(TestState.Failing(total = total, done = done, running = running - 1))
                 }
@@ -264,7 +264,7 @@ class PoolProgressAccumulator(
     }
 
     fun testStarted(device: DeviceInfo, test: Test) {
-        tests[test.toTestName()]?.transition(TestEvent.Started)
+        transition(test, TestEvent.Started)
         println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started")
     }
 
@@ -275,42 +275,43 @@ class PoolProgressAccumulator(
         return when (testResult.status) {
             TestStatus.FAILURE -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} failed")
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Failed(device, testResult)).sideffect()
+                transition(testResult.test, TestEvent.Failed(device, testResult)).sideffect()
             }
 
             TestStatus.PASSED -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} passed")
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult)).sideffect()
+                transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect()
             }
 
             TestStatus.IGNORED, TestStatus.ASSUMPTION_FAILURE -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ignored")
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult)).sideffect()
+                transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect()
             }
 
             TestStatus.INCOMPLETE -> {
                 println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} incomplete")
-                tests[testResult.test.toTestName()]?.transition(TestEvent.Incomplete(device, testResult, final)).sideffect()
+                transition(testResult.test, TestEvent.Incomplete(device, testResult, final)).sideffect()
             }
         }
     }
 
+
     /**
      * Should always be called before testEnded, otherwise the FSM might transition into a terminal state prematurely
      */
     fun retryTest(test: Test): TestAction? {
-        return tests[test.toTestName()]?.transition(TestEvent.AddRetry).sideffect()
+        return transition(test, TestEvent.AddRetry).sideffect()
     }
 
     fun removeTest(test: Test, diff: Int): TestAction? {
-        return tests[test.toTestName()]?.transition(TestEvent.RemoveAttempts(diff)).sideffect()
+        return transition(test, TestEvent.RemoveAttempts(diff)).sideffect()
     }
 
     private fun trackTestTransition(poolId: DevicePoolId, transition: StateMachine.Transition<TestState, TestEvent, TestAction>) {
         val validTransition = transition as? StateMachine.Transition.Valid
         val final = if (validTransition is StateMachine.Transition.Valid) {
             when (validTransition.sideEffect) {
-                is TestAction.Conclude -> true
+                is TestAction.Complete -> true
                 else -> false
             }
         } else false
@@ -362,8 +363,8 @@ class PoolProgressAccumulator(
                     is TestState.Passed -> true
                     is TestState.Passing -> {
                         logger.error { "Expected to run ${it.key} more, but no terminal events received" }
-                        //The test is passing but the execution mode might require all the 
-                        //runs to pass before considering this an actual pass 
+                        //The test is passing but the execution mode might require all the
+                        //runs to pass before considering this an actual pass
                         executionStrategy.mode == ExecutionMode.ANY_SUCCESS
                     }
                 }
@@ -406,6 +407,10 @@ class PoolProgressAccumulator(
         return done.toFloat() / total.toFloat()
     }
 
+    private fun transition(test: Test, transition: TestEvent): StateMachine.Transition<TestState, TestEvent, TestAction>? {
+        return tests[test.toTestName()]?.transition(transition)?.also { logger.warn { "No FSM registered for test ${test.toTestName()}" } }
+    }
+
     private fun toPercent(float: Float): String {
         val percent = (float * HUNDRED_PERCENT_IN_FLOAT).roundToInt()
         val format = "%02d%%"
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
index a0bb72cae..f5fe2517a 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
@@ -2,7 +2,6 @@ package com.malinskiy.marathon.execution.queue
 
 import com.malinskiy.marathon.actor.Actor
 import com.malinskiy.marathon.analytics.external.Analytics
-import com.malinskiy.marathon.analytics.internal.pub.Track
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.device.DeviceInfo
 import com.malinskiy.marathon.device.DevicePoolId
@@ -27,7 +26,6 @@ import kotlinx.coroutines.channels.SendChannel
 import java.util.PriorityQueue
 import java.util.Queue
 import kotlin.coroutines.CoroutineContext
-import kotlin.math.max
 
 class QueueActor(
     private val configuration: Configuration,
@@ -121,7 +119,7 @@ class QueueActor(
             for (testResult in uncompleted) {
                 val testAction = poolProgressAccumulator.testEnded(device, testResult, final = false)
                 when (testAction) {
-                    TestAction.Conclude -> processTestAction(testAction, testResult)
+                    TestAction.Complete -> processTestAction(testAction, testResult)
                     null -> rerunTest(testResult.test)
                 }
             }
@@ -161,7 +159,7 @@ class QueueActor(
 
     private fun processTestAction(testAction: TestAction?, testResult: TestResult) {
         when (testAction) {
-            TestAction.Conclude -> {
+            TestAction.Complete -> {
                 //Test has reached final state. No need to run any of the other retries
                 //This doesn't do anything with retries currently in progress
                 val oldSize = queue.size
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
index 210b47df4..ed2255771 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
@@ -1,5 +1,9 @@
 package com.malinskiy.marathon.execution.queue
 
 sealed class TestAction {
-    object Conclude : TestAction()
+    /**
+     * Indicates that test reached terminal state of no return according to the current execution strategy logic
+     * Test outputs can be produced after this action and some logic about retries left can be executed
+     */
+    object Complete : TestAction()
 }
diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt
index 4b1faacf4..31a7a6e77 100644
--- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt
+++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestResultsListener.kt
@@ -29,7 +29,6 @@ class TestResultsListener(
 ) : AccumulatingResultTestRunListener(timer), AttachmentListener by attachmentCollector {
 
     private val logger = MarathonLogging.logger("TestRunResultsListener")
-    private val progressReporter = ProgressReporter(testBatch, poolId, device.toDeviceInfo())
 
     override suspend fun afterTestRun() {
         val results = mergeParameterisedResults(runResult.temporalTestResults)
diff --git a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt
index a7ede1a9c..ba9c8114a 100644
--- a/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt
+++ b/vendor/vendor-ios/src/main/kotlin/com/malinskiy/marathon/ios/executor/listener/TestResultsListener.kt
@@ -6,7 +6,6 @@ import com.malinskiy.marathon.device.toDeviceInfo
 import com.malinskiy.marathon.execution.TestBatchResults
 import com.malinskiy.marathon.execution.TestResult
 import com.malinskiy.marathon.execution.TestStatus
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.execution.result.TemporalTestResult
 import com.malinskiy.marathon.ios.RemoteFileManager
 import com.malinskiy.marathon.ios.bin.xcrun.xcresulttool.ResultBundleFormat
@@ -38,19 +37,6 @@ class TestResultsListener(
 
     private val logger = MarathonLogging.logger {}
     private val enhanceUsingXcresult = true
-    private val progressReporter = ProgressReporter(testBatch, devicePoolId, device.toDeviceInfo())
-
-    override suspend fun testStarted(test: Test) {
-        progressReporter.testStarted(test)
-    }
-
-    override suspend fun testFailed(test: Test, startTime: Long, endTime: Long, trace: String?) {
-        progressReporter.testFailed(test)
-    }
-
-    override suspend fun testPassed(test: Test, startTime: Long, endTime: Long) {
-        progressReporter.testPassed(test)
-    }
 
     override suspend fun afterTestRun() {
         if(enhanceUsingXcresult) enhanceResults()

From 07b25eaaafe323ac75098c3bb7f06767379c26a8 Mon Sep 17 00:00:00 2001
From: Anton Malinskiy <anton@malinskiy.com>
Date: Mon, 6 Mar 2023 11:44:30 +1000
Subject: [PATCH 7/7] docs(docs): address review comments

---
 docs/docs/intro/configure.md | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/docs/docs/intro/configure.md b/docs/docs/intro/configure.md
index 7233c0771..4ddd32f59 100644
--- a/docs/docs/intro/configure.md
+++ b/docs/docs/intro/configure.md
@@ -377,11 +377,11 @@ marathon {
 
 ### Execution strategy
 When executing tests with retries there are multiple trade-offs to be made. Two execution strategies are supported: any success or all success.
-By default, any success strategy is used with fast execution i.e. if one of the test retries succeeds then the test is considered successfully
+By default, `ANY_SUCCESS` strategy is used with fast execution i.e. if one of the test retries succeeds then the test is considered successfully
 executed and all non-started retries are removed.
 
 #### Any success
-Test passes if any of executions is passing. This mode works only if there is no complex sharding strategy applied. This is the default.
+Test passes if any of its executions are passing. This mode works only if there is no complex sharding strategy applied. This is the default.
 
 <Tabs>
 <TabItem value="YAML" label="Marathonfile">
@@ -414,12 +414,12 @@ marathon {
 
 :::info
 
-Complex sharding with any success mode doesn't make sense when user asks for N number of tests to run explicitly, and we pass on the first one.
+Complex sharding with `ANY_SUCCESS` mode doesn't make sense when user asks for N number of tests to run explicitly, and we pass on the first one.
 
 :::
 
 #### All success
-Test passes if and only if all the executions are passing. This mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy.
+Test passes if and only if all its executions are passing. This mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy.
 
 <Tabs>
 <TabItem value="YAML" label="Marathonfile">
@@ -452,28 +452,28 @@ marathon {
 
 :::info
 
-Adding retries with retry/flakiness strategies means users wants to trade-off cost for reliability, i.e. add more retries and pass if one 
-of test retries passes, so retries only make sense for any success mode. 
+Adding retries with retry/flakiness strategies means users wants to trade off cost for reliability, i.e. add more retries and pass if one 
+of test retries passes, so retries only make sense for `ANY_SUCCESS` mode. 
 
-When we use all success mode it means user wants to verify each test with a number of tries (they are not retries per se) and pass only if 
+When we use `ALL_SUCCESS` mode it means the user want to verify each test with a number of tries (they are not retries per se) and pass only if 
 all of them succeed. This is the case when fixing a flaky test or adding a new test, and we want to have a signal that the test is fixed/not flaky.
 
 :::
 
 #### Fast execution mode
-When test reaches a state when a decision about it's state can be made marathon can remove additional in-progress retries.
+When the test reaches a state where a decision about its state can be made, marathon can remove additional in-progress retries.
 This decision point is different depending on the execution mode used. Let's walk through two examples.
 
-Assume any success strategy is used and 100 retries are scheduled for a test A via flakiness strategy. Let's say first 3 failed and the 4th attempt succeeded. At 
-this point the test should already be considered passed since any success out of all retries leads to the result by definition of any successs
+Assume `ANY_SUCCESS` strategy is used and 100 retries are scheduled for a test A via flakiness strategy. Let's say first 3 failed and the 4th attempt succeeded. At 
+this point the test should already be considered passed since `ANY_SUCCESS` out of all retries leads to the result by definition of `ANY_SUCCESS`
 execution strategy. To save cost one can remove additional non-started retries by using fast execution mode (this is the default behavior for
-any success strategy). On the other hand one could disable fast execution and get much more accurate statistics about this test by executing
+`ANY_SUCCESS` strategy). On the other hand one could disable fast execution and get much more accurate statistics about this test by executing
 more retries and calculating the probability of passing as a measure of flakiness for test A.
 
-Assume all success strategy is used and 100 retries are scheduled using sharding strategy. Let's say first 3 passed and the 4th attempt failed.
-At this point the test should already be considered failed since any failure out of all retries leads to the result by definition of all success
+Assume `ALL_SUCCESS` strategy is used and 100 retries are scheduled using sharding strategy. Let's say first 3 passed and the 4th attempt failed.
+At this point the test should already be considered failed since any failure out of all retries leads to the result by definition of `ALL_SUCCESS`
 execution strategy. You can save cost by removing additional non-started retries by using fast execution mode (this is the default behaviour for
-all success strategy). On the other hand one could disable fast execution and verify the flakiness rate with a defined precision, in this case
+`ALL_SUCCESS` strategy). On the other hand one could disable fast execution and verify the flakiness rate with a defined precision, in this case
 there are 100 retries, so you would get precision up to 1% for test A.
 
 <Tabs>