Compare control to the best variant

PostHog · Jan 23, 2025 · 25a4231 · 25a4231
1 parent 7ff87f2
commit 25a4231
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 10 deletions.
diff --git a/posthog/hogql_queries/experiments/funnels_statistics_v2.py b/posthog/hogql_queries/experiments/funnels_statistics_v2.py
@@ -38,7 +38,7 @@ def calculate_probabilities_v2(
     --------
     list[float]
         A list of probabilities where each element represents:
-        - index 0: probability control variant beats all test variants
+        - index 0: probability control beats the best test variant
         - index i>0: probability test variant i-1 beats control
 
     Notes:
@@ -70,12 +70,15 @@ def calculate_probabilities_v2(
         samples.append(variant_samples)
 
     samples_array = np.array(samples)
-    # Calculate probability of each variant beating the control
     probabilities = []
     control_samples = samples_array[0]  # Control is always first variant
 
-    # Control's probability is of being better than all variants
-    control_prob = np.mean(np.all(control_samples >= samples_array[1:], axis=0))
+    # Find the best test variant at each sample point
+    test_variants_samples = samples_array[1:]
+    best_variant_samples = np.max(test_variants_samples, axis=0)
+
+    # Control's probability is of being better than the best test variant
+    control_prob = np.mean(control_samples >= best_variant_samples)
     probabilities.append(float(control_prob))
 
     # For each test variant, calculate probability of beating control

diff --git a/posthog/hogql_queries/experiments/trends_statistics_v2_continuous.py b/posthog/hogql_queries/experiments/trends_statistics_v2_continuous.py
@@ -45,7 +45,7 @@ def calculate_probabilities_v2_continuous(
     --------
     list[float]
         A list of probabilities where each element represents:
-        - index 0: probability control variant beats all test variants
+        - index 0: probability control variant beats the best test variant
         - index i>0: probability test variant i-1 beats control
 
     Notes:
@@ -116,8 +116,9 @@ def calculate_probabilities_v2_continuous(
     # Calculate probabilities
     probabilities = []
 
-    # Probability control wins (beats all test variants)
-    control_wins = np.all([samples_control > test_sample for test_sample in test_samples], axis=0)
+    # Probability control wins (beats the best test variant)
+    best_test_samples = np.max(test_samples, axis=0)  # Get best test variant at each sample point
+    control_wins = samples_control > best_test_samples
     probabilities.append(float(np.mean(control_wins)))
 
     # Probability each test variant wins (beats control only)

diff --git a/posthog/hogql_queries/experiments/trends_statistics_v2_count.py b/posthog/hogql_queries/experiments/trends_statistics_v2_count.py
@@ -37,7 +37,7 @@ def calculate_probabilities_v2_count(
     --------
     list[float]
         A list of probabilities where each element represents:
-        - index 0: probability control variant beats all test variants
+        - index 0: probability control variant beats the best test variant
         - index i>0: probability test variant i-1 beats control
 
     Notes:
@@ -78,8 +78,9 @@ def calculate_probabilities_v2_count(
     # Calculate probabilities
     probabilities = []
 
-    # Probability control wins (beats all test variants)
-    control_wins = np.all([samples_control > test_sample for test_sample in test_samples], axis=0)
+    # Probability control wins (beats the best test variant)
+    best_test_samples = np.max(test_samples, axis=0)
+    control_wins = samples_control > best_test_samples
     probabilities.append(float(np.mean(control_wins)))
 
     # Probability each test variant wins (beats control only)