fix(specs): a/b testing (#2981)

Co-authored-by: Clément Vannicatte <vannicattec@gmail.com>
Co-authored-by: Thomas Raffray <Fluf22@users.noreply.github.com>

Author: 3 people

.github/workflows/check.yml

@@ -328,6 +328,7 @@ jobs:
         run: |
           set -eo pipefail
           cd clients/algoliasearch-client-swift
+          find Sources  -type f | rev | cut -d '/' -f1 | rev | sort | uniq -d
           [ $(find Sources  -type f | rev | cut -d '/' -f1 | rev | sort | uniq -d | wc -l) -gt 0 ] && echo "Duplicates found" && exit 1 || echo "No duplicate found"
 
       - name: Build clients

generators/src/main/java/com/algolia/codegen/AlgoliaSwiftGenerator.java

@@ -97,6 +97,7 @@ public class AlgoliaSwiftGenerator extends Swift5ClientCodegen {
     "snippetresultoption",
     "sortremainingby",
     "source",
+    "status",
     "supportedlanguage",
     "tagfilters",
     "taskstatus",

specs/abtesting/common/parameters.yml

@@ -2,11 +2,10 @@
 ID:
   in: path
   name: id
-  description: Unique A/B test ID.
+  description: Unique A/B test identifier.
   required: true
   schema:
-    type: integer
-    example: 390
+    $ref: '#/abTestID'
 
 # misc
 index:
@@ -16,37 +15,39 @@ index:
 
 abTestID:
   type: integer
-  description: Unique A/B test ID.
+  description: Unique A/B test identifier.
   example: 224
 
 endAt:
   type: string
-  description: End date timestamp in [ISO-8601](https://wikipedia.org/wiki/ISO_8601) format.
-  example: '2023-06-17T00:00:00Z'
+  description: End date and time of the A/B test, in RFC 3339 format.
+  example: 2023-06-17T00:00:00Z
 
 createdAt:
   type: string
-  description: Creation date timestamp in [ISO-8601](https://wikipedia.org/wiki/ISO_8601) format.
-  example: '2023-06-15T15:06:04.249906Z'
+  description: Date and time when the A/B test was created, in RFC 3339 format.
+  example: 2023-06-15T15:06:04.249906Z
 
 updatedAt:
   type: string
-  description: Update date timestamp in [ISO-8601](https://wikipedia.org/wiki/ISO_8601) format.
-  example: '2023-06-15T15:06:44.400601Z'
+  description: Date and time when the A/B test was last updated, in RFC 3339 format.
+  example: 2023-06-15T15:06:44.400601Z
 
 name:
   type: string
   description: A/B test name.
-  example: 'Custom ranking sales rank test'
+  example: Custom ranking sales rank test
 
 description:
   type: string
-  description: A/B test description.
-  example: 'Current production index'
+  description: Description for this variant.
+  example: Current production index
 
 trafficPercentage:
   type: integer
-  description: A/B test traffic percentage.
+  description: Percentage of search requests each variant receives.
+  minimum: 0
+  maximum: 100
   example: 60
 
 currencies:
@@ -65,6 +66,7 @@ currencies:
       standardDeviation: 10.3
   additionalProperties:
     $ref: '#/currency'
+    x-additionalPropertiesName: currency code
 
 currency:
   type: object

specs/abtesting/common/schemas/ABTest.yml

@@ -5,6 +5,7 @@ ABTests:
       items:
         $ref: '#/ABTest'
     - type: 'null'
+      description: No A/B tests are configured for this application.
 
 ABTest:
   type: object
@@ -13,45 +14,62 @@ ABTest:
     abTestID:
       $ref: '../parameters.yml#/abTestID'
     clickSignificance:
+      description: |
+        A/B test significance calculated from click events.
+
+        Values of 0.95 or higher can be considered significant,
+        that is, the difference between A and B variants is _not_ due to random variations.
+        Lower values have a.
       oneOf:
         - type: number
           format: double
-          description: >
-            [A/B test significance](https://www.algolia.com/doc/guides/ab-testing/what-is-ab-testing/in-depth/how-ab-test-scores-are-calculated/#statistical-significance-or-chance) based on click data. A value of 0.95 or over is considered to be _significant_.
           example: 1
         - type: 'null'
     conversionSignificance:
+      description: |
+        A/B test significance calculated from conversion events.
+
+        Values of 0.95 or higher can be considered significant,
+        that is, the difference between A and B variants is _not_ due to random variations.
       oneOf:
         - type: number
           format: double
-          description: >
-            [A/B test significance](https://www.algolia.com/doc/guides/ab-testing/what-is-ab-testing/in-depth/how-ab-test-scores-are-calculated/#statistical-significance-or-chance) based on conversion. A value of 0.95 or over is considered to be _significant_.
           example: 1
         - type: 'null'
     addToCartSignificance:
+      description: |
+        A/B test significance calculated from add-to-cart events.
+
+        Values of 0.95 or higher can be considered significant,
+        that is, the difference between A and B variants is _not_ due to random variations.
       oneOf:
         - type: number
           format: double
-          description: >
-            [A/B test significance](https://www.algolia.com/doc/guides/ab-testing/what-is-ab-testing/in-depth/how-ab-test-scores-are-calculated/#statistical-significance-or-chance) based on add-to-cart data. A value of 0.95 or over is considered to be _significant_.
           example: 1
         - type: 'null'
     purchaseSignificance:
+      description: |
+        A/B test significance calculated from purchase events.
+
+        Values of 0.95 or higher can be considered significant,
+        that is, the difference between A and B variants is _not_ due to random variations.
       oneOf:
         - type: number
           format: double
-          description: >
-            [A/B test significance](https://www.algolia.com/doc/guides/ab-testing/what-is-ab-testing/in-depth/how-ab-test-scores-are-calculated/#statistical-significance-or-chance) based on purchase data. A value of 0.95 or over is considered to be _significant_.
           example: 1
         - type: 'null'
     revenueSignificance:
+      description: |
+        A/B test significance calculated from revenue data.
+
+        Values of 0.95 or higher can be considered significant,
+        that is, the difference between A and B variants is _not_ due to random variations.
       oneOf:
         - type: object
           additionalProperties:
             type: number
             format: double
-          description: >
-            [A/B test significance](https://www.algolia.com/doc/guides/ab-testing/what-is-ab-testing/in-depth/how-ab-test-scores-are-calculated/#statistical-significance-or-chance) based on revenue data. A value of 0.95 or over is considered to be _significant_.
+            x-additionalPropertiesName: currency code
           example:
             USD: 1
             EUR: 0.87
@@ -65,11 +83,11 @@ ABTest:
     name:
       $ref: '../parameters.yml#/name'
     status:
-      type: string
-      description: A/B test status.
-      example: 'running'
+      $ref: '#/Status'
     variants:
       $ref: 'Variant.yml#/variants'
+    configuration:
+      $ref: '#/ABTestConfiguration'
   required:
     - status
     - name
@@ -83,3 +101,74 @@ ABTest:
     - revenueSignificance
     - abTestID
     - variants
+
+Status:
+  type: string
+  description: |
+    A/B test status.
+
+    - `active`. The A/B test is live and search traffic is split between the two variants.
+    - `stopped`. You stopped the A/B test. The A/B test data is still available for analysis.
+    - `expired`. The A/B test was automatically stopped after reaching its end date.
+    - `failed`. Creating the A/B test failed.
+  example: active
+  enum:
+    - active
+    - stopped
+    - expired
+    - failed
+
+ABTestConfiguration:
+  title: configuration
+  type: object
+  description: A/B test configuration.
+  properties:
+    outliers:
+      $ref: '#/Outliers'
+    emptySearch:
+      $ref: '#/EmptySearch'
+    minimumDetectableEffect:
+      $ref: '#/MinimumDetectableEffect'
+  required:
+    - outliers
+
+Outliers:
+  type: object
+  description: Configuration for handling outliers.
+  properties:
+    exclude:
+      type: boolean
+      description: Whether to exclude outliers when calculating A/B test results.
+      default: true
+
+EmptySearch:
+  type: object
+  description: Configuration for handling empty searches.
+  properties:
+    exclude:
+      type: boolean
+      description: Whether to exclude empty searches when calculating A/B test results.
+
+MinimumDetectableEffect:
+  type: object
+  description: Configuration for the smallest difference between test variants you want to detect.
+  properties:
+    size:
+      type: number
+      format: double
+      minimum: 0
+      maximum: 1
+      description: |
+        Smallest difference in an observable metric between variants.
+        For example, to detect a 10% difference between variants, set this value to 0.1.
+    effect:
+      $ref: '#/Effect'
+
+Effect:
+  type: string
+  description: Metric for which you want to detect the smallest relative difference.
+  enum:
+    - addToCartRate
+    - clickThroughRate
+    - conversionRate
+    - purchaseRate

specs/abtesting/common/schemas/AddABTestsVariant.yml

@@ -24,8 +24,10 @@ abTestsVariant:
 
 customSearchParams:
   type: object
-  description: Applies search parameters from [a restricted set of options](https://www.algolia.com/doc/api-reference/api-methods/add-ab-test/#method-param-customsearchparameters). Only use this parameter if the two variants use the same index.
-  example: {'typoTolerance': false,'synonyms': false}
+  description: |
+    Search parameters to add to the test variant.
+    Only use this parameter if the two variants use the same index.
+  example: {'typoTolerance': false, 'synonyms': false}
   additionalProperties: false
   properties:
     customSearchParameters:

specs/abtesting/common/schemas/Variant.yml

@@ -1,6 +1,10 @@
 variants:
   type: array
-  description: A/B test variants.
+  description: |
+    A/B test variants.
+
+    The first variant is your _control_ index, typically your production index.
+    The second variant is an index with changed settings that you want to test against the control.
   items:
     $ref: '#/variant'
 
@@ -16,13 +20,15 @@ variant:
       oneOf:
         - type: number
           format: double
-          description: Variant's [add-to-cart rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#add-to-cart-rate).
+          description: |
+            [Add-to-cart rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#add-to-cart-rate) for this variant.
           example: 0.0
         - type: 'null'
     averageClickPosition:
       oneOf:
         - type: integer
-          description: Variant's [average click position](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#click-position).
+          description: |
+            [Average click position](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#click-position) for this variant.
           example: 0
         - type: 'null'
     clickCount:
@@ -33,7 +39,8 @@ variant:
       oneOf:
         - type: number
           format: double
-          description: Variant's [click-through rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#click-through-rate).
+          description: |
+            [Click-through rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#click-through-rate) for this variant.
           example: 0.22219857724813036
         - type: 'null'
     conversionCount:
@@ -44,7 +51,8 @@ variant:
       oneOf:
         - type: number
           format: double
-          description: Variant's [conversion rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#conversion-rate).
+          description: |
+            [Conversion rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#conversion-rate) for this variant.
           example: 0.14546725846658964
         - type: 'null'
     currencies:
@@ -53,7 +61,10 @@ variant:
       $ref: '../parameters.yml#/description'
     estimatedSampleSize:
       type: integer
-      description: The estimated number of searches that will need to be run to achieve the desired confidence level and statistical power. A `minimumDetectableEffect` must be set in the `configuration` object for this to be used.
+      description: |
+        Estimated number of searches required to achieve the desired statistical significance.
+
+        The A/B test configuration must include a `mininmumDetectableEffect` setting for this number to be included in the response.
       example: 0
     filterEffects:
       $ref: '../parameters.yml#/filterEffects'
@@ -62,7 +73,7 @@ variant:
     noResultCount:
       oneOf:
         - type: integer
-          description: Number of [searches without results](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#searches-without-results) for that variant.
+          description: Number of [searches without results](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#searches-without-results) for this variant.
           example: 0
         - type: 'null'
     purchaseCount:
@@ -73,13 +84,14 @@ variant:
       oneOf:
         - type: number
           format: double
-          description: Variant's [purchase rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#purchase-rate).
+          description: |
+            [Purchase rate](https://www.algolia.com/doc/guides/search-analytics/concepts/metrics/#purchase-rate) for this variant.
           example: 0.0
         - type: 'null'
     searchCount:
       oneOf:
         - type: integer
-          description: Number of searches carried out during the A/B test.
+          description: Number of searches for this variant.
           example: 86269
         - type: 'null'
     trackedSearchCount:
@@ -89,13 +101,13 @@ variant:
     userCount:
       oneOf:
         - type: integer
-          description: Number of users during the A/B test.
+          description: Number of users that made searches to this variant.
           example: 55501
         - type: 'null'
     trackedUserCount:
       oneOf:
         - type: integer
-          description: Number of users that performed a tracked search during the A/B test.
+          description: Number of users that made tracked searches to this variant.
           example: 55501
         - type: 'null'
   required: