CI: run e2e flaky tests in a separate job (facebook#6365)

2wheeh · Jul 17, 2024 · 1798861 · 1798861
1 parent 0393671
commit 1798861
Show file tree

Hide file tree

Showing 11 changed files with 1,007 additions and 1,375 deletions.
diff --git a/.github/workflows/call-e2e-all-tests.yml b/.github/workflows/call-e2e-all-tests.yml
@@ -141,3 +141,20 @@ jobs:
       editor-mode: ${{ matrix.editor-mode }}
       prod: ${{ matrix.prod }}
       override-react-version: beta
+
+  flaky:
+    strategy:
+      matrix:
+        node-version: [18.18.0]
+        # Currently using single browser & os combination for flaky tests to reduce cost impact
+        browser: ['chromium']
+        editor-mode: ['rich-text', 'plain-text', 'rich-text-with-collab']
+        events-mode: ['modern-events']
+    uses: ./.github/workflows/call-e2e-test.yml
+    with:
+      os: 'macos-latest'
+      flaky: true
+      node-version: ${{ matrix.node-version }}
+      browser: ${{ matrix.browser }}
+      editor-mode: ${{ matrix.editor-mode }}
+      events-mode: ${{ matrix.events-mode }}
diff --git a/.github/workflows/call-e2e-test.yml b/.github/workflows/call-e2e-test.yml
@@ -9,6 +9,7 @@ on:
       editor-mode: {required: true, type: string}
       events-mode: {required: true, type: string}
       prod: {required: false, type: boolean}
+      flaky: {required: false, type: boolean} # Defaults to false: dont run flaky
       override-react-version: {required: false, type: string}
 
 jobs:
@@ -22,7 +23,7 @@ jobs:
       OVERRIDE_REACT_VERSION: ${{ inputs.override-react-version }}
       cache_playwright_path: ${{ inputs.os == 'macos-latest' && '~/Library/Caches/ms-playwright' || inputs.os == 'windows-latest' && 'C:\Users\runneradmin\AppData\Local\ms-playwright' || '~/.cache/ms-playwright' }}
       test_results_path: ${{ inputs.os == 'windows-latest' && '~/.npm/_logs/' || 'test-results/' }}
-      test_script: test-e2e-${{ inputs.editor-mode == 'rich-text-with-collab' && 'collab-' || '' }}${{ inputs.prod && 'prod-' || '' }}ci-${{ inputs.browser }}
+      test_script: test-e2e-${{ inputs.editor-mode == 'rich-text-with-collab' && 'collab-' || '' }}${{ inputs.prod && 'prod-' || '' }}ci-${{ inputs.browser }} ${{ inputs.flaky && '-- -- --grep "@flaky"' || '-- -- --grep-invert "@flaky"' }}
     steps:
       - uses: actions/checkout@v4
       - name: Use Node.js ${{ inputs.node-version }}