Merge pull request #3533 from Emurgo/denis/yoext-1190/separate-failin…

…g-tests Move failing tests to a separate workflow
Emurgo · Jun 14, 2024 · c6616e3 · c6616e3
2 parents 420a6bc + 9085887
commit c6616e3
Show file tree

Hide file tree

Showing 22 changed files with 263 additions and 87 deletions.
diff --git a/.github/workflows/e2e-tests-dapp.yml b/.github/workflows/e2e-tests-dapp.yml
@@ -0,0 +1,162 @@
+name: E2E DApp tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, ready_for_review]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  General:
+    if: github.event.review && (github.event.review.state == 'approved' || contains(github.event.review.body, '/check') || contains(github.event.review.body, '/dapp-general'))
+    runs-on: macos-14
+    strategy:
+      matrix:
+        browser: ["chrome"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: distutils hack
+        run: brew install python-setuptools
+
+      - name: Forcefully update the Chrome browser
+        if: matrix.browser=='chrome'
+        run: brew update && brew upgrade --cask google-chrome
+
+      - name: Forcefully install Firefox for Developers browser
+        if: matrix.browser=='firefox'
+        run: |
+          brew update
+          brew tap homebrew/cask-versions && brew install --cask firefox-developer-edition
+          echo "FIREFOX_BIN=/Applications/Firefox Developer Edition.app/Contents/MacOS/firefox-bin" >> $GITHUB_ENV
+
+      - name: Read .nvmrc
+        id: nvm
+        run: echo "NVMRC=$(cat .nvmrc)" >> $GITHUB_OUTPUT
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "${{ steps.nvm.outputs.NVMRC }}"
+
+      - name:
+          Cache extension node modules
+          # https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-yoroi-extension-node-modules
+        with:
+          # It is recommended to cache the NPM cache (~/.npm) instead of node_modules.
+          # https://github.com/actions/cache/blob/main/examples.md#node---npm
+          # But we put node version into the cache key and cache node_modules.
+          path: packages/yoroi-extension/node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-node-${{ steps.nvm.outputs.NVMRC }}-${{ hashFiles('packages/yoroi-extension/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: npm install
+        run: |
+          . install-all.sh
+
+      - name: Build the test mainnet version
+        working-directory: ./packages/yoroi-extension
+        run: npm run test:build:mainnet
+
+      - name: Run tests
+        working-directory: ./packages/e2e-tests
+        env:
+          FIRST_SMOKE_TEST_WALLET: ${{ secrets.FIRST_SMOKE_TEST_WALLET }}
+          SECOND_STATIC_TEST_WALLET: ${{ secrets.SECOND_STATIC_TEST_WALLET }}
+          SECOND_SMOKE_TEST_WALLET: ${{ secrets.SECOND_SMOKE_TEST_WALLET }}
+          SECOND_SMOKE_TEST_WALLET_FF: ${{ secrets.SECOND_SMOKE_TEST_WALLET_FF }}
+        run: npm run test:${{ matrix.browser }}:dapp
+
+      - name: Archive tests screenshots and logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: testRunsData_E2E_tests_${{ matrix.browser }}
+          path: |
+            ./packages/e2e-tests/mochawesome-report
+            ./packages/e2e-tests/testRunsData_${{ matrix.browser }}
+
+  Failing:
+    if: github.event.review && contains(github.event.review.body, '/dapp-failing')
+    runs-on: macos-14
+    strategy:
+      matrix:
+        browser: ['chrome']
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: distutils hack
+        run: brew install python-setuptools
+
+      - name: Forcefully update the Chrome browser
+        if: matrix.browser=='chrome'
+        run: brew update && brew upgrade --cask google-chrome
+
+      - name: Forcefully install Firefox for Developers browser
+        if: matrix.browser=='firefox'
+        run: |
+          brew update
+          brew tap homebrew/cask-versions && brew install --cask firefox-developer-edition
+          echo "FIREFOX_BIN=/Applications/Firefox Developer Edition.app/Contents/MacOS/firefox-bin" >> $GITHUB_ENV
+
+      - name: Read .nvmrc
+        id: nvm
+        run: echo "NVMRC=$(cat .nvmrc)" >> $GITHUB_OUTPUT
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '${{ steps.nvm.outputs.NVMRC }}'
+
+      - name: Cache extension node modules
+        # https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-yoroi-extension-node-modules
+        with:
+          # https://github.com/actions/cache/blob/main/examples.md#node---npm
+          # It is recommended to cache the NPM cache (~/.npm) instead of node_modules.
+          # But we put node version into the cache key and cache node_modules.
+          path: packages/yoroi-extension/node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-node-${{ steps.nvm.outputs.NVMRC }}-${{ hashFiles('packages/yoroi-extension/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: npm install
+        run: |
+          . install-all.sh
+      
+      - name: Build the test mainnet version
+        working-directory: ./packages/yoroi-extension
+        run: npm run test:build:mainnet
+
+      - name: Run tests
+        working-directory: ./packages/e2e-tests
+        env:
+          FIRST_SMOKE_TEST_WALLET: ${{ secrets.FIRST_SMOKE_TEST_WALLET }}
+          SECOND_STATIC_TEST_WALLET: ${{ secrets.SECOND_STATIC_TEST_WALLET }}
+          SECOND_SMOKE_TEST_WALLET: ${{ secrets.SECOND_SMOKE_TEST_WALLET }}
+          SECOND_SMOKE_TEST_WALLET_FF: ${{ secrets.SECOND_SMOKE_TEST_WALLET_FF }}
+        run: npm run test:${{ matrix.browser }}:dapp:bl
+
+      - name: Archive tests screenshots and logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: testRunsData_E2E_tests_${{ matrix.browser }}
+          path: |
+            ./packages/e2e-tests/mochawesome-report
+            ./packages/e2e-tests/testRunsData_${{ matrix.browser }}
diff --git a/.github/workflows/e2e-tests.yml → .github/workflows/e2e-tests-ext.yml b/.github/workflows/e2e-tests.yml → .github/workflows/e2e-tests-ext.yml
@@ -1,19 +1,19 @@
-name: End-2-end tests
+name: E2E Ext tests
 
 on:
   workflow_dispatch:
   pull_request:
-    types: [ opened, ready_for_review ]
+    types: [opened, ready_for_review]
   pull_request_review:
-    types: [ submitted ]
+    types: [submitted]
 
 jobs:
-  E2E_tests:
-    if: github.event.review  && (github.event.review.state == 'approved' || contains(github.event.review.body, '/check') || contains(github.event.review.body, '/regression-check'))
+  General:
+    if: github.event.review && (github.event.review.state == 'approved' || contains(github.event.review.body, '/check') || contains(github.event.review.body, '/ext-general'))
     runs-on: macos-14
     strategy:
       matrix:
-        browser: ['chrome']
+        browser: ["chrome"]
       fail-fast: false
 
     steps:
@@ -40,7 +40,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v4
         with:
-          node-version: '${{ steps.nvm.outputs.NVMRC }}'
+          node-version: "${{ steps.nvm.outputs.NVMRC }}"
 
       - name: Cache extension node modules
         # https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
@@ -84,14 +84,14 @@ jobs:
             ./packages/e2e-tests/mochawesome-report
             ./packages/e2e-tests/testRunsData_${{ matrix.browser }}
 
-  E2E_dapp_tests:
-    if: github.event.review && (github.event.review.state == 'approved' || contains(github.event.review.body, '/check') || contains(github.event.review.body, '/dapp-check'))
+  Failing:
+    if: github.event.review && contains(github.event.review.body, '/ext-failing')
     runs-on: macos-14
     strategy:
       matrix:
-        browser: ['chrome']
+        browser: ["chrome"]
       fail-fast: false
-    
+
     steps:
       - uses: actions/checkout@v4
 
@@ -116,7 +116,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v4
         with:
-          node-version: '${{ steps.nvm.outputs.NVMRC }}'
+          node-version: "${{ steps.nvm.outputs.NVMRC }}"
 
       - name: Cache extension node modules
         # https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
@@ -137,7 +137,7 @@ jobs:
       - name: npm install
         run: |
           . install-all.sh
-      
+
       - name: Build the test mainnet version
         working-directory: ./packages/yoroi-extension
         run: npm run test:build:mainnet
@@ -149,7 +149,7 @@ jobs:
           SECOND_STATIC_TEST_WALLET: ${{ secrets.SECOND_STATIC_TEST_WALLET }}
           SECOND_SMOKE_TEST_WALLET: ${{ secrets.SECOND_SMOKE_TEST_WALLET }}
           SECOND_SMOKE_TEST_WALLET_FF: ${{ secrets.SECOND_SMOKE_TEST_WALLET_FF }}
-        run: npm run test:${{ matrix.browser }}:dapp
+        run: npm run test:${{ matrix.browser }}:bl
 
       - name: Archive tests screenshots and logs
         if: ${{ failure() }}
@@ -158,4 +158,4 @@ jobs:
           name: testRunsData_E2E_tests_${{ matrix.browser }}
           path: |
             ./packages/e2e-tests/mochawesome-report
-            ./packages/e2e-tests/testRunsData_${{ matrix.browser }}
+            ./packages/e2e-tests/testRunsData_${{ matrix.browser }}
diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json
@@ -7,26 +7,33 @@
   "config": {
     "hooksPath": "./utils/hooks.mjs",
     "testPathGeneral": "./test/*.test.js",
+    "testPathGeneralBl": "./test/blacklist/*.test.js",
     "testPathHW": "./test/hw/*.test.js",
-    "testPathDApp": "./test/dapp/*.test.js"
+    "testPathDApp": "./test/dapp/*.test.js",
+    "testPathDAppBl": "./test/blacklist/dapp/*.test.js"
   },
   "scripts": {
     "test:clean:logs": "find . -type d -name \"testRunsData_*\" -exec rm -r {} +",
     "test:base:sequintel": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathGeneral --reporter mochawesome",
+    "test:base:sequintel:bl": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathGeneralBl --reporter mochawesome",
     "test:hw:base:sequintel": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathHW --reporter mochawesome",
     "test:dapp:base:sequintel": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathDApp --reporter mochawesome",
+    "test:dapp:base:sequintel:bl": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathDAppBl --reporter mochawesome",
     "test:base:parallel": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathGeneral --parallel --reporter mochawesome --require mochawesome/register",
     "test:base:one": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathGeneral -g",
     "test:hw:base:one": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathHW -g",
     "test:dapp:base:one": "npm run test:clean:logs && mocha --require $npm_package_config_hooksPath $npm_package_config_testPathDApp -g",
     "test:chrome": "env TARGETBROWSER=chrome npm run test:base:sequintel",
+    "test:chrome:bl": "env TARGETBROWSER=chrome npm run test:base:sequintel:bl",
     "test:chrome:hw": "env TARGETBROWSER=chrome npm run test:hw:base:sequintel",
     "test:chrome:dapp": "env TARGETBROWSER=chrome npm run test:dapp:base:sequintel",
+    "test:chrome:dapp:bl": "env TARGETBROWSER=chrome npm run test:dapp:base:sequintel:bl",
     "test:chrome:parallel": "env TARGETBROWSER=chrome npm run test:base:parallel",
     "test:chrome:one": "env TARGETBROWSER=chrome npm run test:base:one",
     "test:chrome:hw:one": "env TARGETBROWSER=chrome npm run test:hw:base:one",
     "test:chrome:dapp:one": "env TARGETBROWSER=chrome npm run test:dapp:base:one",
     "test:firefox": "env TARGETBROWSER=firefox npm run test:base:sequintel",
+    "test:firefox:bl": "env TARGETBROWSER=firefox npm run test:base:sequintel:bl",
     "test:firefox:hw": "env TARGETBROWSER=firefox npm run test:hw:base:sequintel",
     "test:firefox:dapp": "env TARGETBROWSER=firefox npm run test:dapp:base:sequintel",
     "test:firefox:parallel": "env TARGETBROWSER=firefox npm run test:base:parallel",

diff --git a/packages/e2e-tests/pages/basepage.js b/packages/e2e-tests/pages/basepage.js
@@ -2,15 +2,20 @@ import { until, Key, logging } from 'selenium-webdriver';
 import path from 'path';
 import * as fs from 'node:fs';
 import { promisify } from 'util';
-import { createTestRunDataDir, getByLocator, isFirefox, isChrome } from '../utils/utils.js';
+import {
+  createTestRunDataDir,
+  getByLocator,
+  getSnapshotObjectFromJSON,
+  isFirefox,
+  isChrome,
+} from '../utils/utils.js';
 import { getExtensionUrl } from '../utils/driverBootstrap.js';
 import {
   defaultRepeatPeriod,
   defaultWaitTimeout,
   halfSecond,
   oneSecond,
 } from '../helpers/timeConstants.js';
-import { getSnapshotObjectFromJSON } from '../utils/utils.js';
 import { dbSnapshotsDir } from '../helpers/constants.js';
 
 const writeFile = promisify(fs.writeFile);
@@ -88,7 +93,8 @@ class BasePage {
   }
   async getText(locator) {
     this.logger.info(`BasePage::getText is called. Locator: ${JSON.stringify(locator)}`);
-    return await this.driver.findElement(getByLocator(locator)).getText();
+    const locatorElem = await this.waitForElement(locator);
+    return await locatorElem.getText();
   }
   async getCssValue(locator, cssStyleProperty) {
     this.logger.info(
@@ -512,7 +518,7 @@ class BasePage {
               console.log('-----> Error happend:', event.target.result);
             };
             const store = tx.objectStore(tableName);
-            const addRequest = store.put(valueItem);
+            store.put(valueItem);
           };
         },
         name,

diff --git a/packages/e2e-tests/pages/walletCommonBase.page.js b/packages/e2e-tests/pages/walletCommonBase.page.js
@@ -121,10 +121,11 @@ class WalletCommonBase extends BasePage {
     const rawNameAndPlateText = await this.getText(this.walletNameAndPlateNumberTextLocator);
     const [walletName, walletPlate] = rawNameAndPlateText.split('\n');
 
-    await this.waitForElement(this.walletBalanceTextLocator);
-    const rawBalanceText = await this.getText(this.walletBalanceTextLocator);
+    const walletBalanceElem = await this.waitForElement(this.walletBalanceTextLocator);
+    const rawBalanceText = await walletBalanceElem.getText();
     const adaBalance = Number(rawBalanceText.split(' ')[0]);
 
+    await this.waitForElement(this.walletFiatBalanceTextLocator);
     const rawFiatBalanceText = await this.getText(this.walletFiatBalanceTextLocator);
     const [fiatBalanceStr, fiatCurrency] = rawFiatBalanceText.split(' ');
     const fiatBalance = fiatBalanceStr === '-' ? 0 : Number(fiatBalanceStr);
@@ -200,7 +201,8 @@ class WalletCommonBase extends BasePage {
   }
   async openChangeWalletModal() {
     this.logger.info(`WalletCommonBase::openChangeWalletModal is called`);
-    await this.click(this.selectedWalletButtonLocator);
+    const selectWalletBtnElem = await this.waitForElement(this.selectedWalletButtonLocator);
+    await selectWalletBtnElem.click();
     await this.waitForElement(this.changeWalletDialogLocator);
   }
   async addNewWallet() {

diff --git a/packages/e2e-tests/test/04_removingWallet.test.js b/packages/e2e-tests/test/04_removingWallet.test.js
@@ -1,11 +1,9 @@
 import BasePage from '../pages/basepage.js';
 import { customAfterEach } from '../utils/customHooks.js';
 import TransactionsSubTab from '../pages/wallet/walletTab/walletTransactions.page.js';
-import { testWallet1 } from '../utils/testWallets.js';
 import { expect } from 'chai';
 import { getTestLogger } from '../utils/utils.js';
 import { oneMinute } from '../helpers/timeConstants.js';
-import { restoreWallet } from '../helpers/restoreWalletHelper.js';
 import SettingsTab from '../pages/wallet/settingsTab/settingsTab.page.js';
 import WalletSubTab from '../pages/wallet/settingsTab/walletSubTab.page.js';
 import AddNewWallet from '../pages/addNewWallet.page.js';

diff --git a/packages/e2e-tests/test/06_renamingWallet.test.js b/packages/e2e-tests/test/06_renamingWallet.test.js
@@ -4,7 +4,6 @@ import { testWallet1 } from '../utils/testWallets.js';
 import { expect } from 'chai';
 import { getTestLogger, walletNameShortener } from '../utils/utils.js';
 import { oneMinute } from '../helpers/timeConstants.js';
-import { restoreWallet } from '../helpers/restoreWalletHelper.js';
 import SettingsTab from '../pages/wallet/settingsTab/settingsTab.page.js';
 import WalletSubTab from '../pages/wallet/settingsTab/walletSubTab.page.js';
 import TransactionsSubTab from '../pages/wallet/walletTab/walletTransactions.page.js';

diff --git a/packages/e2e-tests/test/07_changingPassword.test.js b/packages/e2e-tests/test/07_changingPassword.test.js
@@ -5,7 +5,6 @@ import { testWallet1 } from '../utils/testWallets.js';
 import { expect } from 'chai';
 import { getTestLogger } from '../utils/utils.js';
 import { oneMinute } from '../helpers/timeConstants.js';
-import { restoreWallet } from '../helpers/restoreWalletHelper.js';
 import SettingsTab from '../pages/wallet/settingsTab/settingsTab.page.js';
 import WalletSubTab from '../pages/wallet/settingsTab/walletSubTab.page.js';
 import { getPassword } from '../helpers/constants.js';

diff --git a/packages/e2e-tests/test/08_downloadingLogs.test.js b/packages/e2e-tests/test/08_downloadingLogs.test.js
@@ -1,7 +1,6 @@
 import BasePage from '../pages/basepage.js';
 import { customAfterEach } from '../utils/customHooks.js';
 import TransactionsSubTab from '../pages/wallet/walletTab/walletTransactions.page.js';
-import { testWallet1 } from '../utils/testWallets.js';
 import { expect } from 'chai';
 import {
   cleanDownloads,
@@ -10,7 +9,6 @@ import {
   getTestLogger,
 } from '../utils/utils.js';
 import { oneMinute } from '../helpers/timeConstants.js';
-import { restoreWallet } from '../helpers/restoreWalletHelper.js';
 import SettingsTab from '../pages/wallet/settingsTab/settingsTab.page.js';
 import SupportSubTab from '../pages/wallet/settingsTab/supportSubTab.page.js';
 import driversPoolsManager from '../utils/driversPool.js';