Upgrade to cypress 12 (#668)

* Upgrade to cypress 12

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Freeze cypress version

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add changes from ftrepo PR #674

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Remove cypress.json

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Regenerate package-lock

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Run vanilla snapshot on feature branches

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Run vanilla snapshot on feature branches

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update vanilla test to be headless

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Go through upgrade wizard to rename specs

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add specPattern

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* integration to e2e

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update more references to /integration in dev guide

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Typo fix

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Set numTestsKeptInMemory for vanilla tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Re-run CI

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Try experimentMemoryManagement

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Pass via CLI

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Do testIsolation as describe level

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Add cy wait 5s

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update notifications tests

Signed-off-by: Craig Perkins <craig5008@gmail.com>

* Undo testIsolation

Signed-off-by: Craig Perkins <craig5008@gmail.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>
Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
Signed-off-by: Craig Perkins <craig5008@gmail.com>
Co-authored-by: Peter Zhu <zhujiaxi@amazon.com>

.eslintrc.json

@@ -18,7 +18,8 @@
       "prettier/prettier": "error",
       "header/header": [2, "block", ["", " * Copyright OpenSearch Contributors", " * SPDX-License-Identifier: Apache-2.0", " "]],
       // turn off no-unnecessary-waiting for now due to the issue: https://github.com/cypress-io/eslint-plugin-cypress/issues/43
-      "cypress/no-unnecessary-waiting": "off"
+      "cypress/no-unnecessary-waiting": "off",
+      "cypress/unsafe-to-chain-command": "off"
     },
     "ignorePatterns": ["**/*.d.ts"]
 }

.github/workflows/alerting-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Alerting
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/alerting-dashboards-plugin/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/alerting-dashboards-plugin/*'

.github/workflows/anomaly-detection-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Anomaly Detection
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/anomaly-detection-dashboards-plugin/*'

.github/workflows/custom-import-map-dashboards-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Observability
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/custom-import-map-dashboards/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/custom-import-map-dashboards/*'

.github/workflows/cypress-workflow-bundle-snapshot-based-windows.yml

@@ -24,7 +24,7 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-with-security --browser chrome --spec 'cypress/e2e/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
 
@@ -34,7 +34,7 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template-windows.yml
     with:
       test-name: Core Dashboards using Bundle Snapshot
-      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/integration/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
+      test-command: env CYPRESS_VISBUILDER_ENABLED=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true yarn cypress:run-without-security --browser chrome --spec 'cypress/e2e/core-opensearch-dashboards/opensearch-dashboards/**/*.js'
       # not useful now as the windows e2e template currently do not allow serving parameters
       #osd-serve-args: --data_source.enabled=true --vis_builder.enabled=true
       security-enabled: false

.github/workflows/cypress-workflow-vanilla-snapshot-based.yml

@@ -4,10 +4,12 @@ on:
     branches:
       - main
       - dev-*
+      - feature/*
   push:
     branches:
       - main
       - dev-*
+      - feature/*
 jobs:
   tests:
     name: Run Cypress E2E tests
@@ -71,7 +73,7 @@ jobs:
         uses: cypress-io/github-action@v2
         with:
           working-directory: monetery-test
-          command: yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/core-opensearch-dashboards/vanilla-opensearch-dashboards/*.js'
+          command: yarn cypress:run-without-security --browser chromium --spec 'cypress/e2e/core-opensearch-dashboards/vanilla-opensearch-dashboards/*.js'
           wait-on: 'http://localhost:5601'
       # Screenshots are only captured on failure, will change this once we do visual regression tests
       - uses: actions/upload-artifact@v1

.github/workflows/gantt-chart-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Gantt Chart
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/gantt-chart-dashboards/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/gantt-chart-dashboards/*'

.github/workflows/index-management-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Index Management
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/index-management-dashboards-plugin/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/index-management-dashboards-plugin/*'

.github/workflows/ml-commons-release-e2e-workflow.yml

@@ -21,5 +21,5 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: ML Commons
-      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/ml-commons-dashboards/*'
+      test-command: env CYPRESS_ML_COMMONS_DASHBOARDS_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/ml-commons-dashboards/*'
       osd-serve-args: --ml_commons_dashboards.enabled=true

.github/workflows/notifications-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Notifications
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/notifications-dashboards/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/notifications-dashboards/*'

.github/workflows/observability-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Observability
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/observability-dashboards/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/observability-dashboards/*'

.github/workflows/query-workbench-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Query Workbench
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/query-workbench-dashboards/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/query-workbench-dashboards/*'

.github/workflows/reports-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Reports
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/reports-dashboards/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/reports-dashboards/*'

.github/workflows/search-relevance-release-e2e-workflow.yml

@@ -20,4 +20,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Search Relevance
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/search-relevance-dashboards/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/search-relevance-dashboards/*'

.github/workflows/security-analytics-release-e2e-workflow.yml

@@ -24,4 +24,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Security Analytics
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/security-analytics-dashboards-plugin/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/security-analytics-dashboards-plugin/*'

.github/workflows/security-release-e2e-workflow.yml

@@ -21,4 +21,4 @@ jobs:
     uses: ./.github/workflows/release-e2e-workflow-template.yml
     with:
       test-name: Security
-      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/security/*'
+      test-command: yarn cypress:run-with-security --browser chromium --spec 'cypress/e2e/plugins/security/*'

DEVELOPER_GUIDE.md

@@ -76,21 +76,21 @@ To run tests against a local cluster
 without security:
 
 ```
-$ yarn cypress run-without-security --spec "cypress/integration/core-opensearch-dashboards/vanilla-opensearch-dashboards/*.js"
+$ yarn cypress run-without-security --spec "cypress/e2e/core-opensearch-dashboards/vanilla-opensearch-dashboards/*.js"
 ```
 
 with security:
 
 ```
-$ yarn cypress run-with-security --spec "cypress/integration/core-opensearch-dashboards/opensearch-dashboards/*.js"
+$ yarn cypress run-with-security --spec "cypress/e2e/core-opensearch-dashboards/opensearch-dashboards/*.js"
 ```
 
 These tests run in headless mode by default. 
 
 And you can override certain [cypress config or environment variable](cypress.json) by applying additional cli arguments, for example to override the baseUrl and openSearchUrl to test a remote OpenSearch endpoint:
 
 ```
-$ yarn cypress run --spec "cypress/integration/core-opensearch-dashboards/opensearch-dashboards/*.js" --config "baseUrl=https://<endpoint>/_dashboards" --env "openSearchUrl=https://<endpoint>,SECURITY_ENABLED=true,username=admin,password=xxxxxxxx,ENDPOINT_WITH_PROXY=true"
+$ yarn cypress run --spec "cypress/e2e/core-opensearch-dashboards/opensearch-dashboards/*.js" --config "baseUrl=https://<endpoint>/_dashboards" --env "openSearchUrl=https://<endpoint>,SECURITY_ENABLED=true,username=admin,password=xxxxxxxx,ENDPOINT_WITH_PROXY=true"
 ```
 
 `SECURITY_ENABLED`: if true, the `username` and `password` passing in are used as basic authentication credentials during `cy.visit` and `cy.request`. Also, please notice security enabled endpoint normally uses https protocol, so you may want to pass in different urls.
@@ -102,15 +102,15 @@ $ yarn cypress run --spec "cypress/integration/core-opensearch-dashboards/opense
 
 ## Writing tests
 
-The testing library uses [Cypress](https://www.cypress.io/) as its testing framework and follow its high level folder structure. All tests are written under the `./cypress/integration` folder.
+The testing library uses [Cypress](https://www.cypress.io/) as its testing framework and follow its high level folder structure. All tests are written under the `./cypress/e2e` folder.
 
 ### Tests for OpenSearch Dashboards
 
 Tests for core features specific to [OpenSearch Dashboards](https://github.com/opensearch-project/OpenSearch-Dashboards) can be written under
 
 ```
 /cypress
-    /integration
+    /e2e
         /core-opensearch-dashboards
             /opensearch-dashboards
 ```
@@ -121,7 +121,7 @@ Tests for plugins that are not a part of the [OpenSearch Dashboards](https://git
 
 ```
 /cypress
-    /integration
+    /e2e
         /plugins
             /<YOUR_PLUGIN_NAME>
 ```
@@ -131,7 +131,7 @@ When writing tests for experimental features, please follow these steps.
 
 1. Figure out the folder location to put the tests
 
-Similar to the regular tests, OSD Core tests go to the [folder](integration/core-opensearch-dashboards/opensearch-dashboards/) and OSD plugin tests go to the [folder](cypress/integration/plugins/).
+Similar to the regular tests, OSD Core tests go to the [folder](e2e/core-opensearch-dashboards/opensearch-dashboards/) and OSD plugin tests go to the [folder](cypress/e2e/plugins/).
 
 2. Develop tests with a flag to turn on and off
 

ONBOARDING.md

@@ -6,7 +6,7 @@ To onboard your release tests (Dashboards plugins) onto this repo test function,
 
 The dir name shall be descriptive to identify your plugin. You can use the same name defined in build repo https://github.com/opensearch-project/opensearch-build/tree/main/scripts/components
 
-1. Place test files under `cypress/integration/plugins/<plugin-name>`
+1. Place test files under `cypress/e2e/plugins/<plugin-name>`
 
 2. Place fixtures under `cypress/fixtures/plugins/<plugin-name>`
 
@@ -21,13 +21,13 @@ Start OpenSearch and OpenSearch Dashboards. Then refer to the [test execution fi
 E.g if you want to run all plugin tests with security enabled.
 
 ```
-npx cypress run --env SECURITY_ENABLED=true --spec "cypress/integration/plugins/*/*"
+npx cypress run --env SECURITY_ENABLED=true --spec "cypress/e2e/plugins/*/*"
 ```
 
 E.g if you want to run AD plugin tests with security enabled.
 
 ```
-npx cypress run --env SECURITY_ENABLED=true --spec "cypress/integration/plugins/anomaly-detection-dashboards-plugin"
+npx cypress run --env SECURITY_ENABLED=true --spec "cypress/e2e/plugins/anomaly-detection-dashboards-plugin"
 ```
 
 For the complete ways to run Cypress, you can refer to the Cypress official site https://docs.cypress.io/guides/getting-started/.installing-cypress#Opening-Cypress.

cypress.config.js

@@ -0,0 +1,38 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+const { defineConfig } = require('cypress');
+
+module.exports = defineConfig({
+  defaultCommandTimeout: 60000,
+  requestTimeout: 60000,
+  responseTimeout: 60000,
+  reporter: 'cypress-multi-reporters',
+  reporterOptions: {
+    configFile: 'reporter-config.json',
+  },
+  viewportWidth: 2000,
+  viewportHeight: 1320,
+  env: {
+    openSearchUrl: 'http://localhost:9200',
+    SECURITY_ENABLED: false,
+    AGGREGATION_VIEW: false,
+    username: 'admin',
+    password: 'admin',
+    ENDPOINT_WITH_PROXY: false,
+    MANAGED_SERVICE_ENDPOINT: false,
+    VISBUILDER_ENABLED: true,
+    DATASOURCE_MANAGEMENT_ENABLED: false,
+    ML_COMMONS_DASHBOARDS_ENABLED: true,
+  },
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.js')(on, config);
+    },
+    baseUrl: 'http://localhost:5601',
+    specPattern: '**/*.(js|ts|tsx)',
+  },
+});

cypress/integration/core-opensearch-dashboards/opensearch-dashboards/dashboard_sanity_test_spec.js → cypress/e2e/core-opensearch-dashboards/opensearch-dashboards/dashboard_sanity_test.cy.js

@@ -3,6 +3,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { dashboardSanityTests } from '../../common/dashboard_sample_data_spec.js';
+import { dashboardSanityTests } from '../../common/dashboard_sample_data.cy.js';
 
 dashboardSanityTests();

cypress/integration/core-opensearch-dashboards/vanilla-opensearch-dashboards/dashboard_filtering_spec.js → cypress/e2e/core-opensearch-dashboards/vanilla-opensearch-dashboards/dashboard_filtering.cy.js

@@ -29,7 +29,7 @@ const testFixtureHandler = new TestFixtureHandler(
 const commonUI = new CommonUI(cy);
 const dashboardPage = new DashboardPage(cy);
 const miscUtils = new MiscUtils(cy);
-describe('dashboard filtering', () => {
+describe('dashboard filtering', { testIsolation: false }, () => {
   before(() => {
     testFixtureHandler.clearJSONMapping(
       'cypress/fixtures/dashboard/data/mappings.json.txt'
@@ -111,7 +111,7 @@ describe('dashboard filtering', () => {
         commonUI.checkElementDoesNotExist('[data-test-subj="dataGridRowCell"]');
       });
 
-      it('Nonpinned filter: goal and guages are filtered', () => {
+      it('Nonpinned filter: goal and gauges are filtered', () => {
         // Goal label should be 0, gauge label should be 0%
         commonUI.checkValuesExistInComponent('svg > g > g > text.chart-label', [
           '0',
@@ -189,7 +189,7 @@ describe('dashboard filtering', () => {
         commonUI.checkElementDoesNotExist('[data-test-subj="dataGridRowCell"]');
       });
 
-      it('Pinned filter: goal and guages are filtered', () => {
+      it('Pinned filter: goal and gauges are filtered', () => {
         // Goal label should be 0, gauge label should be 0%
         commonUI.checkValuesExistInComponent('svg > g > g > text.chart-label', [
           '0',
@@ -263,7 +263,7 @@ describe('dashboard filtering', () => {
         commonUI.checkElementExists('[data-test-subj="dataGridRowCell"]', 20);
       });
 
-      it.skip('Filter disabled: goal and guages', () => {
+      it.skip('Filter disabled: goal and gauges', () => {
         // Goal label should be 7,544, and the gauge label should be 39.958%%
         // Inconsistency: original code says that the goal label should have "7,544",
         // but sometimes the goal displays "7,565". It may have been related to a
@@ -355,6 +355,8 @@ describe('dashboard filtering', () => {
       );
       dashboardPage.clickEditVisualization();
 
+      cy.wait(5000);
+
       miscUtils.setQuery('weightLbs:>50');
       commonUI.checkElementExists('svg > g > g.arcs > path.slice', 3);