E2e cypress

.circleci/docker-compose.cypress.yml → .ci/docker-compose.cypress.yml

@@ -44,7 +44,7 @@ services:
     ipc: host
     build:
       context: ../
-      dockerfile: .circleci/Dockerfile.cypress
+      dockerfile: .ci/Dockerfile.cypress
     depends_on:
       - server
       - worker

.github/workflows/ci.yml

@@ -0,0 +1,146 @@
+name: Tests
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+env:
+  NODE_VERSION: 14.17
+jobs:
+  backend-lint:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.7'
+      - run: sudo pip install flake8
+      - run: ./bin/flake8_tests.sh
+
+  backend-unit-tests:
+    runs-on: ubuntu-22.04
+    needs: backend-lint
+    env:
+      COMPOSE_FILE: .ci/docker-compose.ci.yml
+      COMPOSE_PROJECT_NAME: redash
+      COMPOSE_DOCKER_CLI_BUILD: 1
+      DOCKER_BUILDKIT: 1
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Build Docker Images
+        run: |
+          set -x
+          docker-compose build --build-arg skip_ds_deps=true --build-arg skip_frontend_build=true
+          docker-compose up -d
+          sleep 10
+      - name: Create Test Database
+        run: docker-compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;"
+      - name: List Enabled Query Runners
+        run: docker-compose -p redash run --rm redash manage ds list_types
+      - name: Run Tests
+        run: docker-compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report xml --cov=redash --cov-config .coveragerc tests/
+      - name: Copy Test Results
+        run: |
+          mkdir -p /tmp/test-results/unit-tests
+          docker cp tests:/app/coverage.xml ./coverage.xml
+          docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
+      - name: Store Test Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: /tmp/test-results
+      - name: Store Coverage Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: coverage.xml
+
+  frontend-lint:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+      - name: Install Dependencies
+        run: |
+          npm install --global --force yarn@1.22.10
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+      - name: Run Lint
+        run: yarn lint:ci
+      - name: Store Test Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: /tmp/test-results
+
+  frontend-unit-tests:
+    runs-on: ubuntu-22.04
+    needs: frontend-lint
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+      - name: Install Dependencies
+        run: |
+          npm install --global --force yarn@1.22.10
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+      - name: Run App Tests
+        run: yarn test
+      - name: Run Visualizations Tests
+        run: cd viz-lib && yarn test
+      - run: yarn lint
+
+  frontend-e2e-tests:
+    runs-on: ubuntu-22.04
+    needs: frontend-lint
+    env:
+      COMPOSE_FILE: .ci/docker-compose.cypress.yml
+      COMPOSE_PROJECT_NAME: cypress
+      CYPRESS_INSTALL_BINARY: 0
+      PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+      - name: Enable Code Coverage Report For Master Branch
+        if: endsWith(github.ref, '/master')
+        run: |
+          echo "CODE_COVERAGE=true" >> $GITHUB_ENV
+      - name: Install Dependencies
+        run: |
+          npm install --global --force yarn@1.22.10
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+      - name: Setup Redash Server
+        run: |
+          set -x
+          yarn cypress build
+          yarn cypress start -- --skip-db-seed
+          docker-compose run cypress yarn cypress db-seed
+      - name: Execute Cypress Tests
+        run: yarn cypress run-ci
+      - name: "Failure: output container logs to console"
+        if: failure()
+        run: docker-compose logs
+      - name: Copy Code Coverage Results
+        run: docker cp cypress:/usr/src/app/coverage ./coverage || true
+      - name: Store Coverage Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: coverage

client/cypress/cypress.js

@@ -1,6 +1,5 @@
 /* eslint-disable import/no-extraneous-dependencies, no-console */
 const { find } = require("lodash");
-const atob = require("atob");
 const { execSync } = require("child_process");
 const { get, post } = require("request").defaults({ jar: true });
 const { seedData } = require("./seed-data");
@@ -59,27 +58,8 @@ function stopServer() {
 }
 
 function runCypressCI() {
-  const {
-    PERCY_TOKEN_ENCODED,
-    CYPRESS_PROJECT_ID_ENCODED,
-    CYPRESS_RECORD_KEY_ENCODED,
-    CIRCLE_REPOSITORY_URL,
-  } = process.env;
-
-  if (CIRCLE_REPOSITORY_URL && CIRCLE_REPOSITORY_URL.includes("getredash/redash")) {
-    if (PERCY_TOKEN_ENCODED) {
-      process.env.PERCY_TOKEN = atob(`${PERCY_TOKEN_ENCODED}`);
-    }
-    if (CYPRESS_PROJECT_ID_ENCODED) {
-      process.env.CYPRESS_PROJECT_ID = atob(`${CYPRESS_PROJECT_ID_ENCODED}`);
-    }
-    if (CYPRESS_RECORD_KEY_ENCODED) {
-      process.env.CYPRESS_RECORD_KEY = atob(`${CYPRESS_RECORD_KEY_ENCODED}`);
-    }
-  }
-
   execSync(
-    "COMMIT_INFO_MESSAGE=$(git show -s --format=%s) docker-compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run --record",
+    "COMMIT_INFO_MESSAGE=$(git show -s --format=%s) docker-compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run",
     { stdio: "inherit" }
   );
 }

requirements.txt

@@ -20,7 +20,7 @@ blinker==1.4
 psycopg2==2.8.3
 python-dateutil==2.8.0
 pytz>=2019.3
-PyYAML==5.1.2
+PyYAML==5.4
 redis==3.5.0
 requests==2.21.0
 SQLAlchemy==1.3.10
@@ -37,7 +37,7 @@ rq==1.5.0
 rq-scheduler==0.9.1
 jsonschema==3.1.1
 RestrictedPython==5.0
-pysaml2==6.1.0
+pysaml2==6.5.0
 pycrypto==2.6.1
 python-dotenv==0.19.2
 funcy==1.13

requirements_dev.txt

@@ -6,6 +6,7 @@ mock==3.0.5
 # PyMongo and Athena dependencies are needed for some of the unit tests:
 # (this is not perfect and we should resolve this in a different way)
 pymongo[srv,tls]==3.9.0
+boto3>=1.10.0,<1.11.0
 botocore>=1.13,<1.14.0
 PyAthena>=1.5.0,<=1.11.5
 ptvsd==4.3.2