From a67beb1c24bb975f8ec5da7cd627177b968ab3c3 Mon Sep 17 00:00:00 2001 From: Ye Sijun Date: Thu, 27 Apr 2023 21:14:53 +0900 Subject: [PATCH 1/4] ci: migrate to github workflow ci Signed-off-by: Ye Sijun --- {.circleci => .ci}/Dockerfile.cypress | 0 .../docker-compose.ci.yml | 0 {.circleci => .ci}/docker-compose.cypress.yml | 2 +- {.circleci => .ci}/docker_build | 0 {.circleci => .ci}/pack | 0 {.circleci => .ci}/update_version | 0 .circleci/config.yml | 179 ------------------ .github/workflows/ci.yml | 149 +++++++++++++++ client/cypress/cypress.js | 4 +- requirements_dev.txt | 1 + 10 files changed, 153 insertions(+), 182 deletions(-) rename {.circleci => .ci}/Dockerfile.cypress (100%) rename .circleci/docker-compose.circle.yml => .ci/docker-compose.ci.yml (100%) rename {.circleci => .ci}/docker-compose.cypress.yml (97%) rename {.circleci => .ci}/docker_build (100%) rename {.circleci => .ci}/pack (100%) rename {.circleci => .ci}/update_version (100%) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/ci.yml diff --git a/.circleci/Dockerfile.cypress b/.ci/Dockerfile.cypress similarity index 100% rename from .circleci/Dockerfile.cypress rename to .ci/Dockerfile.cypress diff --git a/.circleci/docker-compose.circle.yml b/.ci/docker-compose.ci.yml similarity index 100% rename from .circleci/docker-compose.circle.yml rename to .ci/docker-compose.ci.yml diff --git a/.circleci/docker-compose.cypress.yml b/.ci/docker-compose.cypress.yml similarity index 97% rename from .circleci/docker-compose.cypress.yml rename to .ci/docker-compose.cypress.yml index da57925f11..2a6feaf261 100644 --- a/.circleci/docker-compose.cypress.yml +++ b/.ci/docker-compose.cypress.yml @@ -44,7 +44,7 @@ services: ipc: host build: context: ../ - dockerfile: .circleci/Dockerfile.cypress + dockerfile: .ci/Dockerfile.cypress depends_on: - server - worker diff --git a/.circleci/docker_build b/.ci/docker_build similarity index 100% rename from .circleci/docker_build rename to .ci/docker_build diff --git a/.circleci/pack b/.ci/pack similarity index 100% rename from .circleci/pack rename to .ci/pack diff --git a/.circleci/update_version b/.ci/update_version similarity index 100% rename from .circleci/update_version rename to .ci/update_version diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0109051768..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,179 +0,0 @@ -version: 2.0 - -build-docker-image-job: &build-docker-image-job - docker: - - image: circleci/node:14.17 - steps: - - setup_remote_docker: - version: 19.03.13 - - checkout - - run: sudo apt update - - run: sudo apt install python3-pip - - run: .circleci/update_version - - run: sudo npm install --global --force yarn@1.22.10 - - run: .circleci/docker_build -jobs: - backend-lint: - docker: - - image: circleci/python:3.7.0 - steps: - - checkout - - run: sudo pip install flake8 - - run: ./bin/flake8_tests.sh - backend-unit-tests: - environment: - COMPOSE_FILE: .circleci/docker-compose.circle.yml - COMPOSE_PROJECT_NAME: redash - docker: - - image: circleci/buildpack-deps:xenial - steps: - - setup_remote_docker: - version: 19.03.13 - - checkout - - run: - name: Build Docker Images - command: | - set -x - COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build --build-arg skip_ds_deps=true --build-arg skip_frontend_build=true - docker-compose up -d - sleep 10 - - run: - name: Create Test Database - command: docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests;" - - run: - name: List Enabled Query Runners - command: docker-compose run --rm redash manage ds list_types - - run: - name: Run Tests - command: docker-compose run --name tests redash tests --junitxml=junit.xml --cov-report xml --cov=redash --cov-config .coveragerc tests/ - - run: - name: Copy Test Results - command: | - mkdir -p /tmp/test-results/unit-tests - docker cp tests:/app/coverage.xml ./coverage.xml - docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml - when: always - - store_test_results: - path: /tmp/test-results - - store_artifacts: - path: coverage.xml - frontend-lint: - environment: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - docker: - - image: circleci/node:14.17 - steps: - - checkout - - run: mkdir -p /tmp/test-results/eslint - - run: sudo npm install --global --force yarn@1.22.10 - - run: yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - - run: yarn lint:ci - - store_test_results: - path: /tmp/test-results - frontend-unit-tests: - environment: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - docker: - - image: circleci/node:14.17 - steps: - - checkout - - run: sudo apt update - - run: sudo apt install python3-pip - - run: sudo npm install --global --force yarn@1.22.10 - - run: yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - - run: - name: Run App Tests - command: yarn test - - run: - name: Run Visualizations Tests - command: (cd viz-lib && yarn test) - - run: yarn lint - frontend-e2e-tests: - environment: - COMPOSE_FILE: .circleci/docker-compose.cypress.yml - COMPOSE_PROJECT_NAME: cypress - PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA== - CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th - CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - docker: - - image: circleci/node:14.17 - steps: - - setup_remote_docker: - version: 19.03.13 - - checkout - - run: - name: Enable Code Coverage report for master branch - command: | - if [ "$CIRCLE_BRANCH" = "master" ]; then - echo 'export CODE_COVERAGE=true' >> $BASH_ENV - source $BASH_ENV - fi - - run: sudo npm install --global --force yarn@1.22.10 - - run: - name: Install frontend dependencies - command: yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - - run: - name: Setup Redash server - command: | - yarn cypress build - yarn cypress start -- --skip-db-seed - docker-compose run cypress yarn cypress db-seed - - run: - name: Execute Cypress tests - command: yarn cypress run-ci - - run: - name: "Failure: output container logs to console" - command: | - docker-compose logs - when: on_fail - - run: - name: Copy Code Coverage results - command: | - docker cp cypress:/usr/src/app/coverage ./coverage || true - when: always - - store_artifacts: - path: coverage - build-docker-image: *build-docker-image-job - build-preview-docker-image: *build-docker-image-job -workflows: - version: 2 - build: - jobs: - - backend-lint - - backend-unit-tests: - requires: - - backend-lint - - frontend-lint - - frontend-unit-tests: - requires: - - backend-lint - - frontend-lint - - frontend-e2e-tests: - requires: - - frontend-lint - - build-preview-docker-image: - requires: - - backend-unit-tests - - frontend-unit-tests - - frontend-e2e-tests - filters: - branches: - only: - - master - - hold: - type: approval - requires: - - backend-unit-tests - - frontend-unit-tests - - frontend-e2e-tests - filters: - branches: - only: - - /release\/.*/ - - build-docker-image: - requires: - - hold diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..7d00b665b9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,149 @@ +name: Tests +on: + push: + branches: + - master + pull_request: +env: + NODE_VERSION: 14.17 +jobs: + backend-lint: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + - run: sudo pip install flake8 + - run: ./bin/flake8_tests.sh + + backend-unit-tests: + runs-on: ubuntu-22.04 + needs: backend-lint + env: + COMPOSE_FILE: .ci/docker-compose.ci.yml + COMPOSE_PROJECT_NAME: redash + COMPOSE_DOCKER_CLI_BUILD: 1 + DOCKER_BUILDKIT: 1 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Build Docker Images + run: | + set -x + docker-compose build --build-arg skip_ds_deps=true --build-arg skip_frontend_build=true + docker-compose up -d + sleep 10 + - name: Create Test Database + run: docker-compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;" + - name: List Enabled Query Runners + run: docker-compose -p redash run --rm redash manage ds list_types + - name: Run Tests + run: docker-compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report xml --cov=redash --cov-config .coveragerc tests/ + - name: Copy Test Results + run: | + mkdir -p /tmp/test-results/unit-tests + docker cp tests:/app/coverage.xml ./coverage.xml + docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml + - name: Store Test Results + uses: actions/upload-artifact@v3 + with: + name: test-results + path: /tmp/test-results + - name: Store Coverage Results + uses: actions/upload-artifact@v3 + with: + name: coverage + path: coverage.xml + + frontend-lint: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + - name: Install Dependencies + run: | + npm install --global --force yarn@1.22.10 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + - name: Run Lint + run: yarn lint:ci + - name: Store Test Results + uses: actions/upload-artifact@v3 + with: + name: test-results + path: /tmp/test-results + + frontend-unit-tests: + runs-on: ubuntu-22.04 + needs: frontend-lint + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + - name: Install Dependencies + run: | + npm install --global --force yarn@1.22.10 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + - name: Run App Tests + run: yarn test + - name: Run Visualizations Tests + run: cd viz-lib && yarn test + - run: yarn lint + + frontend-e2e-tests: + runs-on: ubuntu-22.04 + needs: frontend-lint + env: + COMPOSE_FILE: .ci/docker-compose.cypress.yml + COMPOSE_PROJECT_NAME: cypress + PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA== + CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th + CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx + CYPRESS_INSTALL_BINARY: 0 + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + - name: Enable Code Coverage Report For Master Branch + if: endsWith(github.ref, '/master') + run: | + echo "CODE_COVERAGE=true" >> $GITHUB_ENV + - name: Install Dependencies + run: | + npm install --global --force yarn@1.22.10 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + - name: Setup Redash Server + run: | + set -x + yarn cypress build + yarn cypress start -- --skip-db-seed + docker-compose run cypress yarn cypress db-seed + - name: Execute Cypress Tests + run: yarn cypress run-ci + - name: "Failure: output container logs to console" + if: failure() + run: docker-compose logs + - name: Copy Code Coverage Results + run: docker cp cypress:/usr/src/app/coverage ./coverage || true + - name: Store Coverage Results + uses: actions/upload-artifact@v3 + with: + name: coverage + path: coverage diff --git a/client/cypress/cypress.js b/client/cypress/cypress.js index 5fd080bb31..583346e85a 100644 --- a/client/cypress/cypress.js +++ b/client/cypress/cypress.js @@ -63,10 +63,10 @@ function runCypressCI() { PERCY_TOKEN_ENCODED, CYPRESS_PROJECT_ID_ENCODED, CYPRESS_RECORD_KEY_ENCODED, - CIRCLE_REPOSITORY_URL, + GITHUB_REPOSITORY, } = process.env; - if (CIRCLE_REPOSITORY_URL && CIRCLE_REPOSITORY_URL.includes("getredash/redash")) { + if (GITHUB_REPOSITORY === "getredash/redash") { if (PERCY_TOKEN_ENCODED) { process.env.PERCY_TOKEN = atob(`${PERCY_TOKEN_ENCODED}`); } diff --git a/requirements_dev.txt b/requirements_dev.txt index 1c32a3d30f..f169eb53a2 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,6 +6,7 @@ mock==3.0.5 # PyMongo and Athena dependencies are needed for some of the unit tests: # (this is not perfect and we should resolve this in a different way) pymongo[srv,tls]==3.9.0 +boto3>=1.10.0,<1.11.0 botocore>=1.13,<1.14.0 PyAthena>=1.5.0,<=1.11.5 ptvsd==4.3.2 From c46d66afec7717199c0e90a81d080f6208de03a7 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 28 Apr 2023 10:35:30 -0700 Subject: [PATCH 2/4] Bump pysaml2 to 6.5.0 (#6013) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 407e2aeb2b..46d898010e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ rq==1.5.0 rq-scheduler==0.9.1 jsonschema==3.1.1 RestrictedPython==5.0 -pysaml2==6.1.0 +pysaml2==6.5.0 pycrypto==2.6.1 python-dotenv==0.19.2 funcy==1.13 From 3444f2b06c159f6a18d88fcd058b50bf89377963 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Apr 2023 08:33:18 -0700 Subject: [PATCH 3/4] Bump pyyaml from 5.1.2 to 5.4 (#5442) Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.1.2 to 5.4. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/5.1.2...5.4) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 46d898010e..72b27dd2e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ blinker==1.4 psycopg2==2.8.3 python-dateutil==2.8.0 pytz>=2019.3 -PyYAML==5.1.2 +PyYAML==5.4 redis==3.5.0 requests==2.21.0 SQLAlchemy==1.3.10 From 13399eca3a83ff318752b4c410403a305c083ffc Mon Sep 17 00:00:00 2001 From: Ye Sijun Date: Wed, 10 May 2023 20:14:08 +0900 Subject: [PATCH 4/4] ci: remove --record flag for cypress Signed-off-by: Ye Sijun --- .github/workflows/ci.yml | 3 --- client/cypress/cypress.js | 22 +--------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d00b665b9..7a287a15e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,9 +108,6 @@ jobs: env: COMPOSE_FILE: .ci/docker-compose.cypress.yml COMPOSE_PROJECT_NAME: cypress - PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA== - CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th - CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx CYPRESS_INSTALL_BINARY: 0 PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 steps: diff --git a/client/cypress/cypress.js b/client/cypress/cypress.js index 583346e85a..320400aa2f 100644 --- a/client/cypress/cypress.js +++ b/client/cypress/cypress.js @@ -1,6 +1,5 @@ /* eslint-disable import/no-extraneous-dependencies, no-console */ const { find } = require("lodash"); -const atob = require("atob"); const { execSync } = require("child_process"); const { get, post } = require("request").defaults({ jar: true }); const { seedData } = require("./seed-data"); @@ -59,27 +58,8 @@ function stopServer() { } function runCypressCI() { - const { - PERCY_TOKEN_ENCODED, - CYPRESS_PROJECT_ID_ENCODED, - CYPRESS_RECORD_KEY_ENCODED, - GITHUB_REPOSITORY, - } = process.env; - - if (GITHUB_REPOSITORY === "getredash/redash") { - if (PERCY_TOKEN_ENCODED) { - process.env.PERCY_TOKEN = atob(`${PERCY_TOKEN_ENCODED}`); - } - if (CYPRESS_PROJECT_ID_ENCODED) { - process.env.CYPRESS_PROJECT_ID = atob(`${CYPRESS_PROJECT_ID_ENCODED}`); - } - if (CYPRESS_RECORD_KEY_ENCODED) { - process.env.CYPRESS_RECORD_KEY = atob(`${CYPRESS_RECORD_KEY_ENCODED}`); - } - } - execSync( - "COMMIT_INFO_MESSAGE=$(git show -s --format=%s) docker-compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run --record", + "COMMIT_INFO_MESSAGE=$(git show -s --format=%s) docker-compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run", { stdio: "inherit" } ); }