forked from DonJayamanne/pythonVSCode
-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Move coverage to nightly and simplify reporting (#15776)
* Enable coverage for venv and single workspace tests * Upload coverage for single workspace * Try with spawn wrap * Fix coverage artifact upload * Try running tests with coverage directly * Run coverage separately * Remove start page from coverage * Add back codecov * No need to move codecov.yml * Ensure nightly coverage runs on schedule only * Nightly should only run coverage * Address comments * Remove setting node version env for each job
- Loading branch information
1 parent
6bb8c69
commit 9b881fc
Showing
8 changed files
with
375 additions
and
218 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -29,6 +29,3 @@ comment: | |
| layout: 'diff, files' | ||
| behavior: default | ||
| require_changes: no | ||
|
|
||
| fixes: | ||
| - 'path with spaces/::' | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,188 @@ | ||
| name: Coverage | ||
|
|
||
| on: | ||
| schedule: | ||
| # Run daily at 2:00 AM | ||
| - cron: '0 2 * * *' | ||
|
|
||
| env: | ||
| NODE_VERSION: 12.15.0 | ||
| PYTHON_VERSION: 3.9 | ||
| MOCHA_REPORTER_JUNIT: true # Use the mocha-multi-reporters and send output to both console (spec) and JUnit (mocha-junit-reporter). Also enables a reporter which exits the process running the tests if it haven't already. | ||
| ARTIFACT_NAME_VSIX: ms-python-insiders-vsix | ||
| VSIX_NAME: ms-python-insiders.vsix | ||
| TEST_RESULTS_DIRECTORY: . | ||
|
|
||
| jobs: | ||
| tests: | ||
| name: Tests with Coverage | ||
| # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. | ||
| runs-on: ${{ matrix.os }} | ||
|
|
||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, | ||
| # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. | ||
| os: [ubuntu-latest, windows-latest] | ||
|
|
||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v2 | ||
|
|
||
| - name: Install Node | ||
| uses: actions/setup-node@v2.1.5 | ||
| with: | ||
| node-version: ${{env.NODE_VERSION}} | ||
|
|
||
| - name: Install dependencies (npm ci) | ||
| run: npm ci | ||
|
|
||
| - name: Compile | ||
| run: npx gulp prePublishNonBundle | ||
|
|
||
| - name: Use Python ${{env.PYTHON_VERSION}} | ||
| uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: ${{env.PYTHON_VERSION}} | ||
|
|
||
| - name: Install Python requirements | ||
| run: | | ||
| python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt --no-user | ||
| # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. | ||
| python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy | ||
| - name: Install test requirements | ||
| run: python -m pip install --upgrade -r build/test-requirements.txt | ||
|
|
||
| - name: Install functional test requirements | ||
| run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt | ||
|
|
||
| - name: Prepare pipenv for venv tests | ||
| env: | ||
| TEST_FILES_SUFFIX: testvirtualenvs | ||
| PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' | ||
| shell: pwsh | ||
| run: | | ||
| python -m pip install pipenv | ||
| python -m pipenv run python ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} pipenvPath | ||
| - name: Prepare virtualenv for venv tests | ||
| env: | ||
| TEST_FILES_SUFFIX: testvirtualenvs | ||
| PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' | ||
| shell: pwsh | ||
| run: | | ||
| python -m pip install virtualenv | ||
| python -m virtualenv .virtualenv/ | ||
| if ('${{matrix.os}}' -match 'windows-latest') { | ||
| & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} virtualEnvPath | ||
| } else { | ||
| & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} virtualEnvPath | ||
| } | ||
| - name: Prepare venv for venv tests | ||
| env: | ||
| TEST_FILES_SUFFIX: testvirtualenvs | ||
| PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' | ||
| shell: pwsh | ||
| run: | | ||
| python -m venv .venv | ||
| if ('${{matrix.os}}' -match 'windows-latest') { | ||
| & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} venvPath | ||
| } else { | ||
| & ".venv/bin/python" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} venvPath | ||
| } | ||
| - name: Prepare conda for venv tests | ||
| env: | ||
| TEST_FILES_SUFFIX: testvirtualenvs | ||
| PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' | ||
| shell: pwsh | ||
| run: | | ||
| # 1. For `terminalActivation.testvirtualenvs.test.ts` | ||
| if ('${{matrix.os}}' -match 'windows-latest') { | ||
| $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe | ||
| $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda | ||
| } else{ | ||
| $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python | ||
| $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda | ||
| } | ||
| & $condaPythonPath ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} condaExecPath $condaExecPath | ||
| & $condaPythonPath ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} condaPath | ||
| # 2. For `interpreterLocatorService.testvirtualenvs.ts` | ||
| & $condaExecPath create -n "test_env1" -y python | ||
| & $condaExecPath create -p "./test_env2" -y python | ||
| & $condaExecPath create -p "~/test_env3" -y python | ||
| - name: Run TypeScript unit tests | ||
| run: npm run test:unittests:cover | ||
|
|
||
| - name: Run Python unit tests | ||
| run: | | ||
| python pythonFiles/tests/run_all.py | ||
| # The virtual environment based tests use the `testSingleWorkspace` set of tests | ||
| # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, | ||
| # which is set in the "Prepare environment for venv tests" step. | ||
| # We also use a third-party GitHub Action to install xvfb on Linux, | ||
| # run tests and then clean up the process once the tests ran. | ||
| # See https://github.com/GabrielBB/xvfb-action | ||
| - name: Run venv tests | ||
| env: | ||
| TEST_FILES_SUFFIX: testvirtualenvs | ||
| CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}} | ||
| CI_DISABLE_AUTO_SELECTION: 1 | ||
| uses: GabrielBB/xvfb-action@v1.4 | ||
| with: | ||
| run: npm run testSingleWorkspace:cover | ||
|
|
||
| - name: Run single-workspace tests | ||
| env: | ||
| CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}} | ||
| CI_DISABLE_AUTO_SELECTION: 1 | ||
| uses: GabrielBB/xvfb-action@v1.4 | ||
| with: | ||
| run: npm run testSingleWorkspace:cover | ||
|
|
||
| # Enable these tests when coverage is setup for multiroot workspace tests | ||
| # - name: Run multi-workspace tests | ||
| # env: | ||
| # CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}} | ||
| # CI_DISABLE_AUTO_SELECTION: 1 | ||
| # uses: GabrielBB/xvfb-action@v1.4 | ||
| # with: | ||
| # run: npm run testMultiWorkspace:cover | ||
|
|
||
| # Enable these tests when coverage is setup for debugger tests | ||
| # - name: Run debugger tests | ||
| # env: | ||
| # CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}} | ||
| # CI_DISABLE_AUTO_SELECTION: 1 | ||
| # uses: GabrielBB/xvfb-action@v1.4 | ||
| # with: | ||
| # run: npm run testDebugger:cover | ||
|
|
||
| - name: Run TypeScript functional tests | ||
| env: | ||
| CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}} | ||
| CI_DISABLE_AUTO_SELECTION: 1 | ||
| run: npm run test:functional:cover | ||
|
|
||
| - name: Generate coverage reports | ||
| run: npm run test:cover:report | ||
|
|
||
| - name: Upload HTML report | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: ${{ runner.os }}-coverage-report-html | ||
| path: ./coverage | ||
| retention-days: 1 | ||
|
|
||
| - name: Upload coverage to codecov | ||
| uses: codecov/codecov-action@v1 | ||
| with: | ||
| token: ${{ secrets.CODECOV_TOKEN }} | ||
| file: ./coverage/cobertura-coverage.xml |
Oops, something went wrong.