From 9ebe59331d9e2430d349178aed2bbe7ad6c31c3e Mon Sep 17 00:00:00 2001 From: Callahan Date: Thu, 15 Jun 2023 17:42:51 -0500 Subject: [PATCH] ci: add tox (#295) Signed-off-by: Callahan Kovacs --- .github/PULL_REQUEST_TEMPLATE.md | 2 + .github/workflows/tests.yaml | 129 ++++++++++++++++++------------- Makefile | 104 ------------------------- docs/conf.py | 2 +- pyproject.toml | 72 +++++++++++------ pyrightconfig.json | 11 --- tox.ini | 123 +++++++++++++++++++++++++++++ 7 files changed, 249 insertions(+), 194 deletions(-) delete mode 100644 Makefile delete mode 100644 pyrightconfig.json create mode 100644 tox.ini diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 440f2add..da2c9731 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,5 @@ +- [ ] Have you followed the guidelines for contributing? - [ ] Have you signed the [CLA](http://www.ubuntu.com/legal/contributors/)? +- [ ] Have you successfully run `tox`? ----- diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d206bfe1..6c41fac2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,17 +1,14 @@ -name: Tests +name: tests on: push: branches: - "main" - "feature/*" - - "release/*" - "hotfix/*" + - "release/*" pull_request: - # Allows triggering the workflow manually from the Actions tab - workflow_dispatch: - jobs: linters: runs-on: ubuntu-latest @@ -23,7 +20,8 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.10' + cache: 'pip' - name: Configure environment run: | echo "::group::Begin snap install" @@ -32,61 +30,68 @@ jobs: sudo snap install --no-wait shellcheck echo "::endgroup::" echo "::group::pip install" - python -m pip install -U .[dev,lint,types] + python -m pip install 'tox>=4' tox-gh + echo "::endgroup::" + echo "::group::Create virtual environments for linting processes." + tox run -m lint --notest echo "::endgroup::" echo "::group::Wait for snap to complete" snap watch --last=install echo "::endgroup::" - - name: Run black - run: | - make test-black - - name: Run ruff - run: | - make test-ruff - - name: Run codespell - run: | - make test-codespell - - name: Run mypy - run: | - make test-mypy - - name: Run pyright - run: | - make test-pyright - - name: Run shellcheck - run: | - make test-shellcheck - - name: Run yamllint - run: | - make test-yaml - + - name: Run Linters + run: tox run --skip-pkg-install --no-list-dependencies -m lint unit-tests: strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.8", "3.10", "3.11", "3.12-dev"] - runs-on: ${{ matrix.os }} + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} steps: - - name: Checkout code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python on ${{ matrix.platform }} + - name: Set up Python versions on ${{ matrix.platform }} uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: | + 3.8 + 3.9 + 3.10 + 3.11 + 3.12-dev + cache: 'pip' - name: Configure environment run: | echo "::group::pip install" - python -m pip install -U .[dev] - python -m pip install -U -e . + python -m pip install 'tox>=4.6' tox-gh echo "::endgroup::" - - name: Run unit tests - run: make test-units - + mkdir -p results + - name: Setup Tox environments + run: tox run -m unit-tests --notest + - name: Test with tox + # use `tox` instead of `.tox/.tox/bin/tox` to support Windows + run: tox run --skip-pkg-install --no-list-dependencies --result-json results/tox-${{ matrix.platform }}.json -m unit-tests + env: + PYTEST_ADDOPTS: "--no-header -vv -rN" + - name: Upload code coverage + uses: codecov/codecov-action@v3 + with: + directory: ./results/ + files: coverage*.xml + - name: Upload test results + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: test-results-${{ matrix.platform }} + path: results/ integration-tests-linux: strategy: matrix: - python-version: ["3.8", "3.10", "3.11", "3.12-dev"] + python: [ + {system-version: "3.8", tox-version: "py38"}, + {system-version: "3.10", tox-version: "py310"}, + {system-version: "3.11", tox-version: "py311"}, + {system-version: "3.12-dev", tox-version: "py312"}, + ] # does not work with canonical/setup-lxd github action (see https://github.com/canonical/craft-providers/issues/271) runs-on: ubuntu-latest steps: @@ -97,11 +102,12 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python.system-version }} + cache: 'pip' - name: Configure environment run: | echo "::group::pip install" - python -m pip install -U .[dev] + python -m pip install 'tox>=4.6' echo "::endgroup::" echo "::group::Configure LXD" sudo groupadd --force --system lxd @@ -114,18 +120,26 @@ jobs: sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT echo "::endgroup::" + - name: Setup Tox environments + run: tox run -e integration-${{ matrix.python.tox-version }} --notest - name: Run integration tests on Linux + env: + CRAFT_PROVIDERS_TESTS_ENABLE_SNAP_INSTALL: 1 + CRAFT_PROVIDERS_TESTS_ENABLE_LXD_INSTALL: 1 + CRAFT_PROVIDERS_TESTS_ENABLE_LXD_UNINSTALL: 1 + PYTEST_ADDOPTS: "--no-header -vv -rN" run: | - export CRAFT_PROVIDERS_TESTS_ENABLE_SNAP_INSTALL=1 - export CRAFT_PROVIDERS_TESTS_ENABLE_LXD_INSTALL=1 - export CRAFT_PROVIDERS_TESTS_ENABLE_LXD_UNINSTALL=1 sg lxd -c "lxc version" - sg lxd -c "make test-integrations" - + sg lxd -c ".tox/.tox/bin/tox run --skip-pkg-install --no-list-dependencies -e integration-${{ matrix.python.tox-version }}" integration-tests-macos: strategy: matrix: - python-version: ["3.8", "3.10", "3.11", "3.12-dev"] + python: [ + {system-version: "3.8", tox-version: "py38"}, + {system-version: "3.10", tox-version: "py310"}, + {system-version: "3.11", tox-version: "py311"}, + {system-version: "3.12-dev", tox-version: "py312"}, + ] runs-on: macos-latest steps: - name: Checkout code @@ -135,11 +149,12 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python.system-version }} + cache: 'pip' - name: Configure environment run: | echo "::group::pip install" - python -m pip install -U -e .[dev] + python -m pip install 'tox>=4.6' echo "::endgroup::" echo "::group::Install Multipass" brew update @@ -151,8 +166,12 @@ jobs: sleep 20 multipass version echo "::endgroup::" + - name: Setup Tox environments + run: tox run -e integration-${{ matrix.python.tox-version }} --notest - name: Run integration tests on MacOS + env: + CRAFT_PROVIDERS_TESTS_ENABLE_MULTIPASS_INSTALL: 1 + CRAFT_PROVIDERS_TESTS_ENABLE_MULTIPASS_UNINSTALL: 1 + PYTEST_ADDOPTS: "--no-header -vv -rN" run: | - export CRAFT_PROVIDERS_TESTS_ENABLE_MULTIPASS_INSTALL=1 - export CRAFT_PROVIDERS_TESTS_ENABLE_MULTIPASS_UNINSTALL=1 - make test-integrations + .tox/.tox/bin/tox run --skip-pkg-install --no-list-dependencies -e integration-${{ matrix.python.tox-version }} diff --git a/Makefile b/Makefile deleted file mode 100644 index ed4a862a..00000000 --- a/Makefile +++ /dev/null @@ -1,104 +0,0 @@ -SOURCES=$(wildcard *.py) craft_providers tests - -.PHONY: help -help: ## Show this help. - @printf "%-40s %s\n" "Target" "Description" - @printf "%-40s %s\n" "------" "-----------" - @fgrep " ## " $(MAKEFILE_LIST) | fgrep -v grep | awk -F ': .*## ' '{$$1 = sprintf("%-40s", $$1)} 1' - -.PHONY: autoformat -autoformat: ## Run automatic code formatters. - ruff --fix --respect-gitignore . - black $(SOURCES) - -.PHONY: clean -clean: ## Clean artifacts from building, testing, etc. - rm -rf build/ - rm -rf dist/ - rm -rf .eggs/ - find . -name '*.egg-info' -exec rm -rf {} + - find . -name '*.egg' -exec rm -f {} + - rm -rf docs/_build/ - rm -f docs/craft_providers.* - rm -f docs/modules.rst - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -rf {} + - rm -f .coverage - rm -rf htmlcov/ - rm -rf .pytest_cache - -.PHONY: coverage -coverage: ## Run pytest with coverage report. - coverage run --source craft_providers -m pytest - coverage report -m - coverage html - -.PHONY: docs -docs: ## Generate documentation. - rm -f docs/craft_providers.rst - rm -f docs/modules.rst - pip install -r docs/requirements.txt - $(MAKE) -C docs clean - $(MAKE) -C docs html - -.PHONY: dist -dist: clean ## Build python package. - python setup.py sdist - python setup.py bdist_wheel - ls -l dist - -.PHONY: freeze-requirements -freeze-requirements: ## Re-freeze requirements. - tools/freeze-requirements.sh - -.PHONY: install -install: clean ## Install python package. - python setup.py install - -.PHONY: lint -lint: test-black test-ruff test-codespell test-mypy test-pyright test-shellcheck test-yaml ## Run all linting tests. - -.PHONY: release -release: dist ## Release with twine. - twine upload dist/* - -.PHONY: test-black -test-black: - black --check --diff $(SOURCES) - -.PHONY: test-codespell -test-codespell: - codespell $(SOURCES) - -.PHONY: test-integrations -test-integrations: ## Run integration tests. - pytest tests/integration - -.PHONY: test-mypy -test-mypy: - mypy $(SOURCES) - -.PHONY: test-pyright -test-pyright: - pyright $(SOURCES) - -.PHONY: test-ruff -test-ruff: - ruff check --respect-gitignore . - -.PHONY: test-shellcheck -test-shellcheck: - git ls-files | file --mime-type -Nnf- | grep shellscript | cut -f1 -d: | xargs shellcheck - -.PHONY: test-yaml -test-yaml: - yamllint . - -.PHONY: test-units -test-units: ## Run unit tests. - pytest tests/unit - -.PHONY: tests -tests: lint test-integrations test-units ## Run all tests. diff --git a/docs/conf.py b/docs/conf.py index 884f5e56..9d2f3ef7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,7 +88,7 @@ def run_apidoc(_): - from sphinx.ext.apidoc import main + from sphinx.ext.apidoc import main # type: ignore import os import sys diff --git a/pyproject.toml b/pyproject.toml index b0ba4b0a..ef6f972a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,28 +29,28 @@ craft-providers = "craft_providers.cli:main" [project.optional-dependencies] dev = [ - "twine", - "coverage", - "freezegun", - "logassert", - "pyfakefs", - "pytest", - "pytest-mock", - "pytest-subprocess", - "responses", - "types-requests", - "types-setuptools", - "types-pyyaml", + "coverage[toml]==7.2.7", + "freezegun==1.2.2", + "logassert==7", + "pyfakefs==5.2.2", + "pytest==7.3.1", + "pytest-cov==4.1.0", + "pytest-mock==3.10.0", + "pytest-subprocess==1.5.0", + "responses==0.23.1", + "types-requests==2.31.0.1", + "types-setuptools==67.8.0.0", + "types-pyyaml==6.0.12.10", ] lint = [ - "black", - "codespell", + "black==23.3.0", + "codespell[toml]==2.2.4", "ruff==0.0.270", - "yamllint", + "yamllint==1.32.0", ] types = [ - "mypy", - "pyright", + "mypy[reports]==1.3.0", + "pyright==1.1.311", ] docs = [ "sphinx", @@ -74,25 +74,51 @@ write_to = "craft_providers/_version.py" [tool.setuptools.packages.find] exclude = [ - "dist", - "docs", - "results", - "tests", + "dist*", + "docs*", + "results*", + "tests*", ] - - [tool.distutils.bdist_wheel] universal = true +[tool.black] +target-version = ["py38"] + [tool.codespell] ignore-words-list = "buildd,crate,keyserver,comandos,ro,dedent,dedented" skip = ".tox,.git,build,.*_cache,__pycache__,*.tar,*.snap,*.png,./node_modules,./docs/_build,.direnv,.venv,venv,.vscode" quiet-level = 3 check-filenames = true +[tool.pytest.ini_options] +minversion = "7.0" +testpaths = "tests" +xfail_strict = true + +[tool.coverage.run] +branch = true +parallel = true +omit = ["tests/**"] + +[tool.coverage.report] +skip_empty = true +fail_under = 95 + +[tool.pyright] +# strict type-checking with pyright is disabled (see #287) +# strict = ["craft_providers"] +pythonVersion = "3.8" +pythonPlatform = "Linux" + [tool.mypy] python_version = "3.8" +exclude = [ + "build", + "tests", + "results", +] warn_unused_configs = true warn_redundant_casts = true strict_equality = true diff --git a/pyrightconfig.json b/pyrightconfig.json deleted file mode 100644 index 016a5409..00000000 --- a/pyrightconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "exclude": [ - "**/__pycache__", - "**/.mypy_cache", - "**/.pytest_cache", - ".direnv", - "build", - "docs", - "venv" - ] -} diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..386b1ed7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,123 @@ +[tox] +env_list = # Environments to run when called with no parameters. + lint-{black,ruff,pyright,shellcheck,codespell} + test-{py38,py310,py311} +minversion = 4.6 +# Tox will use these requirements to bootstrap a venv if necessary. +# tox-igore-env-name-mismatch allows us to have one virtualenv for all linting. +# By setting requirements here, we make this INI file compatible with older +# versions of tox. Tox >= 3.8 will automatically provision the version provided +# inside of a virtual environment, so users of Ubuntu >= focal can simply +# install tox from apt. Older than that, the user gets an upgrade warning. +requires = + # renovate: datasource=pypi + tox-ignore-env-name-mismatch>=0.2.0.post2 +# Allow tox to access the user's $TMPDIR environment variable if set. +# This workaround is required to avoid circular dependencies for TMPDIR, +# since tox will otherwise attempt to use the environment's TMPDIR variable. +user_tmp_dir = {env:TMPDIR} + +[testenv] # Default config for all environments. Overridable in each env. +# We have many tests that create temporary files. Unless the user has set a +# TMPDIR, this will prefer putting those temp files in $XDG_RUNTIME_DIR, +# which will speed up those tests since they'll run on a ramdisk. +env_tmp_dir = {user_tmp_dir:{env:XDG_RUNTIME_DIR:{work_dir}}}/tox_tmp/{env_name} +set_env = + TMPDIR={env_tmp_dir} + COVERAGE_FILE={env_tmp_dir}/.coverage_{env_name} +pass_env = CRAFT_PROVIDERS_TESTS_* + +[test] # Base configuration for unit and integration tests +package = editable +extras = dev +allowlist_externals = mkdir +commands_pre = mkdir -p results + +[testenv:test-{py38,py39,py310,py311,py312}] # Configuration for all tests using pytest +base = testenv, test +description = Run unit tests with pytest +labels = + py38, py310, py311: tests, unit-tests +commands = pytest {tty:--color=yes} --cov --cov-report=xml:results/coverage-{env_name}.xml --junit-xml=results/test-results-{env_name}.xml tests/unit {posargs} + +[testenv:integration-{py38,py39,py310,py311,py312}] +base = testenv, test +description = Run integration tests with pytest +labels = + py38, py310, py311: tests, integration-tests +commands = pytest {tty:--color=yes} --junit-xml=results/test-results-{env_name}.xml tests/integration {posargs} + +[lint] # Standard linting configuration +package = editable +extras = lint +env_dir = {work_dir}/linting +runner = ignore_env_name_mismatch + +[shellcheck] +find = git ls-files +filter = file --mime-type -Nnf- | grep shellscript | cut -f1 -d: + +[testenv:lint-{black,ruff,shellcheck,codespell,yaml}] +description = Lint the source code +base = testenv, lint +labels = lint +allowlist_externals = + shellcheck: bash, xargs +commands_pre = + shellcheck: bash -c '{[shellcheck]find} | {[shellcheck]filter} > {env_tmp_dir}/shellcheck_files' +commands = + black: black --check --diff {tty:--color} {posargs} . + ruff: ruff check --respect-gitignore {posargs} . + shellcheck: xargs -ra {env_tmp_dir}/shellcheck_files shellcheck + codespell: codespell --toml {tox_root}/pyproject.toml {posargs} + yaml: yamllint {posargs} . + +[testenv:lint-{mypy,pyright}] +description = Static type checking +base = testenv, lint +env_dir = {work_dir}/typing +extras = dev, types +labels = lint, type +allowlist_externals = + mypy: mkdir +commands_pre = + mypy: mkdir -p .mypy_cache +commands = + pyright: pyright {posargs} + mypy: mypy --install-types --non-interactive {posargs:.} + +[testenv:format-{black,ruff,codespell}] +description = Automatically format source code +base = testenv, lint +labels = format +commands = + black: black {tty:--color} {posargs} . + ruff: ruff --fix --respect-gitignore {posargs} . + codespell: codespell --toml {tox_root}/pyproject.toml --write-changes {posargs} + +# docs are not yet configured (see #287 or CRAFT-1799) +# [docs] # Sphinx documentation configuration +# extras = docs +# package = editable +# no_package = true +# env_dir = {work_dir}/docs +# runner = ignore_env_name_mismatch +# +# [testenv:build-docs] +# description = Build sphinx documentation +# base = docs +# allowlist_externals = bash +# commands_pre = bash -c 'if [[ ! -e docs ]];then echo "No docs directory. Run `tox run -e sphinx-quickstart` to create one.;";return 1;fi' +# # "-W" is to treat warnings as errors +# commands = sphinx-build {posargs:-b html} -W {tox_root}/docs {tox_root}/docs/_build +# +# [testenv:autobuild-docs] +# description = Build documentation with an autoupdating server +# base = docs +# commands = sphinx-autobuild {posargs:-b html --open-browser --port 8080} -W --watch {tox_root}/starcraft {tox_root}/docs {tox_root}/docs/_build +# +# [testenv:lint-docs] +# description = Lint the documentation with sphinx-lint +# base = docs +# commands = sphinx-lint --ignore docs/_build --max-line-length 80 -e all {posargs} docs/ +# labels = lint