From 245e7bf4b54a000d8a4d258429cc9d0e6fe049ab Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 22 Sep 2023 22:52:54 +0000 Subject: [PATCH] build: Demo of no-setup Makefile (standardized virtualenv handling) This allows a developer to run `make test` or other targets without having to manually set up a virtualenv or install requirements. The virtualenv is set up for them as a hidden subdirectory and requirements are automatically installed as needed. The goal here is larger: The new base.mk would be pulled in via git subtree from some centralized repository. Over time, this standardized base Makefile would grow and the Makefiles in regular repositories would shrink. By centralizing Makefile logic, we would gain the following: - Standardized names for make targets. Currently there is no universal way to install Python dependencies, to run all tests and quality checks, or to build docs. Developers have to learn what is used in each repo. We have standards for some of this, but they're not always followed. By composing Makefiles, we can ensure new repos always follow the standard, even as the standard changes. - Faster onboarding of developers who have not yet set up their own personalized virtualenv toolchains. Experienced Python developers already have these tools set up, but newer developers stumble on virtualenv creation. - A way to update repos to the latest Makefile improvements without having to manually update several hundreds of repos by hand. Automating the update of a git subtree reference still requires making and merging PRs but it does not require manual review of each PR, let alone manual PR creation. --- .gitignore | 2 ++ Makefile | 66 +++++++++++++++++++++++----------------------- repo-tools/base.mk | 38 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 33 deletions(-) create mode 100644 repo-tools/base.mk diff --git a/.gitignore b/.gitignore index ef2b4aea..38aff747 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ __pycache__ .pytest_cache +/.venv-3.8 + # C extensions *.so diff --git a/Makefile b/Makefile index e1c0ee09..4f17ecf7 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ extract_translations fake_translations help pull_translations push_translations \ quality requirements selfcheck test test-all upgrade validate +include repo-tools/base.mk + .DEFAULT_GOAL := help define BROWSER_PYSCRIPT @@ -20,63 +22,61 @@ help: ## display this help message @echo "Please use \`make ' where is one of" @perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' -clean: ## remove generated byte code, coverage reports, and build artifacts +clean: $(COVERAGE) ## remove generated byte code, coverage reports, and build artifacts find . -name '__pycache__' -exec rm -rf {} + find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + - coverage erase + $(COVERAGE) erase rm -fr build/ rm -fr dist/ rm -fr *.egg-info -coverage: clean ## generate and view HTML coverage report - pytest --cov-report html +coverage: clean $(PYTEST) ## generate and view HTML coverage report + $(PYTEST) --cov-report html $(BROWSER) htmlcov/index.html -docs: ## generate Sphinx HTML documentation, including API docs - tox -e docs +docs: $(TOX) ## generate Sphinx HTML documentation, including API docs + $(TOX) -e docs $(BROWSER) docs/_build/html/index.html upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade -upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in - pip install -qr requirements/pip-tools.txt +upgrade: $(PIP) $(PIP-SYNC) ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in + $(PIP) install -qr requirements/pip-tools.txt # Make sure to compile files after any other files they include! - pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in - pip-compile --no-emit-trusted-host --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in - pip install -qr requirements/pip.txt - pip install -qr requirements/pip-tools.txt - pip-compile --no-emit-trusted-host --upgrade -o requirements/base.txt requirements/base.in - pip-compile --no-emit-trusted-host --upgrade -o requirements/test.txt requirements/test.in - pip-compile --no-emit-trusted-host --upgrade -o requirements/doc.txt requirements/doc.in - pip-compile --no-emit-trusted-host --upgrade -o requirements/quality.txt requirements/quality.in - pip-compile --no-emit-trusted-host --upgrade -o requirements/ci.txt requirements/ci.in - pip-compile --no-emit-trusted-host --upgrade -o requirements/dev.txt requirements/dev.in + $(PIP-COMPILE) --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in + $(PIP-COMPILE) --no-emit-trusted-host --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in + $(PIP) install -qr requirements/pip.txt + $(PIP) install -qr requirements/pip-tools.txt + $(PIP-COMPILE) --no-emit-trusted-host --upgrade -o requirements/base.txt requirements/base.in + $(PIP-COMPILE) --no-emit-trusted-host --upgrade -o requirements/test.txt requirements/test.in + $(PIP-COMPILE) --no-emit-trusted-host --upgrade -o requirements/doc.txt requirements/doc.in + $(PIP-COMPILE) --no-emit-trusted-host --upgrade -o requirements/quality.txt requirements/quality.in + $(PIP-COMPILE) --no-emit-trusted-host --upgrade -o requirements/ci.txt requirements/ci.in + $(PIP-COMPILE) --no-emit-trusted-host --upgrade -o requirements/dev.txt requirements/dev.in # Let tox control the Django version for tests sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp mv requirements/test.tmp requirements/test.txt -isort: ## fixes isort issues found during quality check - tox -e isort +isort: $(TOX) ## fixes isort issues found during quality check + $(TOX) -e isort quality: ## check coding style with pycodestyle and pylint - tox -e quality + $(TOX) -e quality -requirements: ## install development environment requirements - pip install -r requirements/pip.txt - pip install -qr requirements/pip-tools.txt - pip-sync requirements/dev.txt requirements/private.* - pip install . # CLI entry points +requirements: $(PIP) $(PIP-SYNC) ## install development environment requirements + $(PIP-SYNC) requirements/dev.txt requirements/private.* + $(PIP) install . # CLI entry points -test: clean ## run tests in the current virtualenv - pytest +test: clean $(PYTEST) ## run tests in the current virtualenv + $(PYTEST) -diff_cover: test ## find diff lines that need test coverage - diff-cover coverage.xml +diff_cover: test $(DIFF-COVER) ## find diff lines that need test coverage + $(DIFF-COVER) coverage.xml -test-all: ## run tests on every supported Python/Django combination - tox -e quality - tox +test-all: $(TOX) ## run tests on every supported Python/Django combination + $(TOX) -e quality + $(TOX) validate: quality test ## run tests and quality checks diff --git a/repo-tools/base.mk b/repo-tools/base.mk new file mode 100644 index 00000000..9efe8088 --- /dev/null +++ b/repo-tools/base.mk @@ -0,0 +1,38 @@ +# Base Makefile components to build on. +# +# Ideally, this would be pulled in from another repo by a `git subtree` +# reference and would include most of the Makefile so that upgrading +# repos to the latest functionality (and keeping them standardized) +# would be fast and simple. + +PYTHON_VER := 3.8 + +# These are required for bootstrapping a virtualenv to the point where +# requirements can be installed. + +VENV := .venv-$(PYTHON_VER) +PIP := $(VENV)/bin/pip +PIP-COMPILE := $(VENV)/bin/pip-compile +PIP-SYNC := $(VENV)/bin/pip-sync + +$(VENV): ## Create the virtualenv, or recreate it if it already exists + python$(PYTHON_VER) -m venv $(VENV) --clear + +$(PIP): $(VENV) + pip install -r requirements/pip.txt + +$(PIP-COMPILE) $(PIP-SYNC): $(VENV) + $(PIP) install -r requirements/pip-tools.txt + +# The rest of the Python tools are listed here, and all are fulfilled +# by the same target. By calling them from their explicit virtualenv +# paths, we ensure that the virtualenv will be used (including by +# other tools they call). + +COVERAGE := $(VENV)/bin/coverage +DIFF-COVER := $(VENV)/bin/diff-cover +PYTEST := $(VENV)/bin/pytest +TOX := $(VENV)/bin/tox + +$(VENV)/bin/%: $(VENV) + make requirements