diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 106b452abd3..88e217b547e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,14 +32,14 @@ jobs: - name: Cache PyPI uses: actions/cache@v2 with: - key: pip-lint-${{ hashFiles('requirements/*.txt') }} + key: pip-lint-${{ hashFiles('requirements/pinned/*.txt') }} path: ~/.cache/pip restore-keys: | pip-lint- - name: Install dependencies uses: py-actions/py-dependency-install@v2 with: - path: requirements/lint.txt + path: requirements/pinned/linux-lint.txt - name: Pre-Commit hooks uses: pre-commit/action@v2.0.0 - name: Install itself @@ -53,7 +53,7 @@ jobs: - name: Install spell checker run: | sudo apt install libenchant-dev - pip install -r requirements/doc-spelling.txt + pip install -r requirements/pinned/linux-3.8-doc-spelling.txt - name: Run docs spelling run: | # towncrier --yes # uncomment me after publishing a release @@ -111,7 +111,7 @@ jobs: - name: Cache PyPI uses: actions/cache@v2 with: - key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }} + key: pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/pinned/*.txt') }} path: ${{ steps.pip-cache.outputs.dir }} restore-keys: | pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}- diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1688c3dcb4..94fd2ea76d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: ^docs/[^/]*\.svg$ - id: requirements-txt-fixer exclude: >- - ^requirements/dev.txt$ + ^requirements/pinned/.*\.txt$ - id: trailing-whitespace - id: file-contents-sorter files: | diff --git a/CHANGES/5486.misc b/CHANGES/5486.misc new file mode 100644 index 00000000000..bc8fa84a4e3 --- /dev/null +++ b/CHANGES/5486.misc @@ -0,0 +1 @@ +Complete pip-tools setup. diff --git a/Makefile b/Makefile index 6a617042960..c72dd3fe584 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,10 @@ CYS := $(wildcard aiohttp/*.pyx) $(wildcard aiohttp/*.pyi) $(wildcard aiohttp/* PYXS := $(wildcard aiohttp/*.pyx) CS := $(wildcard aiohttp/*.c) PYS := $(wildcard aiohttp/*.py) -REQS := $(wildcard requirements/*.txt) +REQS := $(wildcard requirements/pinned/*.txt) ALLS := $(sort $(CYS) $(CS) $(PYS) $(REQS)) + .PHONY: all all: test @@ -45,9 +46,11 @@ endif # Enumerate intermediate files to don't remove them automatically. .SECONDARY: $(call to-hash,$(ALLS)) +.update-pip: + @pip install -U 'pip' -.install-cython: $(call to-hash,requirements/cython.txt) - pip install -r requirements/cython.txt +.install-cython: .update-pip $(call to-hash,requirements/pinned/*cython.txt) + @python tools/deps.py install cython @touch .install-cython aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py) @@ -62,7 +65,7 @@ aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c cythonize: .install-cython $(PYXS:.pyx=.c) .install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS)) - pip install -r requirements/dev.txt + @python tools/deps.py install dev @touch .install-deps .PHONY: lint @@ -135,17 +138,14 @@ doc: doc-spelling: @make -C docs spelling SPHINXOPTS="-W -E" -.update-pip: - @pip install -U 'pip' - .PHONY: compile-deps compile-deps: .update-pip @pip install pip-tools - @pip-compile --allow-unsafe -q requirements/dev.in + @python tools/deps.py compile .PHONY: install install: .update-pip - @pip install -r requirements/dev.in -c requirements/dev.txt + @python tools/deps.py install dev .PHONY: install-dev install-dev: .develop diff --git a/requirements/cython.txt b/requirements/cython.in similarity index 100% rename from requirements/cython.txt rename to requirements/cython.in diff --git a/requirements/dev.in b/requirements/dev.in index fc7aee6945c..c5cb20f7102 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -1,4 +1,4 @@ --r lint.txt +-r lint.in -r test.txt -r doc.txt cherry_picker==1.3.2; python_version>="3.6" diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index 6c3e3f33d68..00000000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,252 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --allow-unsafe requirements/dev.in -# -aiodns==2.0.0 ; sys_platform == "linux" or sys_platform == "darwin" and python_version >= "3.7" - # via -r requirements/base.txt -aiohttp-theme==0.1.6 - # via -r requirements/doc.txt -aiosignal==1.1.2 - # via -r requirements/base.txt -alabaster==0.7.12 - # via sphinx -appdirs==1.4.4 - # via - # black - # virtualenv -async-generator==1.10 - # via -r requirements/base.txt -async-timeout==4.0.0a3 - # via -r requirements/base.txt -attrs==20.3.0 - # via - # -r requirements/base.txt - # flake8-pyi - # pytest -babel==2.9.0 - # via sphinx -black==20.8b1 ; implementation_name == "cpython" - # via -r requirements/lint.txt -blockdiag==2.0.1 - # via sphinxcontrib-blockdiag -brotli==1.0.9 - # via -r requirements/base.txt -cchardet==2.1.7 - # via -r requirements/base.txt -certifi==2020.12.5 - # via requests -cffi==1.14.4 - # via - # cryptography - # pycares -cfgv==3.2.0 - # via pre-commit -chardet==4.0.0 - # via - # -r requirements/base.txt - # requests -cherry_picker==1.3.2 ; python_version >= "3.6" - # via -r requirements/dev.in -click==7.1.2 - # via - # black - # cherry-picker - # towncrier -coverage==5.3.1 - # via - # -r requirements/test.txt - # pytest-cov -cryptography==3.3.1 - # via - # pyjwt - # trustme -distlib==0.3.1 - # via virtualenv -docutils==0.16 - # via sphinx -filelock==3.0.12 - # via virtualenv -flake8-pyi==20.10.0 - # via -r requirements/lint.txt -flake8==3.8.4 - # via - # -r requirements/lint.txt - # flake8-pyi -freezegun==1.0.0 - # via -r requirements/test.txt -frozenlist==1.1.1 - # via - # -r requirements/base.txt - # aiosignal -funcparserlib==0.3.6 - # via blockdiag -gidgethub==5.0.0 - # via cherry-picker -gunicorn==20.0.4 - # via -r requirements/base.txt -identify==1.5.12 - # via pre-commit -idna==2.10 - # via - # requests - # trustme - # yarl -imagesize==1.2.0 - # via sphinx -incremental==17.5.0 - # via towncrier -iniconfig==1.1.1 - # via pytest -isort==5.6.4 - # via -r requirements/lint.txt -jinja2==2.11.2 - # via - # sphinx - # towncrier -markupsafe==1.1.1 - # via jinja2 -mccabe==0.6.1 - # via flake8 -multidict==5.1.0 - # via - # -r requirements/multidict.txt - # yarl -mypy-extensions==0.4.3 ; implementation_name == "cpython" - # via - # -r requirements/test.txt - # black - # mypy -mypy==0.790 ; implementation_name == "cpython" - # via - # -r requirements/lint.txt - # -r requirements/test.txt -nodeenv==1.5.0 - # via pre-commit -packaging==20.8 - # via - # pytest - # sphinx -pathspec==0.8.1 - # via black -pillow==8.1.0 - # via blockdiag -pluggy==0.13.1 - # via pytest -pre-commit==2.9.3 - # via -r requirements/lint.txt -py==1.10.0 - # via pytest -pycares==3.1.1 - # via aiodns -pycodestyle==2.6.0 - # via flake8 -pycparser==2.20 - # via cffi -pyflakes==2.2.0 - # via - # flake8 - # flake8-pyi -pygments==2.8.0 - # via - # -r requirements/doc.txt - # sphinx -pyjwt[crypto]==2.0.0 - # via gidgethub -pyparsing==2.4.7 - # via packaging -pytest-cov==2.10.1 - # via -r requirements/test.txt -pytest-mock==3.5.1 - # via -r requirements/test.txt -pytest==6.1.2 - # via - # -r requirements/lint.txt - # -r requirements/test.txt - # pytest-cov - # pytest-mock -python-dateutil==2.8.1 - # via freezegun -pytz==2020.5 - # via babel -pyyaml==5.3.1 - # via pre-commit -re-assert==1.1.0 - # via -r requirements/test.txt -regex==2020.11.13 - # via - # black - # re-assert -requests==2.25.1 - # via - # cherry-picker - # sphinx -setuptools-git==1.2 - # via -r requirements/test.txt -six==1.15.0 - # via - # cryptography - # python-dateutil - # virtualenv -snowballstemmer==2.0.0 - # via sphinx -sphinx==3.5.0 - # via - # -r requirements/doc.txt - # sphinxcontrib-asyncio - # sphinxcontrib-blockdiag -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-asyncio==0.3.0 - # via -r requirements/doc.txt -sphinxcontrib-blockdiag==2.0.0 - # via -r requirements/doc.txt -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==1.0.3 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.4 - # via sphinx -toml==0.10.2 - # via - # black - # cherry-picker - # pre-commit - # pytest - # towncrier -towncrier==19.2.0 - # via -r requirements/doc.txt -trustme==0.7.0 ; platform_machine != "i686" - # via -r requirements/test.txt -typed-ast==1.4.2 - # via - # black - # mypy -typing-extensions==3.7.4.3 - # via - # -r requirements/base.txt - # async-timeout - # black - # mypy -uritemplate==3.0.1 - # via gidgethub -urllib3==1.26.2 - # via requests -virtualenv==20.3.1 - # via pre-commit -webcolors==1.11.1 - # via blockdiag -yarl==1.6.3 - # via -r requirements/base.txt - -# The following packages are considered to be unsafe in a requirements file: -setuptools==51.3.1 - # via - # blockdiag - # gunicorn - # sphinx diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.in similarity index 100% rename from requirements/doc-spelling.txt rename to requirements/doc-spelling.in diff --git a/requirements/lint.txt b/requirements/lint.in similarity index 71% rename from requirements/lint.txt rename to requirements/lint.in index 71ddfd1f8b0..ccfe30aac8c 100644 --- a/requirements/lint.txt +++ b/requirements/lint.in @@ -1,7 +1,8 @@ black==20.8b1; implementation_name=="cpython" flake8==3.8.4 flake8-pyi==20.10.0 -isort==5.6.4 +importlib-metadata==3.7.0; python_version < "3.8" +isort==5.7.0 mypy==0.790; implementation_name=="cpython" pre-commit==2.9.3 pytest==6.1.2 diff --git a/tools/deps.py b/tools/deps.py new file mode 100755 index 00000000000..e0d8a45044d --- /dev/null +++ b/tools/deps.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import argparse +import collections +import hashlib +import os +import pathlib +import platform +import subprocess +import sys + +PLATFORM_NAME = sys.platform if platform.python_implementation() != "PyPy" else "pypy" +ROOT = pathlib.Path(__file__).parent.parent +REQ_FOLDER = ROOT / "requirements" +PIN_FOLDER = REQ_FOLDER / "pinned" + + +def txt_file_name(in_file_name): + components = ( + PLATFORM_NAME, + f"{sys.version_info.major}.{sys.version_info.minor}", + in_file_name, + ) + txt_name = f"{'-'.join(components)}.txt" + return PIN_FOLDER / txt_name + + +def compile_dependencies(): + for in_file in REQ_FOLDER.glob("*.in"): + output_file = txt_file_name(in_file.stem) + + subprocess.call( + ( + "pip-compile", + "--allow-unsafe", + "--generate-hashes", + "--no-header", + "--no-annotate", + "--quiet", + "--output-file", + str(output_file), + str(in_file), + ) + ) + + +def install_dependencies(in_name): + txt_file = txt_file_name(in_name) + + subprocess.call( + ( + "pip", + "install", + "--require-hashes", + "-r", + str(txt_file), + ) + ) + + +def replace_duplicates_with_symlinks(): + hash_to_file = collections.defaultdict(list) + + orig_dir = os.getcwd() + os.chdir(PIN_FOLDER) + try: + for txt_fname in pathlib.Path(".").glob("*.txt"): + if not txt_fname.is_symlink(): + with open(txt_fname, "rb") as f: + digest = hashlib.sha1(f.read()).hexdigest() + hash_to_file[digest].append(txt_fname) + + for digest, files in hash_to_file.items(): + if len(files) > 1: + base_file, *other_files = files + for other_file in other_files: + os.unlink(other_file) + os.symlink(base_file, other_file) + finally: + os.chdir(orig_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="cmd", help="sub-command help") + + install_parser = subparsers.add_parser("install", help="install dependencies") + compile_parser = subparsers.add_parser("compile", help="compile dependencies") + remove_duplicates = subparsers.add_parser( + "rdup", help="remove duplicate .txt files" + ) + + install_parser.add_argument("file", type=str, help="dep to install", default="dev") + + args = parser.parse_args() + if args.cmd == "install": + install_dependencies(args.file) + elif args.cmd == "compile": + compile_dependencies() + elif args.cmd == "rdup": + replace_duplicates_with_symlinks()