Complete pip-tools setup (backport #5486)

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
       ^docs/[^/]*\.svg$
   - id: requirements-txt-fixer
     exclude: >-
-      ^requirements/dev.txt$
+      ^requirements/(cython|dev|doc-spelling|lint).txt$
   - id: trailing-whitespace
   - id: file-contents-sorter
     files: |

CHANGES/5486.misc

@@ -0,0 +1 @@
+Complete pip-tools setup.

Makefile

@@ -9,6 +9,9 @@ CS := $(wildcard aiohttp/*.c)
 PYS := $(wildcard aiohttp/*.py)
 REQS := $(wildcard requirements/*.txt)
 ALLS := $(sort $(CYS) $(CS) $(PYS) $(REQS))
+IN := doc-spelling lint cython dev
+REQIN := $(foreach fname,$(IN),requirements/$(fname).in)
+
 
 .PHONY: all
 all: test
@@ -45,9 +48,11 @@ endif
 # Enumerate intermediate files to don't remove them automatically.
 .SECONDARY: $(call to-hash,$(ALLS))
 
+.update-pip:
+	@pip install -U 'pip'
 
-.install-cython: $(call to-hash,requirements/cython.txt)
-	pip install -r requirements/cython.txt
+.install-cython: .update-pip $(call to-hash,requirements/cython.txt)
+	@pip install -r requirements/cython.txt
 	@touch .install-cython
 
 aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py)
@@ -62,7 +67,7 @@ aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c
 cythonize: .install-cython $(PYXS:.pyx=.c)
 
 .install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS))
-	pip install -r requirements/dev.txt
+	@pip install -r requirements/dev.txt
 	@touch .install-deps
 
 .PHONY: lint
@@ -135,17 +140,14 @@ doc:
 doc-spelling:
 	@make -C docs spelling SPHINXOPTS="-W -E"
 
-.update-pip:
-	@pip install -U 'pip'
-
 .PHONY: compile-deps
 compile-deps: .update-pip
 	@pip install pip-tools
-	@pip-compile --allow-unsafe -q requirements/dev.in
+	@$(foreach fname,$(REQIN),pip-compile --allow-unsafe -q $(fname);)
 
 .PHONY: install
 install: .update-pip
-	@pip install -r requirements/dev.in -c requirements/dev.txt
+	@pip install -r requirements/dev.txt
 
 .PHONY: install-dev
 install-dev: .develop

requirements/base.txt

@@ -12,6 +12,8 @@ chardet==4.0.0
 frozenlist==1.1.1
 gunicorn==20.0.4
 idna-ssl==1.1.0; python_version<"3.7"
+typing==3.7.4.3; python_version<"3.7"
 typing_extensions==3.7.4.3
 uvloop==0.14.0; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.9" # MagicStack/uvloop#14
 yarl==1.6.3
+zipp==3.4.1; python_version<"3.7"

requirements/cython.in

@@ -0,0 +1,3 @@
+-r multidict.txt
+cython==0.29.21
+typing_extensions==3.7.4.3  # required for parsing aiohttp/hdrs.py by tools/gen.py

requirements/cython.txt

@@ -1,3 +1,12 @@
--r multidict.txt
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --allow-unsafe requirements/cython.in
+#
 cython==0.29.21
-typing_extensions==3.7.4.3  # required for parsing aiohttp/hdrs.py by tools/gen.py
+    # via -r requirements/cython.in
+multidict==5.1.0
+    # via -r requirements/multidict.txt
+typing_extensions==3.7.4.3
+    # via -r requirements/cython.in

requirements/dev.txt

@@ -14,15 +14,19 @@ alabaster==0.7.12
     # via sphinx
 appdirs==1.4.4
     # via
+    #   -r requirements/lint.txt
     #   black
     #   virtualenv
 async-generator==1.10
     # via -r requirements/base.txt
 async-timeout==4.0.0a3
     # via -r requirements/base.txt
+asynctest==0.13.0 ; python_version < "3.8"
+    # via -r requirements/base.txt
 attrs==20.3.0
     # via
     #   -r requirements/base.txt
+    #   -r requirements/lint.txt
     #   flake8-pyi
     #   pytest
 babel==2.9.0
@@ -42,7 +46,9 @@ cffi==1.14.4
     #   cryptography
     #   pycares
 cfgv==3.2.0
-    # via pre-commit
+    # via
+    #   -r requirements/lint.txt
+    #   pre-commit
 chardet==4.0.0
     # via
     #   -r requirements/base.txt
@@ -51,23 +57,33 @@ cherry_picker==1.3.2 ; python_version >= "3.6"
     # via -r requirements/dev.in
 click==7.1.2
     # via
+    #   -r requirements/lint.txt
     #   black
     #   cherry-picker
     #   towncrier
 coverage==5.3.1
     # via
     #   -r requirements/test.txt
     #   pytest-cov
-cryptography==3.3.1
+cryptography==3.3.1 ; platform_machine != "i686" and python_version < "3.9"
     # via
+    #   -r requirements/test.txt
     #   pyjwt
     #   trustme
+dataclasses==0.8 ; python_version < "3.7"
+    # via
+    #   -r requirements/lint.txt
+    #   black
 distlib==0.3.1
-    # via virtualenv
+    # via
+    #   -r requirements/lint.txt
+    #   virtualenv
 docutils==0.16
     # via sphinx
 filelock==3.0.12
-    # via virtualenv
+    # via
+    #   -r requirements/lint.txt
+    #   virtualenv
 flake8-pyi==20.10.0
     # via -r requirements/lint.txt
 flake8==3.8.4
@@ -86,20 +102,40 @@ gidgethub==5.0.0
     # via cherry-picker
 gunicorn==20.0.4
     # via -r requirements/base.txt
-identify==1.5.12
-    # via pre-commit
+identify==2.1.1
+    # via
+    #   -r requirements/lint.txt
+    #   pre-commit
+idna-ssl==1.1.0 ; python_version < "3.7"
+    # via -r requirements/base.txt
 idna==2.10
     # via
+    #   idna-ssl
     #   requests
     #   trustme
     #   yarl
 imagesize==1.2.0
     # via sphinx
+importlib-metadata==3.7.0 ; python_version < "3.8"
+    # via
+    #   -r requirements/lint.txt
+    #   flake8
+    #   pluggy
+    #   pre-commit
+    #   pytest
+    #   virtualenv
+importlib-resources==5.1.2 ; python_version < "3.9"
+    # via
+    #   -r requirements/lint.txt
+    #   pre-commit
+    #   virtualenv
 incremental==17.5.0
     # via towncrier
 iniconfig==1.1.1
-    # via pytest
-isort==5.6.4
+    # via
+    #   -r requirements/lint.txt
+    #   pytest
+isort==5.7.0
     # via -r requirements/lint.txt
 jinja2==2.11.2
     # via
@@ -108,13 +144,16 @@ jinja2==2.11.2
 markupsafe==1.1.1
     # via jinja2
 mccabe==0.6.1
-    # via flake8
+    # via
+    #   -r requirements/lint.txt
+    #   flake8
 multidict==5.1.0
     # via
     #   -r requirements/multidict.txt
     #   yarl
 mypy-extensions==0.4.3 ; implementation_name == "cpython"
     # via
+    #   -r requirements/lint.txt
     #   -r requirements/test.txt
     #   black
     #   mypy
@@ -123,29 +162,41 @@ mypy==0.790 ; implementation_name == "cpython"
     #   -r requirements/lint.txt
     #   -r requirements/test.txt
 nodeenv==1.5.0
-    # via pre-commit
-packaging==20.8
     # via
+    #   -r requirements/lint.txt
+    #   pre-commit
+packaging==20.9
+    # via
+    #   -r requirements/lint.txt
     #   pytest
     #   sphinx
 pathspec==0.8.1
-    # via black
+    # via
+    #   -r requirements/lint.txt
+    #   black
 pillow==8.1.0
     # via blockdiag
 pluggy==0.13.1
-    # via pytest
+    # via
+    #   -r requirements/lint.txt
+    #   pytest
 pre-commit==2.9.3
     # via -r requirements/lint.txt
 py==1.10.0
-    # via pytest
+    # via
+    #   -r requirements/lint.txt
+    #   pytest
 pycares==3.1.1
     # via aiodns
 pycodestyle==2.6.0
-    # via flake8
+    # via
+    #   -r requirements/lint.txt
+    #   flake8
 pycparser==2.20
     # via cffi
 pyflakes==2.2.0
     # via
+    #   -r requirements/lint.txt
     #   flake8
     #   flake8-pyi
 pygments==2.8.0
@@ -155,7 +206,9 @@ pygments==2.8.0
 pyjwt[crypto]==2.0.0
     # via gidgethub
 pyparsing==2.4.7
-    # via packaging
+    # via
+    #   -r requirements/lint.txt
+    #   packaging
 pytest-cov==2.10.1
     # via -r requirements/test.txt
 pytest-mock==3.5.1
@@ -170,12 +223,15 @@ python-dateutil==2.8.1
     # via freezegun
 pytz==2020.5
     # via babel
-pyyaml==5.3.1
-    # via pre-commit
+pyyaml==5.4.1
+    # via
+    #   -r requirements/lint.txt
+    #   pre-commit
 re-assert==1.1.0
     # via -r requirements/test.txt
 regex==2020.11.13
     # via
+    #   -r requirements/lint.txt
     #   black
     #   re-assert
 requests==2.25.1
@@ -186,12 +242,13 @@ setuptools-git==1.2
     # via -r requirements/test.txt
 six==1.15.0
     # via
+    #   -r requirements/lint.txt
     #   cryptography
     #   python-dateutil
     #   virtualenv
 snowballstemmer==2.0.0
     # via sphinx
-sphinx==3.5.0
+sphinx==3.5.1
     # via
     #   -r requirements/doc.txt
     #   sphinxcontrib-asyncio
@@ -214,6 +271,7 @@ sphinxcontrib-serializinghtml==1.1.4
     # via sphinx
 toml==0.10.2
     # via
+    #   -r requirements/lint.txt
     #   black
     #   cherry-picker
     #   pre-commit
@@ -223,26 +281,44 @@ towncrier==19.2.0
     # via -r requirements/doc.txt
 trustme==0.7.0 ; platform_machine != "i686"
     # via -r requirements/test.txt
-typed-ast==1.4.2
+typed-ast==1.4.2 ; implementation_name == "cpython"
     # via
+    #   -r requirements/lint.txt
     #   black
     #   mypy
 typing-extensions==3.7.4.3
     # via
     #   -r requirements/base.txt
+    #   -r requirements/lint.txt
     #   async-timeout
     #   black
+    #   importlib-metadata
     #   mypy
+    #   yarl
+typing==3.7.4.3 ; python_version < "3.7"
+    # via
+    #   -r requirements/base.txt
+    #   aiodns
 uritemplate==3.0.1
     # via gidgethub
 urllib3==1.26.2
     # via requests
-virtualenv==20.3.1
-    # via pre-commit
+uvloop==0.14.0 ; platform_system != "Windows" and implementation_name == "cpython" and python_version < "3.9"
+    # via -r requirements/base.txt
+virtualenv==20.4.2
+    # via
+    #   -r requirements/lint.txt
+    #   pre-commit
 webcolors==1.11.1
     # via blockdiag
 yarl==1.6.3
     # via -r requirements/base.txt
+zipp==3.4.1 ; python_version < "3.7"
+    # via
+    #   -r requirements/base.txt
+    #   -r requirements/lint.txt
+    #   importlib-metadata
+    #   importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 setuptools==51.3.1

requirements/doc-spelling.in

@@ -0,0 +1,2 @@
+-r doc.txt
+sphinxcontrib-spelling==7.1.0; platform_system!="Windows"  # We only use it in Travis CI