[3.8] Merge PR #7460

(cherry picked from commit 4b59d55)
(cherry picked from commit 7a6b0d7)

.github/workflows/ci-cd.yml

@@ -34,7 +34,14 @@ jobs:
       uses: actions/checkout@v2.4.0
       with:
         submodules: true
-    - name: Setup Python 3.8
+    - name: >-
+        Verify that `requirements/runtime-deps.in`
+        is in sync with `setup.cfg`
+      run: |
+        set -eEuo pipefail
+        make sync-direct-runtime-deps
+        git diff --exit-code -- requirements/runtime-deps.in
+    - name: Setup Python
       uses: actions/setup-python@v4
       with:
         python-version: 3.8
@@ -50,10 +57,10 @@ jobs:
         python -m pip install -U pip wheel setuptools build twine
     - name: Install dependencies
       run: |
-        python -m pip install -r requirements/lint.txt -c requirements/constraints.txt
+        python -m pip install -r requirements/lint.in -c requirements/lint.txt
     - name: Install self
       run: |
-        python -m pip install .
+        python -m pip install . -c requirements/runtime-deps.txt
       env:
         AIOHTTP_NO_EXTENSIONS: 1
     - name: Run linters
@@ -64,7 +71,7 @@ jobs:
         sudo apt install libenchant-2-dev
     - name: Install spell checker
       run: |
-        pip install -r requirements/doc-spelling.txt -c requirements/constraints.txt
+        pip install -r requirements/doc-spelling.in -c requirements/doc-spelling.txt
     - name: Run docs spelling
       run: |
         # towncrier --yes  # uncomment me after publishing a release
@@ -176,7 +183,7 @@ jobs:
         python -m pip install -U pip wheel setuptools build twine
     - name: Install dependencies
       run: |
-        python -m pip install -r requirements/test.txt -c requirements/constraints.txt
+        python -m pip install -r requirements/test.in -c requirements/test.txt
     - name: Cythonize
       if: ${{ matrix.no-extensions == '' }}
       run: |
@@ -264,8 +271,9 @@ jobs:
       run: |
         python -m pip install -U pip wheel setuptools build twine
     - name: Install cython
-      run: |
-        python -m pip install -r requirements/cython.txt -c requirements/constraints.txt
+      run: >-
+        python -m
+        pip install -r requirements/cython.in -c requirements/cython.txt
     - name: Restore llhttp generated files
       uses: actions/download-artifact@v2
       with:
@@ -326,8 +334,9 @@ jobs:
       run: |
         python -m pip install -U pip wheel setuptools build twine
     - name: Install cython
-      run: |
-        python -m pip install -r requirements/cython.txt -c requirements/constraints.txt
+      run: >-
+        python -m
+        pip install -r requirements/cython.in -c requirements/cython.txt
     - name: Restore llhttp generated files
       uses: actions/download-artifact@v2
       with:
@@ -363,9 +372,6 @@ jobs:
       uses: actions/checkout@v2.4.0
       with:
         submodules: true
-    - name: Update pip, wheel, setuptools, build, twine
-      run: |
-        python -m pip install -U pip wheel setuptools build twine
     - name: Login
       run: |
         echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token

.pip-tools.toml

@@ -0,0 +1,4 @@
+[pip-tools]
+allow-unsafe = true
+resolver = "backtracking"
+strip-extras = true

.pre-commit-config.yaml

@@ -46,7 +46,7 @@ repos:
       ^docs/[^/]*\.svg$
   - id: requirements-txt-fixer
     exclude: >-
-      ^requirements/constraints[.]txt$
+      ^requirements/.*\.txt$
   - id: trailing-whitespace
   - id: file-contents-sorter
     files: |

.readthedocs.yml

@@ -11,11 +11,20 @@ submodules:
   recursive: true
 
 build:
-  image: latest
-python:
-  version: 3.8
-  install:
-  - method: pip
-    path: .
-  - requirements: requirements/doc.txt
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+  jobs:
+    post_create_environment:
+    - >-
+      pip install
+      . -c requirements/runtime-deps.txt
+      -r requirements/doc.in -c requirements/doc.txt
+    install:
+    - >-
+      pip install
+      . -c requirements/runtime-deps.txt
+      -r requirements/doc.in -c requirements/doc.txt
+
 ...

Makefile

@@ -50,7 +50,7 @@ endif
 	@python -m pip install --upgrade pip
 
 .install-cython: .update-pip $(call to-hash,requirements/cython.txt)
-	@python -m pip install -r requirements/cython.txt -c requirements/constraints.txt
+	@python -m pip install -r requirements/cython.in -c requirements/cython.txt
 	@touch .install-cython
 
 aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py)
@@ -74,7 +74,7 @@ generate-llhttp: .llhttp-gen
 cythonize: .install-cython $(PYXS:.pyx=.c)
 
 .install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS))
-	@python -m pip install -r requirements/dev.txt -c requirements/constraints.txt
+	@python -m pip install -r requirements/dev.in -c requirements/dev.txt
 	@touch .install-deps
 
 .PHONY: lint
@@ -89,7 +89,7 @@ mypy:
 	mypy
 
 .develop: .install-deps generate-llhttp $(call to-hash,$(PYS) $(CYS) $(CS))
-	python -m pip install -e . -c requirements/constraints.txt
+	python -m pip install -e . -c requirements/runtime-deps.txt
 	@touch .develop
 
 .PHONY: test
@@ -174,15 +174,19 @@ doc:
 doc-spelling:
 	@make -C docs spelling SPHINXOPTS="-W --keep-going -n -E"
 
-.PHONY: compile-deps
-compile-deps: .update-pip $(REQS)
-	pip-compile --no-header --allow-unsafe -q --strip-extras \
-		-o requirements/constraints.txt \
-		requirements/constraints.in
-
 .PHONY: install
 install: .update-pip
-	@python -m pip install -r requirements/dev.txt -c requirements/constraints.txt
+	@python -m pip install -r requirements/dev.in -c requirements/dev.txt
 
 .PHONY: install-dev
 install-dev: .develop
+
+.PHONY: sync-direct-runtime-deps
+sync-direct-runtime-deps:
+	@echo Updating 'requirements/runtime-deps.in' from 'setup.cfg'... >&2
+	@echo '# Extracted from `setup.cfg` via `make sync-direct-runtime-deps`' > requirements/runtime-deps.in
+	@echo >> requirements/runtime-deps.in
+	@echo '-c broken-projects.in' >> requirements/runtime-deps.in
+	@echo >> requirements/runtime-deps.in
+	@python -c 'from configparser import ConfigParser; from itertools import chain; from pathlib import Path; cfg = ConfigParser(); cfg.read_string(Path("setup.cfg").read_text()); print("\n".join(line.strip() for line in chain(cfg["options"].get("install_requires").splitlines(), "\n".join(cfg["options.extras_require"].values()).splitlines()) if line.strip()))' \
+		>> requirements/runtime-deps.in

requirements/base.in

@@ -0,0 +1,5 @@
+-r typing-extensions.in
+-r runtime-deps.in
+
+gunicorn
+uvloop; platform_system != "Windows" and implementation_name == "cpython"  # MagicStack/uvloop#14

requirements/base.txt

@@ -1,21 +1,48 @@
--r multidict.txt
--r typing-extensions.txt
-# required c-ares will not build on windows and has build problems on Macos Python<3.7
-aiodns==3.0.0; sys_platform=="linux" or sys_platform=="darwin" and python_version>="3.7"
-aiosignal==1.2.0
-async-generator==1.10
-async-timeout==4.0.1
-asynctest==0.13.0; python_version<"3.8"
-attrs==21.2.0
-Brotli==1.0.9
-cchardet==2.1.7; python_version < "3.10"  # Unmaintained: aio-libs/aiohttp#6819
-charset-normalizer==2.0.12
-frozenlist==1.3.1; python_version >= "3.7"
-frozenlist==1.2.0; python_version < "3.7"  # The last release supporting Python 3.6
-gunicorn==20.1.0
-idna-ssl==1.1.0; python_version<"3.7"
-typing==3.7.4.3; python_version<"3.7"
-uvloop==0.14.0; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.9" # MagicStack/uvloop#14
-yarl==1.8.1; python_version >= "3.7"
-yarl==1.7.2; python_version < "3.7"  # The last release supporting Python 3.6
-zipp==3.4.1; python_version<"3.7"
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    pip-compile --allow-unsafe --output-file=requirements/base.txt --resolver=backtracking --strip-extras requirements/base.in
+#
+aiodns==3.0.0 ; sys_platform == "linux" or sys_platform == "darwin"
+    # via -r requirements/runtime-deps.in
+aiosignal==1.3.1
+    # via -r requirements/runtime-deps.in
+async-timeout==4.0.2
+    # via -r requirements/runtime-deps.in
+attrs==22.2.0
+    # via -r requirements/runtime-deps.in
+brotli==1.0.9
+    # via -r requirements/runtime-deps.in
+cchardet==2.1.7 ; python_version < "3.10"
+    # via -r requirements/runtime-deps.in
+cffi==1.15.1
+    # via pycares
+charset-normalizer==3.2.0
+    # via -r requirements/runtime-deps.in
+frozenlist==1.4.0
+    # via
+    #   -r requirements/runtime-deps.in
+    #   aiosignal
+gunicorn==21.2.0
+    # via -r requirements/base.in
+idna==3.4
+    # via yarl
+multidict==6.0.4
+    # via
+    #   -r requirements/runtime-deps.in
+    #   yarl
+packaging==23.1
+    # via gunicorn
+pycares==4.3.0
+    # via aiodns
+pycparser==2.21
+    # via cffi
+typing-extensions==4.1.1
+    # via
+    #   -c requirements/broken-projects.in
+    #   -r requirements/typing-extensions.in
+uvloop==0.17.0 ; platform_system != "Windows" and implementation_name == "cpython"
+    # via -r requirements/base.in
+yarl==1.9.2
+    # via -r requirements/runtime-deps.in

requirements/broken-projects.in

@@ -0,0 +1,7 @@
+attrs < 23  # Python 3.6 compat
+funcparserlib == 1.0.0a0  # https://github.com/blockdiag/blockdiag/pull/148
+Jinja2 < 3.1  # https://github.com/readthedocs/readthedocs.org/issues/9038
+Pillow < 10  # https://github.com/blockdiag/sphinxcontrib-blockdiag/issues/26
+pydantic < 2  # looks like `python-on-whales` hasn't upgraded to support v2
+Sphinx < 5.3  # compat with `sphinxcontrib-asyncio`
+typing-extensions <= 4.1.1  # last version supporting Python 3.6

requirements/constraints.in

@@ -1,4 +1,4 @@
--r cython.txt
--r dev.txt
--r doc-spelling.txt
--r lint.txt
+-r cython.in
+-r dev.in
+-r doc-spelling.in
+-r lint.in