From e06ccfa7370058495ac8009a06d05a1fc1b2662e Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Sat, 13 Jul 2024 12:11:11 +0100 Subject: [PATCH 1/4] Deprecate Python 3.7, switch to Python 3.11 in CI --- .github/workflows/check-style.yml | 2 +- .github/workflows/run-benchmarks.yml | 4 ++-- .github/workflows/run-tests.yml | 10 +++++----- .readthedocs.yml | 2 +- README.md | 2 +- docs/user/quickstart.md | 2 +- hivemind/averaging/matchmaking.py | 9 ++++----- hivemind/utils/asyncio.py | 4 +--- hivemind/utils/mpfuture.py | 10 +--------- setup.py | 3 ++- tests/test_util_modules.py | 3 +-- 11 files changed, 20 insertions(+), 31 deletions(-) diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml index 0c8d03ea0..d12ca8eb3 100644 --- a/.github/workflows/check-style.yml +++ b/.github/workflows/check-style.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.11 - uses: isort/isort-action@master with: isortVersion: "5.10.1" diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 3073f6c50..926f1e8f5 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -15,12 +15,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: 3.11 - name: Cache dependencies uses: actions/cache@v3 with: path: ~/.cache/pip - key: Key-v1-3.9-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }} + key: Key-v1-3.11-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }} - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index feaeb0142..99358bfd9 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: [ '3.8', '3.9', '3.10', '3.11' ] timeout-minutes: 15 steps: - uses: actions/checkout@v3 @@ -52,12 +52,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.8' + python-version: '3.11' - name: Cache dependencies uses: actions/cache@v3 with: path: ~/.cache/pip - key: Key-v1-3.8-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }} + key: Key-v1-3.11-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }} - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel @@ -80,12 +80,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.8' + python-version: '3.11' - name: Cache dependencies uses: actions/cache@v3 with: path: ~/.cache/pip - key: Key-v1-3.8-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }} + key: Key-v1-3.11-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }} - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel diff --git a/.readthedocs.yml b/.readthedocs.yml index a65b37e6f..c833a9eb8 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -13,4 +13,4 @@ python: build: os: ubuntu-22.04 tools: - python: "3.7" + python: "3.11" diff --git a/README.md b/README.md index f35b77322..81c4bf85b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ feel free to submit a pull request that adds your project to this list. ## Installation -Before installing, make sure that your environment has Python 3.7+ +Before installing, make sure that your environment has Python 3.8+ and [PyTorch](https://pytorch.org/get-started/locally/#start-locally) 1.9.0 or newer. They can be installed either natively or with [Anaconda](https://www.anaconda.com/products/individual). diff --git a/docs/user/quickstart.md b/docs/user/quickstart.md index 3142b841a..5b9f4b64c 100644 --- a/docs/user/quickstart.md +++ b/docs/user/quickstart.md @@ -4,7 +4,7 @@ This tutorial will teach you how to install `hivemind`, host your own experts an ## Installation -Just `pip install hivemind` to get the latest release (requires Python 3.7 or newer). +Just `pip install hivemind` to get the latest release (requires Python 3.8 or newer). You can also install the bleeding edge version from GitHub: diff --git a/hivemind/averaging/matchmaking.py b/hivemind/averaging/matchmaking.py index 1bf4e7c47..b5a5f1146 100644 --- a/hivemind/averaging/matchmaking.py +++ b/hivemind/averaging/matchmaking.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import concurrent.futures import contextlib import random from math import isfinite @@ -168,8 +167,8 @@ async def _request_join_potential_leaders(self, step: StepControl) -> GroupInfo: elif len(self.current_followers) > 0: await self.leader_disband_group() continue - except (concurrent.futures.CancelledError, asyncio.CancelledError): - break # note: this is a compatibility layer for python3.7 + except asyncio.CancelledError: + break except Exception as e: if not self.assembled_group.done(): self.assembled_group.set_exception(e) @@ -321,8 +320,8 @@ async def rpc_join_group( ordered_peer_ids=[item.to_bytes() for item in group_info.peer_ids], gathered=group_info.gathered, ) - except (concurrent.futures.CancelledError, asyncio.CancelledError): - return # note: this is a compatibility layer for python3.7 + except asyncio.CancelledError: + return except Exception as e: logger.exception(e) yield averaging_pb2.MessageFromLeader(code=averaging_pb2.INTERNAL_ERROR) diff --git a/hivemind/utils/asyncio.py b/hivemind/utils/asyncio.py index af2af1aca..36901a2d3 100644 --- a/hivemind/utils/asyncio.py +++ b/hivemind/utils/asyncio.py @@ -84,9 +84,7 @@ async def await_cancelled(awaitable: Awaitable) -> bool: try: await awaitable return False - except (asyncio.CancelledError, concurrent.futures.CancelledError): - # In Python 3.7, awaiting a cancelled asyncio.Future raises concurrent.futures.CancelledError - # instead of asyncio.CancelledError + except asyncio.CancelledError: return True except BaseException: logger.exception(f"Exception in {awaitable}:") diff --git a/hivemind/utils/mpfuture.py b/hivemind/utils/mpfuture.py index 11952811a..454d4777c 100644 --- a/hivemind/utils/mpfuture.py +++ b/hivemind/utils/mpfuture.py @@ -6,6 +6,7 @@ import os import threading import uuid +from concurrent.futures import InvalidStateError from contextlib import nullcontext from enum import Enum, auto from multiprocessing.reduction import ForkingPickler @@ -26,15 +27,6 @@ ALL_STATES = base.PENDING, base.RUNNING, base.FINISHED, base.CANCELLED, base.CANCELLED_AND_NOTIFIED TERMINAL_STATES = {base.FINISHED, base.CANCELLED, base.CANCELLED_AND_NOTIFIED} -try: - from concurrent.futures import InvalidStateError -except ImportError: - # Python 3.7 doesn't raise concurrent.futures.InvalidStateError for repeating set_result/set_exception calls and - # doesn't even define this error. In this module, we simulate the Python 3.8+ behavior, - # defining and raising this error if necessary. - class InvalidStateError(Exception): - """Raised when attempting to change state of a future in a terminal state (e.g. finished)""" - class SharedBytes: """ diff --git a/setup.py b/setup.py index 8f0dcedbe..422c0bd56 100644 --- a/setup.py +++ b/setup.py @@ -183,9 +183,10 @@ def run(self): "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Mathematics", "Topic :: Scientific/Engineering :: Artificial Intelligence", diff --git a/tests/test_util_modules.py b/tests/test_util_modules.py index 0243b91d7..f245b777e 100644 --- a/tests/test_util_modules.py +++ b/tests/test_util_modules.py @@ -528,8 +528,7 @@ async def coro(): async with enter_asynchronously(lock2): await asyncio.sleep(1e-2) - num_coros = max(100, mp.cpu_count() * 5 + 1) - # note: if we deprecate py3.7, this can be reduced to max(33, cpu + 5); see https://bugs.python.org/issue35279 + num_coros = max(33, mp.cpu_count() * 5 + 1) await asyncio.wait({asyncio.create_task(coro()) for _ in range(num_coros)}) From 7f2e5fd1214b5f303d7fd36c71433b4fdf43e8c2 Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Sat, 13 Jul 2024 12:20:47 +0100 Subject: [PATCH 2/4] Update Sphinx --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 25df3d480..eef82573a 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ recommonmark==0.5.0 sphinx_rtd_theme==0.4.3 docutils==0.16 -sphinx==4.2.0 +sphinx==5.0.0 From e9ee2400d900ed82b2ce037f61f65f57b74449f0 Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Sat, 13 Jul 2024 12:26:47 +0100 Subject: [PATCH 3/4] Set Sphinx language to "en" --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index fdf11c0da..11afc8bdf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -73,7 +73,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From cdb87a3acb6c913592056598eb3998d0a7634d2c Mon Sep 17 00:00:00 2001 From: Max Ryabinin Date: Mon, 15 Jul 2024 15:21:09 +0100 Subject: [PATCH 4/4] Increase the timeout for codecov --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 99358bfd9..74d778bd0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -74,7 +74,7 @@ jobs: codecov_in_develop_mode: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: actions/checkout@v3 - name: Set up Python