Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Align connection pool size with installer.max-workers config option #8559

Merged
merged 5 commits into from
Oct 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/poetry/config/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,22 @@ def virtualenvs_path(self) -> Path:
path = Path(self.get("cache-dir")) / "virtualenvs"
return Path(path).expanduser()

@property
def installer_max_workers(self) -> int:
# This should be directly handled by ThreadPoolExecutor
# however, on some systems the number of CPUs cannot be determined
# (it raises a NotImplementedError), so, in this case, we assume
# that the system only has one CPU.
try:
default_max_workers = (os.cpu_count() or 1) + 4
except NotImplementedError:
default_max_workers = 5

desired_max_workers = self.get("installer.max-workers")
if desired_max_workers is None:
return default_max_workers
return min(default_max_workers, int(desired_max_workers))

def get(self, setting_name: str, default: Any = None) -> Any:
"""
Retrieve a setting value.
Expand Down
3 changes: 2 additions & 1 deletion src/poetry/console/commands/init.py
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,7 @@ def _get_pool(self) -> RepositoryPool:

if self._pool is None:
self._pool = RepositoryPool()
self._pool.add_repository(PyPiRepository())
pool_size = self.poetry.config.installer_max_workers
self._pool.add_repository(PyPiRepository(pool_size=pool_size))

return self._pool
5 changes: 4 additions & 1 deletion src/poetry/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,14 @@ def create_package_source(
except KeyError:
raise InvalidSourceError("Missing [name] in source.")

pool_size = config.installer_max_workers

if name.lower() == "pypi":
if "url" in source:
raise InvalidSourceError(
"The PyPI repository cannot be configured with a custom url."
)
return PyPiRepository(disable_cache=disable_cache)
return PyPiRepository(disable_cache=disable_cache, pool_size=pool_size)

try:
url = source["url"]
Expand All @@ -246,6 +248,7 @@ def create_package_source(
url,
config=config,
disable_cache=disable_cache,
pool_size=pool_size,
)

@classmethod
Expand Down
20 changes: 1 addition & 19 deletions src/poetry/installation/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import csv
import itertools
import json
import os
import threading

from concurrent.futures import ThreadPoolExecutor
Expand Down Expand Up @@ -73,9 +72,7 @@ def __init__(
parallel = config.get("installer.parallel", True)

if parallel:
self._max_workers = self._get_max_workers(
desired_max_workers=config.get("installer.max-workers")
)
self._max_workers = config.installer_max_workers
else:
self._max_workers = 1

Expand Down Expand Up @@ -223,21 +220,6 @@ def execute(self, operations: list[Operation]) -> int:

return 1 if self._shutdown else 0

@staticmethod
def _get_max_workers(desired_max_workers: int | None = None) -> int:
# This should be directly handled by ThreadPoolExecutor
# however, on some systems the number of CPUs cannot be determined
# (it raises a NotImplementedError), so, in this case, we assume
# that the system only has one CPU.
try:
default_max_workers = (os.cpu_count() or 1) + 4
except NotImplementedError:
default_max_workers = 5

if desired_max_workers is None:
return default_max_workers
return min(default_max_workers, desired_max_workers)

def _write(self, operation: Operation, line: str) -> None:
if not self.supports_fancy_output() or not self._should_write_operation(
operation
Expand Down
3 changes: 3 additions & 0 deletions src/poetry/repositories/http_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from typing import Iterator

import requests
import requests.adapters

from poetry.core.constraints.version import parse_constraint
from poetry.core.packages.dependency import Dependency
Expand Down Expand Up @@ -44,13 +45,15 @@ def __init__(
url: str,
config: Config | None = None,
disable_cache: bool = False,
pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
) -> None:
super().__init__(name, disable_cache, config)
self._url = url
self._authenticator = Authenticator(
config=config,
cache_id=name,
disable_cache=disable_cache,
pool_size=pool_size,
)
self._authenticator.add_repository(name, url)
self.get_page = functools.lru_cache(maxsize=None)(self._get_page)
Expand Down
5 changes: 4 additions & 1 deletion src/poetry/repositories/legacy_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
from typing import TYPE_CHECKING
from typing import Any

import requests.adapters

from poetry.core.packages.package import Package

from poetry.inspection.info import PackageInfo
Expand All @@ -27,11 +29,12 @@ def __init__(
url: str,
config: Config | None = None,
disable_cache: bool = False,
pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
) -> None:
if name == "pypi":
raise ValueError("The name [pypi] is reserved for repositories")

super().__init__(name, url.rstrip("/"), config, disable_cache)
super().__init__(name, url.rstrip("/"), config, disable_cache, pool_size)

@property
def packages(self) -> list[Package]:
Expand Down
7 changes: 6 additions & 1 deletion src/poetry/repositories/pypi_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from typing import Any

import requests
import requests.adapters

from cachecontrol.controller import logger as cache_control_logger
from poetry.core.packages.package import Package
Expand Down Expand Up @@ -38,9 +39,13 @@ def __init__(
url: str = "https://pypi.org/",
disable_cache: bool = False,
fallback: bool = True,
pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
) -> None:
super().__init__(
"PyPI", url.rstrip("/") + "/simple/", disable_cache=disable_cache
"PyPI",
url.rstrip("/") + "/simple/",
disable_cache=disable_cache,
pool_size=pool_size,
)

self._base_url = url
Expand Down
3 changes: 2 additions & 1 deletion src/poetry/utils/authenticator.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from typing import Any

import requests
import requests.adapters
import requests.auth
import requests.exceptions

Expand Down Expand Up @@ -110,7 +111,7 @@ def __init__(
io: IO | None = None,
cache_id: str | None = None,
disable_cache: bool = False,
pool_size: int = 10,
pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
) -> None:
self._config = config or Config.create()
self._io = io
Expand Down