Skip to content

Commit

Permalink
Merge pull request #8912 from uranusjr/cache-found-candidates
Browse files Browse the repository at this point in the history
  • Loading branch information
pradyunsg authored Sep 27, 2020
2 parents 75befb5 + dedecf0 commit 703d6ac
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 25 deletions.
3 changes: 3 additions & 0 deletions news/8905.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cache package listings on index packages so they are guarenteed to stay stable
during a pip command session. This also improves performance when a index page
is accessed multiple times during the command session.
26 changes: 2 additions & 24 deletions src/pip/_internal/index/collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from pip._internal.models.link import Link
from pip._internal.models.search_scope import SearchScope
from pip._internal.network.utils import raise_for_status
from pip._internal.utils.compat import lru_cache
from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS
from pip._internal.utils.misc import pairwise, redact_auth_from_url
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
Expand All @@ -36,10 +37,8 @@
List,
MutableMapping,
Optional,
Protocol,
Sequence,
Tuple,
TypeVar,
Union,
)

Expand All @@ -50,31 +49,10 @@
HTMLElement = xml.etree.ElementTree.Element
ResponseHeaders = MutableMapping[str, str]

# Used in the @lru_cache polyfill.
F = TypeVar('F')

class LruCache(Protocol):
def __call__(self, maxsize=None):
# type: (Optional[int]) -> Callable[[F], F]
raise NotImplementedError


logger = logging.getLogger(__name__)


# Fallback to noop_lru_cache in Python 2
# TODO: this can be removed when python 2 support is dropped!
def noop_lru_cache(maxsize=None):
# type: (Optional[int]) -> Callable[[F], F]
def _wrapper(f):
# type: (F) -> F
return f
return _wrapper


_lru_cache = getattr(functools, "lru_cache", noop_lru_cache) # type: LruCache


def _match_vcs_scheme(url):
# type: (str) -> Optional[str]
"""Look for VCS schemes in the URL.
Expand Down Expand Up @@ -344,7 +322,7 @@ def with_cached_html_pages(
`page` has `page.cache_link_parsing == False`.
"""

@_lru_cache(maxsize=None)
@lru_cache(maxsize=None)
def wrapper(cacheable_page):
# type: (CacheablePageContent) -> List[Link]
return list(fn(cacheable_page.page))
Expand Down
2 changes: 2 additions & 0 deletions src/pip/_internal/index/package_finder.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from pip._internal.models.selection_prefs import SelectionPreferences
from pip._internal.models.target_python import TargetPython
from pip._internal.models.wheel import Wheel
from pip._internal.utils.compat import lru_cache
from pip._internal.utils.filetypes import WHEEL_EXTENSION
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import build_netloc
Expand Down Expand Up @@ -799,6 +800,7 @@ def process_project_url(self, project_url, link_evaluator):

return package_links

@lru_cache(maxsize=None)
def find_all_candidates(self, project_name):
# type: (str) -> List[InstallationCandidate]
"""Find all available InstallationCandidate for project_name
Expand Down
24 changes: 23 additions & 1 deletion src/pip/_internal/utils/compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from __future__ import absolute_import, division

import codecs
import functools
import locale
import logging
import os
Expand All @@ -18,7 +19,15 @@
from pip._internal.utils.typing import MYPY_CHECK_RUNNING

if MYPY_CHECK_RUNNING:
from typing import Optional, Text, Tuple, Union
from typing import Callable, Optional, Protocol, Text, Tuple, TypeVar, Union

# Used in the @lru_cache polyfill.
F = TypeVar('F')

class LruCache(Protocol):
def __call__(self, maxsize=None):
# type: (Optional[int]) -> Callable[[F], F]
raise NotImplementedError

try:
import ipaddress
Expand Down Expand Up @@ -269,3 +278,16 @@ def ioctl_GWINSZ(fd):
if not cr:
cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
return int(cr[1]), int(cr[0])


# Fallback to noop_lru_cache in Python 2
# TODO: this can be removed when python 2 support is dropped!
def noop_lru_cache(maxsize=None):
# type: (Optional[int]) -> Callable[[F], F]
def _wrapper(f):
# type: (F) -> F
return f
return _wrapper


lru_cache = getattr(functools, "lru_cache", noop_lru_cache) # type: LruCache

0 comments on commit 703d6ac

Please sign in to comment.