diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index b70048778b8..71331861e24 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -252,6 +252,65 @@ def _get_html_page(link, session=None): return None +def group_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] + """ + Divide a list of locations into two groups: "files" (archives) and "urls." + + :return: A pair of lists (files, urls). + """ + files = [] + urls = [] + + # puts the url for the given file path into the appropriate list + def sort_path(path): + url = path_to_url(path) + if mimetypes.guess_type(url, strict=False)[0] == 'text/html': + urls.append(url) + else: + files.append(url) + + for url in locations: + + is_local_path = os.path.exists(url) + is_file_url = url.startswith('file:') + + if is_local_path or is_file_url: + if is_local_path: + path = url + else: + path = url_to_path(url) + if os.path.isdir(path): + if expand_dir: + path = os.path.realpath(path) + for item in os.listdir(path): + sort_path(os.path.join(path, item)) + elif is_file_url: + urls.append(url) + else: + logger.warning( + "Path '{0}' is ignored: " + "it is a directory.".format(path), + ) + elif os.path.isfile(path): + sort_path(path) + else: + logger.warning( + "Url '%s' is ignored: it is neither a file " + "nor a directory.", url, + ) + elif is_url(url): + # Only add url with clear scheme + urls.append(url) + else: + logger.warning( + "Url '%s' is ignored. It is either a non-existing " + "path or lacks a specific scheme.", url, + ) + + return files, urls + + def _check_link_requires_python( link, # type: Link version_info, # type: Tuple[int, int, int] @@ -899,64 +958,6 @@ def set_allow_all_prereleases(self): # type: () -> None self._candidate_prefs.allow_all_prereleases = True - @staticmethod - def _sort_locations(locations, expand_dir=False): - # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] - """ - Sort locations into "files" (archives) and "urls", and return - a pair of lists (files,urls) - """ - files = [] - urls = [] - - # puts the url for the given file path into the appropriate list - def sort_path(path): - url = path_to_url(path) - if mimetypes.guess_type(url, strict=False)[0] == 'text/html': - urls.append(url) - else: - files.append(url) - - for url in locations: - - is_local_path = os.path.exists(url) - is_file_url = url.startswith('file:') - - if is_local_path or is_file_url: - if is_local_path: - path = url - else: - path = url_to_path(url) - if os.path.isdir(path): - if expand_dir: - path = os.path.realpath(path) - for item in os.listdir(path): - sort_path(os.path.join(path, item)) - elif is_file_url: - urls.append(url) - else: - logger.warning( - "Path '{0}' is ignored: " - "it is a directory.".format(path), - ) - elif os.path.isfile(path): - sort_path(path) - else: - logger.warning( - "Url '%s' is ignored: it is neither a file " - "nor a directory.", url, - ) - elif is_url(url): - # Only add url with clear scheme - urls.append(url) - else: - logger.warning( - "Url '%s' is ignored. It is either a non-existing " - "path or lacks a specific scheme.", url, - ) - - return files, urls - def make_link_evaluator(self, project_name): # type: (str) -> LinkEvaluator canonical_name = canonicalize_name(project_name) @@ -971,6 +972,63 @@ def make_link_evaluator(self, project_name): ignore_requires_python=self._ignore_requires_python, ) + def _sort_links(self, links): + # type: (Iterable[Link]) -> List[Link] + """ + Returns elements of links in order, non-egg links first, egg links + second, while eliminating duplicates + """ + eggs, no_eggs = [], [] + seen = set() # type: Set[Link] + for link in links: + if link not in seen: + seen.add(link) + if link.egg_fragment: + eggs.append(link) + else: + no_eggs.append(link) + return no_eggs + eggs + + def _log_skipped_link(self, link, reason): + # type: (Link, Text) -> None + if link not in self._logged_links: + # Mark this as a unicode string to prevent "UnicodeEncodeError: + # 'ascii' codec can't encode character" in Python 2 when + # the reason contains non-ascii characters. + # Also, put the link at the end so the reason is more visible + # and because the link string is usually very long. + logger.debug(u'Skipping link: %s: %s', reason, link) + self._logged_links.add(link) + + def get_install_candidate(self, link_evaluator, link): + # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] + """ + If the link is a candidate for install, convert it to an + InstallationCandidate and return it. Otherwise, return None. + """ + is_candidate, result = link_evaluator.evaluate_link(link) + if not is_candidate: + if result: + self._log_skipped_link(link, reason=result) + return None + + return InstallationCandidate( + project=link_evaluator.project_name, + link=link, + # Convert the Text result to str since InstallationCandidate + # accepts str. + version=str(result), + ) + + def _package_versions(self, link_evaluator, links): + # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] + result = [] + for link in self._sort_links(links): + candidate = self.get_install_candidate(link_evaluator, link) + if candidate is not None: + result.append(candidate) + return result + def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name @@ -983,8 +1041,8 @@ def find_all_candidates(self, project_name): """ search_scope = self.search_scope index_locations = search_scope.get_index_urls_locations(project_name) - index_file_loc, index_url_loc = self._sort_locations(index_locations) - fl_file_loc, fl_url_loc = self._sort_locations( + index_file_loc, index_url_loc = group_locations(index_locations) + fl_file_loc, fl_url_loc = group_locations( self.find_links, expand_dir=True, ) @@ -1177,63 +1235,6 @@ def _get_pages(self, locations, project_name): yield page - def _sort_links(self, links): - # type: (Iterable[Link]) -> List[Link] - """ - Returns elements of links in order, non-egg links first, egg links - second, while eliminating duplicates - """ - eggs, no_eggs = [], [] - seen = set() # type: Set[Link] - for link in links: - if link not in seen: - seen.add(link) - if link.egg_fragment: - eggs.append(link) - else: - no_eggs.append(link) - return no_eggs + eggs - - def _log_skipped_link(self, link, reason): - # type: (Link, Text) -> None - if link not in self._logged_links: - # Mark this as a unicode string to prevent "UnicodeEncodeError: - # 'ascii' codec can't encode character" in Python 2 when - # the reason contains non-ascii characters. - # Also, put the link at the end so the reason is more visible - # and because the link string is usually very long. - logger.debug(u'Skipping link: %s: %s', reason, link) - self._logged_links.add(link) - - def get_install_candidate(self, link_evaluator, link): - # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] - """ - If the link is a candidate for install, convert it to an - InstallationCandidate and return it. Otherwise, return None. - """ - is_candidate, result = link_evaluator.evaluate_link(link) - if not is_candidate: - if result: - self._log_skipped_link(link, reason=result) - return None - - return InstallationCandidate( - project=link_evaluator.project_name, - link=link, - # Convert the Text result to str since InstallationCandidate - # accepts str. - version=str(result), - ) - - def _package_versions(self, link_evaluator, links): - # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] - result = [] - for link in self._sort_links(links): - candidate = self.get_install_candidate(link_evaluator, link) - if candidate is not None: - result.append(candidate) - return result - def _find_name_version_sep(fragment, canonical_name): # type: (str, str) -> int diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index f3d6366bc4d..cc8fd8fc19b 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -23,6 +23,24 @@ from tests.lib import make_test_finder +def make_no_network_finder( + find_links, + allow_all_prereleases=False, # type: bool +): + """ + Create and return a PackageFinder instance for test purposes that + doesn't make any network requests when _get_pages() is called. + """ + finder = make_test_finder( + find_links=find_links, + allow_all_prereleases=allow_all_prereleases, + ) + # Replace the PackageFinder object's _get_pages() with a no-op. + finder._get_pages = lambda locations, project_name: [] + + return finder + + def test_no_mpkg(data): """Finder skips zipfiles with "macosx10" in the name.""" finder = make_test_finder(find_links=[data.find_links]) @@ -279,25 +297,22 @@ def test_finder_priority_nonegg_over_eggfragments(): req = install_req_from_line('bar==1.0', None) links = ['http://foo/bar.py#egg=bar-1.0', 'http://foo/bar-1.0.tar.gz'] - finder = make_test_finder(find_links=links) - - with patch.object(finder, "_get_pages", lambda x, y: []): - all_versions = finder.find_all_candidates(req.name) - assert all_versions[0].link.url.endswith('tar.gz') - assert all_versions[1].link.url.endswith('#egg=bar-1.0') + finder = make_no_network_finder(links) + all_versions = finder.find_all_candidates(req.name) + assert all_versions[0].link.url.endswith('tar.gz') + assert all_versions[1].link.url.endswith('#egg=bar-1.0') - link = finder.find_requirement(req, False) + link = finder.find_requirement(req, False) assert link.url.endswith('tar.gz') links.reverse() - finder = make_test_finder(find_links=links) - with patch.object(finder, "_get_pages", lambda x, y: []): - all_versions = finder.find_all_candidates(req.name) - assert all_versions[0].link.url.endswith('tar.gz') - assert all_versions[1].link.url.endswith('#egg=bar-1.0') - link = finder.find_requirement(req, False) + finder = make_no_network_finder(links) + all_versions = finder.find_all_candidates(req.name) + assert all_versions[0].link.url.endswith('tar.gz') + assert all_versions[1].link.url.endswith('#egg=bar-1.0') + link = finder.find_requirement(req, False) assert link.url.endswith('tar.gz') @@ -316,18 +331,16 @@ def test_finder_only_installs_stable_releases(data): # using find-links links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"] - finder = make_test_finder(find_links=links) - with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) - assert link.url == "https://foo/bar-1.0.tar.gz" + finder = make_no_network_finder(links) + link = finder.find_requirement(req, False) + assert link.url == "https://foo/bar-1.0.tar.gz" links.reverse() - finder = make_test_finder(find_links=links) - with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) - assert link.url == "https://foo/bar-1.0.tar.gz" + finder = make_no_network_finder(links) + link = finder.find_requirement(req, False) + assert link.url == "https://foo/bar-1.0.tar.gz" def test_finder_only_installs_data_require(data): @@ -371,18 +384,16 @@ def test_finder_installs_pre_releases(data): # using find-links links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"] - finder = make_test_finder(find_links=links, allow_all_prereleases=True) - with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) - assert link.url == "https://foo/bar-2.0b1.tar.gz" + finder = make_no_network_finder(links, allow_all_prereleases=True) + link = finder.find_requirement(req, False) + assert link.url == "https://foo/bar-2.0b1.tar.gz" links.reverse() - finder = make_test_finder(find_links=links, allow_all_prereleases=True) - with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) - assert link.url == "https://foo/bar-2.0b1.tar.gz" + finder = make_no_network_finder(links, allow_all_prereleases=True) + link = finder.find_requirement(req, False) + assert link.url == "https://foo/bar-2.0b1.tar.gz" def test_finder_installs_dev_releases(data): @@ -408,18 +419,15 @@ def test_finder_installs_pre_releases_with_version_spec(): req = install_req_from_line("bar>=0.0.dev0", None) links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"] - finder = make_test_finder(find_links=links) - - with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) - assert link.url == "https://foo/bar-2.0b1.tar.gz" + finder = make_no_network_finder(links) + link = finder.find_requirement(req, False) + assert link.url == "https://foo/bar-2.0b1.tar.gz" links.reverse() - finder = make_test_finder(find_links=links) - with patch.object(finder, "_get_pages", lambda x, y: []): - link = finder.find_requirement(req, False) - assert link.url == "https://foo/bar-2.0b1.tar.gz" + finder = make_no_network_finder(links) + link = finder.find_requirement(req, False) + assert link.url == "https://foo/bar-2.0b1.tar.gz" class TestLinkEvaluator(object): diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index 71bfd2a163b..c27b3589d86 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -22,6 +22,7 @@ _find_name_version_sep, _get_html_page, filter_unallowed_hashes, + group_locations, ) from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.search_scope import SearchScope @@ -29,7 +30,7 @@ from pip._internal.models.target_python import TargetPython from pip._internal.pep425tags import get_supported from pip._internal.utils.hashes import Hashes -from tests.lib import CURRENT_PY_VERSION_INFO, make_test_finder +from tests.lib import CURRENT_PY_VERSION_INFO def make_mock_candidate(version, yanked_reason=None, hex_digest=None): @@ -748,34 +749,31 @@ def test_make_candidate_evaluator( assert evaluator._supported_tags == [('py36', 'none', 'any')] -def test_sort_locations_file_expand_dir(data): +def test_group_locations__file_expand_dir(data): """ Test that a file:// dir gets listdir run with expand_dir """ - finder = make_test_finder(find_links=[data.find_links]) - files, urls = finder._sort_locations([data.find_links], expand_dir=True) + files, urls = group_locations([data.find_links], expand_dir=True) assert files and not urls, ( "files and not urls should have been found at find-links url: %s" % data.find_links ) -def test_sort_locations_file_not_find_link(data): +def test_group_locations__file_not_find_link(data): """ Test that a file:// url dir that's not a find-link, doesn't get a listdir run """ - finder = make_test_finder() - files, urls = finder._sort_locations([data.index_url("empty_with_pkg")]) + files, urls = group_locations([data.index_url("empty_with_pkg")]) assert urls and not files, "urls, but not files should have been found" -def test_sort_locations_non_existing_path(): +def test_group_locations__non_existing_path(): """ Test that a non-existing path is ignored. """ - finder = make_test_finder() - files, urls = finder._sort_locations( + files, urls = group_locations( [os.path.join('this', 'doesnt', 'exist')]) assert not urls and not files, "nothing should have been found"