From 786ff8c60714609cbdc91f029069cee46bcd03fd Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 10 Apr 2015 15:15:00 +1200 Subject: [PATCH 1/3] Break out version extraction for reuse. This is needed for determining the wheel-cachability of a requirement. --- pip/index.py | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/pip/index.py b/pip/index.py index d21ccbe08e7..abf9e959319 100644 --- a/pip/index.py +++ b/pip/index.py @@ -648,7 +648,6 @@ def _get_pages(self, locations, project_name): all_locations.append(link) - _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I) _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') def _sort_links(self, links): @@ -742,7 +741,7 @@ def _link_package_versions(self, link, search_name): version = wheel.version if not version: - version = self._egg_info_matches(egg_info, search_name, link) + version = egg_info_matches(egg_info, search_name, link) if version is None: self._log_skipped_link( link, 'wrong project name (not %s)' % search_name) @@ -783,25 +782,39 @@ def _link_package_versions(self, link, search_name): return InstallationCandidate(search_name, version, link) - def _egg_info_matches(self, egg_info, search_name, link): - match = self._egg_info_re.search(egg_info) - if not match: - logger.debug('Could not parse version from link: %s', link) - return None - name = match.group(0).lower() - # To match the "safe" name that pkg_resources creates: - name = name.replace('_', '-') - # project name and version must be separated by a dash - look_for = search_name.lower() + "-" - if name.startswith(look_for): - return match.group(0)[len(look_for):] - else: - return None - def _get_page(self, link): return HTMLPage.get_page(link, session=self.session) +def egg_info_matches( + egg_info, search_name, link, + _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): + """Pull the version part out of a string. + + :param egg_info: The string to parse. E.g. foo-2.1 + :param search_name: The name of the package this belongs to. None to + infer the name. Note that this cannot unambiguously parse strings + like foo-2-2 which might be foo, 2-2 or foo-2, 2. + :param link: The link the string came from, for logging on failure. + """ + match = _egg_info_re.search(egg_info) + if not match: + logger.debug('Could not parse version from link: %s', link) + return None + if search_name is None: + full_match = match.group(0) + return full_match[full_match.index('-'):] + name = match.group(0).lower() + # To match the "safe" name that pkg_resources creates: + name = name.replace('_', '-') + # project name and version must be separated by a dash + look_for = search_name.lower() + "-" + if name.startswith(look_for): + return match.group(0)[len(look_for):] + else: + return None + + class HTMLPage(object): """Represents one page, along with its URL""" From 4926409340569b7c2c10cc0373f2b75025b9279d Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 31 Mar 2015 10:44:02 +1300 Subject: [PATCH 2/3] Issue #2563: Read cached wheels from ~/.cache/pip This won't put wheels into that directory, but will read them if they are there. --no-cache-dir will disable reading such wheels. --- docs/reference/pip_install.rst | 17 ++++++++++++ pip/basecommand.py | 2 ++ pip/commands/freeze.py | 3 ++- pip/commands/install.py | 1 + pip/commands/list.py | 1 + pip/commands/uninstall.py | 5 +++- pip/commands/wheel.py | 1 + pip/operations/freeze.py | 5 +++- pip/req/req_file.py | 19 +++++++------- pip/req/req_install.py | 25 ++++++++++++++---- pip/req/req_set.py | 7 ++++- pip/wheel.py | 48 ++++++++++++++++++++++++++++++++++ tests/unit/test_req_file.py | 2 +- 13 files changed, 116 insertions(+), 20 deletions(-) diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 5f11cba73f9..29612593e88 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -388,6 +388,23 @@ Windows :file:`\\pip\\Cache` +Wheel cache +*********** + +Pip will read from the subdirectory ``wheels`` within the pip cache dir and use +any packages found there. This is disabled via the same ``no-cache-dir`` option +that disables the HTTP cache. The internal structure of that cache is not part +of the Pip API. As of 7.0 pip uses a subdirectory per sdist that wheels were +built from, and wheels within that subdirectory. + +Pip attempts to choose the best wheels from those built in preference to +building a new wheel. Note that this means when a package has both optional +C extensions and builds `py` tagged wheels when the C extension can't be built +that pip will not attempt to build a better wheel for Python's that would have +supported it, once any generic wheel is built. To correct this, make sure that +the wheel's are built with Python specific tags - e.g. pp on Pypy. + + Hash Verification +++++++++++++++++ diff --git a/pip/basecommand.py b/pip/basecommand.py index 002c25b35dd..7dffe03d565 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -289,6 +289,7 @@ def populate_requirement_set(requirement_set, args, options, finder, requirement_set.add_requirement( InstallRequirement.from_line( name, None, isolated=options.isolated_mode, + cache_root=options.cache_dir ) ) @@ -298,6 +299,7 @@ def populate_requirement_set(requirement_set, args, options, finder, name, default_vcs=options.default_vcs, isolated=options.isolated_mode, + cache_root=options.cache_dir ) ) diff --git a/pip/commands/freeze.py b/pip/commands/freeze.py index 42261b2958b..7f87c340394 100644 --- a/pip/commands/freeze.py +++ b/pip/commands/freeze.py @@ -60,7 +60,8 @@ def run(self, options, args): local_only=options.local, user_only=options.user, skip_regex=options.skip_requirements_regex, - isolated=options.isolated_mode) + isolated=options.isolated_mode, + cache_root=options.cache_dir) for line in freeze(**freeze_kwargs): sys.stdout.write(line + '\n') diff --git a/pip/commands/install.py b/pip/commands/install.py index 5b4489019ef..44b1e5351a7 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -239,6 +239,7 @@ def run(self, options, args): delete=build_delete) as build_dir: requirement_set = RequirementSet( build_dir=build_dir, + cache_root=options.cache_dir, src_dir=options.src_dir, download_dir=options.download_dir, upgrade=options.upgrade, diff --git a/pip/commands/list.py b/pip/commands/list.py index dbc17b2f62d..cd46abd785c 100644 --- a/pip/commands/list.py +++ b/pip/commands/list.py @@ -131,6 +131,7 @@ def find_packages_latest_versions(self, options): for dist in installed_packages: req = InstallRequirement.from_line( dist.key, None, isolated=options.isolated_mode, + cache_root=options.cache_dir, ) typ = 'unknown' try: diff --git a/pip/commands/uninstall.py b/pip/commands/uninstall.py index c2af5f73055..2194f32217a 100644 --- a/pip/commands/uninstall.py +++ b/pip/commands/uninstall.py @@ -45,6 +45,7 @@ def run(self, options, args): requirement_set = RequirementSet( build_dir=None, + cache_root=options.cache_dir, src_dir=None, download_dir=None, isolated=options.isolated_mode, @@ -54,13 +55,15 @@ def run(self, options, args): requirement_set.add_requirement( InstallRequirement.from_line( name, isolated=options.isolated_mode, + cache_root=options.cache_dir, ) ) for filename in options.requirements: for req in parse_requirements( filename, options=options, - session=session): + session=session, + cache_root=options.cache_dir): requirement_set.add_requirement(req) if not requirement_set.has_requirements: raise InstallationError( diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index 98834b645d2..ccfa1c19c73 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -159,6 +159,7 @@ def run(self, options, args): delete=build_delete) as build_dir: requirement_set = RequirementSet( build_dir=build_dir, + cache_root=options.cache_dir, src_dir=options.src_dir, download_dir=None, ignore_dependencies=options.ignore_dependencies, diff --git a/pip/operations/freeze.py b/pip/operations/freeze.py index d61a84c72ee..fb18c151b99 100644 --- a/pip/operations/freeze.py +++ b/pip/operations/freeze.py @@ -21,7 +21,8 @@ def freeze( find_links=None, local_only=None, user_only=None, skip_regex=None, find_tags=False, default_vcs=None, - isolated=False): + isolated=False, + cache_root=None): find_links = find_links or [] skip_match = None @@ -75,11 +76,13 @@ def freeze( line, default_vcs=default_vcs, isolated=isolated, + cache_root=cache_root, ) else: line_req = InstallRequirement.from_line( line, isolated=isolated, + cache_root=cache_root, ) if not line_req.name: diff --git a/pip/req/req_file.py b/pip/req/req_file.py index f5c6c330940..3d1df1dbbcb 100644 --- a/pip/req/req_file.py +++ b/pip/req/req_file.py @@ -77,7 +77,7 @@ def parser_exit(self, msg): def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None): + session=None, cache_root=None): """ Parse a requirements file and yield InstallRequirement instances. @@ -87,7 +87,6 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, :param options: Global options. :param session: Instance of pip.download.PipSession. """ - if session is None: raise TypeError( "parse_requirements() missing 1 required keyword argument: " @@ -99,7 +98,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, ) parser = parse_content( - filename, content, finder, comes_from, options, session + filename, content, finder, comes_from, options, session, cache_root ) for item in parser: @@ -107,7 +106,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, def parse_content(filename, content, finder=None, comes_from=None, - options=None, session=None): + options=None, session=None, cache_root=None): # Split, sanitize and join lines with continuations. content = content.splitlines() @@ -129,8 +128,8 @@ def parse_content(filename, content, finder=None, comes_from=None, comes_from = '-r %s (line %s)' % (filename, line_number) isolated = options.isolated_mode if options else False yield InstallRequirement.from_line( - req, comes_from, isolated=isolated, options=opts - ) + req, comes_from, isolated=isolated, options=opts, + cache_root=cache_root) # --------------------------------------------------------------------- elif linetype == REQUIREMENT_EDITABLE: @@ -139,8 +138,8 @@ def parse_content(filename, content, finder=None, comes_from=None, default_vcs = options.default_vcs if options else None yield InstallRequirement.from_editable( value, comes_from=comes_from, - default_vcs=default_vcs, isolated=isolated - ) + default_vcs=default_vcs, isolated=isolated, + cache_root=cache_root) # --------------------------------------------------------------------- elif linetype == REQUIREMENT_FILE: @@ -152,8 +151,8 @@ def parse_content(filename, content, finder=None, comes_from=None, req_url = os.path.join(os.path.dirname(filename), value) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? parser = parse_requirements( - req_url, finder, comes_from, options, session - ) + req_url, finder, comes_from, options, session, + cache_root=cache_root) for req in parser: yield req diff --git a/pip/req/req_install.py b/pip/req/req_install.py index e103c579fba..6d910e2bbe0 100644 --- a/pip/req/req_install.py +++ b/pip/req/req_install.py @@ -73,7 +73,8 @@ class InstallRequirement(object): def __init__(self, req, comes_from, source_dir=None, editable=False, link=None, as_egg=False, update=True, editable_options=None, - pycompile=True, markers=None, isolated=False, options=None): + pycompile=True, markers=None, isolated=False, options=None, + cache_root=None): self.extras = () if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse(req) @@ -88,6 +89,7 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, editable_options = {} self.editable_options = editable_options + self._cache_root = cache_root self.link = link self.as_egg = as_egg self.markers = markers @@ -118,7 +120,7 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, @classmethod def from_editable(cls, editable_req, comes_from=None, default_vcs=None, - isolated=False, options=None): + isolated=False, options=None, cache_root=None): from pip.index import Link name, url, extras_override, editable_options = parse_editable( @@ -133,7 +135,8 @@ def from_editable(cls, editable_req, comes_from=None, default_vcs=None, link=Link(url), editable_options=editable_options, isolated=isolated, - options=options if options else {}) + options=options if options else {}, + cache_root=cache_root) if extras_override is not None: res.extras = extras_override @@ -141,7 +144,9 @@ def from_editable(cls, editable_req, comes_from=None, default_vcs=None, return res @classmethod - def from_line(cls, name, comes_from=None, isolated=False, options=None): + def from_line( + cls, name, comes_from=None, isolated=False, options=None, + cache_root=None): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -208,7 +213,7 @@ def from_line(cls, name, comes_from=None, isolated=False, options=None): options = options if options else {} return cls(req, comes_from, link=link, markers=markers, - isolated=isolated, options=options) + isolated=isolated, options=options, cache_root=cache_root) def __str__(self): if self.req: @@ -241,6 +246,16 @@ def populate_link(self, finder, upgrade): if self.link is None: self.link = finder.find_requirement(self, upgrade) + @property + def link(self): + return self._link + + @link.setter + def link(self, link): + # Lookup a cached wheel, if possible. + link = pip.wheel.cached_wheel(self._cache_root, link) + self._link = link + @property def specifier(self): return self.req.specifier diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 660c9eb9f0f..751c85bd123 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -139,7 +139,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False, ignore_installed=False, as_egg=False, target_dir=None, ignore_dependencies=False, force_reinstall=False, use_user_site=False, session=None, pycompile=True, - isolated=False, wheel_download_dir=None): + isolated=False, wheel_download_dir=None, + cache_root=None): """Create a RequirementSet. :param wheel_download_dir: Where still-packed .whl files should be @@ -149,6 +150,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False, :param download_dir: Where still packed archives should be written to. If None they are not saved, and are deleted immediately after unpacking. + :param cache_root: The root of the pip cache, for passing to + InstallRequirement. """ if session is None: raise TypeError( @@ -181,6 +184,7 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False, if wheel_download_dir: wheel_download_dir = normalize_path(wheel_download_dir) self.wheel_download_dir = wheel_download_dir + self._cache_root = cache_root # Maps from install_req -> dependencies_of_install_req self._dependencies = defaultdict(list) @@ -512,6 +516,7 @@ def add_req(subreq): str(subreq), req_to_install, isolated=self.isolated, + cache_root=self._cache_root, ) more_reqs.extend(self.add_requirement( sub_install_req, req_to_install.name)) diff --git a/pip/wheel.py b/pip/wheel.py index d689903fc57..6b9129c2944 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -5,6 +5,7 @@ import compileall import csv +import errno import functools import hashlib import logging @@ -20,6 +21,8 @@ from pip._vendor.six import StringIO +import pip +from pip.download import path_to_url from pip.exceptions import InvalidWheelFilename, UnsupportedWheel from pip.locations import distutils_scheme from pip import pep425tags @@ -39,6 +42,51 @@ logger = logging.getLogger(__name__) +def _cache_for_filename(cache_dir, sdistfilename): + """Return a directory to store cached wheels in for sdistfilename. + + Because there are M wheels for any one sdist, we provide a directory + to cache them in, and then consult that directory when looking up + cache hits. + + :param cache_dir: The cache_dir being used by pip. + :param sdistfilename: The filename of the sdist for which this will cache + wheels. + """ + return os.path.join(cache_dir, 'wheels', sdistfilename) + + +def cached_wheel(cache_dir, link): + if not cache_dir: + return link + if not link: + return link + if link.is_wheel: + return link + root = _cache_for_filename(cache_dir, link.filename) + try: + wheel_names = os.listdir(root) + except OSError as e: + if e.errno == errno.ENOENT: + return link + raise + candidates = [] + for wheel_name in wheel_names: + try: + wheel = Wheel(wheel_name) + except InvalidWheelFilename: + continue + if not wheel.supported(): + # Built for a different python/arch/etc + continue + candidates.append((wheel.support_index_min(), wheel_name)) + if not candidates: + return link + candidates.sort() + path = os.path.join(root, candidates[0][1]) + return pip.index.Link(path_to_url(path), trusted=True) + + def rehash(path, algo='sha256', blocksize=1 << 20): """Return (hash, length) for path using hashlib.new(algo)""" h = hashlib.new(algo) diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index e5eebf277e3..0f9fcd0a5cf 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -182,7 +182,7 @@ def test_parse_content_requirements_file(self, monkeypatch): import pip.req.req_file def stub_parse_requirements(req_url, finder, comes_from, options, - session): + session, cache_root): return [req] parse_requirements_stub = stub(call=stub_parse_requirements) monkeypatch.setattr(pip.req.req_file, 'parse_requirements', From 08acb6670dfde30f4f42c221860a80557ea53a22 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 1 Apr 2015 14:39:45 +1300 Subject: [PATCH 3/3] Issue #2140: Build wheels automatically Building wheels before installing elminates a cause of broken environments - where install fails after we've already installed one or more packages. If a package fails to wheel, we run setup.py install as normally. --- CHANGES.txt | 3 + docs/reference/pip_install.rst | 16 +++ pip/commands/install.py | 25 +++- pip/req/req_set.py | 3 +- pip/wheel.py | 111 +++++++++++++++--- tests/conftest.py | 7 +- tests/data/packages/README.txt | 5 + .../requires_wheelbroken_upper/__init__.py | 0 .../requires_wheelbroken_upper/setup.py | 5 + tests/functional/test_install.py | 51 +++++++- 10 files changed, 199 insertions(+), 27 deletions(-) create mode 100644 tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py create mode 100644 tests/data/packages/requires_wheelbroken_upper/setup.py diff --git a/CHANGES.txt b/CHANGES.txt index c0dfe90e253..0c95af1d037 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,6 +22,9 @@ * Support ``--install-option`` and ``--global-option`` per requirement in requirement files (:pull:`2537`) +* Build Wheels prior to installing from sdist, caching them in the pip cache + directory to speed up subsequent installs. (:pull:`2618`) + **6.1.1 (2015-04-07)** * No longer ignore dependencies which have been added to the standard library, diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 29612593e88..6f3ac11b11e 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -16,6 +16,15 @@ Description .. pip-command-description:: install +Overview +++++++++ + +Pip install has several stages: + +1. Resolve dependencies. What will be installed is determined here. +2. Build wheels. All the dependencies that can be are built into wheels. +3. Install the packages (and uninstall anything being upgraded/replaced). + Installation Order ++++++++++++++++++ @@ -404,6 +413,9 @@ that pip will not attempt to build a better wheel for Python's that would have supported it, once any generic wheel is built. To correct this, make sure that the wheel's are built with Python specific tags - e.g. pp on Pypy. +When no wheels are found for an sdist, pip will attempt to build a wheel +automatically and insert it into the wheel cache. + Hash Verification +++++++++++++++++ @@ -499,6 +511,10 @@ implement the following command:: This should implement the complete process of installing the package in "editable" mode. +All packages will be attempted to built into wheels:: + + setup.py bdist_wheel -d XXX + One further ``setup.py`` command is invoked by ``pip install``:: setup.py clean diff --git a/pip/commands/install.py b/pip/commands/install.py index 44b1e5351a7..09b554a4c85 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -6,10 +6,14 @@ import tempfile import shutil import warnings +try: + import wheel +except ImportError: + wheel = None from pip.req import RequirementSet -from pip.locations import virtualenv_no_global, distutils_scheme from pip.basecommand import RequirementCommand +from pip.locations import virtualenv_no_global, distutils_scheme from pip.index import PackageFinder from pip.exceptions import ( InstallationError, CommandError, PreviousBuildDirError, @@ -18,6 +22,7 @@ from pip.utils import ensure_dir from pip.utils.build import BuildDirectory from pip.utils.deprecation import RemovedInPip8Warning +from pip.wheel import WheelBuilder logger = logging.getLogger(__name__) @@ -233,7 +238,6 @@ def run(self, options, args): with self._build_session(options) as session: finder = self._build_package_finder(options, index_urls, session) - build_delete = (not (options.no_clean or options.build_dir)) with BuildDirectory(options.build_dir, delete=build_delete) as build_dir: @@ -262,7 +266,22 @@ def run(self, options, args): return try: - requirement_set.prepare_files(finder) + if options.download_dir or not wheel: + # on -d don't do complex things like building + # wheels, and don't try to build wheels when wheel is + # not installed. + requirement_set.prepare_files(finder) + else: + # build wheels before install. + wb = WheelBuilder( + requirement_set, + finder, + build_options=[], + global_options=[], + ) + # Ignore the result: a failed wheel will be + # installed from the sdist/vcs whatever. + wb.build(autobuilding=True) if not options.download_dir: requirement_set.install( diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 751c85bd123..d601c17a661 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -162,7 +162,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False, self.build_dir = build_dir self.src_dir = src_dir # XXX: download_dir and wheel_download_dir overlap semantically and may - # be combinable. + # be combined if we're willing to have non-wheel archives present in + # the wheelhouse output by 'pip wheel'. self.download_dir = download_dir self.upgrade = upgrade self.ignore_installed = ignore_installed diff --git a/pip/wheel.py b/pip/wheel.py index 6b9129c2944..24b4244d6e5 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -14,6 +14,7 @@ import shutil import stat import sys +import tempfile import warnings from base64 import urlsafe_b64encode @@ -22,12 +23,13 @@ from pip._vendor.six import StringIO import pip -from pip.download import path_to_url +from pip.download import path_to_url, unpack_url from pip.exceptions import InvalidWheelFilename, UnsupportedWheel -from pip.locations import distutils_scheme +from pip.locations import distutils_scheme, PIP_DELETE_MARKER_FILENAME from pip import pep425tags from pip.utils import ( - call_subprocess, ensure_dir, make_path_relative, captured_stdout) + call_subprocess, ensure_dir, make_path_relative, captured_stdout, + rmtree) from pip.utils.logging import indent_log from pip._vendor.distlib.scripts import ScriptMaker from pip._vendor import pkg_resources @@ -49,6 +51,12 @@ def _cache_for_filename(cache_dir, sdistfilename): to cache them in, and then consult that directory when looking up cache hits. + We only insert things into the cache if they have plausible version + numbers, so that we don't contaminate the cache with things that were not + unique. E.g. ./package might have dozens of installs done for it and build + a version of 0.0...and if we built and cached a wheel, we'd end up using + the same wheel even if the source has been edited. + :param cache_dir: The cache_dir being used by pip. :param sdistfilename: The filename of the sdist for which this will cache wheels. @@ -590,16 +598,35 @@ class WheelBuilder(object): """Build wheels from a RequirementSet.""" def __init__(self, requirement_set, finder, build_options=None, - global_options=None): + global_options=None, cache_root=None): self.requirement_set = requirement_set self.finder = finder - self.wheel_dir = requirement_set.wheel_download_dir + self._cache_root = requirement_set._cache_root + self._wheel_dir = requirement_set.wheel_download_dir self.build_options = build_options or [] self.global_options = global_options or [] - def _build_one(self, req): - """Build one wheel.""" + def _build_one(self, req, output_dir): + """Build one wheel. + :return: The filename of the built wheel, or None if the build failed. + """ + tempd = tempfile.mkdtemp('pip-wheel-') + try: + if self.__build_one(req, tempd): + try: + wheel_name = os.listdir(tempd)[0] + wheel_path = os.path.join(output_dir, wheel_name) + os.rename(os.path.join(tempd, wheel_name), wheel_path) + logger.info('Stored in directory: %s', output_dir) + return wheel_path + except: + return None + return None + finally: + rmtree(tempd) + + def __build_one(self, req, tempd): base_args = [ sys.executable, '-c', "import setuptools;__file__=%r;" @@ -608,8 +635,8 @@ def _build_one(self, req): ] + list(self.global_options) logger.info('Running setup.py bdist_wheel for %s', req.name) - logger.info('Destination directory: %s', self.wheel_dir) - wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] \ + logger.debug('Destination directory: %s', tempd) + wheel_args = base_args + ['bdist_wheel', '-d', tempd] \ + self.build_options try: call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False) @@ -618,10 +645,15 @@ def _build_one(self, req): logger.error('Failed building wheel for %s', req.name) return False - def build(self): - """Build wheels.""" + def build(self, autobuilding=False): + """Build wheels. - # unpack and constructs req set + :param unpack: If True, replace the sdist we built from the with the + newly built wheel, in preparation for installation. + :return: True if all the wheels built correctly. + """ + assert self._wheel_dir or (autobuilding and self._cache_root) + # unpack sdists and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() @@ -629,14 +661,24 @@ def build(self): buildset = [] for req in reqset: if req.is_wheel: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, - ) + if not autobuilding: + logger.info( + 'Skipping %s, due to already being wheel.', req.name) elif req.editable: - logger.info( - 'Skipping %s, due to being editable', req.name, - ) + if not autobuilding: + logger.info( + 'Skipping bdist_wheel for %s, due to being editable', + req.name) + elif autobuilding and not req.source_dir: + pass else: + if autobuilding: + link = req.link + base, ext = link.splitext() + if pip.index.egg_info_matches(base, None, link) is None: + # Doesn't look like a package - don't autobuild a wheel + # because we'll have no way to lookup the result sanely + continue buildset.append(req) if not buildset: @@ -650,8 +692,39 @@ def build(self): with indent_log(): build_success, build_failure = [], [] for req in buildset: - if self._build_one(req): + if autobuilding: + output_dir = _cache_for_filename( + self._cache_root, req.link.filename) + ensure_dir(output_dir) + else: + output_dir = self._wheel_dir + wheel_file = self._build_one(req, output_dir) + if wheel_file: build_success.append(req) + if autobuilding: + # XXX: This is mildly duplicative with prepare_files, + # but not close enough to pull out to a single common + # method. + # The code below assumes temporary source dirs - + # prevent it doing bad things. + if req.source_dir and not os.path.exists(os.path.join( + req.source_dir, PIP_DELETE_MARKER_FILENAME)): + raise AssertionError( + "bad source dir - missing marker") + # Delete the source we built the wheel from + req.remove_temporary_source() + # set the build directory again - name is known from + # the work prepare_files did. + req.source_dir = req.build_location( + self.requirement_set.build_dir) + # Update the link for this. + req.link = pip.index.Link( + path_to_url(wheel_file), trusted=True) + assert req.link.is_wheel + # extract the wheel into the dir + unpack_url( + req.link, req.source_dir, None, False, + session=self.requirement_set.session) else: build_failure.append(req) diff --git a/tests/conftest.py b/tests/conftest.py index 84019dd06bc..3e05bff420f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,8 @@ import py import pytest +from pip.utils import appdirs + from tests.lib import SRC_DIR, TestData from tests.lib.path import Path from tests.lib.scripttest import PipTestEnvironment @@ -119,7 +121,7 @@ def isolate(tmpdir): @pytest.fixture -def virtualenv(tmpdir, monkeypatch): +def virtualenv(tmpdir, monkeypatch, isolate): """ Return a virtual environment which is unique to each test function invocation created inside of a sub directory of the test function's @@ -148,6 +150,9 @@ def virtualenv(tmpdir, monkeypatch): pip_source_dir=pip_src, ) + # Clean out our cache: creating the venv injects wheels into it. + shutil.rmtree(appdirs.user_cache_dir("pip")) + # Undo our monkeypatching of shutil monkeypatch.undo() diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt index b93d92f2332..7fd5cd3a7fc 100644 --- a/tests/data/packages/README.txt +++ b/tests/data/packages/README.txt @@ -103,3 +103,8 @@ Is an empty package which install_requires the simple and simple2 packages. requires_simple_extra-0.1-py2.py3-none-any.whl ---------------------------------------------- requires_simple_extra[extra] requires simple==1.0 + +requires_wheelbroken_upper +-------------------------- +Requires wheelbroken and upper - used for testing implicit wheel building +during install. diff --git a/tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py b/tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/requires_wheelbroken_upper/setup.py b/tests/data/packages/requires_wheelbroken_upper/setup.py new file mode 100644 index 00000000000..255cf2219eb --- /dev/null +++ b/tests/data/packages/requires_wheelbroken_upper/setup.py @@ -0,0 +1,5 @@ +import setuptools +setuptools.setup( + name="requires_wheelbroken_upper", + version="0", + install_requires=['wheelbroken', 'upper']) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index a294d598fcd..21703858352 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -7,7 +7,7 @@ import pytest -from pip.utils import rmtree +from pip.utils import appdirs, rmtree from tests.lib import (pyversion, pyversion_tuple, _create_test_package, _create_svn_repo, path_to_url) from tests.lib.local_repos import local_checkout @@ -144,14 +144,24 @@ def test_install_dev_version_from_pypi(script): ) -def test_install_editable_from_git(script, tmpdir): +def _test_install_editable_from_git(script, tmpdir, wheel): """Test cloning from Git.""" + if wheel: + script.pip('install', 'wheel') pkg_path = _create_test_package(script, name='testpackage', vcs='git') args = ['install', '-e', 'git+%s#egg=testpackage' % path_to_url(pkg_path)] result = script.pip(*args, **{"expect_error": True}) result.assert_installed('testpackage', with_files=['.git']) +def test_install_editable_from_git(script, tmpdir): + _test_install_editable_from_git(script, tmpdir, False) + + +def test_install_editable_from_git_autobuild_wheel(script, tmpdir): + _test_install_editable_from_git(script, tmpdir, True) + + def test_install_editable_from_hg(script, tmpdir): """Test cloning from Mercurial.""" pkg_path = _create_test_package(script, name='testpackage', vcs='hg') @@ -667,5 +677,40 @@ def test_install_topological_sort(script, data): def test_install_wheel_broken(script, data): script.pip('install', 'wheel') res = script.pip( - 'install', '--no-index', '-f', data.find_links, 'wheelbroken') + 'install', '--no-index', '-f', data.find_links, 'wheelbroken', + expect_stderr=True) assert "Successfully installed wheelbroken-0.1" in str(res), str(res) + + +def test_install_builds_wheels(script, data): + # NB This incidentally tests a local tree + tarball inputs + # see test_install_editable_from_git_autobuild_wheel for editable + # vcs coverage. + script.pip('install', 'wheel') + to_install = data.packages.join('requires_wheelbroken_upper') + res = script.pip( + 'install', '--no-index', '-f', data.find_links, + to_install, expect_stderr=True) + expected = ("Successfully installed requires-wheelbroken-upper-0" + " upper-2.0 wheelbroken-0.1") + # Must have installed it all + assert expected in str(res), str(res) + root = appdirs.user_cache_dir('pip') + wheels = [] + for top, dirs, files in os.walk(root): + wheels.extend(files) + # and built wheels for upper and wheelbroken + assert "Running setup.py bdist_wheel for upper" in str(res), str(res) + assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) + # But not requires_wheel... which is a local dir and thus uncachable. + assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # wheelbroken has to run install + # into the cache + assert wheels != [], str(res) + # and installed from the wheel + assert "Running setup.py install for upper" not in str(res), str(res) + # the local tree can't build a wheel (because we can't assume that every + # build will have a suitable unique key to cache on). + assert "Running setup.py install for requires-wheel" in str(res), str(res) + # wheelbroken has to run install + assert "Running setup.py install for wheelb" in str(res), str(res)