Issue pypa#2140: Build wheels automatically

Building wheels before installing elminates a cause of broken environments -
where install fails after we've already installed one or more packages.

If a package fails to wheel, we run setup.py install as normally.

CHANGES.txt

@@ -19,6 +19,8 @@
 * Ignores bz2 archives if Python wasn't compiled with bz2 support.
   Fixes :issue:`497`
 
+* Build Wheels prior to installing from sdist, caching them in the pip cache
+  directory to speed up subsequent installs. (:pull:`2618`)
 
 **6.1.1 (2015-04-07)**
 

docs/reference/pip_install.rst

@@ -16,6 +16,15 @@ Description
 .. pip-command-description:: install
 
 
+Overview
+++++++++
+
+Pip install has several stages:
+
+1. Resolve dependencies. What will be installed is determined here.
+2. Build wheels. All the dependencies that can be are built into wheels.
+3. Install the packages (and uninstall anything being upgraded/replaced).
+
 Installation Order
 ++++++++++++++++++
 
@@ -454,6 +463,10 @@ implement the following command::
 This should implement the complete process of installing the package in
 "editable" mode.
 
+All packages will be attempted to built into wheels::
+
+    setup.py bdist_wheel -d XXX
+
 One further ``setup.py`` command is invoked by ``pip install``::
 
     setup.py clean

pip/commands/install.py

@@ -6,10 +6,14 @@
 import tempfile
 import shutil
 import warnings
+try:
+    import wheel
+except ImportError:
+    wheel = None
 
 from pip.req import RequirementSet
-from pip.locations import virtualenv_no_global, distutils_scheme
 from pip.basecommand import RequirementCommand
+from pip.locations import virtualenv_no_global, distutils_scheme
 from pip.index import PackageFinder
 from pip.exceptions import (
     InstallationError, CommandError, PreviousBuildDirError,
@@ -18,6 +22,7 @@
 from pip.utils import ensure_dir
 from pip.utils.build import BuildDirectory
 from pip.utils.deprecation import RemovedInPip8Warning
+from pip.wheel import WheelBuilder
 
 
 logger = logging.getLogger(__name__)
@@ -233,7 +238,6 @@ def run(self, options, args):
         with self._build_session(options) as session:
 
             finder = self._build_package_finder(options, index_urls, session)
-
             build_delete = (not (options.no_clean or options.build_dir))
             with BuildDirectory(options.build_dir,
                                 delete=build_delete) as build_dir:
@@ -262,7 +266,22 @@ def run(self, options, args):
                     return
 
                 try:
-                    requirement_set.prepare_files(finder)
+                    if options.download_dir or not wheel:
+                        # on -d don't do complex things like building
+                        # wheels, and don't try to build wheels when wheel is
+                        # not installed.
+                        requirement_set.prepare_files(finder)
+                    else:
+                        # build wheels before install.
+                        wb = WheelBuilder(
+                            requirement_set,
+                            finder,
+                            build_options=[],
+                            global_options=[],
+                        )
+                        # Ignore the result: a failed wheel will be
+                        # installed from the sdist/vcs whatever.
+                        wb.build(autobuilding=True)
 
                     if not options.download_dir:
                         requirement_set.install(

pip/req/req_install.py

@@ -247,7 +247,7 @@ def _get_link(self):
 
     def _set_link(self, link):
         # Lookup a cached wheel, if possible.
-        link = wheel.cached_wheel(self._cache_root, link)
+        link = pip.wheel.cached_wheel(self._cache_root, link)
         self._link = link
 
     link = property(_get_link, _set_link)

pip/req/req_set.py

@@ -162,7 +162,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False,
         self.build_dir = build_dir
         self.src_dir = src_dir
         # XXX: download_dir and wheel_download_dir overlap semantically and may
-        # be combinable.
+        # be combined if we're willing to have non-wheel archives present in
+        # the wheelhouse output by 'pip wheel'.
         self.download_dir = download_dir
         self.upgrade = upgrade
         self.ignore_installed = ignore_installed

pip/wheel.py

@@ -5,6 +5,7 @@
 
 import compileall
 import csv
+import errno
 import functools
 import hashlib
 import logging
@@ -13,18 +14,22 @@
 import shutil
 import stat
 import sys
+import tempfile
 import warnings
 
 from base64 import urlsafe_b64encode
 from email.parser import Parser
 
 from pip._vendor.six import StringIO
 
+import pip
+from pip.download import path_to_url, unpack_url
 from pip.exceptions import InvalidWheelFilename, UnsupportedWheel
-from pip.locations import distutils_scheme
+from pip.locations import distutils_scheme, PIP_DELETE_MARKER_FILENAME
 from pip import pep425tags
 from pip.utils import (
-    call_subprocess, ensure_dir, make_path_relative, captured_stdout)
+    call_subprocess, ensure_dir, make_path_relative, captured_stdout,
+    rmtree)
 from pip.utils.logging import indent_log
 from pip._vendor.distlib.scripts import ScriptMaker
 from pip._vendor import pkg_resources
@@ -56,14 +61,17 @@ def _cache_for_filename(cache_dir, sdistfilename):
 def cached_wheel(cache_dir, link):
     if not cache_dir:
         return link
+    if not link:
+        return link
     if link.is_wheel:
         return link
     root = _cache_for_filename(cache_dir, link.filename)
     try:
         candidates = os.listdir(root)
     except OSError as e:
-        if e.errno == errno.EEXIST:
+        if e.errno == errno.ENOENT:
             return link
+        raise
     for candidate in candidates:
         try:
             wheel = Wheel(candidate)
@@ -579,16 +587,35 @@ class WheelBuilder(object):
     """Build wheels from a RequirementSet."""
 
     def __init__(self, requirement_set, finder, build_options=None,
-                 global_options=None):
+                 global_options=None, cache_root=None):
         self.requirement_set = requirement_set
         self.finder = finder
-        self.wheel_dir = requirement_set.wheel_download_dir
+        self._cache_root = requirement_set._cache_root
+        self._wheel_dir = requirement_set.wheel_download_dir
         self.build_options = build_options or []
         self.global_options = global_options or []
 
-    def _build_one(self, req):
-        """Build one wheel."""
+    def _build_one(self, req, output_dir):
+        """Build one wheel.
 
+        :return: The filename of the built wheel, or None if the build failed.
+        """
+        tempd = tempfile.mkdtemp('pip-wheel-')
+        try:
+            if self.__build_one(req, tempd):
+                try:
+                    wheel_name = os.listdir(tempd)[0]
+                    wheel_path = os.path.join(output_dir, wheel_name)
+                    os.rename(os.path.join(tempd, wheel_name), wheel_path)
+                    logger.info('Stored in directory: %s', output_dir)
+                    return wheel_path
+                except:
+                    return None
+            return None
+        finally:
+            rmtree(tempd)
+
+    def __build_one(self, req, tempd):
         base_args = [
             sys.executable, '-c',
             "import setuptools;__file__=%r;"
@@ -597,8 +624,8 @@ def _build_one(self, req):
         ] + list(self.global_options)
 
         logger.info('Running setup.py bdist_wheel for %s', req.name)
-        logger.info('Destination directory: %s', self.wheel_dir)
-        wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] \
+        logger.debug('Destination directory: %s', tempd)
+        wheel_args = base_args + ['bdist_wheel', '-d', tempd] \
             + self.build_options
         try:
             call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False)
@@ -607,24 +634,31 @@ def _build_one(self, req):
             logger.error('Failed building wheel for %s', req.name)
             return False
 
-    def build(self):
-        """Build wheels."""
+    def build(self, autobuilding=False):
+        """Build wheels.
 
-        # unpack and constructs req set
+        :param unpack: If True, replace the sdist we built from the with the
+            newly built wheel, in preparation for installation.
+        :return: True if all the wheels built correctly.
+        """
+        assert self._wheel_dir or (autobuilding and self._cache_root)
+        # unpack sdists and constructs req set
         self.requirement_set.prepare_files(self.finder)
 
         reqset = self.requirement_set.requirements.values()
 
         buildset = []
         for req in reqset:
             if req.is_wheel:
-                logger.info(
-                    'Skipping %s, due to already being wheel.', req.name,
-                )
+                if not autobuilding:
+                    logger.info(
+                        'Skipping %s, due to already being wheel.', req.name)
             elif req.editable:
                 logger.info(
-                    'Skipping %s, due to being editable', req.name,
-                )
+                    'Skipping bdist_wheel for %s, due to being editable',
+                    req.name)
+            elif autobuilding and not req.source_dir:
+                pass
             else:
                 buildset.append(req)
 
@@ -639,8 +673,38 @@ def build(self):
         with indent_log():
             build_success, build_failure = [], []
             for req in buildset:
-                if self._build_one(req):
+                if autobuilding:
+                    output_dir = _cache_for_filename(
+                        self._cache_root, req.link.filename)
+                    ensure_dir(output_dir)
+                else:
+                    output_dir = self._wheel_dir
+                wheel_file = self._build_one(req, output_dir)
+                if wheel_file:
                     build_success.append(req)
+                    if autobuilding:
+                        # XXX: This is mildly duplicative with prepare_files,
+                        # but not close enough to pull out to a single common
+                        # method.
+                        # The code below assumes temporary source dirs -
+                        # prevent it doing bad things.
+                        if req.source_dir and not os.path.exists(os.path.join(
+                                req.source_dir, PIP_DELETE_MARKER_FILENAME)):
+                            raise Exception("bad source dir - missing marker")
+                        # Delete the source we built the wheel from
+                        req.remove_temporary_source()
+                        # set the build directory again - name is known from
+                        # the work prepare_files did.
+                        req.source_dir = req.build_location(
+                            self.requirement_set.build_dir)
+                        # Update the link for this.
+                        req.link = pip.index.Link(
+                            path_to_url(wheel_file), trusted=True)
+                        assert req.link.is_wheel
+                        # extract the wheel into the dir
+                        unpack_url(
+                            req.link, req.source_dir, None, False,
+                            session=self.requirement_set.session)
                 else:
                     build_failure.append(req)
 

tests/conftest.py

@@ -4,6 +4,8 @@
 import py
 import pytest
 
+from pip.utils import appdirs
+
 from tests.lib import SRC_DIR, TestData
 from tests.lib.path import Path
 from tests.lib.scripttest import PipTestEnvironment
@@ -119,7 +121,7 @@ def isolate(tmpdir):
 
 
 @pytest.fixture
-def virtualenv(tmpdir, monkeypatch):
+def virtualenv(tmpdir, monkeypatch, isolate):
     """
     Return a virtual environment which is unique to each test function
     invocation created inside of a sub directory of the test function's
@@ -148,6 +150,9 @@ def virtualenv(tmpdir, monkeypatch):
         pip_source_dir=pip_src,
     )
 
+    # Clean out our cache: creating the venv injects wheels into it.
+    shutil.rmtree(appdirs.user_cache_dir("pip"))
+
     # Undo our monkeypatching of shutil
     monkeypatch.undo()
 

tests/data/packages/README.txt

@@ -104,4 +104,7 @@ requires_simple_extra-0.1-py2.py3-none-any.whl
 ----------------------------------------------
 requires_simple_extra[extra] requires simple==1.0
 
-
+requires_wheelbroken_upper
+--------------------------
+Requires wheelbroken and upper - used for testing implicit wheel building
+during install.

tests/data/packages/requires_wheelbroken_upper/setup.py

@@ -0,0 +1,5 @@
+import setuptools
+setuptools.setup(
+    name="requires_wheelbroken_upper",
+    version="0",
+    install_requires=['wheelbroken', 'upper'])