Issue pypa#2140: Build wheels automatically

Building wheels before installing elminates a cause of broken environments -
where install fails after we've already installed one or more packages.

If a package fails to wheel, we run setup.py install as normally.

CHANGES.txt

@@ -19,6 +19,8 @@
 * Ignores bz2 archives if Python wasn't compiled with bz2 support.
   Fixes :issue:`497`
 
+* Build Wheels prior to installing from sdist, caching them in the pip cache
+  directory to speed up subsequent installs. (:pull:`2618`)
 
 **6.1.1 (2015-04-07)**
 

docs/reference/pip_install.rst

@@ -16,6 +16,15 @@ Description
 .. pip-command-description:: install
 
 
+Overview
+++++++++
+
+Pip install has several stages:
+
+1. Resolve dependencies. What will be installed is determined here.
+2. Build wheels. All the dependencies that can be are built into wheels.
+3. Install the packages (and uninstall anything being upgraded/replaced).
+
 Installation Order
 ++++++++++++++++++
 
@@ -452,6 +461,10 @@ implement the following command::
 This should implement the complete process of installing the package in
 "editable" mode.
 
+All packages will be attempted to built into wheels::
+
+    setup.py bdist_wheel -d XXX
+
 One further ``setup.py`` command is invoked by ``pip install``::
 
     setup.py clean

pip/commands/install.py

@@ -6,10 +6,15 @@
 import tempfile
 import shutil
 import warnings
+try:
+    import wheel
+except ImportError:
+    wheel = None
 
 from pip.req import RequirementSet
-from pip.locations import virtualenv_no_global, distutils_scheme
 from pip.basecommand import RequirementCommand
+from pip.locations import (
+    virtualenv_no_global, distutils_scheme, WHEEL_CACHE_DIR)
 from pip.index import PackageFinder
 from pip.exceptions import (
     InstallationError, CommandError, PreviousBuildDirError,
@@ -18,6 +23,7 @@
 from pip.utils import ensure_dir
 from pip.utils.build import BuildDirectory
 from pip.utils.deprecation import RemovedInPip8Warning
+from pip.wheel import WheelBuilder
 
 
 logger = logging.getLogger(__name__)
@@ -234,16 +240,20 @@ def run(self, options, args):
         with self._build_session(options) as session:
 
             finder = self._build_package_finder(options, index_urls, session)
-
             build_delete = (not (options.no_clean or options.build_dir))
             with BuildDirectory(options.build_dir,
                                 delete=build_delete) as build_dir:
+                if not options.cache_dir or options.download_dir:
+                    wheel_download_dir = None
+                else:
+                    wheel_download_dir = WHEEL_CACHE_DIR()
                 requirement_set = RequirementSet(
                     build_dir=build_dir,
                     src_dir=options.src_dir,
                     download_dir=options.download_dir,
                     upgrade=options.upgrade,
                     as_egg=options.as_egg,
+                    installing_wheels=wheel_download_dir is not None,
                     ignore_installed=options.ignore_installed,
                     ignore_dependencies=options.ignore_dependencies,
                     force_reinstall=options.force_reinstall,
@@ -252,6 +262,7 @@ def run(self, options, args):
                     session=session,
                     pycompile=options.compile,
                     isolated=options.isolated_mode,
+                    wheel_download_dir=wheel_download_dir,
                 )
 
                 self.populate_requirement_set(
@@ -262,7 +273,22 @@ def run(self, options, args):
                     return
 
                 try:
-                    requirement_set.prepare_files(finder)
+                    if options.download_dir or not wheel:
+                        # on -d don't do complex things like building
+                        # wheels, and don't try to build wheels when wheel is
+                        # not installed.
+                        requirement_set.prepare_files(finder)
+                    else:
+                        # build wheels before install.
+                        wb = WheelBuilder(
+                            requirement_set,
+                            finder,
+                            build_options=[],
+                            global_options=[],
+                        )
+                        # Ignore the result: a failed wheel will be
+                        # installed from the sdist/vcs whatever.
+                        wb.build(autobuilding=True)
 
                     if not options.download_dir:
                         requirement_set.install(

pip/req/req_set.py

@@ -139,7 +139,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False,
                  ignore_installed=False, as_egg=False, target_dir=None,
                  ignore_dependencies=False, force_reinstall=False,
                  use_user_site=False, session=None, pycompile=True,
-                 isolated=False, wheel_download_dir=None):
+                 isolated=False, wheel_download_dir=None,
+                 installing_wheels=False):
         """Create a RequirementSet.
 
         :param wheel_download_dir: Where still-packed .whl files should be
@@ -149,6 +150,8 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False,
         :param download_dir: Where still packed archives should be written to.
             If None they are not saved, and are deleted immediately after
             unpacking.
+        :param installing_wheels: If True, wheels will be getting installed and
+            should not be marked for pip deletion.
         """
         if session is None:
             raise TypeError(
@@ -159,10 +162,12 @@ def __init__(self, build_dir, src_dir, download_dir, upgrade=False,
         self.build_dir = build_dir
         self.src_dir = src_dir
         # XXX: download_dir and wheel_download_dir overlap semantically and may
-        # be combinable.
+        # be combined if we're willing to have non-wheel archives present in
+        # the wheelhouse output by 'pip wheel'.
         self.download_dir = download_dir
         self.upgrade = upgrade
         self.ignore_installed = ignore_installed
+        self.installing_wheels = installing_wheels
         self.force_reinstall = force_reinstall
         self.requirements = Requirements()
         # Mapping of alias: real_name
@@ -446,11 +451,13 @@ def _prepare_file(self, finder, req_to_install):
                             self.wheel_download_dir:
                         # when doing 'pip wheel`
                         download_dir = self.wheel_download_dir
+                        only_download = not self.installing_wheels
                     else:
                         download_dir = self.download_dir
+                        only_download = True
                     unpack_url(
                         req_to_install.link, req_to_install.source_dir,
-                        download_dir, True, session=self.session,
+                        download_dir, only_download, session=self.session,
                     )
                 except requests.HTTPError as exc:
                     logger.critical(

pip/wheel.py

@@ -13,18 +13,22 @@
 import shutil
 import stat
 import sys
+import tempfile
 import warnings
 
 from base64 import urlsafe_b64encode
 from email.parser import Parser
 
 from pip._vendor.six import StringIO
 
+import pip
+from pip.download import path_to_url, unpack_url
 from pip.exceptions import InvalidWheelFilename, UnsupportedWheel
-from pip.locations import distutils_scheme
+from pip.locations import distutils_scheme, PIP_DELETE_MARKER_FILENAME
 from pip import pep425tags
 from pip.utils import (
-    call_subprocess, ensure_dir, make_path_relative, captured_stdout)
+    call_subprocess, ensure_dir, make_path_relative, captured_stdout,
+    rmtree)
 from pip.utils.logging import indent_log
 from pip._vendor.distlib.scripts import ScriptMaker
 from pip._vendor import pkg_resources
@@ -550,8 +554,25 @@ def __init__(self, requirement_set, finder, build_options=None,
         self.global_options = global_options or []
 
     def _build_one(self, req):
-        """Build one wheel."""
+        """Build one wheel.
 
+        :return: The filename of the built wheel, or None if the build failed.
+        """
+        tempd = tempfile.mkdtemp('pip-wheel-')
+        try:
+            if self.__build_one(req, tempd):
+                try:
+                    wheel_name = os.listdir(tempd)[0]
+                    wheel_path = os.path.join(self.wheel_dir, wheel_name)
+                    os.rename(os.path.join(tempd, wheel_name), wheel_path)
+                    return wheel_path
+                except:
+                    return None
+            return None
+        finally:
+            rmtree(tempd)
+
+    def __build_one(self, req, tempd):
         base_args = [
             sys.executable, '-c',
             "import setuptools;__file__=%r;"
@@ -561,7 +582,7 @@ def _build_one(self, req):
 
         logger.info('Running setup.py bdist_wheel for %s', req.name)
         logger.info('Destination directory: %s', self.wheel_dir)
-        wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] \
+        wheel_args = base_args + ['bdist_wheel', '-d', tempd] \
             + self.build_options
         try:
             call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False)
@@ -570,24 +591,29 @@ def _build_one(self, req):
             logger.error('Failed building wheel for %s', req.name)
             return False
 
-    def build(self):
-        """Build wheels."""
+    def build(self, autobuilding=True):
+        """Build wheels.
 
-        # unpack and constructs req set
+        :param unpack: If True, replace the sdist we built from the with the
+            newly built wheel, in preparation for installation.
+        :return: True if all the wheels built correctly.
+        """
+        # unpack sdists and constructs req set
         self.requirement_set.prepare_files(self.finder)
 
         reqset = self.requirement_set.requirements.values()
 
         buildset = []
         for req in reqset:
-            if req.is_wheel:
+            if req.is_wheel and not autobuilding:
                 logger.info(
-                    'Skipping %s, due to already being wheel.', req.name,
-                )
+                    'Skipping %s, due to already being wheel.', req.name)
             elif req.editable:
                 logger.info(
-                    'Skipping %s, due to being editable', req.name,
-                )
+                    'Skipping bdist_wheel for %s, due to being editable',
+                    req.name)
+            elif autobuilding and not req.source_dir:
+                pass
             else:
                 buildset.append(req)
 
@@ -602,8 +628,32 @@ def build(self):
         with indent_log():
             build_success, build_failure = [], []
             for req in buildset:
-                if self._build_one(req):
+                wheel_file = self._build_one(req)
+                if wheel_file:
                     build_success.append(req)
+                    if autobuilding:
+                        # XXX: This is mildly duplicative with prepare_files,
+                        # but not close enough to pull out to a single common
+                        # method.
+                        # The code below assumes temporary source dirs -
+                        # prevent it doing bad things.
+                        if req.source_dir and not os.path.exists(os.path.join(
+                                req.source_dir, PIP_DELETE_MARKER_FILENAME)):
+                            raise Exception("bad source dir - missing marker")
+                        # Delete the source we built the wheel from
+                        req.remove_temporary_source()
+                        # set the build directory again - name is known from
+                        # the work prepare_files did.
+                        req.source_dir = req.build_location(
+                            self.requirement_set.build_dir)
+                        # Update the link for this.
+                        req.link = pip.index.Link(
+                            path_to_url(wheel_file), trusted=True)
+                        assert req.link.is_wheel
+                        # extract the wheel into the dir
+                        unpack_url(
+                            req.link, req.source_dir, None, False,
+                            session=self.requirement_set.session)
                 else:
                     build_failure.append(req)
 

tests/conftest.py

@@ -4,6 +4,8 @@
 import py
 import pytest
 
+from pip import locations
+
 from tests.lib import SRC_DIR, TestData
 from tests.lib.path import Path
 from tests.lib.scripttest import PipTestEnvironment
@@ -119,7 +121,7 @@ def isolate(tmpdir):
 
 
 @pytest.fixture
-def virtualenv(tmpdir, monkeypatch):
+def virtualenv(tmpdir, monkeypatch, isolate):
     """
     Return a virtual environment which is unique to each test function
     invocation created inside of a sub directory of the test function's
@@ -148,6 +150,9 @@ def virtualenv(tmpdir, monkeypatch):
         pip_source_dir=pip_src,
     )
 
+    # Clean out our cache.
+    shutil.rmtree(locations.WHEEL_CACHE_DIR())
+
     # Undo our monkeypatching of shutil
     monkeypatch.undo()
 

tests/data/packages/README.txt

@@ -104,4 +104,7 @@ requires_simple_extra-0.1-py2.py3-none-any.whl
 ----------------------------------------------
 requires_simple_extra[extra] requires simple==1.0
 
-
+requires_wheelbroken_upper
+--------------------------
+Requires wheelbroken and upper - used for testing implicit wheel building
+during install.

tests/data/packages/requires_wheelbroken_upper/setup.py

@@ -0,0 +1,5 @@
+import setuptools
+setuptools.setup(
+    name="requires_wheelbroken_upper",
+    version="0",
+    install_requires=['wheelbroken', 'upper'])

tests/functional/test_install.py

@@ -7,6 +7,7 @@
 
 import pytest
 
+from pip.locations import WHEEL_CACHE_DIR
 from pip.utils import rmtree
 from tests.lib import (pyversion, pyversion_tuple,
                        _create_test_package, _create_svn_repo, path_to_url)
@@ -144,14 +145,24 @@ def test_install_dev_version_from_pypi(script):
     )
 
 
-def test_install_editable_from_git(script, tmpdir):
+def _test_install_editable_from_git(script, tmpdir, wheel):
     """Test cloning from Git."""
+    if wheel:
+        script.pip('install', 'wheel')
     pkg_path = _create_test_package(script, name='testpackage', vcs='git')
     args = ['install', '-e', 'git+%s#egg=testpackage' % path_to_url(pkg_path)]
     result = script.pip(*args, **{"expect_error": True})
     result.assert_installed('testpackage', with_files=['.git'])
 
 
+def test_install_editable_from_git(script, tmpdir):
+    _test_install_editable_from_git(script, tmpdir, False)
+
+
+def test_install_editable_from_git_autobuild_wheel(script, tmpdir):
+    _test_install_editable_from_git(script, tmpdir, True)
+
+
 def test_install_editable_from_hg(script, tmpdir):
     """Test cloning from Mercurial."""
     pkg_path = _create_test_package(script, name='testpackage', vcs='hg')
@@ -669,3 +680,26 @@ def test_install_wheel_broken(script, data):
     res = script.pip(
         'install', '--no-index', '-f', data.find_links, 'wheelbroken')
     assert "Successfully installed wheelbroken-0.1" in str(res), str(res)
+
+
+def test_install_builds_wheels(script, data):
+    # NB This incidentally tests a local tree + tarball inputs
+    # see test_install_editable_from_git_autobuild_wheel for editable
+    # vcs coverage.
+    script.pip('install', 'wheel')
+    to_install = data.packages.join('requires_wheelbroken_upper')
+    res = script.pip(
+        'install', '--no-index', '-f', data.find_links,
+        to_install)
+    expected = ("Successfully installed requires-wheelbroken-upper-0"
+                " upper-2.0 wheelbroken-0.1")
+    # Must have installed it all
+    assert expected in str(res), str(res)
+    wheels = os.listdir(WHEEL_CACHE_DIR())
+    # and built wheels into the cache
+    assert wheels != [], str(res)
+    # and installed from the wheels
+    assert "Running setup.py install for upper" not in str(res), str(res)
+    assert "Running setup.py install for requires" not in str(res), str(res)
+    # wheelbroken has to run install
+    assert "Running setup.py install for wheelb" in str(res), str(res)