From acd4aae08c9f6f640cbcba77f4b4e5a16c018eb1 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 1 Apr 2015 14:39:45 +1300 Subject: [PATCH] Issue #2140: Build wheels automatically Building wheels before installing elminates a cause of broken environments - where install fails after we've already installed one or more packages. If a package fails to wheel, we run setup.py install as normally. --- docs/reference/pip_install.rst | 13 ++++ pip/commands/install.py | 26 ++++++- pip/wheel.py | 70 +++++++++++++++---- tests/data/packages/README.txt | 5 +- .../requires_wheelbroken_upper/__init__.py | 0 .../requires_wheelbroken_upper/setup.py | 5 ++ tests/functional/test_install.py | 21 ++++++ 7 files changed, 124 insertions(+), 16 deletions(-) create mode 100644 tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py create mode 100644 tests/data/packages/requires_wheelbroken_upper/setup.py diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 82fd0d5d1e1..b6f3bbac698 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -18,6 +18,15 @@ Description .. _`Requirements File Format`: +Overview +++++++++ + +Pip install has several stages: + +1. Resolve dependencies. What will be installed is determined here. +2. Build wheels. All the dependencies that can be are built into wheels. +3. Install the packages (and uninstall anything being upgraded/replaced). + Installation Order ++++++++++++++++++ @@ -421,6 +430,10 @@ implement the following command:: This should implement the complete process of installing the package in "editable" mode. +All packages will be attempted to built into wheels:: + + setup.py bdist_wheel -d XXX + One further ``setup.py`` command is invoked by ``pip install``:: setup.py clean diff --git a/pip/commands/install.py b/pip/commands/install.py index 458772c4556..5ff03442200 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -9,7 +9,8 @@ from pip.commands._utils import populate_requirement_set, BadOptions from pip.req import RequirementSet -from pip.locations import build_prefix, virtualenv_no_global, distutils_scheme +from pip.locations import ( + build_prefix, virtualenv_no_global, distutils_scheme, WHEEL_CACHE_DIR) from pip.basecommand import Command from pip.index import PackageFinder from pip.exceptions import ( @@ -19,6 +20,7 @@ from pip.utils import ensure_dir from pip.utils.build import BuildDirectory from pip.utils.deprecation import RemovedInPip7Warning, RemovedInPip8Warning +from pip.wheel import WheelBuilder logger = logging.getLogger(__name__) @@ -285,10 +287,13 @@ def run(self, options, args): with self._build_session(options) as session: finder = self._build_package_finder(options, index_urls, session) - build_delete = (not (options.no_clean or options.build_dir)) with BuildDirectory(options.build_dir, delete=build_delete) as build_dir: + if not options.cache_dir: + wheel_download_dir = None + else: + wheel_download_dir = WHEEL_CACHE_DIR() requirement_set = RequirementSet( build_dir=build_dir, src_dir=options.src_dir, @@ -303,6 +308,7 @@ def run(self, options, args): session=session, pycompile=options.compile, isolated=options.isolated_mode, + wheel_download_dir=wheel_download_dir, ) try: @@ -314,7 +320,21 @@ def run(self, options, args): try: if not options.no_download: - requirement_set.prepare_files(finder) + if options.download_dir: + # on -d don't do complex things like building + # wheels. + requirement_set.prepare_files(finder) + else: + # build wheels before install. + wb = WheelBuilder( + requirement_set, + finder, + build_options=[], + global_options=[], + ) + # Ignore the result: a failed wheel will be + # installed from the sdist/vcs whatever. + wb.build(autobuilding=True) else: # This is the only call site of locate_files. Nuke with # fire. diff --git a/pip/wheel.py b/pip/wheel.py index d689903fc57..ce62f61d182 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -13,6 +13,7 @@ import shutil import stat import sys +import tempfile import warnings from base64 import urlsafe_b64encode @@ -20,11 +21,14 @@ from pip._vendor.six import StringIO +import pip +from pip.download import path_to_url, unpack_url from pip.exceptions import InvalidWheelFilename, UnsupportedWheel from pip.locations import distutils_scheme from pip import pep425tags from pip.utils import ( - call_subprocess, ensure_dir, make_path_relative, captured_stdout) + call_subprocess, ensure_dir, make_path_relative, captured_stdout, + rmtree) from pip.utils.logging import indent_log from pip._vendor.distlib.scripts import ScriptMaker from pip._vendor import pkg_resources @@ -550,8 +554,25 @@ def __init__(self, requirement_set, finder, build_options=None, self.global_options = global_options or [] def _build_one(self, req): - """Build one wheel.""" + """Build one wheel. + :return: The filename of the built wheel, or None if the build failed. + """ + tempd = tempfile.mkdtemp('pip-wheel-') + try: + if self.__build_one(req, tempd): + try: + wheel_name = os.listdir(tempd)[0] + wheel_path = os.path.join(self.wheel_dir, wheel_name) + os.rename(os.path.join(tempd, wheel_name), wheel_path) + return wheel_path + except: + return None + return None + finally: + rmtree(tempd) + + def __build_one(self, req, tempd): base_args = [ sys.executable, '-c', "import setuptools;__file__=%r;" @@ -561,7 +582,7 @@ def _build_one(self, req): logger.info('Running setup.py bdist_wheel for %s', req.name) logger.info('Destination directory: %s', self.wheel_dir) - wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] \ + wheel_args = base_args + ['bdist_wheel', '-d', tempd] \ + self.build_options try: call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False) @@ -570,10 +591,14 @@ def _build_one(self, req): logger.error('Failed building wheel for %s', req.name) return False - def build(self): - """Build wheels.""" + def build(self, autobuilding=True): + """Build wheels. - # unpack and constructs req set + :param unpack: If True, replace the sdist we built from the with the + newly built wheel, in preparation for installation. + :return: True if all the wheels built correctly. + """ + # unpack sdists and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() @@ -581,13 +606,15 @@ def build(self): buildset = [] for req in reqset: if req.is_wheel: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, - ) + if autobuilding: + logger.info( + 'Skipping %s, due to already being wheel.', req.name) elif req.editable: logger.info( - 'Skipping %s, due to being editable', req.name, - ) + 'Skipping bdist_wheel for %s, due to being editable', + req.name) + elif autobuilding and not req.source_dir: + pass else: buildset.append(req) @@ -602,8 +629,27 @@ def build(self): with indent_log(): build_success, build_failure = [], [] for req in buildset: - if self._build_one(req): + wheel_file = self._build_one(req) + if wheel_file: build_success.append(req) + if autobuilding: + # XXX: This is mildly duplicative with prepare_files, + # but not close enough to pull out to a single common + # method. + # Delete the source we build the wheel from + req.remove_temporary_source() + # set the build directory again - name is known from + # the work prepare_files did. + req.source_dir = req.build_location( + self.requirement_set.build_dir) + # Update the link for this. + req.link = pip.index.Link( + path_to_url(wheel_file), trusted=True) + assert req.link.is_wheel + # extract the wheel into the dir + unpack_url( + req.link, req.source_dir, None, False, + session=self.requirement_set.session) else: build_failure.append(req) diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt index 9e89e911439..7fd5cd3a7fc 100644 --- a/tests/data/packages/README.txt +++ b/tests/data/packages/README.txt @@ -104,4 +104,7 @@ requires_simple_extra-0.1-py2.py3-none-any.whl ---------------------------------------------- requires_simple_extra[extra] requires simple==1.0 - +requires_wheelbroken_upper +-------------------------- +Requires wheelbroken and upper - used for testing implicit wheel building +during install. diff --git a/tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py b/tests/data/packages/requires_wheelbroken_upper/requires_wheelbroken_upper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/packages/requires_wheelbroken_upper/setup.py b/tests/data/packages/requires_wheelbroken_upper/setup.py new file mode 100644 index 00000000000..255cf2219eb --- /dev/null +++ b/tests/data/packages/requires_wheelbroken_upper/setup.py @@ -0,0 +1,5 @@ +import setuptools +setuptools.setup( + name="requires_wheelbroken_upper", + version="0", + install_requires=['wheelbroken', 'upper']) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 6099f96569f..f0a20dfebef 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -7,6 +7,7 @@ import pytest +from pip.locations import WHEEL_CACHE_DIR from pip.utils import rmtree from tests.lib import (pyversion, pyversion_tuple, _create_test_package, _create_svn_repo, path_to_url) @@ -748,3 +749,23 @@ def test_install_wheel_broken(script, data): res = script.pip( 'install', '--no-index', '-f', data.find_links, 'wheelbroken') assert "Successfully installed wheelbroken-0.1" in str(res), str(res) + + +def test_install_builds_wheels(script, data): + script.pip('install', 'wheel') + to_install = data.packages.join('requires_wheelbroken_upper') + res = script.pip( + 'install', '--no-index', '-f', data.find_links, + to_install) + expected = ("Successfully installed requires-wheelbroken-upper-0" + " upper-2.0 wheelbroken-0.1") + # Must have installed it all + assert expected in str(res), str(res) + wheels = os.listdir(WHEEL_CACHE_DIR()) + # and built wheels into the cache + assert wheels != [], str(res) + # and installed from the wheels + assert "Running setup.py install for upper" not in str(res), str(res) + assert "Running setup.py install for requires" not in str(res), str(res) + # wheelbroken has to run install + assert "Running setup.py install for wheelb" in str(res), str(res)