Merge pull request #2618 from rbtcollins/issue-2140

Issue #2140: Build wheels during pip install

CHANGES.txt

@@ -22,6 +22,9 @@
 * Support ``--install-option`` and ``--global-option`` per requirement in
   requirement files (:pull:`2537`)
 
+* Build Wheels prior to installing from sdist, caching them in the pip cache
+  directory to speed up subsequent installs. (:pull:`2618`)
+
 **6.1.1 (2015-04-07)**
 
 * No longer ignore dependencies which have been added to the standard library,

docs/reference/pip_install.rst

@@ -16,6 +16,15 @@ Description
 .. pip-command-description:: install
 
 
+Overview
+++++++++
+
+Pip install has several stages:
+
+1. Resolve dependencies. What will be installed is determined here.
+2. Build wheels. All the dependencies that can be are built into wheels.
+3. Install the packages (and uninstall anything being upgraded/replaced).
+
 Installation Order
 ++++++++++++++++++
 
@@ -388,6 +397,26 @@ Windows
   :file:`<CSIDL_LOCAL_APPDATA>\\pip\\Cache`
 
 
+Wheel cache
+***********
+
+Pip will read from the subdirectory ``wheels`` within the pip cache dir and use
+any packages found there. This is disabled via the same ``no-cache-dir`` option
+that disables the HTTP cache. The internal structure of that cache is not part
+of the Pip API. As of 7.0 pip uses a subdirectory per sdist that wheels were
+built from, and wheels within that subdirectory.
+
+Pip attempts to choose the best wheels from those built in preference to
+building a new wheel. Note that this means when a package has both optional
+C extensions and builds `py` tagged wheels when the C extension can't be built
+that pip will not attempt to build a better wheel for Python's that would have
+supported it, once any generic wheel is built. To correct this, make sure that
+the wheel's are built with Python specific tags - e.g. pp on Pypy.
+
+When no wheels are found for an sdist, pip will attempt to build a wheel
+automatically and insert it into the wheel cache.
+
+
 Hash Verification
 +++++++++++++++++
 
@@ -482,6 +511,10 @@ implement the following command::
 This should implement the complete process of installing the package in
 "editable" mode.
 
+All packages will be attempted to built into wheels::
+
+    setup.py bdist_wheel -d XXX
+
 One further ``setup.py`` command is invoked by ``pip install``::
 
     setup.py clean

pip/basecommand.py

@@ -289,6 +289,7 @@ def populate_requirement_set(requirement_set, args, options, finder,
             requirement_set.add_requirement(
                 InstallRequirement.from_line(
                     name, None, isolated=options.isolated_mode,
+                    cache_root=options.cache_dir
                 )
             )
 
@@ -298,6 +299,7 @@ def populate_requirement_set(requirement_set, args, options, finder,
                     name,
                     default_vcs=options.default_vcs,
                     isolated=options.isolated_mode,
+                    cache_root=options.cache_dir
                 )
             )
 

pip/commands/freeze.py

@@ -60,7 +60,8 @@ def run(self, options, args):
             local_only=options.local,
             user_only=options.user,
             skip_regex=options.skip_requirements_regex,
-            isolated=options.isolated_mode)
+            isolated=options.isolated_mode,
+            cache_root=options.cache_dir)
 
         for line in freeze(**freeze_kwargs):
             sys.stdout.write(line + '\n')

pip/commands/install.py

@@ -6,10 +6,14 @@
 import tempfile
 import shutil
 import warnings
+try:
+    import wheel
+except ImportError:
+    wheel = None
 
 from pip.req import RequirementSet
-from pip.locations import virtualenv_no_global, distutils_scheme
 from pip.basecommand import RequirementCommand
+from pip.locations import virtualenv_no_global, distutils_scheme
 from pip.index import PackageFinder
 from pip.exceptions import (
     InstallationError, CommandError, PreviousBuildDirError,
@@ -18,6 +22,7 @@
 from pip.utils import ensure_dir
 from pip.utils.build import BuildDirectory
 from pip.utils.deprecation import RemovedInPip8Warning
+from pip.wheel import WheelBuilder
 
 
 logger = logging.getLogger(__name__)
@@ -233,12 +238,12 @@ def run(self, options, args):
         with self._build_session(options) as session:
 
             finder = self._build_package_finder(options, index_urls, session)
-
             build_delete = (not (options.no_clean or options.build_dir))
             with BuildDirectory(options.build_dir,
                                 delete=build_delete) as build_dir:
                 requirement_set = RequirementSet(
                     build_dir=build_dir,
+                    cache_root=options.cache_dir,
                     src_dir=options.src_dir,
                     download_dir=options.download_dir,
                     upgrade=options.upgrade,
@@ -261,7 +266,22 @@ def run(self, options, args):
                     return
 
                 try:
-                    requirement_set.prepare_files(finder)
+                    if options.download_dir or not wheel:
+                        # on -d don't do complex things like building
+                        # wheels, and don't try to build wheels when wheel is
+                        # not installed.
+                        requirement_set.prepare_files(finder)
+                    else:
+                        # build wheels before install.
+                        wb = WheelBuilder(
+                            requirement_set,
+                            finder,
+                            build_options=[],
+                            global_options=[],
+                        )
+                        # Ignore the result: a failed wheel will be
+                        # installed from the sdist/vcs whatever.
+                        wb.build(autobuilding=True)
 
                     if not options.download_dir:
                         requirement_set.install(

pip/commands/list.py

@@ -131,6 +131,7 @@ def find_packages_latest_versions(self, options):
             for dist in installed_packages:
                 req = InstallRequirement.from_line(
                     dist.key, None, isolated=options.isolated_mode,
+                    cache_root=options.cache_dir,
                 )
                 typ = 'unknown'
                 try:

pip/commands/uninstall.py

@@ -45,6 +45,7 @@ def run(self, options, args):
 
             requirement_set = RequirementSet(
                 build_dir=None,
+                cache_root=options.cache_dir,
                 src_dir=None,
                 download_dir=None,
                 isolated=options.isolated_mode,
@@ -54,13 +55,15 @@ def run(self, options, args):
                 requirement_set.add_requirement(
                     InstallRequirement.from_line(
                         name, isolated=options.isolated_mode,
+                        cache_root=options.cache_dir,
                     )
                 )
             for filename in options.requirements:
                 for req in parse_requirements(
                         filename,
                         options=options,
-                        session=session):
+                        session=session,
+                        cache_root=options.cache_dir):
                     requirement_set.add_requirement(req)
             if not requirement_set.has_requirements:
                 raise InstallationError(

pip/commands/wheel.py

@@ -159,6 +159,7 @@ def run(self, options, args):
                                 delete=build_delete) as build_dir:
                 requirement_set = RequirementSet(
                     build_dir=build_dir,
+                    cache_root=options.cache_dir,
                     src_dir=options.src_dir,
                     download_dir=None,
                     ignore_dependencies=options.ignore_dependencies,

pip/index.py

@@ -648,7 +648,6 @@ def _get_pages(self, locations, project_name):
 
                 all_locations.append(link)
 
-    _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)
     _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$')
 
     def _sort_links(self, links):
@@ -742,7 +741,7 @@ def _link_package_versions(self, link, search_name):
                 version = wheel.version
 
         if not version:
-            version = self._egg_info_matches(egg_info, search_name, link)
+            version = egg_info_matches(egg_info, search_name, link)
         if version is None:
             self._log_skipped_link(
                 link, 'wrong project name (not %s)' % search_name)
@@ -783,25 +782,39 @@ def _link_package_versions(self, link, search_name):
 
         return InstallationCandidate(search_name, version, link)
 
-    def _egg_info_matches(self, egg_info, search_name, link):
-        match = self._egg_info_re.search(egg_info)
-        if not match:
-            logger.debug('Could not parse version from link: %s', link)
-            return None
-        name = match.group(0).lower()
-        # To match the "safe" name that pkg_resources creates:
-        name = name.replace('_', '-')
-        # project name and version must be separated by a dash
-        look_for = search_name.lower() + "-"
-        if name.startswith(look_for):
-            return match.group(0)[len(look_for):]
-        else:
-            return None
-
     def _get_page(self, link):
         return HTMLPage.get_page(link, session=self.session)
 
 
+def egg_info_matches(
+        egg_info, search_name, link,
+        _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)):
+    """Pull the version part out of a string.
+
+    :param egg_info: The string to parse. E.g. foo-2.1
+    :param search_name: The name of the package this belongs to. None to
+        infer the name. Note that this cannot unambiguously parse strings
+        like foo-2-2 which might be foo, 2-2 or foo-2, 2.
+    :param link: The link the string came from, for logging on failure.
+    """
+    match = _egg_info_re.search(egg_info)
+    if not match:
+        logger.debug('Could not parse version from link: %s', link)
+        return None
+    if search_name is None:
+        full_match = match.group(0)
+        return full_match[full_match.index('-'):]
+    name = match.group(0).lower()
+    # To match the "safe" name that pkg_resources creates:
+    name = name.replace('_', '-')
+    # project name and version must be separated by a dash
+    look_for = search_name.lower() + "-"
+    if name.startswith(look_for):
+        return match.group(0)[len(look_for):]
+    else:
+        return None
+
+
 class HTMLPage(object):
     """Represents one page, along with its URL"""
 

pip/operations/freeze.py

@@ -21,7 +21,8 @@ def freeze(
         find_links=None, local_only=None, user_only=None, skip_regex=None,
         find_tags=False,
         default_vcs=None,
-        isolated=False):
+        isolated=False,
+        cache_root=None):
     find_links = find_links or []
     skip_match = None
 
@@ -75,11 +76,13 @@ def freeze(
                         line,
                         default_vcs=default_vcs,
                         isolated=isolated,
+                        cache_root=cache_root,
                     )
                 else:
                     line_req = InstallRequirement.from_line(
                         line,
                         isolated=isolated,
+                        cache_root=cache_root,
                     )
 
                 if not line_req.name:

pip/req/req_file.py

@@ -77,7 +77,7 @@ def parser_exit(self, msg):
 
 
 def parse_requirements(filename, finder=None, comes_from=None, options=None,
-                       session=None):
+                       session=None, cache_root=None):
     """
     Parse a requirements file and yield InstallRequirement instances.
 
@@ -87,7 +87,6 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
     :param options:    Global options.
     :param session:    Instance of pip.download.PipSession.
     """
-
     if session is None:
         raise TypeError(
             "parse_requirements() missing 1 required keyword argument: "
@@ -99,15 +98,15 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
     )
 
     parser = parse_content(
-        filename, content, finder, comes_from, options, session
+        filename, content, finder, comes_from, options, session, cache_root
     )
 
     for item in parser:
         yield item
 
 
 def parse_content(filename, content, finder=None, comes_from=None,
-                  options=None, session=None):
+                  options=None, session=None, cache_root=None):
 
     # Split, sanitize and join lines with continuations.
     content = content.splitlines()
@@ -129,8 +128,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
             comes_from = '-r %s (line %s)' % (filename, line_number)
             isolated = options.isolated_mode if options else False
             yield InstallRequirement.from_line(
-                req, comes_from, isolated=isolated, options=opts
-            )
+                req, comes_from, isolated=isolated, options=opts,
+                cache_root=cache_root)
 
         # ---------------------------------------------------------------------
         elif linetype == REQUIREMENT_EDITABLE:
@@ -139,8 +138,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
             default_vcs = options.default_vcs if options else None
             yield InstallRequirement.from_editable(
                 value, comes_from=comes_from,
-                default_vcs=default_vcs, isolated=isolated
-            )
+                default_vcs=default_vcs, isolated=isolated,
+                cache_root=cache_root)
 
         # ---------------------------------------------------------------------
         elif linetype == REQUIREMENT_FILE:
@@ -152,8 +151,8 @@ def parse_content(filename, content, finder=None, comes_from=None,
                 req_url = os.path.join(os.path.dirname(filename), value)
             # TODO: Why not use `comes_from='-r {} (line {})'` here as well?
             parser = parse_requirements(
-                req_url, finder, comes_from, options, session
-            )
+                req_url, finder, comes_from, options, session,
+                cache_root=cache_root)
             for req in parser:
                 yield req