[WIP] internal wheel cache

pip/cmdoptions.py

@@ -9,7 +9,7 @@
 """
 import copy
 from optparse import OptionGroup, SUPPRESS_HELP, Option
-from pip.locations import build_prefix, default_log_file
+from pip.locations import build_prefix, default_log_file, default_wheel_cache
 
 
 def make_option_group(group, parser):
@@ -286,6 +286,31 @@ def make(self):
           'find-links locations.'),
 )
 
+wheel_cache = OptionMaker(
+    '--wheel-cache',
+    action='store_true',
+    default=False,
+    help="Build and Cache wheels.")
+
+wheel_cache_rebuild = OptionMaker(
+    '--wheel-cache-rebuild',
+    action='store_true',
+    default=False,
+    help="Rebuild pre-existing wheels in the cache.")
+
+wheel_cache_dir = OptionMaker(
+    '--wheel-cache-dir',
+    metavar='dir',
+    default=default_wheel_cache,
+    help='Directory for cached wheels. Defaults to %default.')
+
+wheel_cache_exclude = OptionMaker(
+    '--wheel-cache-exclude',
+    action='append',
+    default=[],
+    metavar='pkg',
+    help="A Package to exclude from wheel building and caching.")
+
 download_cache = OptionMaker(
     '--download-cache',
     dest='download_cache',

pip/commands/install.py

@@ -165,6 +165,10 @@ def __init__(self, *args, **kw):
 
         cmd_opts.add_option(cmdoptions.use_wheel.make())
         cmd_opts.add_option(cmdoptions.no_use_wheel.make())
+        cmd_opts.add_option(cmdoptions.wheel_cache.make())
+        cmd_opts.add_option(cmdoptions.wheel_cache_rebuild.make())
+        cmd_opts.add_option(cmdoptions.wheel_cache_dir.make())
+        cmd_opts.add_option(cmdoptions.wheel_cache_exclude.make())
 
         cmd_opts.add_option(
             '--pre',
@@ -278,6 +282,9 @@ def run(self, options, args):
             target_dir=temp_target_dir,
             session=session,
             pycompile=options.compile,
+            wheel_cache = options.wheel_cache,
+            wheel_cache_dir = options.wheel_cache_dir,
+            wheel_cache_exclude = options.wheel_cache_exclude
         )
         for name in args:
             requirement_set.add_requirement(

pip/commands/wheel.py

@@ -9,7 +9,7 @@
 from pip.exceptions import CommandError, PreviousBuildDirError
 from pip.req import InstallRequirement, RequirementSet, parse_requirements
 from pip.util import normalize_path
-from pip.wheel import WheelBuilder
+from pip.wheel import bdist_wheel
 from pip import cmdoptions
 
 DEFAULT_WHEEL_DIR = os.path.join(normalize_path(os.curdir), 'wheelhouse')
@@ -92,6 +92,44 @@ def __init__(self, *args, **kw):
         self.parser.insert_option_group(0, index_opts)
         self.parser.insert_option_group(0, cmd_opts)
 
+
+    def build(self, requirement_set, finder, wheel_dir,
+              build_options, global_options):
+
+        reqset = requirement_set.requirements.values()
+
+        buildset = [req for req in reqset if not req.is_wheel]
+
+        if not buildset:
+            return
+
+        #build the wheels
+        logger.notify(
+            'Building wheels for collected packages: %s' %
+            ','.join([req.name for req in buildset])
+        )
+        logger.indent += 2
+        build_success, build_failure = [], []
+        for req in buildset:
+            if bdist_wheel(req, wheel_dir, build_options, global_options):
+                build_success.append(req)
+            else:
+                build_failure.append(req)
+        logger.indent -= 2
+
+        #notify sucess/failure
+        if build_success:
+            logger.notify(
+                'Successfully built %s' %
+                ' '.join([req.name for req in build_success])
+            )
+        if build_failure:
+            logger.notify(
+                'Failed to build %s' %
+                ' '.join([req.name for req in build_failure])
+            )
+
+
     def run(self, options, args):
 
         # confirm requirements
@@ -197,15 +235,16 @@ def run(self, options, args):
             return
 
         try:
+            #unpack and constructs req set
+            requirement_set.prepare_files(finder)
             #build wheels
-            wb = WheelBuilder(
+            self.build(
                 requirement_set,
                 finder,
                 options.wheel_dir,
-                build_options=options.build_options or [],
-                global_options=options.global_options or [],
+                options.build_options or [],
+                options.global_options or [],
             )
-            wb.build()
         except PreviousBuildDirError:
             options.no_clean = True
             raise

pip/download.py

@@ -17,6 +17,7 @@
 from pip.util import (splitext, rmtree, format_size, display_path,
                       backup_dir, ask_path_exists, unpack_file,
                       create_download_cache_folder, cache_download)
+from pip.locations import write_delete_marker_file
 from pip.vcs import vcs
 from pip.log import logger
 from pip._vendor import requests, six
@@ -498,6 +499,39 @@ def _copy_file(filename, location, content_type, link):
         logger.notify('Saved %s' % display_path(download_location))
 
 
+def unpack_url(link, location, download_dir=None, only_download=False,
+               download_cache=None, session=None):
+
+    if session is None:
+        session = PipSession()
+
+    # non-editable vcs urls
+    if is_vcs_url(link):
+        if only_download:
+            loc = download_dir
+        else:
+            loc = location
+        unpack_vcs_link(link, loc, only_download)
+
+    # file urls
+    elif is_file_url(link):
+        unpack_file_url(link, location, download_dir)
+        if only_download:
+            write_delete_marker_file(location)
+
+    # http urls
+    else:
+        unpack_http_url(
+            link,
+            location,
+            download_cache,
+            download_dir,
+            session,
+        )
+        if only_download:
+            write_delete_marker_file(location)
+
+
 def unpack_http_url(link, location, download_cache, download_dir=None,
                     session=None):
     if session is None:

pip/locations.py

@@ -157,7 +157,9 @@ def _get_build_prefix():
     if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/':
         bin_py = '/usr/local/bin'
         default_log_file = os.path.join(user_dir, 'Library/Logs/pip.log')
-
+default_wheel_cache = os.path.join(default_storage_dir, 'wheel_cache')
+if not os.path.isdir(default_wheel_cache):
+    os.makedirs(default_wheel_cache)
 
 def distutils_scheme(dist_name, user=False, home=None, root=None):
     """

pip/req/req_install.py

@@ -1,3 +1,4 @@
+import glob
 import os
 import re
 import shutil
@@ -12,7 +13,8 @@
 from pip.backwardcompat import (
     urllib, ConfigParser, string_types, get_python_version,
 )
-from pip.download import is_url, url_to_path, path_to_url, is_archive_file
+from pip.download import (is_url, url_to_path, path_to_url, is_archive_file,
+                          unpack_url)
 from pip.exceptions import (
     InstallationError, UninstallationError, UnsupportedWheel,
 )
@@ -28,7 +30,7 @@
 )
 from pip.req.req_uninstall import UninstallPathSet
 from pip.vcs import vcs
-from pip.wheel import move_wheel_files, Wheel, wheel_ext
+from pip.wheel import move_wheel_files, Wheel, wheel_ext, bdist_wheel
 
 
 class InstallRequirement(object):
@@ -705,6 +707,33 @@ def _clean_zip_name(self, name, prefix):
         name = name.replace(os.path.sep, '/')
         return name
 
+    def build(self, wheel_cache_dir, build_options, global_options):
+        """Build a wheel from sdist src, and place it in the cache"""
+
+        if self.is_wheel:
+            return
+
+        # build into tmp dir (so we know what file we built). knowing the
+        # filename in advance is semi-hard see
+        # https://github.com/pypa/pip/issues/855#issuecomment-350447813
+        wheel_tmp_dir = tempfile.mkdtemp()
+        bdist_wheel(self, wheel_tmp_dir, build_options, global_options)
+        wheel_path = glob.glob(os.path.join(wheel_tmp_dir, '*.whl'))[0]
+        wheel_filename = os.path.basename(wheel_path)
+
+        # move the wheel into the cache
+        dest_path = os.path.join(wheel_cache_dir, wheel_filename)
+        shutil.move(wheel_path, dest_path)
+
+        # unpack the wheel into a new src dir (and set self.source_dir)
+        # i.e. we want to install from the wheel we just built, not the sdist src
+        cache_url = path_to_url(dest_path)
+        self.source_dir = tempfile.mkdtemp()
+        self.url = cache_url
+        unpack_url(Link(cache_url), self.source_dir)
+
+        # TODO: deal with tmp dir cleanup (pip delete files)
+
     def install(self, install_options, global_options=(), root=None):
         if self.editable:
             self.install_editable(install_options, global_options)