diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index 703f731b0c5..2015f09d338 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -9,7 +9,7 @@ """ import copy from optparse import OptionGroup, SUPPRESS_HELP, Option -from pip.locations import build_prefix, default_log_file +from pip.locations import build_prefix, default_log_file, default_wheel_cache def make_option_group(group, parser): @@ -286,6 +286,31 @@ def make(self): 'find-links locations.'), ) +wheel_cache = OptionMaker( + '--wheel-cache', + action='store_true', + default=False, + help="Build and Cache wheels.") + +wheel_cache_rebuild = OptionMaker( + '--wheel-cache-rebuild', + action='store_true', + default=False, + help="Rebuild pre-existing wheels in the cache.") + +wheel_cache_dir = OptionMaker( + '--wheel-cache-dir', + metavar='dir', + default=default_wheel_cache, + help='Directory for cached wheels. Defaults to %default.') + +wheel_cache_exclude = OptionMaker( + '--wheel-cache-exclude', + action='append', + default=[], + metavar='pkg', + help="A Package to exclude from wheel building and caching.") + download_cache = OptionMaker( '--download-cache', dest='download_cache', diff --git a/pip/commands/install.py b/pip/commands/install.py index 803fa262ade..461f3d9c63b 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -165,6 +165,10 @@ def __init__(self, *args, **kw): cmd_opts.add_option(cmdoptions.use_wheel.make()) cmd_opts.add_option(cmdoptions.no_use_wheel.make()) + cmd_opts.add_option(cmdoptions.wheel_cache.make()) + cmd_opts.add_option(cmdoptions.wheel_cache_rebuild.make()) + cmd_opts.add_option(cmdoptions.wheel_cache_dir.make()) + cmd_opts.add_option(cmdoptions.wheel_cache_exclude.make()) cmd_opts.add_option( '--pre', @@ -278,6 +282,9 @@ def run(self, options, args): target_dir=temp_target_dir, session=session, pycompile=options.compile, + wheel_cache = options.wheel_cache, + wheel_cache_dir = options.wheel_cache_dir, + wheel_cache_exclude = options.wheel_cache_exclude ) for name in args: requirement_set.add_requirement( diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index 757891a5b52..d6fc1a2e551 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -9,7 +9,7 @@ from pip.exceptions import CommandError, PreviousBuildDirError from pip.req import InstallRequirement, RequirementSet, parse_requirements from pip.util import normalize_path -from pip.wheel import WheelBuilder +from pip.wheel import bdist_wheel from pip import cmdoptions DEFAULT_WHEEL_DIR = os.path.join(normalize_path(os.curdir), 'wheelhouse') @@ -92,6 +92,44 @@ def __init__(self, *args, **kw): self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, cmd_opts) + + def build(self, requirement_set, finder, wheel_dir, + build_options, global_options): + + reqset = requirement_set.requirements.values() + + buildset = [req for req in reqset if not req.is_wheel] + + if not buildset: + return + + #build the wheels + logger.notify( + 'Building wheels for collected packages: %s' % + ','.join([req.name for req in buildset]) + ) + logger.indent += 2 + build_success, build_failure = [], [] + for req in buildset: + if bdist_wheel(req, wheel_dir, build_options, global_options): + build_success.append(req) + else: + build_failure.append(req) + logger.indent -= 2 + + #notify sucess/failure + if build_success: + logger.notify( + 'Successfully built %s' % + ' '.join([req.name for req in build_success]) + ) + if build_failure: + logger.notify( + 'Failed to build %s' % + ' '.join([req.name for req in build_failure]) + ) + + def run(self, options, args): # confirm requirements @@ -197,15 +235,16 @@ def run(self, options, args): return try: + #unpack and constructs req set + requirement_set.prepare_files(finder) #build wheels - wb = WheelBuilder( + self.build( requirement_set, finder, options.wheel_dir, - build_options=options.build_options or [], - global_options=options.global_options or [], + options.build_options or [], + options.global_options or [], ) - wb.build() except PreviousBuildDirError: options.no_clean = True raise diff --git a/pip/download.py b/pip/download.py index 7458e48e9d3..b70c8dea504 100644 --- a/pip/download.py +++ b/pip/download.py @@ -17,6 +17,7 @@ from pip.util import (splitext, rmtree, format_size, display_path, backup_dir, ask_path_exists, unpack_file, create_download_cache_folder, cache_download) +from pip.locations import write_delete_marker_file from pip.vcs import vcs from pip.log import logger from pip._vendor import requests, six @@ -498,6 +499,39 @@ def _copy_file(filename, location, content_type, link): logger.notify('Saved %s' % display_path(download_location)) +def unpack_url(link, location, download_dir=None, only_download=False, + download_cache=None, session=None): + + if session is None: + session = PipSession() + + # non-editable vcs urls + if is_vcs_url(link): + if only_download: + loc = download_dir + else: + loc = location + unpack_vcs_link(link, loc, only_download) + + # file urls + elif is_file_url(link): + unpack_file_url(link, location, download_dir) + if only_download: + write_delete_marker_file(location) + + # http urls + else: + unpack_http_url( + link, + location, + download_cache, + download_dir, + session, + ) + if only_download: + write_delete_marker_file(location) + + def unpack_http_url(link, location, download_cache, download_dir=None, session=None): if session is None: diff --git a/pip/locations.py b/pip/locations.py index b270349d086..f12c9a5eab6 100644 --- a/pip/locations.py +++ b/pip/locations.py @@ -157,7 +157,9 @@ def _get_build_prefix(): if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': bin_py = '/usr/local/bin' default_log_file = os.path.join(user_dir, 'Library/Logs/pip.log') - +default_wheel_cache = os.path.join(default_storage_dir, 'wheel_cache') +if not os.path.isdir(default_wheel_cache): + os.makedirs(default_wheel_cache) def distutils_scheme(dist_name, user=False, home=None, root=None): """ diff --git a/pip/req/req_install.py b/pip/req/req_install.py index e2f018ab5ed..ebcd40fd47a 100644 --- a/pip/req/req_install.py +++ b/pip/req/req_install.py @@ -1,3 +1,4 @@ +import glob import os import re import shutil @@ -12,7 +13,8 @@ from pip.backwardcompat import ( urllib, ConfigParser, string_types, get_python_version, ) -from pip.download import is_url, url_to_path, path_to_url, is_archive_file +from pip.download import (is_url, url_to_path, path_to_url, is_archive_file, + unpack_url) from pip.exceptions import ( InstallationError, UninstallationError, UnsupportedWheel, ) @@ -28,7 +30,7 @@ ) from pip.req.req_uninstall import UninstallPathSet from pip.vcs import vcs -from pip.wheel import move_wheel_files, Wheel, wheel_ext +from pip.wheel import move_wheel_files, Wheel, wheel_ext, bdist_wheel class InstallRequirement(object): @@ -705,6 +707,33 @@ def _clean_zip_name(self, name, prefix): name = name.replace(os.path.sep, '/') return name + def build(self, wheel_cache_dir, build_options, global_options): + """Build a wheel from sdist src, and place it in the cache""" + + if self.is_wheel: + return + + # build into tmp dir (so we know what file we built). knowing the + # filename in advance is semi-hard see + # https://github.com/pypa/pip/issues/855#issuecomment-350447813 + wheel_tmp_dir = tempfile.mkdtemp() + bdist_wheel(self, wheel_tmp_dir, build_options, global_options) + wheel_path = glob.glob(os.path.join(wheel_tmp_dir, '*.whl'))[0] + wheel_filename = os.path.basename(wheel_path) + + # move the wheel into the cache + dest_path = os.path.join(wheel_cache_dir, wheel_filename) + shutil.move(wheel_path, dest_path) + + # unpack the wheel into a new src dir (and set self.source_dir) + # i.e. we want to install from the wheel we just built, not the sdist src + cache_url = path_to_url(dest_path) + self.source_dir = tempfile.mkdtemp() + self.url = cache_url + unpack_url(Link(cache_url), self.source_dir) + + # TODO: deal with tmp dir cleanup (pip delete files) + def install(self, install_options, global_options=(), root=None): if self.editable: self.install_editable(install_options, global_options) diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 59e611ffb99..298f35384a2 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -5,10 +5,10 @@ from pip._vendor import pkg_resources from pip.backwardcompat import HTTPError from pip.download import (PipSession, url_to_path, unpack_vcs_link, is_vcs_url, - is_file_url, unpack_file_url, unpack_http_url) + is_file_url, unpack_url) from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled, DistributionNotFound, PreviousBuildDirError) -from pip.index import Link +from pip.index import Link, PackageFinder from pip.locations import (PIP_DELETE_MARKER_FILENAME, build_prefix, write_delete_marker_file) from pip.log import logger @@ -53,7 +53,8 @@ def __init__(self, build_dir, src_dir, download_dir, download_cache=None, upgrade=False, ignore_installed=False, as_egg=False, target_dir=None, ignore_dependencies=False, force_reinstall=False, use_user_site=False, session=None, - pycompile=True, wheel_download_dir=None): + pycompile=True, wheel_download_dir=None, wheel_cache=False, + wheel_cache_dir=None, wheel_cache_exclude=[]): self.build_dir = build_dir self.src_dir = src_dir self.download_dir = download_dir @@ -77,6 +78,10 @@ def __init__(self, build_dir, src_dir, download_dir, download_cache=None, self.session = session or PipSession() self.pycompile = pycompile self.wheel_download_dir = wheel_download_dir + self.wheel_cache = wheel_cache + self.wheel_cache_dir = wheel_cache_dir + self.wheel_cache_exclude = wheel_cache_exclude + def __str__(self): reqs = [req for req in self.requirements.values() @@ -147,6 +152,25 @@ def uninstall(self, auto_confirm=False): req.uninstall(auto_confirm=auto_confirm) req.commit_uninstall() + def find_requirement(self, finder, req, upgrade=False): + """Return the url that fulfills a requirement. If caching wheels, try to + fulfill from the cache first.""" + + url = None + if self.wheel_cache: + # TODO: don't do this when --wheel-cache-rebuild + # TODO: quiet the failure logging from the finder + cache_finder = PackageFinder(find_links=[self.wheel_cache_dir], index_urls=[]) + try: + url = cache_finder.find_requirement(req, upgrade=upgrade) + logger.notify("Using wheel from cache: %s" % url_to_path(url.url)) + except DistributionNotFound: + # TODO: need more handling; refactor handling from below + pass + if not url: + url = finder.find_requirement(req, upgrade=upgrade) + return url + def locate_files(self): ## FIXME: duplicates code from prepare_files; relevant code should ## probably be factored out into a separate method @@ -223,8 +247,9 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False): if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: - url = finder.find_requirement( - req_to_install, self.upgrade) + url = self.find_requirement(finder, + req_to_install, + self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False @@ -267,10 +292,6 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False): logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 - ################################## - ## vcs update or unpack archive ## - ################################## - try: is_bundle = False is_wheel = False @@ -325,10 +346,9 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False): if req_to_install.url is None: if not_found: raise not_found - url = finder.find_requirement( - req_to_install, - upgrade=self.upgrade, - ) + url = self.find_requirement(finder, + req_to_install, + self.upgrade) else: ## FIXME: should req_to_install.url already be a # link? @@ -337,19 +357,25 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False): if url: try: - if ( - url.filename.endswith(wheel_ext) + # 'pip wheel' + if (url.filename.endswith(wheel_ext) and self.wheel_download_dir ): - # when doing 'pip wheel` download_dir = self.wheel_download_dir do_download = True + # 'pip install --wheel-cache' + elif (url.filename.endswith(wheel_ext) + and self.wheel_cache_dir + ): + download_dir = self.wheel_cache_dir + do_download = True else: download_dir = self.download_dir do_download = self.is_download - self.unpack_url( + unpack_url( url, location, download_dir, - do_download, + do_download, session = self.session, + download_cache = self.download_cache ) except HTTPError as exc: logger.fatal( @@ -538,36 +564,6 @@ def copy_to_build_dir(self, req_to_install): call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest, command_desc='python setup.py clean') - def unpack_url(self, link, location, download_dir=None, - only_download=False): - if download_dir is None: - download_dir = self.download_dir - - # non-editable vcs urls - if is_vcs_url(link): - if only_download: - loc = download_dir - else: - loc = location - unpack_vcs_link(link, loc, only_download) - - # file urls - elif is_file_url(link): - unpack_file_url(link, location, download_dir) - if only_download: - write_delete_marker_file(location) - - # http urls - else: - unpack_http_url( - link, - location, - self.download_cache, - download_dir, - self.session, - ) - if only_download: - write_delete_marker_file(location) def install(self, install_options, global_options=(), *args, **kwargs): """ @@ -634,6 +630,13 @@ def install(self, install_options, global_options=(), *args, **kwargs): finally: logger.indent -= 2 try: + if self.wheel_cache and not requirement.is_wheel: + # TODO: handle --wheel-cache-exclude + requirement.build( + self.wheel_cache_dir, + install_options, + global_options, + ) requirement.install( install_options, global_options, diff --git a/pip/wheel.py b/pip/wheel.py index 58148a09f33..2f049dc1b3d 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -447,73 +447,24 @@ def supported(self, tags=None): return bool(set(tags).intersection(self.file_tags)) -class WheelBuilder(object): - """Build wheels from a RequirementSet.""" - - def __init__(self, requirement_set, finder, wheel_dir, build_options=[], - global_options=[]): - self.requirement_set = requirement_set - self.finder = finder - self.wheel_dir = normalize_path(wheel_dir) - self.build_options = build_options - self.global_options = global_options - - def _build_one(self, req): - """Build one wheel.""" - - base_args = [ - sys.executable, '-c', - "import setuptools;__file__=%r;" - "exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), " - "__file__, 'exec'))" % req.setup_py - ] + list(self.global_options) - - logger.notify('Running setup.py bdist_wheel for %s' % req.name) - logger.notify('Destination directory: %s' % self.wheel_dir) - wheel_args = base_args + ['bdist_wheel', '-d', self.wheel_dir] \ - + self.build_options - try: - call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False) - return True - except: - logger.error('Failed building wheel for %s' % req.name) - return False - - def build(self): - """Build wheels.""" - - #unpack and constructs req set - self.requirement_set.prepare_files(self.finder) - - reqset = self.requirement_set.requirements.values() - - buildset = [req for req in reqset if not req.is_wheel] - - if not buildset: - return +def bdist_wheel(req, wheel_dir, build_options, global_options): + """Build a wheel from an unpacked `InstallRequirement` using bdist_wheel""" + + base_args = [ + sys.executable, '-c', + "import setuptools;__file__=%r;" + "exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), " + "__file__, 'exec'))" % req.setup_py + ] + list(global_options) + + logger.notify('Running setup.py bdist_wheel for %s' % req.name) + logger.notify('Destination directory: %s' % wheel_dir) + wheel_args = base_args + ['bdist_wheel', '-d', wheel_dir] \ + + build_options + try: + call_subprocess(wheel_args, cwd=req.source_dir, show_stdout=False) + return True + except: + logger.error('Failed building wheel for %s' % req.name) + return False - #build the wheels - logger.notify( - 'Building wheels for collected packages: %s' % - ','.join([req.name for req in buildset]) - ) - logger.indent += 2 - build_success, build_failure = [], [] - for req in buildset: - if self._build_one(req): - build_success.append(req) - else: - build_failure.append(req) - logger.indent -= 2 - - #notify sucess/failure - if build_success: - logger.notify( - 'Successfully built %s' % - ' '.join([req.name for req in build_success]) - ) - if build_failure: - logger.notify( - 'Failed to build %s' % - ' '.join([req.name for req in build_failure]) - )