From 24ffd0cf509acdcce01a715b203838cd8cd12ce8 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 2 Jan 2020 19:15:07 +0000 Subject: [PATCH] interface compatibility with before rewrite Ensure that what ran with virtualenv 17 will continue running in a post rewrite world minus the deprecated flags, plus the relocatable feature. Signed-off-by: Bernat Gabor --- src/virtualenv/activation/bash/__init__.py | 2 +- src/virtualenv/activation/batch/__init__.py | 2 +- src/virtualenv/activation/cshell/__init__.py | 2 +- src/virtualenv/activation/fish/__init__.py | 2 +- .../activation/powershell/__init__.py | 2 +- src/virtualenv/activation/python/__init__.py | 2 +- src/virtualenv/activation/xonosh/__init__.py | 2 +- src/virtualenv/config/ini.py | 10 +- src/virtualenv/info.py | 2 +- .../interpreters/create/cpython/common.py | 2 +- .../interpreters/create/cpython/cpython2.py | 2 +- .../interpreters/create/cpython/cpython3.py | 2 +- src/virtualenv/interpreters/create/creator.py | 4 +- src/virtualenv/interpreters/create/debug.py | 4 +- src/virtualenv/interpreters/create/venv.py | 4 +- .../interpreters/create/via_global_ref.py | 1 + .../interpreters/discovery/builtin.py | 3 +- src/virtualenv/report.py | 10 +- src/virtualenv/run.py | 12 +- src/virtualenv/seed/embed/base_embed.py | 47 +++- src/virtualenv/seed/embed/link_app_data.py | 32 ++- src/virtualenv/seed/embed/pip_invoke.py | 22 +- src/virtualenv/seed/embed/wheels/acquire.py | 207 ++++++++++++------ src/virtualenv/session.py | 9 +- src/virtualenv/util/__init__.py | 70 +----- src/virtualenv/util/path.py | 137 ------------ src/virtualenv/util/path/__init__.py | 12 + src/virtualenv/util/path/_pathlib/__init__.py | 40 ++++ .../util/path/_pathlib/via_os_path.py | 110 ++++++++++ src/virtualenv/util/path/_sync.py | 50 +++++ src/virtualenv/util/subprocess/__init__.py | 23 +- .../{win_subprocess.py => _win_subprocess.py} | 0 tests/conftest.py | 2 +- tests/unit/activation/conftest.py | 2 +- tests/unit/config/test_env_var.py | 2 +- .../unit/interpreters/create/test_creator.py | 13 +- .../discovery/windows/test_windows_pep514.py | 2 +- tests/unit/test_run.py | 20 ++ tests/unit/test_util.py | 3 +- 39 files changed, 527 insertions(+), 346 deletions(-) delete mode 100644 src/virtualenv/util/path.py create mode 100644 src/virtualenv/util/path/__init__.py create mode 100644 src/virtualenv/util/path/_pathlib/__init__.py create mode 100644 src/virtualenv/util/path/_pathlib/via_os_path.py create mode 100644 src/virtualenv/util/path/_sync.py rename src/virtualenv/util/subprocess/{win_subprocess.py => _win_subprocess.py} (100%) diff --git a/src/virtualenv/activation/bash/__init__.py b/src/virtualenv/activation/bash/__init__.py index 7c85564a8..c89b42edb 100644 --- a/src/virtualenv/activation/bash/__init__.py +++ b/src/virtualenv/activation/bash/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from virtualenv.util import Path +from virtualenv.util.path import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/batch/__init__.py b/src/virtualenv/activation/batch/__init__.py index 2c3da4431..89b03e436 100644 --- a/src/virtualenv/activation/batch/__init__.py +++ b/src/virtualenv/activation/batch/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from virtualenv.util import Path +from virtualenv.util.path import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/cshell/__init__.py b/src/virtualenv/activation/cshell/__init__.py index 6a96d5676..b25c602a5 100644 --- a/src/virtualenv/activation/cshell/__init__.py +++ b/src/virtualenv/activation/cshell/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from virtualenv.util import Path +from virtualenv.util.path import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/fish/__init__.py b/src/virtualenv/activation/fish/__init__.py index 0dfe63c8d..3671093a1 100644 --- a/src/virtualenv/activation/fish/__init__.py +++ b/src/virtualenv/activation/fish/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from virtualenv.util import Path +from virtualenv.util.path import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/powershell/__init__.py b/src/virtualenv/activation/powershell/__init__.py index a3c6f9e6c..4fadc63bc 100644 --- a/src/virtualenv/activation/powershell/__init__.py +++ b/src/virtualenv/activation/powershell/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from virtualenv.util import Path +from virtualenv.util.path import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/python/__init__.py b/src/virtualenv/activation/python/__init__.py index 36b2b2cbb..b06af78e4 100644 --- a/src/virtualenv/activation/python/__init__.py +++ b/src/virtualenv/activation/python/__init__.py @@ -3,7 +3,7 @@ import json import os -from virtualenv.util import Path +from virtualenv.util.path import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/xonosh/__init__.py b/src/virtualenv/activation/xonosh/__init__.py index 94546ff9e..0340dcb47 100644 --- a/src/virtualenv/activation/xonosh/__init__.py +++ b/src/virtualenv/activation/xonosh/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from virtualenv.util import Path +from virtualenv.util.path import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/config/ini.py b/src/virtualenv/config/ini.py index 4acb70ffc..a799abe1d 100644 --- a/src/virtualenv/config/ini.py +++ b/src/virtualenv/config/ini.py @@ -4,17 +4,11 @@ import os from virtualenv.info import PY3, get_default_config_dir -from virtualenv.util import Path +from virtualenv.util import ConfigParser +from virtualenv.util.path import Path from .convert import convert -try: - import ConfigParser -except ImportError: - # noinspection PyPep8Naming - import configparser as ConfigParser - - DEFAULT_CONFIG_FILE = get_default_config_dir() / "virtualenv.ini" diff --git a/src/virtualenv/info.py b/src/virtualenv/info.py index 294110eba..96050b037 100644 --- a/src/virtualenv/info.py +++ b/src/virtualenv/info.py @@ -4,7 +4,7 @@ from appdirs import user_config_dir, user_data_dir -from virtualenv.util import Path +from virtualenv.util.path import Path IS_PYPY = hasattr(sys, "pypy_version_info") PY3 = sys.version_info[0] == 3 diff --git a/src/virtualenv/interpreters/create/cpython/common.py b/src/virtualenv/interpreters/create/cpython/common.py index 89a91c6df..b5796de0d 100644 --- a/src/virtualenv/interpreters/create/cpython/common.py +++ b/src/virtualenv/interpreters/create/cpython/common.py @@ -6,7 +6,7 @@ import six from virtualenv.interpreters.create.via_global_ref import ViaGlobalRef -from virtualenv.util import Path, copy, ensure_dir, symlink +from virtualenv.util.path import Path, copy, ensure_dir, symlink @six.add_metaclass(abc.ABCMeta) diff --git a/src/virtualenv/interpreters/create/cpython/cpython2.py b/src/virtualenv/interpreters/create/cpython/cpython2.py index 7c7915882..e41abbb44 100644 --- a/src/virtualenv/interpreters/create/cpython/cpython2.py +++ b/src/virtualenv/interpreters/create/cpython/cpython2.py @@ -4,7 +4,7 @@ import six -from virtualenv.util import Path, copy +from virtualenv.util.path import Path, copy from .common import CPython, CPythonPosix, CPythonWindows diff --git a/src/virtualenv/interpreters/create/cpython/cpython3.py b/src/virtualenv/interpreters/create/cpython/cpython3.py index b14ce0a86..4d833b4d9 100644 --- a/src/virtualenv/interpreters/create/cpython/cpython3.py +++ b/src/virtualenv/interpreters/create/cpython/cpython3.py @@ -4,7 +4,7 @@ import six -from virtualenv.util import Path, copy +from virtualenv.util.path import Path, copy from .common import CPython, CPythonPosix, CPythonWindows diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index b917a3709..fa3caa056 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -13,7 +13,8 @@ from virtualenv.info import IS_WIN from virtualenv.pyenv_cfg import PyEnvCfg -from virtualenv.util import Path, run_cmd +from virtualenv.util.path import Path +from virtualenv.util.subprocess import run_cmd from virtualenv.version import __version__ HERE = Path(__file__).absolute().parent @@ -104,6 +105,7 @@ def non_write_able(dest, value): def run(self): if self.dest_dir.exists() and self.clear: + logging.debug("delete %s", self.dest_dir) shutil.rmtree(str(self.dest_dir), ignore_errors=True) self.create() self.set_pyenv_cfg() diff --git a/src/virtualenv/interpreters/create/debug.py b/src/virtualenv/interpreters/create/debug.py index 37f1a4507..d9f6d525e 100644 --- a/src/virtualenv/interpreters/create/debug.py +++ b/src/virtualenv/interpreters/create/debug.py @@ -7,8 +7,8 @@ def encode_path(value): return None if isinstance(value, bytes): return value.decode(sys.getfilesystemencoding()) - if isinstance(value, type): - return repr(value) + elif not isinstance(value, str): + return repr(value if isinstance(value, type) else type(value)) return value diff --git a/src/virtualenv/interpreters/create/venv.py b/src/virtualenv/interpreters/create/venv.py index e80a45ecb..feb160b4b 100644 --- a/src/virtualenv/interpreters/create/venv.py +++ b/src/virtualenv/interpreters/create/venv.py @@ -5,7 +5,7 @@ from virtualenv.error import ProcessCallFailed from virtualenv.interpreters.discovery.py_info import CURRENT -from virtualenv.util import run_cmd +from virtualenv.util.subprocess import run_cmd from .via_global_ref import ViaGlobalRef @@ -41,7 +41,7 @@ def create_inline(self): def create_via_sub_process(self): cmd = self.get_host_create_cmd() - logging.info("create with venv %s", " ".join(cmd)) + logging.info("using host built-in venv to create via %s", " ".join(cmd)) code, out, err = run_cmd(cmd) if code != 0: raise ProcessCallFailed(code, out, err, cmd) diff --git a/src/virtualenv/interpreters/create/via_global_ref.py b/src/virtualenv/interpreters/create/via_global_ref.py index e574cbd00..240b313a0 100644 --- a/src/virtualenv/interpreters/create/via_global_ref.py +++ b/src/virtualenv/interpreters/create/via_global_ref.py @@ -27,6 +27,7 @@ def add_parser_arguments(cls, parser, interpreter): ) group.add_argument( "--copies", + "--always-copy", default=not symlink, action="store_false", dest="symlinks", diff --git a/src/virtualenv/interpreters/discovery/builtin.py b/src/virtualenv/interpreters/discovery/builtin.py index 166066d4c..b90b05045 100644 --- a/src/virtualenv/interpreters/discovery/builtin.py +++ b/src/virtualenv/interpreters/discovery/builtin.py @@ -38,12 +38,13 @@ def __str__(self): def get_interpreter(key): spec = PythonSpec.from_string_spec(key) - logging.debug("find interpreter for spec %r", spec) + logging.info("find interpreter for spec %r", spec) proposed_paths = set() for interpreter, impl_must_match in propose_interpreters(spec): if interpreter.executable not in proposed_paths: logging.debug("proposed %s", interpreter) if interpreter.satisfies(spec, impl_must_match): + logging.info("accepted target interpreter %s", interpreter) return interpreter proposed_paths.add(interpreter.executable) diff --git a/src/virtualenv/report.py b/src/virtualenv/report.py index d2dc8384c..be7506eaa 100644 --- a/src/virtualenv/report.py +++ b/src/virtualenv/report.py @@ -3,6 +3,8 @@ import logging import sys +import six + LEVELS = { 0: logging.CRITICAL, 1: logging.ERROR, @@ -23,11 +25,11 @@ def setup_report(verbose, quiet): verbosity = MAX_LEVEL # pragma: no cover level = LEVELS[verbosity] msg_format = "%(message)s" - if level >= logging.DEBUG: - locate = "pathname" if level > logging.DEBUG else "module" - msg_format += "[%(asctime)s] %(levelname)s [%({})s:%(lineno)d]".format(locate) + if level <= logging.DEBUG: + locate = "pathname" if level < logging.DEBUG else "module" + msg_format = "%(relativeCreated)d {} [%(levelname)s %({})s:%(lineno)d]".format(msg_format, locate) - formatter = logging.Formatter(str(msg_format)) + formatter = logging.Formatter(six.ensure_str(msg_format)) stream_handler = logging.StreamHandler(stream=sys.stdout) stream_handler.setLevel(level) LOGGER.setLevel(logging.NOTSET) diff --git a/src/virtualenv/run.py b/src/virtualenv/run.py index 6dad18d64..1608e939d 100644 --- a/src/virtualenv/run.py +++ b/src/virtualenv/run.py @@ -8,6 +8,7 @@ from .config.cli.parser import VirtualEnvConfigParser from .report import LEVELS, setup_report from .session import Session +from .version import __version__ def run_via_cli(args): @@ -23,6 +24,7 @@ def run_via_cli(args): def session_via_cli(args): parser = VirtualEnvConfigParser() + add_version_flag(parser) options, verbosity = _do_report_setup(parser, args) discover = _get_discover(parser, args, options) interpreter = discover.interpreter @@ -44,12 +46,20 @@ def session_via_cli(args): return session +def add_version_flag(parser): + import virtualenv + + parser.add_argument( + "--version", action="version", version="%(prog)s {} from {}".format(__version__, virtualenv.__file__) + ) + + def _do_report_setup(parser, args): level_map = ", ".join("{}:{}".format(c, logging.getLevelName(l)) for c, l in sorted(list(LEVELS.items()))) msg = "verbosity = verbose - quiet, default {}, count mapping = {{{}}}" verbosity_group = parser.add_argument_group(msg.format(logging.getLevelName(LEVELS[3]), level_map)) verbosity = verbosity_group.add_mutually_exclusive_group() - verbosity.add_argument("-v", "--verbose", action="count", dest="verbose", help="increase verbosity", default=3) + verbosity.add_argument("-v", "--verbose", action="count", dest="verbose", help="increase verbosity", default=2) verbosity.add_argument("-q", "--quiet", action="count", dest="quiet", help="decrease verbosity", default=0) options, _ = parser.parse_known_args(args) verbosity_value = setup_report(options.verbose, options.quiet) diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index 6220126b5..f3e77de0d 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -4,6 +4,8 @@ import six +from virtualenv.util.path import Path + from ..seeder import Seeder @@ -13,8 +15,11 @@ def __init__(self, options): super(Seeder, self).__init__() self.enabled = options.without_pip is False self.download = options.download - self.pip_version = options.pip - self.setuptools_version = options.setuptools + self.extra_search_dir = [i.resolve() for i in options.extra_search_dir if i.exists()] + self.pip_version = None if options.pip == "latest" else options.pip + self.setuptools_version = None if options.setuptools == "latest" else options.setuptools + self.no_pip = options.no_pip + self.no_setuptools = options.no_setuptools @classmethod def add_parser_arguments(cls, parser): @@ -28,17 +33,49 @@ def add_parser_arguments(cls, parser): ) group.add_argument( "--no-download", + "--never-download", dest="download", action="store_false", help="do not download latest pip/setuptools from PyPi", default=True, ) - + parser.add_argument( + "--extra-search-dir", + metavar="d", + type=Path, + nargs="+", + help="a location containing wheels candidates to install from", + default=[], + ) for package in ["pip", "setuptools"]: parser.add_argument( "--{}".format(package), dest=package, metavar="version", - help="{} version to install, default: latest from cache, bundle for bundled".format(package), - default=None, + help="{} version to install, bundle for bundled".format(package), + default="latest", + ) + for extra, package in [ + ("", "pip"), + ("", "setuptools"), + ("N/A - kept only for backwards compatibility; ", "wheel"), + ]: + parser.add_argument( + "--no-{}".format(package), + dest="no_{}".format(package), + action="store_true", + help="{}do not install {}".format(extra, package), + default=False, + ) + + def __str__(self): + result = self.__class__.__name__ + if self.extra_search_dir: + result += " extra search dirs = {}".format( + ", ".join(six.ensure_text(str(i)) for i in self.extra_search_dir) ) + if self.no_pip is False: + result += " pip{}".format("={}".format(self.pip_version or "latest")) + if self.no_setuptools is False: + result += " setuptools{}".format("={}".format(self.setuptools_version or "latest")) + return result diff --git a/src/virtualenv/seed/embed/link_app_data.py b/src/virtualenv/seed/embed/link_app_data.py index d660574a3..99f73f51a 100644 --- a/src/virtualenv/seed/embed/link_app_data.py +++ b/src/virtualenv/seed/embed/link_app_data.py @@ -9,30 +9,46 @@ from shutil import copytree from textwrap import dedent +import six from six import PY3 from virtualenv.info import get_default_data_dir -from virtualenv.util import Path +from virtualenv.util import ConfigParser +from virtualenv.util.path import Path from .base_embed import BaseEmbed from .wheels.acquire import get_wheel -try: - import ConfigParser -except ImportError: - # noinspection PyPep8Naming - import configparser as ConfigParser - class LinkFromAppData(BaseEmbed): + def __init__(self, options): + super(LinkFromAppData, self).__init__(options) + self.clear_app_data = options.clear_app_data + def run(self, creator): if not self.enabled: return cache = get_default_data_dir() / "seed-v1" + if self.clear_app_data: + logging.debug("delete %s", cache) + shutil.rmtree(six.ensure_text(str(cache))) version = creator.interpreter.version_release_str - name_to_whl = get_wheel(version, cache, self.download, self.pip_version, self.setuptools_version) + name_to_whl = get_wheel( + version, cache, self.extra_search_dir, self.download, self.pip_version, self.setuptools_version, + ) pip_install(name_to_whl, creator, cache) + @classmethod + def add_parser_arguments(cls, parser): + super(LinkFromAppData, cls).add_parser_arguments(parser) + parser.add_argument( + "--clear-app-data", + dest="clear_app_data", + action="store_true", + help="clear the app data folder", + default=False, + ) + def pip_install(wheels, creator, cache): site_package, bin_dir, env_exe = creator.site_packages[0], creator.bin_dir, creator.exe diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index d65e89a77..1ede693b9 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -1,9 +1,7 @@ from __future__ import absolute_import, unicode_literals -import os - from virtualenv.seed.embed.base_embed import BaseEmbed -from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel +from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, pip_wheel_env_run from virtualenv.util.subprocess import Popen @@ -18,26 +16,14 @@ def run(self, creator): version = creator.interpreter.version_release_str cmd = [str(creator.exe), "-m", "pip", "install", "--only-binary", ":all:"] + for folder in {get_bundled_wheel(p, version).parent for p in ("pip", "setuptools")}: cmd.extend(["--find-links", str(folder)]) + cmd.extend(self.extra_search_dir) if not self.download: cmd.append("--no-index") for key, version in {"pip": self.pip_version, "setuptools": self.setuptools_version}.items(): cmd.append("{}{}".format(key, "=={}".format(version) if version is not None else "")) - env = os.environ.copy() - env.update( - { - str(k): str(v) # python 2 requires these to be string only (non-unicode) - for k, v in { - # put the bundled wheel onto the path, and use it to do the bootstrap operation - "PYTHONPATH": get_bundled_wheel("pip", version), - "PIP_USE_WHEEL": "1", - "PIP_USER": "0", - "PIP_NO_INPUT": "1", - }.items() - } - ) - - process = Popen(cmd, env=env) + process = Popen(cmd, env=pip_wheel_env_run(version)) process.communicate() diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index 6460b31a5..9e629dc70 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -1,98 +1,161 @@ """Bootstrap""" from __future__ import absolute_import, unicode_literals +import logging +import os +import sys from collections import defaultdict from shutil import copy2 +from zipfile import ZipFile -from virtualenv.util import Path -from virtualenv.util.subprocess import subprocess +import six + +from virtualenv.util.path import Path +from virtualenv.util.subprocess import Popen, subprocess from . import BUNDLE_SUPPORT, MAX BUNDLE_FOLDER = Path(__file__).parent -def get_wheel(version_release, cache, download, pip, setuptools): - wheel_download = cache / "download" / version_release - wheel_download.mkdir(parents=True, exist_ok=True) +def get_wheel(for_py_version, cache, extra_search_dir, download, pip, setuptools): + # not all wheels are compatible with all python versions, so we need to py version qualify it + wheel_cache_dir = cache / "acquired" / for_py_version + wheel_cache_dir.mkdir(parents=True, exist_ok=True) packages = {"pip": pip, "setuptools": setuptools} - ensure_bundle_cached( - packages, version_release, wheel_download - ) # first ensure all bundled versions area already there - if download is True: - must_download = check_if_must_download(packages, wheel_download) # check what needs downloading - if must_download: # perform download if any of the packages require - download_wheel(version_release, must_download, wheel_download) - return _get_wheels_for_package(wheel_download, packages) + # 1. acquire from bundle + acquire_from_bundle(packages, for_py_version, wheel_cache_dir) + # 2. acquire from extra search dir + acquire_from_dir(packages, for_py_version, wheel_cache_dir, extra_search_dir) + # 3. download from the internet + if download: + download_wheel(packages, for_py_version, wheel_cache_dir) + + # in the end just get the wheels + wheels = _get_wheels(wheel_cache_dir, {"pip": pip, "setuptools": setuptools}) + return {p: next(iter(ver_to_files))[1] for p, ver_to_files in wheels.items()} + + +def acquire_from_bundle(packages, for_py_version, to_folder): + for pkg, version in list(packages.items()): + bundle = get_bundled_wheel(pkg, for_py_version) + if bundle is not None: + pkg_version = bundle.stem.split("-")[1] + exact_version_match = version == pkg_version + if exact_version_match: + del packages[pkg] + if version is None or exact_version_match: + bundled_wheel_file = to_folder / bundle.name + if not bundled_wheel_file.exists(): + logging.debug("get bundled wheel %s", bundle) + copy2(str(bundle), str(bundled_wheel_file)) + + +def get_bundled_wheel(package, version_release): + return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package) -def download_wheel(version_str, must_download, wheel_download): +def acquire_from_dir(packages, for_py_version, to_folder, extra_search_dir): + if not packages: + return + for search_dir in extra_search_dir: + wheels = _get_wheels(search_dir, packages) + for pkg, ver_wheels in wheels.items(): + stop = False + for _, filename in ver_wheels: + dest = to_folder / filename.name + if not dest.exists(): + if wheel_support_py(filename, for_py_version): + logging.debug("get extra search dir wheel %s", filename) + copy2(str(filename), str(dest)) + stop = True + else: + stop = True + if stop and packages[pkg] is not None: + del packages[pkg] + break + + +def wheel_support_py(filename, py_version): + name = "{}.dist-info/METADATA".format("-".join(filename.stem.split("-")[0:2])) + with ZipFile(six.ensure_text(str(filename)), "r") as zip_file: + metadata = zip_file.read(name).decode("utf-8") + marker = "Requires-Python:" + requires = next(i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)) + py_version_int = tuple(int(i) for i in py_version.split(".")) + for require in (i.strip() for i in requires.split(",")): + # https://www.python.org/dev/peps/pep-0345/#version-specifiers + for operator, check in [ + ("!=", lambda v: py_version_int != v), + ("==", lambda v: py_version_int == v), + ("<=", lambda v: py_version_int <= v), + (">=", lambda v: py_version_int >= v), + ("<", lambda v: py_version_int < v), + (">", lambda v: py_version_int > v), + ]: + if require.startswith(operator): + ver_str = require[len(operator) :].strip() + version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2] + if not check(version): + return False + break + return True + + +def _get_wheels(from_folder, packages): + wheels = defaultdict(list) + for filename in from_folder.iterdir(): + if filename.suffix == ".whl": + data = filename.stem.split("-") + if len(data) >= 2: + pkg, version = data[0:2] + if pkg in packages: + pkg_version = packages[pkg] + if pkg_version is None or pkg_version == version: + wheels[pkg].append((version, filename)) + for versions in wheels.values(): + versions.sort( + key=lambda a: tuple(int(i) if i.isdigit() else i for i in a[0].split(".")), reverse=True, + ) + return wheels + + +def download_wheel(packages, for_py_version, to_folder): + to_download = list(p if v is None else "{}={}".format(p, v) for p, v in packages.items()) + logging.debug("download wheels %s", to_download) cmd = [ + sys.executable, + "-m", + "pip", "download", "--disable-pip-version-check", "--only-binary=:all:", "--no-deps", "--python-version", - version_str, + for_py_version, "-d", - str(wheel_download), + str(to_folder), ] - cmd.extend(must_download) + cmd.extend(to_download) # pip has no interface in python - must be a new sub-process - subprocess.call(cmd) - - -def check_if_must_download(packages, wheel_download): - must_download = set() - if any(i is not None for i in packages.values()): - has_version = _get_wheels(wheel_download) - for pkg, version in packages.items(): - if pkg in has_version and version in has_version[pkg]: - continue - must_download.add(pkg) - return must_download - - -def _get_wheels(inside_folder): - has_version = defaultdict(set) - for filename in inside_folder.iterdir(): - if filename.suffix == ".whl": - pkg, version = filename.stem.split("-")[0:2] - has_version[pkg].add(version) - return has_version - - -def _get_wheels_for_package(inside_folder, package): - has_version = defaultdict(dict) - for filename in inside_folder.iterdir(): - if filename.suffix == ".whl": - pkg, version = filename.stem.split("-")[0:2] - has_version[pkg][version] = filename - result = {} - for pkg, version in package.items(): - content = has_version[pkg] - if version in content: - result[pkg] = content[version] - else: - elements = sorted( - content.items(), - key=lambda a: tuple(int(i) if i.isdigit() else i for i in a[0].split(".")), - reverse=True, - ) - result[pkg] = elements[0][1] - return result - - -def ensure_bundle_cached(packages, version_release, wheel_download): - for package in packages: - bundle = get_bundled_wheel(package, version_release) - if bundle is not None: - bundled_wheel_file = wheel_download / bundle.name - if not bundled_wheel_file.exists(): - copy2(str(bundle), str(bundled_wheel_file)) - - -def get_bundled_wheel(package, version_release): - return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package) + process = Popen(cmd, env=pip_wheel_env_run("{}{}".format(*sys.version_info[0:2])), stdout=subprocess.PIPE) + process.communicate() + + +def pip_wheel_env_run(version): + env = os.environ.copy() + env.update( + { + str(k): str(v) # python 2 requires these to be string only (non-unicode) + for k, v in { + # put the bundled wheel onto the path, and use it to do the bootstrap operation + "PYTHONPATH": get_bundled_wheel("pip", version), + "PIP_USE_WHEEL": "1", + "PIP_USER": "0", + "PIP_NO_INPUT": "1", + }.items() + } + ) + return env diff --git a/src/virtualenv/session.py b/src/virtualenv/session.py index d69fc8fd6..b0954fbaa 100644 --- a/src/virtualenv/session.py +++ b/src/virtualenv/session.py @@ -25,11 +25,16 @@ def _create(self): def _seed(self): if self.seeder is not None: + logging.info("add seed packages via %s", self.seeder) self.seeder.run(self.creator) def _activate(self): - for activator in self.activators: - activator.generate(self.creator) + if self.activators: + logging.info( + "add activators for %s", ", ".join(type(i).__name__.replace("Activator", "") for i in self.activators) + ) + for activator in self.activators: + activator.generate(self.creator) _DEBUG_MARKER = "=" * 30 + " target debug " + "=" * 30 diff --git a/src/virtualenv/util/__init__.py b/src/virtualenv/util/__init__.py index 9e7d13bad..39e5eb8ba 100644 --- a/src/virtualenv/util/__init__.py +++ b/src/virtualenv/util/__init__.py @@ -1,68 +1,10 @@ from __future__ import absolute_import, unicode_literals -import logging -import os -import shutil -from functools import partial -from os import makedirs +try: + import ConfigParser +except ImportError: + # noinspection PyPep8Naming + import configparser as ConfigParser -import six -from virtualenv.util.subprocess import Popen, subprocess - -from .path import Path - - -def ensure_dir(path): - if not path.exists(): - logging.debug("created %s", six.ensure_text(str(path))) - makedirs(six.ensure_text(str(path))) - - -HAS_SYMLINK = hasattr(os, "symlink") - - -def symlink_or_copy(do_copy, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a target, and if that fails, fall back to copying. - """ - if do_copy is False and HAS_SYMLINK is False: # if no symlink, always use copy - do_copy = True - if not do_copy: - try: - if not dst.is_symlink(): # can't link to itself! - if relative_symlinks_ok: - assert src.parent == dst.parent - os.symlink(six.ensure_text(src.name), six.ensure_text(str(dst))) - else: - os.symlink(six.ensure_text(str(src)), six.ensure_text(str(dst))) - except OSError as exception: - logging.warning( - "symlink failed %r, for %s to %s, will try copy", - exception, - six.ensure_text(str(src)), - six.ensure_text(str(dst)), - ) - do_copy = True - if do_copy: - copier = shutil.copy2 if src.is_file() else shutil.copytree - copier(six.ensure_text(str(src)), six.ensure_text(str(dst))) - logging.debug("%s %s to %s", "copy" if do_copy else "symlink", six.ensure_text(str(src)), six.ensure_text(str(dst))) - - -def run_cmd(cmd): - try: - process = Popen( - cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE - ) - out, err = process.communicate() # input disabled - code = process.returncode - except OSError as os_error: - code, out, err = os_error.errno, "", os_error.strerror - return code, out, err - - -symlink = partial(symlink_or_copy, False) -copy = partial(symlink_or_copy, True) - -__all__ = ("Path", "symlink", "copy", "run_cmd", "ensure_dir") +__all__ = ("ConfigParser",) diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py deleted file mode 100644 index 5ef51867d..000000000 --- a/src/virtualenv/util/path.py +++ /dev/null @@ -1,137 +0,0 @@ -import sys -from contextlib import contextmanager - -import six - -if six.PY3: - from pathlib import Path - - if sys.version_info[0:2] == (3, 4): - # no read/write text on python3.4 - BuiltinPath = Path - - class Path(type(BuiltinPath())): - def read_text(self, encoding=None, errors=None): - """ - Open the file in text mode, read it, and close the file. - """ - with self.open(mode="r", encoding=encoding, errors=errors) as f: - return f.read() - - def write_text(self, data, encoding=None, errors=None): - """ - Open the file in text mode, write to it, and close the file. - """ - if not isinstance(data, str): - raise TypeError("data must be str, not %s" % data.__class__.__name__) - with self.open(mode="w", encoding=encoding, errors=errors) as f: - return f.write(data) - - -else: - if sys.platform == "win32": - # workaround for https://github.com/mcmtroffaes/pathlib2/issues/56 - import os - - class Path(object): - def __init__(self, path): - self._path = path._path if isinstance(path, Path) else six.ensure_text(path) - - def __repr__(self): - return six.ensure_str(u"Path({})".format(self._path)) - - def __str__(self): - return six.ensure_str(self._path) - - def __div__(self, other): - return Path( - os.path.join(self._path, other._path if isinstance(other, Path) else six.ensure_text(other)) - ) - - def __eq__(self, other): - return self._path == (other._path if isinstance(other, Path) else None) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return hash(self._path) - - def exists(self): - return os.path.exists(self._path) - - def absolute(self): - return Path(os.path.abspath(self._path)) - - @property - def parent(self): - return Path(os.path.abspath(os.path.join(self._path, os.path.pardir))) - - def resolve(self): - return Path(os.path.realpath(self._path)) - - @property - def name(self): - return os.path.basename(self._path) - - @property - def parts(self): - return self._path.split(os.sep) - - def is_file(self): - return os.path.isfile(self._path) - - def is_dir(self): - return os.path.isdir(self._path) - - def mkdir(self, parents=True, exist_ok=True): - if not self.exists() and exist_ok: - os.makedirs(self._path) - - def read_text(self, encoding="utf-8"): - with open(self._path, "rb") as file_handler: - return file_handler.read().decode(encoding) - - def write_text(self, text, encoding="utf-8"): - with open(self._path, "wb") as file_handler: - file_handler.write(text.encode(encoding)) - - def iterdir(self): - for p in os.listdir(self._path): - yield Path(os.path.join(self._path, p)) - - @property - def suffix(self): - _, ext = os.path.splitext(self.name) - return ext - - @property - def stem(self): - base, _ = os.path.splitext(self.name) - return base - - @contextmanager - def open(self, mode="r"): - with open(self._path, mode) as file_handler: - yield file_handler - - @property - def parents(self): - result = [] - parts = self.parts - for i in range(len(parts)): - result.append(Path(os.sep.join(parts[0 : i + 1]))) - return result - - def unlink(self): - os.remove(self._path) - - def with_name(self, name): - return self.parent / name - - def is_symlink(self): - return os.path.islink(self._path) - - else: - from pathlib2 import Path -__all__ = ("Path",) diff --git a/src/virtualenv/util/path/__init__.py b/src/virtualenv/util/path/__init__.py new file mode 100644 index 000000000..ab5db8ef2 --- /dev/null +++ b/src/virtualenv/util/path/__init__.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, unicode_literals + +from ._pathlib import Path +from ._sync import copy, ensure_dir, symlink, symlink_or_copy + +__all__ = ( + "ensure_dir", + "symlink_or_copy", + "symlink", + "copy", + "Path", +) diff --git a/src/virtualenv/util/path/_pathlib/__init__.py b/src/virtualenv/util/path/_pathlib/__init__.py new file mode 100644 index 000000000..3b18d3d4b --- /dev/null +++ b/src/virtualenv/util/path/_pathlib/__init__.py @@ -0,0 +1,40 @@ +from __future__ import absolute_import, unicode_literals + +import sys + +import six + +if six.PY3: + from pathlib import Path + + if sys.version_info[0:2] == (3, 4): + # no read/write text on python3.4 + BuiltinPath = Path + + class Path(type(BuiltinPath())): + def read_text(self, encoding=None, errors=None): + """ + Open the file in text mode, read it, and close the file. + """ + with self.open(mode="r", encoding=encoding, errors=errors) as f: + return f.read() + + def write_text(self, data, encoding=None, errors=None): + """ + Open the file in text mode, write to it, and close the file. + """ + if not isinstance(data, str): + raise TypeError("data must be str, not %s" % data.__class__.__name__) + with self.open(mode="w", encoding=encoding, errors=errors) as f: + return f.write(data) + + +else: + if sys.platform == "win32": + # workaround for https://github.com/mcmtroffaes/pathlib2/issues/56 + from .via_os_path import Path + else: + from pathlib2 import Path + + +__all__ = ("Path",) diff --git a/src/virtualenv/util/path/_pathlib/via_os_path.py b/src/virtualenv/util/path/_pathlib/via_os_path.py new file mode 100644 index 000000000..1dddd7e11 --- /dev/null +++ b/src/virtualenv/util/path/_pathlib/via_os_path.py @@ -0,0 +1,110 @@ +from __future__ import absolute_import, unicode_literals + +import os +from contextlib import contextmanager + +import six + + +class Path(object): + def __init__(self, path): + self._path = path._path if isinstance(path, Path) else six.ensure_text(path) + + def __repr__(self): + return six.ensure_str("Path({})".format(self._path)) + + def __str__(self): + return six.ensure_str(self._path) + + def __div__(self, other): + return Path(os.path.join(self._path, other._path if isinstance(other, Path) else six.ensure_text(other))) + + def __truediv__(self, other): + return self.__div__(other) + + def __eq__(self, other): + return self._path == (other._path if isinstance(other, Path) else None) + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self._path) + + def exists(self): + return os.path.exists(self._path) + + def absolute(self): + return Path(os.path.abspath(self._path)) + + @property + def parent(self): + return Path(os.path.abspath(os.path.join(self._path, os.path.pardir))) + + def resolve(self): + return Path(os.path.realpath(self._path)) + + @property + def name(self): + return os.path.basename(self._path) + + @property + def parts(self): + return self._path.split(os.sep) + + def is_file(self): + return os.path.isfile(self._path) + + def is_dir(self): + return os.path.isdir(self._path) + + def mkdir(self, parents=True, exist_ok=True): + if not self.exists() and exist_ok: + os.makedirs(self._path) + + def read_text(self, encoding="utf-8"): + with open(self._path, "rb") as file_handler: + return file_handler.read().decode(encoding) + + def write_text(self, text, encoding="utf-8"): + with open(self._path, "wb") as file_handler: + file_handler.write(text.encode(encoding)) + + def iterdir(self): + for p in os.listdir(self._path): + yield Path(os.path.join(self._path, p)) + + @property + def suffix(self): + _, ext = os.path.splitext(self.name) + return ext + + @property + def stem(self): + base, _ = os.path.splitext(self.name) + return base + + @contextmanager + def open(self, mode="r"): + with open(self._path, mode) as file_handler: + yield file_handler + + @property + def parents(self): + result = [] + parts = self.parts + for i in range(len(parts)): + result.append(Path(os.sep.join(parts[0 : i + 1]))) + return result + + def unlink(self): + os.remove(self._path) + + def with_name(self, name): + return self.parent / name + + def is_symlink(self): + return os.path.islink(self._path) + + +__all__ = ("Path",) diff --git a/src/virtualenv/util/path/_sync.py b/src/virtualenv/util/path/_sync.py new file mode 100644 index 000000000..c3d611117 --- /dev/null +++ b/src/virtualenv/util/path/_sync.py @@ -0,0 +1,50 @@ +from __future__ import absolute_import, unicode_literals + +import logging +import os +import shutil +from functools import partial + +import six + +HAS_SYMLINK = hasattr(os, "symlink") + + +def ensure_dir(path): + if not path.exists(): + logging.debug("created %s", six.ensure_text(str(path))) + os.makedirs(six.ensure_text(str(path))) + + +def symlink_or_copy(do_copy, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a target, and if that fails, fall back to copying. + """ + if do_copy is False and HAS_SYMLINK is False: # if no symlink, always use copy + do_copy = True + if not do_copy: + try: + if not dst.is_symlink(): # can't link to itself! + if relative_symlinks_ok: + assert src.parent == dst.parent + os.symlink(six.ensure_text(src.name), six.ensure_text(str(dst))) + else: + os.symlink(six.ensure_text(str(src)), six.ensure_text(str(dst))) + except OSError as exception: + logging.warning( + "symlink failed %r, for %s to %s, will try copy", + exception, + six.ensure_text(str(src)), + six.ensure_text(str(dst)), + ) + do_copy = True + if do_copy: + copier = shutil.copy2 if src.is_file() else shutil.copytree + copier(six.ensure_text(str(src)), six.ensure_text(str(dst))) + logging.debug("%s %s to %s", "copy" if do_copy else "symlink", six.ensure_text(str(src)), six.ensure_text(str(dst))) + + +symlink = partial(symlink_or_copy, False) +copy = partial(symlink_or_copy, True) + +__all__ = ("ensure_dir", "symlink", "copy", "symlink_or_copy") diff --git a/src/virtualenv/util/subprocess/__init__.py b/src/virtualenv/util/subprocess/__init__.py index a980cae6c..01076587f 100644 --- a/src/virtualenv/util/subprocess/__init__.py +++ b/src/virtualenv/util/subprocess/__init__.py @@ -6,10 +6,27 @@ import six if six.PY2 and sys.platform == "win32": - from . import win_subprocess + from . import _win_subprocess - Popen = win_subprocess.Popen + Popen = _win_subprocess.Popen else: Popen = subprocess.Popen -__all__ = ("subprocess", "Popen") + +def run_cmd(cmd): + try: + process = Popen( + cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) + out, err = process.communicate() # input disabled + code = process.returncode + except OSError as os_error: + code, out, err = os_error.errno, "", os_error.strerror + return code, out, err + + +__all__ = ( + "subprocess", + "Popen", + "run_cmd", +) diff --git a/src/virtualenv/util/subprocess/win_subprocess.py b/src/virtualenv/util/subprocess/_win_subprocess.py similarity index 100% rename from src/virtualenv/util/subprocess/win_subprocess.py rename to src/virtualenv/util/subprocess/_win_subprocess.py diff --git a/tests/conftest.py b/tests/conftest.py index c451c2bf8..3ca6dccfa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ import six from virtualenv.interpreters.discovery.py_info import PythonInfo -from virtualenv.util import Path +from virtualenv.util.path import Path @pytest.fixture(scope="session") diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index ab8c28159..c689c76bd 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -12,7 +12,7 @@ import six from virtualenv.run import run_via_cli -from virtualenv.util import Path +from virtualenv.util.path import Path from virtualenv.util.subprocess import Popen diff --git a/tests/unit/config/test_env_var.py b/tests/unit/config/test_env_var.py index 39c32c7e8..2caae9114 100644 --- a/tests/unit/config/test_env_var.py +++ b/tests/unit/config/test_env_var.py @@ -33,7 +33,7 @@ def _exc(of): def test_value_bad(monkeypatch, caplog, empty_conf): monkeypatch.setenv(str("VIRTUALENV_VERBOSE"), str("a")) result = parse_cli([]) - assert result.verbosity == 3 + assert result.verbosity == 2 msg = "env var VIRTUALENV_VERBOSE failed to convert 'a' as {!r} because {!r}".format(int, _exc("a")) # one for the core parse, one for the normal one assert caplog.messages == [msg], "{}{}".format(caplog.text, msg) diff --git a/tests/unit/interpreters/create/test_creator.py b/tests/unit/interpreters/create/test_creator.py index 4db7a0bfb..b26ec1536 100644 --- a/tests/unit/interpreters/create/test_creator.py +++ b/tests/unit/interpreters/create/test_creator.py @@ -14,7 +14,7 @@ from virtualenv.interpreters.discovery.py_info import CURRENT, PythonInfo from virtualenv.pyenv_cfg import PyEnvCfg from virtualenv.run import run_via_cli, session_via_cli -from virtualenv.util import Path +from virtualenv.util.path import Path def test_os_path_sep_not_allowed(tmp_path, capsys): @@ -78,7 +78,16 @@ def cleanup_sys_path(paths): ) def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env, special_name_dir): dest = special_name_dir - cmd = ["-v", "-v", "-p", six.ensure_text(python), six.ensure_text(str(dest)), "--without-pip", "--activators", ""] + cmd = [ + "-v", + "-v", + "-p", + six.ensure_text(python), + six.ensure_text(str(dest)), + "--without-pip", + "--activators", + "", + ] if global_access: cmd.append("--system-site-packages") if use_venv: diff --git a/tests/unit/interpreters/discovery/windows/test_windows_pep514.py b/tests/unit/interpreters/discovery/windows/test_windows_pep514.py index 0a2807785..604d0a980 100644 --- a/tests/unit/interpreters/discovery/windows/test_windows_pep514.py +++ b/tests/unit/interpreters/discovery/windows/test_windows_pep514.py @@ -8,7 +8,7 @@ import pytest import six -from virtualenv.util import Path +from virtualenv.util.path import Path @pytest.mark.skipif(sys.platform != "win32", reason="Windows registry only on Windows platform") diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index f11748e18..95d50ebf0 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -1,5 +1,9 @@ +from __future__ import absolute_import, unicode_literals + import pytest +import six +from virtualenv import __version__ from virtualenv.run import run_via_cli @@ -11,3 +15,19 @@ def test_help(capsys): out, err = capsys.readouterr() assert not err assert out + + +def test_version(capsys): + with pytest.raises(SystemExit) as context: + run_via_cli(args=["--version"]) + assert context.value.code == 0 + + out, err = capsys.readouterr() + extra = out if six.PY2 else err + content = out if six.PY3 else err + assert not extra + + assert __version__ in content + import virtualenv + + assert virtualenv.__file__ in content diff --git a/tests/unit/test_util.py b/tests/unit/test_util.py index 3ee63991d..7eb3dc8f4 100644 --- a/tests/unit/test_util.py +++ b/tests/unit/test_util.py @@ -5,7 +5,8 @@ import pytest -from virtualenv.util import run_cmd, symlink_or_copy +from virtualenv.util.path import symlink_or_copy +from virtualenv.util.subprocess import run_cmd @pytest.mark.skipif(not hasattr(os, "symlink"), reason="requires symlink support")