diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 761df8692..5ab9f21f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,11 +58,21 @@ jobs: dev: null before: - script: 'sudo apt-get update -y && sudo apt-get install fish csh' - condition: and(succeeded(), eq(variables['image_name'], 'linux'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py34', 'py27')) + condition: and(succeeded(), eq(variables['image_name'], 'linux'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py27')) displayName: install fish and csh via apt-get - script: 'brew update -vvv && brew install fish tcsh' - condition: and(succeeded(), eq(variables['image_name'], 'macOs'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py34', 'py27')) + condition: and(succeeded(), eq(variables['image_name'], 'macOs'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py27')) displayName: install fish and csh via brew + - task: UsePythonVersion@0 + condition: and(succeeded(), in(variables['TOXENV'], 'py27')) + displayName: provision python 3 + inputs: + versionSpec: '3.8' + - task: UsePythonVersion@0 + condition: and(succeeded(), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35')) + displayName: provision python 2 + inputs: + versionSpec: '2.7' coverage: with_toxenv: 'coverage' # generate .tox/.coverage, .tox/coverage.xml after test run for_envs: [py38, py37, py36, py35, py27] diff --git a/src/virtualenv/interpreters/discovery/builtin.py b/src/virtualenv/interpreters/discovery/builtin.py index b86f11e1f..1dc80c030 100644 --- a/src/virtualenv/interpreters/discovery/builtin.py +++ b/src/virtualenv/interpreters/discovery/builtin.py @@ -1,8 +1,10 @@ from __future__ import absolute_import, unicode_literals +import logging import os import sys -from distutils.spawn import find_executable + +from pathlib2 import Path from virtualenv.info import IS_WIN @@ -36,16 +38,21 @@ def __str__(self): def get_interpreter(key): spec = PythonSpec.from_string_spec(key) + logging.debug("find interpreter for spec %r", spec) + proposed_paths = set() for interpreter, impl_must_match in propose_interpreters(spec): - if interpreter.satisfies(spec, impl_must_match): - return interpreter + if interpreter.executable not in proposed_paths: + logging.debug("proposed %s", interpreter) + if interpreter.satisfies(spec, impl_must_match): + return interpreter + proposed_paths.add(interpreter.executable) def propose_interpreters(spec): # 1. we always try with the lowest hanging fruit first, the current interpreter yield CURRENT, True - # 2. if it's an absolut path and exists, use that + # 2. if it's an absolute path and exists, use that if spec.is_abs and os.path.exists(spec.path): yield PythonInfo.from_exe(spec.path), True @@ -56,21 +63,75 @@ def propose_interpreters(spec): for interpreter in propose_interpreters(spec): yield interpreter, True - # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts - interpreter = find_on_path(spec.str_spec) - if interpreter is not None: - yield interpreter, False + paths = get_paths() + for path in paths: # find on path, the path order matters (as the candidates are less easy to control by end user) + for candidate, match in possible_specs(spec): + found = check_path(candidate, path) + if found is not None: + exe = os.path.abspath(found) + interpreter = PathPythonInfo.from_exe(exe, raise_on_error=False) + if interpreter is not None: + yield interpreter, match + + +def get_paths(): + path = os.environ.get(str("PATH"), None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + path = os.defpath + if not path: + paths = [] + else: + paths = [p for p in path.split(os.pathsep) if os.path.exists(p)] + logging.debug(LazyPathDump(paths)) + return paths + + +class LazyPathDump(object): + def __init__(self, paths): + self.paths = paths + def __str__(self): + content = "PATH =>{}".format(os.linesep) + for i, p in enumerate(self.paths): + files = [] + for file in Path(p).iterdir(): + try: + if file.is_dir(): + continue + except OSError: + pass + files.append(file.name) + content += str(i) + content += " " + content += str(p) + content += " with " + content += " ".join(files) + content += os.linesep + return content + + +def check_path(candidate, path): + _, ext = os.path.splitext(candidate) + if sys.platform == "win32" and ext != ".exe": + candidate = candidate + ".exe" + if os.path.isfile(candidate): + return candidate + candidate = os.path.join(path, candidate) + if os.path.isfile(candidate): + return candidate + return None + + +def possible_specs(spec): + # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts + yield spec.str_spec, False # 5. or from the spec we can deduce a name on path that matches for exe, match in spec.generate_names(): - interpreter = find_on_path(exe) - if interpreter is not None: - yield interpreter, match + yield exe, match -def find_on_path(key): - exe = find_executable(key) - if exe is not None: - exe = os.path.abspath(exe) - interpreter = PythonInfo.from_exe(str(exe), raise_on_error=False) - return interpreter +class PathPythonInfo(PythonInfo): + """""" diff --git a/src/virtualenv/interpreters/discovery/py_info.py b/src/virtualenv/interpreters/discovery/py_info.py index 824a2be14..2af239412 100644 --- a/src/virtualenv/interpreters/discovery/py_info.py +++ b/src/virtualenv/interpreters/discovery/py_info.py @@ -84,13 +84,34 @@ def is_venv(self): return self.base_prefix is not None and self.version_info.major == 3 def __repr__(self): - return "PythonInfo({!r})".format(self.__dict__) + return "{}({!r})".format(self.__class__.__name__, self.__dict__) def __str__(self): - content = copy.copy(self.__dict__) - for elem in ["path", "prefix", "base_prefix", "exec_prefix", "real_prefix", "base_exec_prefix"]: - del content[elem] - return "PythonInfo({!r})".format(content) + return "{}({})".format( + self.__class__.__name__, + ", ".join( + "{}={}".format(k, v) + for k, v in ( + ( + "spec", + "{}{}-{}".format( + self.implementation, ".".join(str(i) for i in self.version_info), self.architecture + ), + ), + ("exe", self.executable), + ("original" if self.original_executable != self.executable else None, self.original_executable), + ( + "base" + if self.base_executable is not None and self.base_executable != self.executable + else None, + self.base_executable, + ), + ("platform", self.platform), + ("version", repr(self.version)), + ) + if k is not None + ), + ) def to_json(self): data = copy.deepcopy(self.__dict__) @@ -117,12 +138,15 @@ def system_exec_prefix(self): @property def system_executable(self): env_prefix = self.real_prefix or self.base_prefix - if env_prefix: - if self.real_prefix is None and self.base_executable is not None: + if env_prefix: # if this is a virtual environment + if self.real_prefix is None and self.base_executable is not None: # use the saved host if present return self.base_executable + # otherwise fallback to discovery mechanism return self.find_exe_based_of(inside_folder=env_prefix) else: - return self.executable + # need original executable here, as if we need to copy we want to copy the interpreter itself, not the + # setup script things may be wrapped up in + return self.original_executable def find_exe_based_of(self, inside_folder): # we don't know explicitly here, do some guess work - our executable name should tell @@ -162,16 +186,16 @@ def _find_possible_exe_names(self): name_candidate[candidate] = None return list(name_candidate.keys()) - __cache_from_exe = {} + _cache_from_exe = {} @classmethod def from_exe(cls, exe, raise_on_error=True): key = os.path.realpath(exe) - if key in cls.__cache_from_exe: - result, failure = cls.__cache_from_exe[key] + if key in cls._cache_from_exe: + result, failure = cls._cache_from_exe[key] else: failure, result = cls._load_for_exe(exe) - cls.__cache_from_exe[key] = result, failure + cls._cache_from_exe[key] = result, failure if failure is not None: if raise_on_error: raise failure diff --git a/src/virtualenv/interpreters/discovery/py_spec.py b/src/virtualenv/interpreters/discovery/py_spec.py index 382bf2e9f..ab0849140 100644 --- a/src/virtualenv/interpreters/discovery/py_spec.py +++ b/src/virtualenv/interpreters/discovery/py_spec.py @@ -6,7 +6,7 @@ import sys from collections import OrderedDict -PATTERN = re.compile(r"^(?P[a-zA-Z]+)(?P[0-9.]+)?(?:-(?P32|64))?$") +PATTERN = re.compile(r"^(?P[a-zA-Z]+)?(?P[0-9.]+)?(?:-(?P32|64))?$") IS_WIN = sys.platform == "win32" diff --git a/src/virtualenv/interpreters/discovery/windows/__init__.py b/src/virtualenv/interpreters/discovery/windows/__init__.py index 42ec22168..f321d7f82 100644 --- a/src/virtualenv/interpreters/discovery/windows/__init__.py +++ b/src/virtualenv/interpreters/discovery/windows/__init__.py @@ -5,13 +5,17 @@ from .pep514 import discover_pythons +class Pep514PythonInfo(PythonInfo): + """""" + + def propose_interpreters(spec): # see if PEP-514 entries are good for name, major, minor, arch, exe, _ in discover_pythons(): # pre-filter registry_spec = PythonSpec(None, name, major, minor, None, arch, exe) if registry_spec.satisfies(spec): - interpreter = PythonInfo.from_exe(exe, raise_on_error=False) + interpreter = Pep514PythonInfo.from_exe(exe, raise_on_error=False) if interpreter is not None: if interpreter.satisfies(spec, impl_must_match=True): yield interpreter diff --git a/src/virtualenv/run.py b/src/virtualenv/run.py index 96ad7d708..6dad18d64 100644 --- a/src/virtualenv/run.py +++ b/src/virtualenv/run.py @@ -26,6 +26,7 @@ def session_via_cli(args): options, verbosity = _do_report_setup(parser, args) discover = _get_discover(parser, args, options) interpreter = discover.interpreter + logging.debug("target interpreter %r", interpreter) if interpreter is None: raise RuntimeError("failed to find interpreter for {}".format(discover)) elements = [ @@ -83,8 +84,9 @@ def _get_creator(interpreter, parser, options): creator_parser = parser.add_argument_group("creator options") creator_parser.add_argument( "--creator", - choices=list(creators.keys()), - default=next((c for c in creators if c != "venv"), None), + choices=list(creators), + # prefer the built-in venv if present, otherwise fallback to first defined type + default="venv" if "venv" in creators else next(iter(creators), None), required=False, help="create environment via", ) diff --git a/tests/conftest.py b/tests/conftest.py index bad2e166b..515141093 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,8 @@ import pytest from pathlib2 import Path +from virtualenv.interpreters.discovery.py_info import PythonInfo + @pytest.fixture(scope="session") def has_symlink_support(tmp_path_factory): @@ -79,6 +81,12 @@ def check_cwd_not_changed_by_test(): pytest.fail("tests changed cwd: {!r} => {!r}".format(old, new)) +@pytest.fixture(autouse=True) +def ensure_py_info_cache_empty(): + yield + PythonInfo._cache_from_exe.clear() + + @pytest.fixture(autouse=True) def clean_data_dir(tmp_path, monkeypatch): from virtualenv import info @@ -207,3 +215,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): clean() assert not self.cov_pth.exists() assert self._COV_FILE.exists() + + +@pytest.fixture(scope="session") +def is_inside_ci(): + yield "CI_RUN" in os.environ diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 42312741f..13eb7d369 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -172,16 +172,13 @@ def activation_python(tmp_path_factory): return session -IS_INSIDE_CI = "CI_RUN" in os.environ - - @pytest.fixture() -def activation_tester(activation_python, monkeypatch, tmp_path): +def activation_tester(activation_python, monkeypatch, tmp_path, is_inside_ci): def _tester(tester_class): tester = tester_class(activation_python) if not tester.of_class.supports(activation_python.creator.interpreter): pytest.skip("{} not supported on current environment".format(tester.of_class.__name__)) - version = tester.get_version(raise_on_fail=IS_INSIDE_CI) + version = tester.get_version(raise_on_fail=is_inside_ci) if not isinstance(version, six.string_types): pytest.skip(msg=six.text_type(version)) return tester(monkeypatch, tmp_path) diff --git a/tests/unit/interpreters/create/test_creator.py b/tests/unit/interpreters/create/test_creator.py index a85d158f5..5e35748d5 100644 --- a/tests/unit/interpreters/create/test_creator.py +++ b/tests/unit/interpreters/create/test_creator.py @@ -11,7 +11,8 @@ from virtualenv.__main__ import run from virtualenv.interpreters.create.creator import DEBUG_SCRIPT, get_env_debug_info -from virtualenv.interpreters.discovery.py_info import CURRENT +from virtualenv.interpreters.discovery.builtin import get_interpreter +from virtualenv.interpreters.discovery.py_info import CURRENT, PythonInfo from virtualenv.pyenv_cfg import PyEnvCfg from virtualenv.run import run_via_cli, session_via_cli @@ -169,7 +170,7 @@ def test_debug_bad_virtualenv(tmp_path): @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"]) def test_create_clear_resets(tmp_path, use_venv, clear): marker = tmp_path / "magic" - cmd = [str(tmp_path), "--without-pip"] + cmd = [str(tmp_path), "--seeder", "none"] if use_venv: cmd.extend(["--creator", "venv"]) run_via_cli(cmd) @@ -181,16 +182,15 @@ def test_create_clear_resets(tmp_path, use_venv, clear): assert marker.exists() is not clear -@pytest.mark.skip @pytest.mark.parametrize( "use_venv", [False, True] if six.PY3 else [False], ids=["no_venv", "venv"] if six.PY3 else ["no_venv"] ) @pytest.mark.parametrize("prompt", [None, "magic"]) def test_prompt_set(tmp_path, use_venv, prompt): - cmd = [str(tmp_path), "--without-pip"] + cmd = [str(tmp_path), "--seeder", "none"] if prompt is not None: cmd.extend(["--prompt", "magic"]) - if not use_venv: + if not use_venv and six.PY3: cmd.extend(["--creator", "venv"]) result = run_via_cli(cmd) @@ -202,3 +202,23 @@ def test_prompt_set(tmp_path, use_venv, prompt): if use_venv is False: assert "prompt" in cfg, list(cfg.content.keys()) assert cfg["prompt"] == actual_prompt + + +@pytest.fixture(scope="session") +def cross_python(is_inside_ci): + spec = "{}{}".format(CURRENT.implementation, 2 if CURRENT.version_info.major == 3 else 3) + interpreter = get_interpreter(spec) + if interpreter is None: + msg = "could not find {}".format(spec) + if is_inside_ci: + raise RuntimeError(msg) + pytest.skip(msg=msg) + yield interpreter + + +def test_cross_major(cross_python, coverage_env, tmp_path): + cmd = ["-v", "-v", "-p", str(cross_python.executable), str(tmp_path), "--seeder", "none", "--activators", ""] + result = run_via_cli(cmd) + coverage_env() + env = PythonInfo.from_exe(str(result.creator.exe)) + assert env.version_info.major != CURRENT.version_info.major diff --git a/tests/unit/interpreters/discovery/test_discovery.py b/tests/unit/interpreters/discovery/test_discovery.py index 4e9d928bb..4863252ef 100644 --- a/tests/unit/interpreters/discovery/test_discovery.py +++ b/tests/unit/interpreters/discovery/test_discovery.py @@ -11,13 +11,13 @@ @pytest.mark.skipif(sys.platform == "win32", reason="symlink is not guaranteed to work on windows") -@pytest.mark.parametrize("lower", [None, True, False]) -def test_discovery_via_path(tmp_path, monkeypatch, lower): +@pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) +def test_discovery_via_path(tmp_path, monkeypatch, case): core = "somethingVeryCryptic{}".format(".".join(str(i) for i in CURRENT.version_info[0:3])) name = "somethingVeryCryptic" - if lower is True: + if case == "lower": name = name.lower() - elif lower is False: + elif case == "upper": name = name.upper() exe_name = "{}{}{}".format(name, CURRENT.version_info.major, ".exe" if sys.platform == "win32" else "") executable = tmp_path / exe_name diff --git a/tests/unit/interpreters/discovery/test_py_spec.py b/tests/unit/interpreters/discovery/test_py_spec.py index cc10431bf..1bb5f946b 100644 --- a/tests/unit/interpreters/discovery/test_py_spec.py +++ b/tests/unit/interpreters/discovery/test_py_spec.py @@ -22,7 +22,7 @@ def test_bad_py_spec(): def test_py_spec_first_digit_only_major(): - spec = PythonSpec.from_string_spec("python278") + spec = PythonSpec.from_string_spec("278") assert spec.major == 2 assert spec.minor == 78