Cross version compatibility

azure-pipelines.yml

@@ -58,11 +58,21 @@ jobs:
       dev: null
     before:
       - script: 'sudo apt-get update -y && sudo apt-get install fish csh'
-        condition: and(succeeded(), eq(variables['image_name'], 'linux'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py34', 'py27'))
+        condition: and(succeeded(), eq(variables['image_name'], 'linux'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py27'))
         displayName: install fish and csh via apt-get
       - script: 'brew update -vvv && brew install fish tcsh'
-        condition: and(succeeded(), eq(variables['image_name'], 'macOs'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py34', 'py27'))
+        condition: and(succeeded(), eq(variables['image_name'], 'macOs'), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35', 'py27'))
         displayName: install fish and csh via brew
+      - task: UsePythonVersion@0
+        condition: and(succeeded(), in(variables['TOXENV'], 'py27'))
+        displayName: provision python 3
+        inputs:
+          versionSpec: '3.8'
+      - task: UsePythonVersion@0
+        condition: and(succeeded(), in(variables['TOXENV'], 'py38', 'py37', 'py36', 'py35'))
+        displayName: provision python 2
+        inputs:
+          versionSpec: '2.7'
     coverage:
       with_toxenv: 'coverage' # generate .tox/.coverage, .tox/coverage.xml after test run
       for_envs: [py38, py37, py36, py35, py27]

src/virtualenv/interpreters/discovery/builtin.py

@@ -1,8 +1,10 @@
 from __future__ import absolute_import, unicode_literals
 
+import logging
 import os
 import sys
-from distutils.spawn import find_executable
+
+from pathlib2 import Path
 
 from virtualenv.info import IS_WIN
 
@@ -36,16 +38,21 @@ def __str__(self):
 
 def get_interpreter(key):
     spec = PythonSpec.from_string_spec(key)
+    logging.debug("find interpreter for spec %r", spec)
+    proposed_paths = set()
     for interpreter, impl_must_match in propose_interpreters(spec):
-        if interpreter.satisfies(spec, impl_must_match):
-            return interpreter
+        if interpreter.executable not in proposed_paths:
+            logging.debug("proposed %s", interpreter)
+            if interpreter.satisfies(spec, impl_must_match):
+                return interpreter
+            proposed_paths.add(interpreter.executable)
 
 
 def propose_interpreters(spec):
     # 1. we always try with the lowest hanging fruit first, the current interpreter
     yield CURRENT, True
 
-    # 2. if it's an absolut path and exists, use that
+    # 2. if it's an absolute path and exists, use that
     if spec.is_abs and os.path.exists(spec.path):
         yield PythonInfo.from_exe(spec.path), True
 
@@ -56,21 +63,75 @@ def propose_interpreters(spec):
         for interpreter in propose_interpreters(spec):
             yield interpreter, True
 
-    # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts
-    interpreter = find_on_path(spec.str_spec)
-    if interpreter is not None:
-        yield interpreter, False
+    paths = get_paths()
+    for path in paths:  # find on path, the path order matters (as the candidates are less easy to control by end user)
+        for candidate, match in possible_specs(spec):
+            found = check_path(candidate, path)
+            if found is not None:
+                exe = os.path.abspath(found)
+                interpreter = PathPythonInfo.from_exe(exe, raise_on_error=False)
+                if interpreter is not None:
+                    yield interpreter, match
+
+
+def get_paths():
+    path = os.environ.get(str("PATH"), None)
+    if path is None:
+        try:
+            path = os.confstr("CS_PATH")
+        except (AttributeError, ValueError):
+            path = os.defpath
+    if not path:
+        paths = []
+    else:
+        paths = [p for p in path.split(os.pathsep) if os.path.exists(p)]
+    logging.debug(LazyPathDump(paths))
+    return paths
+
+
+class LazyPathDump(object):
+    def __init__(self, paths):
+        self.paths = paths
 
+    def __str__(self):
+        content = "PATH =>{}".format(os.linesep)
+        for i, p in enumerate(self.paths):
+            files = []
+            for file in Path(p).iterdir():
+                try:
+                    if file.is_dir():
+                        continue
+                except OSError:
+                    pass
+                files.append(file.name)
+            content += str(i)
+            content += " "
+            content += str(p)
+            content += " with "
+            content += " ".join(files)
+            content += os.linesep
+        return content
+
+
+def check_path(candidate, path):
+    _, ext = os.path.splitext(candidate)
+    if sys.platform == "win32" and ext != ".exe":
+        candidate = candidate + ".exe"
+    if os.path.isfile(candidate):
+        return candidate
+    candidate = os.path.join(path, candidate)
+    if os.path.isfile(candidate):
+        return candidate
+    return None
+
+
+def possible_specs(spec):
+    # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts
+    yield spec.str_spec, False
     # 5. or from the spec we can deduce a name on path  that matches
     for exe, match in spec.generate_names():
-        interpreter = find_on_path(exe)
-        if interpreter is not None:
-            yield interpreter, match
+        yield exe, match
 
 
-def find_on_path(key):
-    exe = find_executable(key)
-    if exe is not None:
-        exe = os.path.abspath(exe)
-        interpreter = PythonInfo.from_exe(str(exe), raise_on_error=False)
-        return interpreter
+class PathPythonInfo(PythonInfo):
+    """"""

src/virtualenv/interpreters/discovery/py_info.py

@@ -84,13 +84,34 @@ def is_venv(self):
         return self.base_prefix is not None and self.version_info.major == 3
 
     def __repr__(self):
-        return "PythonInfo({!r})".format(self.__dict__)
+        return "{}({!r})".format(self.__class__.__name__, self.__dict__)
 
     def __str__(self):
-        content = copy.copy(self.__dict__)
-        for elem in ["path", "prefix", "base_prefix", "exec_prefix", "real_prefix", "base_exec_prefix"]:
-            del content[elem]
-        return "PythonInfo({!r})".format(content)
+        return "{}({})".format(
+            self.__class__.__name__,
+            ", ".join(
+                "{}={}".format(k, v)
+                for k, v in (
+                    (
+                        "spec",
+                        "{}{}-{}".format(
+                            self.implementation, ".".join(str(i) for i in self.version_info), self.architecture
+                        ),
+                    ),
+                    ("exe", self.executable),
+                    ("original" if self.original_executable != self.executable else None, self.original_executable),
+                    (
+                        "base"
+                        if self.base_executable is not None and self.base_executable != self.executable
+                        else None,
+                        self.base_executable,
+                    ),
+                    ("platform", self.platform),
+                    ("version", repr(self.version)),
+                )
+                if k is not None
+            ),
+        )
 
     def to_json(self):
         data = copy.deepcopy(self.__dict__)
@@ -117,12 +138,15 @@ def system_exec_prefix(self):
     @property
     def system_executable(self):
         env_prefix = self.real_prefix or self.base_prefix
-        if env_prefix:
-            if self.real_prefix is None and self.base_executable is not None:
+        if env_prefix:  # if this is a virtual environment
+            if self.real_prefix is None and self.base_executable is not None:  # use the saved host if present
                 return self.base_executable
+            # otherwise fallback to discovery mechanism
             return self.find_exe_based_of(inside_folder=env_prefix)
         else:
-            return self.executable
+            # need original executable here, as if we need to copy we want to copy the interpreter itself, not the
+            # setup script things may be wrapped up in
+            return self.original_executable
 
     def find_exe_based_of(self, inside_folder):
         # we don't know explicitly here, do some guess work - our executable name should tell
@@ -162,16 +186,16 @@ def _find_possible_exe_names(self):
                         name_candidate[candidate] = None
         return list(name_candidate.keys())
 
-    __cache_from_exe = {}
+    _cache_from_exe = {}
 
     @classmethod
     def from_exe(cls, exe, raise_on_error=True):
         key = os.path.realpath(exe)
-        if key in cls.__cache_from_exe:
-            result, failure = cls.__cache_from_exe[key]
+        if key in cls._cache_from_exe:
+            result, failure = cls._cache_from_exe[key]
         else:
             failure, result = cls._load_for_exe(exe)
-            cls.__cache_from_exe[key] = result, failure
+            cls._cache_from_exe[key] = result, failure
         if failure is not None:
             if raise_on_error:
                 raise failure

src/virtualenv/interpreters/discovery/py_spec.py

@@ -6,7 +6,7 @@
 import sys
 from collections import OrderedDict
 
-PATTERN = re.compile(r"^(?P<impl>[a-zA-Z]+)(?P<version>[0-9.]+)?(?:-(?P<arch>32|64))?$")
+PATTERN = re.compile(r"^(?P<impl>[a-zA-Z]+)?(?P<version>[0-9.]+)?(?:-(?P<arch>32|64))?$")
 IS_WIN = sys.platform == "win32"
 
 

src/virtualenv/interpreters/discovery/windows/__init__.py

@@ -5,13 +5,17 @@
 from .pep514 import discover_pythons
 
 
+class Pep514PythonInfo(PythonInfo):
+    """"""
+
+
 def propose_interpreters(spec):
     # see if PEP-514 entries are good
     for name, major, minor, arch, exe, _ in discover_pythons():
         # pre-filter
         registry_spec = PythonSpec(None, name, major, minor, None, arch, exe)
         if registry_spec.satisfies(spec):
-            interpreter = PythonInfo.from_exe(exe, raise_on_error=False)
+            interpreter = Pep514PythonInfo.from_exe(exe, raise_on_error=False)
             if interpreter is not None:
                 if interpreter.satisfies(spec, impl_must_match=True):
                     yield interpreter

src/virtualenv/run.py

@@ -26,6 +26,7 @@ def session_via_cli(args):
     options, verbosity = _do_report_setup(parser, args)
     discover = _get_discover(parser, args, options)
     interpreter = discover.interpreter
+    logging.debug("target interpreter %r", interpreter)
     if interpreter is None:
         raise RuntimeError("failed to find interpreter for {}".format(discover))
     elements = [
@@ -83,8 +84,9 @@ def _get_creator(interpreter, parser, options):
     creator_parser = parser.add_argument_group("creator options")
     creator_parser.add_argument(
         "--creator",
-        choices=list(creators.keys()),
-        default=next((c for c in creators if c != "venv"), None),
+        choices=list(creators),
+        # prefer the built-in venv if present, otherwise fallback to first defined type
+        default="venv" if "venv" in creators else next(iter(creators), None),
         required=False,
         help="create environment via",
     )

tests/conftest.py

@@ -9,6 +9,8 @@
 import pytest
 from pathlib2 import Path
 
+from virtualenv.interpreters.discovery.py_info import PythonInfo
+
 
 @pytest.fixture(scope="session")
 def has_symlink_support(tmp_path_factory):
@@ -79,6 +81,12 @@ def check_cwd_not_changed_by_test():
         pytest.fail("tests changed cwd: {!r} => {!r}".format(old, new))
 
 
+@pytest.fixture(autouse=True)
+def ensure_py_info_cache_empty():
+    yield
+    PythonInfo._cache_from_exe.clear()
+
+
 @pytest.fixture(autouse=True)
 def clean_data_dir(tmp_path, monkeypatch):
     from virtualenv import info
@@ -207,3 +215,8 @@ def __exit__(self, exc_type, exc_val, exc_tb):
                 clean()
         assert not self.cov_pth.exists()
         assert self._COV_FILE.exists()
+
+
+@pytest.fixture(scope="session")
+def is_inside_ci():
+    yield "CI_RUN" in os.environ

tests/unit/activation/conftest.py

@@ -172,16 +172,13 @@ def activation_python(tmp_path_factory):
     return session
 
 
-IS_INSIDE_CI = "CI_RUN" in os.environ
-
-
 @pytest.fixture()
-def activation_tester(activation_python, monkeypatch, tmp_path):
+def activation_tester(activation_python, monkeypatch, tmp_path, is_inside_ci):
     def _tester(tester_class):
         tester = tester_class(activation_python)
         if not tester.of_class.supports(activation_python.creator.interpreter):
             pytest.skip("{} not supported on current environment".format(tester.of_class.__name__))
-        version = tester.get_version(raise_on_fail=IS_INSIDE_CI)
+        version = tester.get_version(raise_on_fail=is_inside_ci)
         if not isinstance(version, six.string_types):
             pytest.skip(msg=six.text_type(version))
         return tester(monkeypatch, tmp_path)

tests/unit/interpreters/create/test_creator.py

@@ -11,7 +11,8 @@
 
 from virtualenv.__main__ import run
 from virtualenv.interpreters.create.creator import DEBUG_SCRIPT, get_env_debug_info
-from virtualenv.interpreters.discovery.py_info import CURRENT
+from virtualenv.interpreters.discovery.builtin import get_interpreter
+from virtualenv.interpreters.discovery.py_info import CURRENT, PythonInfo
 from virtualenv.pyenv_cfg import PyEnvCfg
 from virtualenv.run import run_via_cli, session_via_cli
 
@@ -169,7 +170,7 @@ def test_debug_bad_virtualenv(tmp_path):
 @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"])
 def test_create_clear_resets(tmp_path, use_venv, clear):
     marker = tmp_path / "magic"
-    cmd = [str(tmp_path), "--without-pip"]
+    cmd = [str(tmp_path), "--seeder", "none"]
     if use_venv:
         cmd.extend(["--creator", "venv"])
     run_via_cli(cmd)
@@ -181,16 +182,15 @@ def test_create_clear_resets(tmp_path, use_venv, clear):
     assert marker.exists() is not clear
 
 
-@pytest.mark.skip
 @pytest.mark.parametrize(
     "use_venv", [False, True] if six.PY3 else [False], ids=["no_venv", "venv"] if six.PY3 else ["no_venv"]
 )
 @pytest.mark.parametrize("prompt", [None, "magic"])
 def test_prompt_set(tmp_path, use_venv, prompt):
-    cmd = [str(tmp_path), "--without-pip"]
+    cmd = [str(tmp_path), "--seeder", "none"]
     if prompt is not None:
         cmd.extend(["--prompt", "magic"])
-    if not use_venv:
+    if not use_venv and six.PY3:
         cmd.extend(["--creator", "venv"])
 
     result = run_via_cli(cmd)
@@ -202,3 +202,23 @@ def test_prompt_set(tmp_path, use_venv, prompt):
         if use_venv is False:
             assert "prompt" in cfg, list(cfg.content.keys())
             assert cfg["prompt"] == actual_prompt
+
+
+@pytest.fixture(scope="session")
+def cross_python(is_inside_ci):
+    spec = "{}{}".format(CURRENT.implementation, 2 if CURRENT.version_info.major == 3 else 3)
+    interpreter = get_interpreter(spec)
+    if interpreter is None:
+        msg = "could not find {}".format(spec)
+        if is_inside_ci:
+            raise RuntimeError(msg)
+        pytest.skip(msg=msg)
+    yield interpreter
+
+
+def test_cross_major(cross_python, coverage_env, tmp_path):
+    cmd = ["-v", "-v", "-p", str(cross_python.executable), str(tmp_path), "--seeder", "none", "--activators", ""]
+    result = run_via_cli(cmd)
+    coverage_env()
+    env = PythonInfo.from_exe(str(result.creator.exe))
+    assert env.version_info.major != CURRENT.version_info.major