env: default to enabling pip/wheels/setuptools

For project virtual environments, default to enabling pip, setuptools
and wheel packages to retain existing stable behaviour to prevent
unexpected breakages caused by development environments making
assumptions of base package availability in virtual environments.

Poetry itself does not require the use of these packages and will
execute correctly within environments that do not have these packages.

This change retains the ability to manage these packages as direct
project dependency as introduced in python-poetry#2826. All poetry internal
execution of pip is retaining the use of the wheel embedded within
the virtualenv package used by poetry.

In cases where a one of these reserved packages are being managed as a
project dependency, the will be treated as any other project
dependency. Executing `poetry install --remove-untracked` will not
remove any of these reserved packages. However, `poetry add pip` and
`poetry remove pip` will trigger the update and removal of `pip`
respectively.

Relates-to: python-poetry#2826
Relates-to: python-poetry#3916

poetry/console/commands/debug/resolve.py

@@ -84,7 +84,14 @@ def handle(self) -> Optional[int]:
 
         pool = self.poetry.pool
 
-        solver = Solver(package, pool, Repository(), Repository(), self._io)
+        solver = Solver(
+            package,
+            pool,
+            Repository(),
+            Repository(),
+            self._io,
+            config=self.poetry.config,
+        )
 
         ops = solver.solve()
 
@@ -121,7 +128,9 @@ def handle(self) -> Optional[int]:
 
             pool.add_repository(locked_repository)
 
-            solver = Solver(package, pool, Repository(), Repository(), NullIO())
+            solver = Solver(
+                package, pool, Repository(), Repository(), NullIO(), poetry=self.poetry
+            )
             with solver.use_environment(env):
                 ops = solver.solve()
 

poetry/console/commands/show.py

@@ -90,6 +90,7 @@ def handle(self) -> Optional[int]:
             installed=Repository(),
             locked=locked_repo,
             io=NullIO(),
+            config=self.poetry.config,
         )
         solver.provider.load_deferred(False)
         with solver.use_environment(self.env):

poetry/installation/installer.py

@@ -74,6 +74,7 @@ def __init__(
             installed = self._get_installed()
 
         self._installed_repository = installed
+        self._poetry_config = config
 
     @property
     def executor(self) -> Executor:
@@ -209,6 +210,7 @@ def _do_refresh(self) -> int:
             locked_repository,
             locked_repository,
             self._io,
+            config=self._poetry_config,
         )
 
         ops = solver.solve(use_latest=[])
@@ -247,6 +249,7 @@ def _do_install(self, local_repo: Repository) -> int:
                 locked_repository,
                 self._io,
                 remove_untracked=self._remove_untracked,
+                config=self._poetry_config,
             )
 
             ops = solver.solve(use_latest=self._whitelist)
@@ -316,6 +319,7 @@ def _do_install(self, local_repo: Repository) -> int:
             locked_repository,
             NullIO(),
             remove_untracked=self._remove_untracked,
+            config=self._poetry_config,
         )
         # Everything is resolved at this point, so we no longer need
         # to load deferred dependencies (i.e. VCS, URL and path dependencies)

poetry/puzzle/solver.py

@@ -31,6 +31,7 @@
 
 
 if TYPE_CHECKING:
+    from poetry.config.config import Config
     from poetry.core.packages.dependency import Dependency
     from poetry.core.packages.directory_dependency import DirectoryDependency
     from poetry.core.packages.file_dependency import FileDependency
@@ -49,6 +50,7 @@ def __init__(
         io: IO,
         remove_untracked: bool = False,
         provider: Optional[Provider] = None,
+        config: Optional["Config"] = None,
     ):
         self._package = package
         self._pool = pool
@@ -63,10 +65,32 @@ def __init__(
         self._overrides = []
         self._remove_untracked = remove_untracked
 
+        self._poetry_config = config
+        self._preserved_package_names = None
+
     @property
     def provider(self) -> Provider:
         return self._provider
 
+    @property
+    def preserved_package_names(self):
+        if self._preserved_package_names is None:
+            self._preserved_package_names = {
+                self._package.name,
+                *Provider.UNSAFE_PACKAGES,
+            }
+
+            deps = {package.name for package in self._locked.packages}
+
+            # preserve pip/setuptools/wheel when not managed by poetry, this is so
+            # to avoid externally managed virtual environments causing unnecessary
+            # removals.
+            for name in {"pip", "wheel", "setuptools"}:
+                if name not in deps:
+                    self._preserved_package_names.add(name)
+
+        return self._preserved_package_names
+
     @contextmanager
     def use_environment(self, env: Env) -> None:
         with self.provider.use_environment(env):
@@ -190,11 +214,9 @@ def solve(self, use_latest: List[str] = None) -> List["OperationTypes"]:
             locked_names = {locked.name for locked in self._locked.packages}
 
             for installed in self._installed.packages:
-                if installed.name == self._package.name:
-                    continue
-                if installed.name in Provider.UNSAFE_PACKAGES:
-                    # Never remove pip, setuptools etc.
+                if installed.name in self.preserved_package_names:
                     continue
+
                 if installed.name not in locked_names:
                     operations.append(Uninstall(installed))
 

poetry/utils/env.py

@@ -877,13 +877,8 @@ def create_venv(
             io.write_line(
                 "Creating virtualenv <c1>{}</> in {}".format(name, str(venv_path))
             )
-
-            self.build_venv(
-                venv,
-                executable=executable,
-                flags=self._poetry.config.get("virtualenvs.options"),
-            )
         else:
+            create_venv = False
             if force:
                 if not env.is_sane():
                     io.write_line(
@@ -895,14 +890,23 @@ def create_venv(
                     "Recreating virtualenv <c1>{}</> in {}".format(name, str(venv))
                 )
                 self.remove_venv(venv)
-                self.build_venv(
-                    venv,
-                    executable=executable,
-                    flags=self._poetry.config.get("virtualenvs.options"),
-                )
+                create_venv = True
             elif io.is_very_verbose():
                 io.write_line(f"Virtualenv <c1>{name}</> already exists.")
 
+        if create_venv:
+            self.build_venv(
+                venv,
+                executable=executable,
+                flags=self._poetry.config.get("virtualenvs.options"),
+                # TODO: in a future version switch remove pip/setuptools/wheel
+                # poetry does not need them these exists today to not break developer
+                # environment assumptions
+                with_pip=True,
+                with_setuptools=True,
+                with_wheel=True,
+            )
+
         # venv detection:
         # stdlib venv may symlink sys.executable, so we can't use realpath.
         # but others can symlink *to* the venv Python,
@@ -927,12 +931,29 @@ def build_venv(
         path: Union[Path, str],
         executable: Optional[Union[str, Path]] = None,
         flags: Dict[str, bool] = None,
-        with_pip: bool = False,
+        with_pip: Optional[bool] = None,
         with_wheel: Optional[bool] = None,
         with_setuptools: Optional[bool] = None,
     ) -> virtualenv.run.session.Session:
         flags = flags or {}
 
+        flags["no-pip"] = (
+            not with_pip if with_pip is not None else flags.pop("no-pip", True)
+        )
+
+        flags["no-setuptools"] = (
+            not with_setuptools
+            if with_setuptools is not None
+            else flags.pop("no-setuptools", True)
+        )
+
+        # we want wheels to be enabled when pip is required and it has not been explicitly disabled
+        flags["no-wheel"] = (
+            not with_wheel
+            if with_wheel is not None
+            else flags.pop("no-wheel", flags["no-pip"])
+        )
+
         if isinstance(executable, Path):
             executable = executable.resolve().as_posix()
 
@@ -943,20 +964,6 @@ def build_venv(
             executable or sys.executable,
         ]
 
-        if not with_pip:
-            args.append("--no-pip")
-        else:
-            if with_wheel is None:
-                # we want wheels to be enabled when pip is required and it has
-                # not been explicitly disabled
-                with_wheel = True
-
-        if with_wheel is None or not with_wheel:
-            args.append("--no-wheel")
-
-        if with_setuptools is None or not with_setuptools:
-            args.append("--no-setuptools")
-
         for flag, value in flags.items():
             if value is True:
                 args.append(f"--{flag}")
@@ -1039,6 +1046,8 @@ def __init__(self, path: Path, base: Optional[Path] = None) -> None:
         self._platlib = None
         self._script_dirs = None
 
+        self._embedded_pip_path = None
+
     @property
     def path(self) -> Path:
         return self._path
@@ -1074,6 +1083,12 @@ def get_embedded_wheel(self, distribution):
             distribution, "{}.{}".format(self.version_info[0], self.version_info[1])
         ).path
 
+    @property
+    def pip_embedded(self) -> str:
+        if self._embedded_pip_path is None:
+            self._embedded_pip_path = str(self.get_embedded_wheel("pip") / "pip")
+        return self._embedded_pip_path
+
     @property
     def pip(self) -> str:
         """
@@ -1082,7 +1097,7 @@ def pip(self) -> str:
         # we do not use as_posix() here due to issues with windows pathlib2 implementation
         path = self._bin("pip")
         if not Path(path).exists():
-            return str(self.get_embedded_wheel("pip") / "pip")
+            return str(self.pip_embedded)
         return path
 
     @property
@@ -1187,7 +1202,7 @@ def get_python_implementation(self) -> str:
     def get_marker_env(self) -> Dict[str, Any]:
         raise NotImplementedError()
 
-    def get_pip_command(self) -> List[str]:
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
         raise NotImplementedError()
 
     def get_supported_tags(self) -> List[Tag]:
@@ -1208,16 +1223,20 @@ def is_sane(self) -> bool:
         """
         return True
 
-    def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]:
+    def get_command_from_bin(self, bin: str) -> List[str]:
         if bin == "pip":
-            return self.run_pip(*args, **kwargs)
+            # when pip is required we need to ensure that we fallback to
+            # embedded pip when pip is not available in the environment
+            return self.get_pip_command()
+
+        return [self._bin(bin)]
 
-        bin = self._bin(bin)
-        cmd = [bin] + list(args)
+    def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]:
+        cmd = self.get_command_from_bin(bin) + list(args)
         return self._run(cmd, **kwargs)
 
     def run_pip(self, *args: str, **kwargs: Any) -> Union[int, str]:
-        pip = self.get_pip_command()
+        pip = self.get_pip_command(embedded=True)
         cmd = pip + list(args)
         return self._run(cmd, **kwargs)
 
@@ -1260,17 +1279,13 @@ def _run(self, cmd: List[str], **kwargs: Any) -> Union[int, str]:
         return decode(output)
 
     def execute(self, bin: str, *args: str, **kwargs: Any) -> Optional[int]:
-        if bin == "pip":
-            return self.run_pip(*args, **kwargs)
-
-        bin = self._bin(bin)
+        command = self.get_command_from_bin(bin) + list(args)
         env = kwargs.pop("env", {k: v for k, v in os.environ.items()})
 
         if not self._is_windows:
-            args = [bin] + list(args)
-            return os.execvpe(bin, args, env=env)
+            return os.execvpe(command[0], command, env=env)
         else:
-            exe = subprocess.Popen([bin] + list(args), env=env, **kwargs)
+            exe = subprocess.Popen([command[0]] + command[1:], env=env, **kwargs)
             exe.communicate()
             return exe.returncode
 
@@ -1338,10 +1353,10 @@ def get_version_info(self) -> Tuple[int]:
     def get_python_implementation(self) -> str:
         return platform.python_implementation()
 
-    def get_pip_command(self) -> List[str]:
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
         # If we're not in a venv, assume the interpreter we're running on
         # has a pip and use that
-        return [sys.executable, self.pip]
+        return [sys.executable, self.pip_embedded if embedded else self.pip]
 
     def get_paths(self) -> Dict[str, str]:
         # We can't use sysconfig.get_paths() because
@@ -1445,10 +1460,10 @@ def get_version_info(self) -> Tuple[int]:
     def get_python_implementation(self) -> str:
         return self.marker_env["platform_python_implementation"]
 
-    def get_pip_command(self) -> List[str]:
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
         # We're in a virtualenv that is known to be sane,
         # so assume that we have a functional pip
-        return [self._bin("python"), self.pip]
+        return [self._bin("python"), self.pip_embedded if embedded else self.pip]
 
     def get_supported_tags(self) -> List[Tag]:
         file_path = Path(packaging.tags.__file__)
@@ -1560,8 +1575,8 @@ def __init__(
         self._execute = execute
         self.executed = []
 
-    def get_pip_command(self) -> List[str]:
-        return [self._bin("python"), self.pip]
+    def get_pip_command(self, embedded: bool = False) -> List[str]:
+        return [self._bin("python"), self.pip_embedded if embedded else self.pip]
 
     def _run(self, cmd: List[str], **kwargs: Any) -> int:
         self.executed.append(cmd)

poetry/utils/pip.py

@@ -53,7 +53,7 @@ def pip_install(
                 executable=environment.python, with_pip=True, with_setuptools=True
             ) as env:
                 return environment.run(
-                    env._bin("pip"),
+                    *env.get_pip_command(),
                     *args,
                     env={**os.environ, "PYTHONPATH": str(env.purelib)},
                 )

tests/console/commands/env/test_use.py

@@ -55,6 +55,9 @@ def test_activate_activates_non_existing_virtualenv_no_envs_file(
         venv_py37,
         executable="python3.7",
         flags={"always-copy": False, "system-site-packages": False},
+        with_pip=True,
+        with_setuptools=True,
+        with_wheel=True,
     )
 
     envs_file = TOMLFile(venv_cache / "envs.toml")

tests/inspection/test_info.py

@@ -217,7 +217,7 @@ def test_info_setup_missing_mandatory_should_trigger_pep517(
     except PackageInfoError:
         assert spy.call_count == 3
     else:
-        assert spy.call_count == 1
+        assert spy.call_count == 2
 
 
 def test_info_prefer_poetry_config_over_egg_info():