From 341f04e33d7416ea80216f8663cfe3760343f1da Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Apr 2021 20:08:06 +0200 Subject: [PATCH 1/6] tests: use env.python instead of env._bin("python") --- tests/masonry/builders/test_editable_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/masonry/builders/test_editable_builder.py b/tests/masonry/builders/test_editable_builder.py index 86ae73ece5b..6fb0a07a445 100644 --- a/tests/masonry/builders/test_editable_builder.py +++ b/tests/masonry/builders/test_editable_builder.py @@ -153,7 +153,7 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_ if __name__ == '__main__': baz.boom.bim() """.format( - python=tmp_venv._bin("python") + python=tmp_venv.python ) assert baz_script == tmp_venv._bin_dir.joinpath("baz").read_text() @@ -165,7 +165,7 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_ if __name__ == '__main__': bar() """.format( - python=tmp_venv._bin("python") + python=tmp_venv.python ) assert foo_script == tmp_venv._bin_dir.joinpath("foo").read_text() @@ -177,7 +177,7 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_ if __name__ == '__main__': bar.baz() """.format( - python=tmp_venv._bin("python") + python=tmp_venv.python ) assert fox_script == tmp_venv._bin_dir.joinpath("fox").read_text() From e25d7845d081ba22fda51df59437624b0b7bb825 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Apr 2021 20:12:10 +0200 Subject: [PATCH 2/6] env: do not modify os.environ Replace updates of os.environ with explicit passing of `env` to subprocess calls in `Env.execute()`. Relates-to: #3199 --- poetry/utils/env.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index ba9519cbc2e..8d5fa861c1b 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -1230,6 +1230,7 @@ def _run(self, cmd: List[str], **kwargs: Any) -> Union[int, str]: """ call = kwargs.pop("call", False) input_ = kwargs.pop("input_", None) + env = kwargs.pop("env", {k: v for k, v in os.environ.items()}) try: if self._is_windows: @@ -1248,10 +1249,10 @@ def _run(self, cmd: List[str], **kwargs: Any) -> Union[int, str]: **kwargs, ).stdout elif call: - return subprocess.call(cmd, stderr=subprocess.STDOUT, **kwargs) + return subprocess.call(cmd, stderr=subprocess.STDOUT, env=env, **kwargs) else: output = subprocess.check_output( - cmd, stderr=subprocess.STDOUT, **kwargs + cmd, stderr=subprocess.STDOUT, env=env, **kwargs ) except CalledProcessError as e: raise EnvCommandError(e, input=input_) From 92d92bb4b3ab36708f4d65fbfdb37b16f519367c Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Wed, 28 Apr 2021 01:56:03 +0200 Subject: [PATCH 3/6] tests: ensure ephemeral config usage Previously, pytest execution was influenced by poetry user configuration. This change ensures that a new config.toml is used each test case. --- tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 9481c31457b..d66ed9ba659 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,6 +99,15 @@ def config(config_source, auth_config_source, mocker): return c +@pytest.fixture(autouse=True) +def mock_user_config_dir(mocker): + config_dir = tempfile.mkdtemp(prefix="poetry_config_") + mocker.patch("poetry.locations.CONFIG_DIR", new=config_dir) + mocker.patch("poetry.factory.CONFIG_DIR", new=config_dir) + yield + shutil.rmtree(config_dir, ignore_errors=True) + + @pytest.fixture(autouse=True) def download_mock(mocker): # Patch download to not download anything but to just copy from fixtures From ccf1322129e832e8293ac532656aed9a716d2f33 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 30 Apr 2021 01:02:26 +0200 Subject: [PATCH 4/6] executor: ensure path is used when generating hash --- poetry/installation/executor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 146efeb2be0..0a3f8d58a9e 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -670,7 +670,13 @@ def _download_link(self, operation: Union[Install, Update], link: Link) -> Link: archive = self._chef.prepare(archive) if package.files: - archive_hash = "sha256:" + FileDependency(package.name, archive).hash() + archive_hash = ( + "sha256:" + + FileDependency( + package.name, + Path(archive.path) if isinstance(archive, Link) else archive, + ).hash() + ) if archive_hash not in {f["hash"] for f in package.files}: raise RuntimeError( f"Invalid hash for {package} using archive {archive.name}" From 3dceee386a3c616d23b301d8ebe9adee30c6f172 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 30 Apr 2021 01:03:28 +0200 Subject: [PATCH 5/6] upgrade dependencies --- poetry.lock | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/poetry.lock b/poetry.lock index ded07f4134e..d187aaf4f1e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -161,7 +161,7 @@ python-versions = ">=3.6, <3.7" [[package]] name = "deepdiff" -version = "5.2.3" +version = "5.5.0" description = "Deep Difference and Search of any Python object/data." category = "dev" optional = false @@ -171,7 +171,7 @@ python-versions = ">=3.6" ordered-set = "4.0.2" [package.extras] -cli = ["click (==7.1.2)", "pyyaml (==5.3.1)", "toml (==0.10.2)", "clevercsv (==0.6.6)"] +cli = ["click (==7.1.2)", "pyyaml (==5.4)", "toml (==0.10.2)", "clevercsv (==0.6.7)"] [[package]] name = "distlib" @@ -225,7 +225,7 @@ python-versions = ">=3" [[package]] name = "identify" -version = "2.2.1" +version = "2.2.4" description = "File identification library for Python" category = "dev" optional = false @@ -327,7 +327,7 @@ python-versions = "*" [[package]] name = "nodeenv" -version = "1.5.0" +version = "1.6.0" description = "Node.js virtual environment builder" category = "dev" optional = false @@ -390,7 +390,7 @@ dev = ["pre-commit", "tox"] [[package]] name = "poetry-core" -version = "1.1.0a2" +version = "1.1.0a3" description = "Poetry PEP 517 Build Backend" category = "main" optional = false @@ -398,18 +398,18 @@ python-versions = "^3.6" develop = false [package.dependencies] -dataclasses = {version = "^0.8", markers = "python_version >= \"3.6\" and python_version < \"3.7\""} -importlib-metadata = {version = "^1.7.0", markers = "python_version >= \"3.5\" and python_version < \"3.8\""} +dataclasses = {version = ">=0.8", markers = "python_version >= \"3.6\" and python_version < \"3.7\""} +importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} [package.source] type = "git" url = "https://github.com/python-poetry/poetry-core.git" reference = "master" -resolved_reference = "d3e60732ce9bd4f30dee3e594405fe6a80163b7e" +resolved_reference = "3f718c55fcda63d9bd88b8fc612970c24fc9af25" [[package]] name = "pre-commit" -version = "2.11.1" +version = "2.12.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false @@ -665,7 +665,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.4.3" +version = "20.4.4" description = "Virtual Python Environment builder" category = "main" optional = false @@ -873,8 +873,8 @@ dataclasses = [ {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, ] deepdiff = [ - {file = "deepdiff-5.2.3-py3-none-any.whl", hash = "sha256:3d3da4bd7e01fb5202088658ed26427104c748dda56a0ecfac9ce9a1d2d00844"}, - {file = "deepdiff-5.2.3.tar.gz", hash = "sha256:ae2cb98353309f93fbfdda4d77adb08fb303314d836bb6eac3d02ed71a10b40e"}, + {file = "deepdiff-5.5.0-py3-none-any.whl", hash = "sha256:e054fed9dfe0d83d622921cbb3a3d0b3a6dd76acd2b6955433a0a2d35147774a"}, + {file = "deepdiff-5.5.0.tar.gz", hash = "sha256:dd79b81c2d84bfa33aa9d94d456b037b68daff6bb87b80dfaa1eca04da68b349"}, ] distlib = [ {file = "distlib-0.3.1-py2.py3-none-any.whl", hash = "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb"}, @@ -896,8 +896,8 @@ httpretty = [ {file = "httpretty-1.0.5.tar.gz", hash = "sha256:e53c927c4d3d781a0761727f1edfad64abef94e828718e12b672a678a8b3e0b5"}, ] identify = [ - {file = "identify-2.2.1-py2.py3-none-any.whl", hash = "sha256:9cc5f58996cd359b7b72f0a5917d8639de5323917e6952a3bfbf36301b576f40"}, - {file = "identify-2.2.1.tar.gz", hash = "sha256:1cfb05b578de996677836d5a2dde14b3dffde313cf7d2b3e793a0787a36e26dd"}, + {file = "identify-2.2.4-py2.py3-none-any.whl", hash = "sha256:ad9f3fa0c2316618dc4d840f627d474ab6de106392a4f00221820200f490f5a8"}, + {file = "identify-2.2.4.tar.gz", hash = "sha256:9bcc312d4e2fa96c7abebcdfb1119563b511b5e3985ac52f60d9116277865b2e"}, ] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, @@ -958,8 +958,8 @@ msgpack = [ {file = "msgpack-1.0.2.tar.gz", hash = "sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984"}, ] nodeenv = [ - {file = "nodeenv-1.5.0-py2.py3-none-any.whl", hash = "sha256:5304d424c529c997bc888453aeaa6362d242b6b4631e90f3d4bf1b290f1c84a9"}, - {file = "nodeenv-1.5.0.tar.gz", hash = "sha256:ab45090ae383b716c4ef89e690c41ff8c2b257b85b309f01f3654df3d084bd7c"}, + {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, + {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, ] ordered-set = [ {file = "ordered-set-4.0.2.tar.gz", hash = "sha256:ba93b2df055bca202116ec44b9bead3df33ea63a7d5827ff8e16738b97f33a95"}, @@ -982,8 +982,8 @@ pluggy = [ ] poetry-core = [] pre-commit = [ - {file = "pre_commit-2.11.1-py2.py3-none-any.whl", hash = "sha256:94c82f1bf5899d56edb1d926732f4e75a7df29a0c8c092559c77420c9d62428b"}, - {file = "pre_commit-2.11.1.tar.gz", hash = "sha256:de55c5c72ce80d79106e48beb1b54104d16495ce7f95b0c7b13d4784193a00af"}, + {file = "pre_commit-2.12.1-py2.py3-none-any.whl", hash = "sha256:70c5ec1f30406250b706eda35e868b87e3e4ba099af8787e3e8b4b01e84f4712"}, + {file = "pre_commit-2.12.1.tar.gz", hash = "sha256:900d3c7e1bf4cf0374bb2893c24c23304952181405b4d88c9c40b72bda1bb8a9"}, ] ptyprocess = [ {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, @@ -1087,8 +1087,8 @@ urllib3 = [ {file = "urllib3-1.25.10.tar.gz", hash = "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a"}, ] virtualenv = [ - {file = "virtualenv-20.4.3-py2.py3-none-any.whl", hash = "sha256:83f95875d382c7abafe06bd2a4cdd1b363e1bb77e02f155ebe8ac082a916b37c"}, - {file = "virtualenv-20.4.3.tar.gz", hash = "sha256:49ec4eb4c224c6f7dd81bb6d0a28a09ecae5894f4e593c89b0db0885f565a107"}, + {file = "virtualenv-20.4.4-py2.py3-none-any.whl", hash = "sha256:a935126db63128861987a7d5d30e23e8ec045a73840eeccb467c148514e29535"}, + {file = "virtualenv-20.4.4.tar.gz", hash = "sha256:09c61377ef072f43568207dc8e46ddeac6bcdcaf288d49011bda0e7f4d38c4a2"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, From f0408d61c55f43add7a6ce78ff63f8dd2ee34126 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 30 Apr 2021 17:08:47 +0200 Subject: [PATCH 6/6] env: default to enabling pip/wheels/setuptools For project virtual environments, default to enabling pip, setuptools and wheel packages to retain existing stable behaviour to prevent unexpected breakages caused by development environments making assumptions of base package availability in virtual environments. Poetry itself does not require the use of these packages and will execute correctly within environments that do not have these packages. This change retains the ability to manage these packages as direct project dependency as introduced in #2826. All poetry internal execution of pip is retaining the use of the wheel embedded within the virtualenv package used by poetry. In cases where a one of these reserved packages are being managed as a project dependency, the will be treated as any other project dependency. Executing `poetry install --remove-untracked` will not remove any of these reserved packages. However, `poetry add pip` and `poetry remove pip` will trigger the update and removal of `pip` respectively. Relates-to: #2826 Relates-to: #3916 --- poetry/puzzle/solver.py | 27 +++++- poetry/utils/env.py | 107 +++++++++++++---------- poetry/utils/pip.py | 2 +- tests/console/commands/env/test_use.py | 3 + tests/inspection/test_info.py | 2 +- tests/installation/test_installer.py | 89 +++++++++++++------ tests/installation/test_installer_old.py | 83 ++++++++++++------ tests/utils/test_env.py | 30 ++++++- 8 files changed, 234 insertions(+), 109 deletions(-) diff --git a/poetry/puzzle/solver.py b/poetry/puzzle/solver.py index b5a413f15e0..2e864bc9b20 100644 --- a/poetry/puzzle/solver.py +++ b/poetry/puzzle/solver.py @@ -63,10 +63,31 @@ def __init__( self._overrides = [] self._remove_untracked = remove_untracked + self._preserved_package_names = None + @property def provider(self) -> Provider: return self._provider + @property + def preserved_package_names(self): + if self._preserved_package_names is None: + self._preserved_package_names = { + self._package.name, + *Provider.UNSAFE_PACKAGES, + } + + deps = {package.name for package in self._locked.packages} + + # preserve pip/setuptools/wheel when not managed by poetry, this is so + # to avoid externally managed virtual environments causing unnecessary + # removals. + for name in {"pip", "wheel", "setuptools"}: + if name not in deps: + self._preserved_package_names.add(name) + + return self._preserved_package_names + @contextmanager def use_environment(self, env: Env) -> None: with self.provider.use_environment(env): @@ -190,11 +211,9 @@ def solve(self, use_latest: List[str] = None) -> List["OperationTypes"]: locked_names = {locked.name for locked in self._locked.packages} for installed in self._installed.packages: - if installed.name == self._package.name: - continue - if installed.name in Provider.UNSAFE_PACKAGES: - # Never remove pip, setuptools etc. + if installed.name in self.preserved_package_names: continue + if installed.name not in locked_names: operations.append(Uninstall(installed)) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 8d5fa861c1b..195d08162de 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -877,13 +877,8 @@ def create_venv( io.write_line( "Creating virtualenv {} in {}".format(name, str(venv_path)) ) - - self.build_venv( - venv, - executable=executable, - flags=self._poetry.config.get("virtualenvs.options"), - ) else: + create_venv = False if force: if not env.is_sane(): io.write_line( @@ -895,14 +890,23 @@ def create_venv( "Recreating virtualenv {} in {}".format(name, str(venv)) ) self.remove_venv(venv) - self.build_venv( - venv, - executable=executable, - flags=self._poetry.config.get("virtualenvs.options"), - ) + create_venv = True elif io.is_very_verbose(): io.write_line(f"Virtualenv {name} already exists.") + if create_venv: + self.build_venv( + venv, + executable=executable, + flags=self._poetry.config.get("virtualenvs.options"), + # TODO: in a future version switch remove pip/setuptools/wheel + # poetry does not need them these exists today to not break developer + # environment assumptions + with_pip=True, + with_setuptools=True, + with_wheel=True, + ) + # venv detection: # stdlib venv may symlink sys.executable, so we can't use realpath. # but others can symlink *to* the venv Python, @@ -927,12 +931,29 @@ def build_venv( path: Union[Path, str], executable: Optional[Union[str, Path]] = None, flags: Dict[str, bool] = None, - with_pip: bool = False, + with_pip: Optional[bool] = None, with_wheel: Optional[bool] = None, with_setuptools: Optional[bool] = None, ) -> virtualenv.run.session.Session: flags = flags or {} + flags["no-pip"] = ( + not with_pip if with_pip is not None else flags.pop("no-pip", True) + ) + + flags["no-setuptools"] = ( + not with_setuptools + if with_setuptools is not None + else flags.pop("no-setuptools", True) + ) + + # we want wheels to be enabled when pip is required and it has not been explicitly disabled + flags["no-wheel"] = ( + not with_wheel + if with_wheel is not None + else flags.pop("no-wheel", flags["no-pip"]) + ) + if isinstance(executable, Path): executable = executable.resolve().as_posix() @@ -943,20 +964,6 @@ def build_venv( executable or sys.executable, ] - if not with_pip: - args.append("--no-pip") - else: - if with_wheel is None: - # we want wheels to be enabled when pip is required and it has - # not been explicitly disabled - with_wheel = True - - if with_wheel is None or not with_wheel: - args.append("--no-wheel") - - if with_setuptools is None or not with_setuptools: - args.append("--no-setuptools") - for flag, value in flags.items(): if value is True: args.append(f"--{flag}") @@ -1039,6 +1046,8 @@ def __init__(self, path: Path, base: Optional[Path] = None) -> None: self._platlib = None self._script_dirs = None + self._embedded_pip_path = None + @property def path(self) -> Path: return self._path @@ -1074,6 +1083,12 @@ def get_embedded_wheel(self, distribution): distribution, "{}.{}".format(self.version_info[0], self.version_info[1]) ).path + @property + def pip_embedded(self) -> str: + if self._embedded_pip_path is None: + self._embedded_pip_path = str(self.get_embedded_wheel("pip") / "pip") + return self._embedded_pip_path + @property def pip(self) -> str: """ @@ -1082,7 +1097,7 @@ def pip(self) -> str: # we do not use as_posix() here due to issues with windows pathlib2 implementation path = self._bin("pip") if not Path(path).exists(): - return str(self.get_embedded_wheel("pip") / "pip") + return str(self.pip_embedded) return path @property @@ -1187,7 +1202,7 @@ def get_python_implementation(self) -> str: def get_marker_env(self) -> Dict[str, Any]: raise NotImplementedError() - def get_pip_command(self) -> List[str]: + def get_pip_command(self, embedded: bool = False) -> List[str]: raise NotImplementedError() def get_supported_tags(self) -> List[Tag]: @@ -1208,16 +1223,20 @@ def is_sane(self) -> bool: """ return True - def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]: + def get_command_from_bin(self, bin: str) -> List[str]: if bin == "pip": - return self.run_pip(*args, **kwargs) + # when pip is required we need to ensure that we fallback to + # embedded pip when pip is not available in the environment + return self.get_pip_command() + + return [self._bin(bin)] - bin = self._bin(bin) - cmd = [bin] + list(args) + def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]: + cmd = self.get_command_from_bin(bin) + list(args) return self._run(cmd, **kwargs) def run_pip(self, *args: str, **kwargs: Any) -> Union[int, str]: - pip = self.get_pip_command() + pip = self.get_pip_command(embedded=True) cmd = pip + list(args) return self._run(cmd, **kwargs) @@ -1260,17 +1279,13 @@ def _run(self, cmd: List[str], **kwargs: Any) -> Union[int, str]: return decode(output) def execute(self, bin: str, *args: str, **kwargs: Any) -> Optional[int]: - if bin == "pip": - return self.run_pip(*args, **kwargs) - - bin = self._bin(bin) + command = self.get_command_from_bin(bin) + list(args) env = kwargs.pop("env", {k: v for k, v in os.environ.items()}) if not self._is_windows: - args = [bin] + list(args) - return os.execvpe(bin, args, env=env) + return os.execvpe(command[0], command, env=env) else: - exe = subprocess.Popen([bin] + list(args), env=env, **kwargs) + exe = subprocess.Popen([command[0]] + command[1:], env=env, **kwargs) exe.communicate() return exe.returncode @@ -1338,10 +1353,10 @@ def get_version_info(self) -> Tuple[int]: def get_python_implementation(self) -> str: return platform.python_implementation() - def get_pip_command(self) -> List[str]: + def get_pip_command(self, embedded: bool = False) -> List[str]: # If we're not in a venv, assume the interpreter we're running on # has a pip and use that - return [sys.executable, self.pip] + return [sys.executable, self.pip_embedded if embedded else self.pip] def get_paths(self) -> Dict[str, str]: # We can't use sysconfig.get_paths() because @@ -1445,10 +1460,10 @@ def get_version_info(self) -> Tuple[int]: def get_python_implementation(self) -> str: return self.marker_env["platform_python_implementation"] - def get_pip_command(self) -> List[str]: + def get_pip_command(self, embedded: bool = False) -> List[str]: # We're in a virtualenv that is known to be sane, # so assume that we have a functional pip - return [self._bin("python"), self.pip] + return [self._bin("python"), self.pip_embedded if embedded else self.pip] def get_supported_tags(self) -> List[Tag]: file_path = Path(packaging.tags.__file__) @@ -1560,8 +1575,8 @@ def __init__( self._execute = execute self.executed = [] - def get_pip_command(self) -> List[str]: - return [self._bin("python"), self.pip] + def get_pip_command(self, embedded: bool = False) -> List[str]: + return [self._bin("python"), self.pip_embedded if embedded else self.pip] def _run(self, cmd: List[str], **kwargs: Any) -> int: self.executed.append(cmd) diff --git a/poetry/utils/pip.py b/poetry/utils/pip.py index 5267cf435af..416c56b97ae 100644 --- a/poetry/utils/pip.py +++ b/poetry/utils/pip.py @@ -53,7 +53,7 @@ def pip_install( executable=environment.python, with_pip=True, with_setuptools=True ) as env: return environment.run( - env._bin("pip"), + *env.get_pip_command(), *args, env={**os.environ, "PYTHONPATH": str(env.purelib)}, ) diff --git a/tests/console/commands/env/test_use.py b/tests/console/commands/env/test_use.py index 93e96c02523..8b37c8e726e 100644 --- a/tests/console/commands/env/test_use.py +++ b/tests/console/commands/env/test_use.py @@ -55,6 +55,9 @@ def test_activate_activates_non_existing_virtualenv_no_envs_file( venv_py37, executable="python3.7", flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) envs_file = TOMLFile(venv_cache / "envs.toml") diff --git a/tests/inspection/test_info.py b/tests/inspection/test_info.py index 215109d551f..3a15e605a6f 100644 --- a/tests/inspection/test_info.py +++ b/tests/inspection/test_info.py @@ -217,7 +217,7 @@ def test_info_setup_missing_mandatory_should_trigger_pep517( except PackageInfoError: assert spy.call_count == 3 else: - assert spy.call_count == 1 + assert spy.call_count == 2 def test_info_prefer_poetry_config_over_egg_info(): diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index e9bc0e4e1c9..00986905984 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import itertools import json import sys @@ -35,6 +36,9 @@ from tests.repositories.test_pypi_repository import MockRepository +RESERVED_PACKAGES = ("pip", "setuptools", "wheel") + + class Installer(BaseInstaller): def _get_installer(self): return NoopInstaller() @@ -367,59 +371,88 @@ def test_run_install_no_dev_and_dev_only(installer, locker, repo, package, insta assert 1 == installer.executor.removals_count -def test_run_install_remove_untracked(installer, locker, repo, package, installed): +@pytest.mark.parametrize( + "managed_reserved_package_names", + [ + i + for i in itertools.chain( + [tuple()], + itertools.permutations(RESERVED_PACKAGES, 1), + itertools.permutations(RESERVED_PACKAGES, 2), + [RESERVED_PACKAGES], + ) + ], +) +def test_run_install_remove_untracked( + managed_reserved_package_names, installer, locker, repo, package, installed +): + package_a = get_package("a", "1.0") + package_b = get_package("b", "1.1") + package_c = get_package("c", "1.2") + package_pip = get_package("pip", "20.0.0") + package_setuptools = get_package("setuptools", "20.0.0") + package_wheel = get_package("wheel", "20.0.0") + + all_packages = [ + package_a, + package_b, + package_c, + package_pip, + package_setuptools, + package_wheel, + ] + + managed_reserved_packages = [ + pkg for pkg in all_packages if pkg.name in managed_reserved_package_names + ] + locked_packages = [package_a, *managed_reserved_packages] + + for pkg in all_packages: + repo.add_package(pkg) + installed.add_package(pkg) + + installed.add_package(package) # Root package never removed. + + package.add_dependency(Factory.create_dependency(package_a.name, package_a.version)) + locker.locked(True) locker.mock_lock_data( { "package": [ { - "name": "a", - "version": "1.0", + "name": pkg.name, + "version": pkg.version, "category": "main", "optional": False, "platform": "*", "python-versions": "*", "checksum": [], } + for pkg in locked_packages ], "metadata": { "python-versions": "*", "platform": "*", "content-hash": "123456789", - "hashes": {"a": []}, + "hashes": {pkg.name: [] for pkg in locked_packages}, }, } ) - package_a = get_package("a", "1.0") - package_b = get_package("b", "1.1") - package_c = get_package("c", "1.2") - package_pip = get_package("pip", "20.0.0") - package_setuptools = get_package("setuptools", "20.0.0") - - repo.add_package(package_a) - repo.add_package(package_b) - repo.add_package(package_c) - repo.add_package(package_pip) - repo.add_package(package_setuptools) - - installed.add_package(package_a) - installed.add_package(package_b) - installed.add_package(package_c) - installed.add_package(package_pip) - installed.add_package(package_setuptools) - installed.add_package(package) # Root package never removed. - - package.add_dependency(Factory.create_dependency("A", "~1.0")) installer.dev_mode(True).remove_untracked(True) installer.run() assert 0 == installer.executor.installations_count assert 0 == installer.executor.updates_count - assert 4 == installer.executor.removals_count - assert {"b", "c", "pip", "setuptools"} == set( - r.name for r in installer.executor.removals - ) + assert 2 + len(managed_reserved_packages) == installer.executor.removals_count + + expected_removals = { + package_b.name, + package_c.name, + *managed_reserved_package_names, + } + + assert expected_removals == set(r.name for r in installer.executor.removals) def test_run_whitelist_add(installer, locker, repo, package): diff --git a/tests/installation/test_installer_old.py b/tests/installation/test_installer_old.py index a8e7a9c1877..20936fe5f09 100644 --- a/tests/installation/test_installer_old.py +++ b/tests/installation/test_installer_old.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import itertools import sys from pathlib import Path @@ -28,6 +29,9 @@ from tests.repositories.test_pypi_repository import MockRepository +RESERVED_PACKAGES = ("pip", "setuptools", "wheel") + + class Installer(BaseInstaller): def _get_installer(self): return NoopInstaller() @@ -292,49 +296,73 @@ def test_run_install_no_dev(installer, locker, repo, package, installed): assert len(removals) == 1 -def test_run_install_remove_untracked(installer, locker, repo, package, installed): +@pytest.mark.parametrize( + "managed_reserved_package_names", + [ + i + for i in itertools.chain( + [tuple()], + itertools.permutations(RESERVED_PACKAGES, 1), + itertools.permutations(RESERVED_PACKAGES, 2), + [RESERVED_PACKAGES], + ) + ], +) +def test_run_install_remove_untracked( + managed_reserved_package_names, installer, locker, repo, package, installed +): + package_a = get_package("a", "1.0") + package_b = get_package("b", "1.1") + package_c = get_package("c", "1.2") + package_pip = get_package("pip", "20.0.0") + package_setuptools = get_package("setuptools", "20.0.0") + package_wheel = get_package("wheel", "20.0.0") + + all_packages = [ + package_a, + package_b, + package_c, + package_pip, + package_setuptools, + package_wheel, + ] + + managed_reserved_packages = [ + pkg for pkg in all_packages if pkg.name in managed_reserved_package_names + ] + locked_packages = [package_a, *managed_reserved_packages] + + for pkg in all_packages: + repo.add_package(pkg) + installed.add_package(pkg) + + installed.add_package(package) # Root package never removed. + + package.add_dependency(Factory.create_dependency(package_a.name, package_a.version)) + locker.locked(True) locker.mock_lock_data( { "package": [ { - "name": "a", - "version": "1.0", + "name": pkg.name, + "version": pkg.version, "category": "main", "optional": False, "platform": "*", "python-versions": "*", "checksum": [], } + for pkg in locked_packages ], "metadata": { "python-versions": "*", "platform": "*", "content-hash": "123456789", - "hashes": {"a": []}, + "hashes": {pkg.name: [] for pkg in locked_packages}, }, } ) - package_a = get_package("a", "1.0") - package_b = get_package("b", "1.1") - package_c = get_package("c", "1.2") - package_pip = get_package("pip", "20.0.0") - package_setuptools = get_package("setuptools", "20.0.0") - - repo.add_package(package_a) - repo.add_package(package_b) - repo.add_package(package_c) - repo.add_package(package_pip) - repo.add_package(package_setuptools) - - installed.add_package(package_a) - installed.add_package(package_b) - installed.add_package(package_c) - installed.add_package(package_pip) - installed.add_package(package_setuptools) - installed.add_package(package) # Root package never removed. - - package.add_dependency(Factory.create_dependency("A", "~1.0")) installer.dev_mode(True).remove_untracked(True) installer.run() @@ -346,7 +374,12 @@ def test_run_install_remove_untracked(installer, locker, repo, package, installe assert len(updates) == 0 removals = installer.installer.removals - assert set(r.name for r in removals) == {"b", "c", "pip", "setuptools"} + expected_removals = { + package_b.name, + package_c.name, + *managed_reserved_package_names, + } + assert set(r.name for r in removals) == expected_removals def test_run_whitelist_add(installer, locker, repo, package): diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index 1cfc9ad5099..d09012c821f 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -from typing import Optional +from typing import Any from typing import Union import pytest @@ -118,9 +118,7 @@ def test_env_get_venv_with_venv_folder_present( assert venv.path == in_project_venv_dir -def build_venv( - path: Union[Path, str], executable: Optional[str] = None, flags: bool = None -) -> (): +def build_venv(path: Union[Path, str], **__: Any) -> (): os.mkdir(str(path)) @@ -161,6 +159,9 @@ def test_activate_activates_non_existing_virtualenv_no_envs_file( Path(tmp_dir) / "{}-py3.7".format(venv_name), executable="python3.7", flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) envs_file = TOMLFile(Path(tmp_dir) / "envs.toml") @@ -281,6 +282,9 @@ def test_activate_activates_different_virtualenv_with_envs_file( Path(tmp_dir) / "{}-py3.6".format(venv_name), executable="python3.6", flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) assert envs_file.exists() @@ -335,6 +339,9 @@ def test_activate_activates_recreates_for_different_patch( Path(tmp_dir) / "{}-py3.7".format(venv_name), executable="python3.7", flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) remove_venv_m.assert_called_with(Path(tmp_dir) / "{}-py3.7".format(venv_name)) @@ -715,6 +722,9 @@ def test_create_venv_tries_to_find_a_compatible_python_executable_using_generic_ config_virtualenvs_path / "{}-py3.7".format(venv_name), executable="python3", flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) @@ -739,6 +749,9 @@ def test_create_venv_tries_to_find_a_compatible_python_executable_using_specific config_virtualenvs_path / "{}-py3.9".format(venv_name), executable="python3.9", flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) @@ -823,6 +836,9 @@ def test_create_venv_uses_patch_version_to_detect_compatibility( / "{}-py{}.{}".format(venv_name, version.major, version.minor), executable=None, flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) @@ -858,6 +874,9 @@ def test_create_venv_uses_patch_version_to_detect_compatibility_with_executable( / "{}-py{}.{}".format(venv_name, version.major, version.minor - 1), executable="python{}.{}".format(version.major, version.minor - 1), flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) @@ -892,6 +911,9 @@ def test_activate_with_in_project_setting_does_not_fail_if_no_venvs_dir( poetry.file.parent / ".venv", executable="python3.7", flags={"always-copy": False, "system-site-packages": False}, + with_pip=True, + with_setuptools=True, + with_wheel=True, ) envs_file = TOMLFile(Path(tmp_dir) / "virtualenvs" / "envs.toml")