Improve detection of installed packages in an environment

poetry/repositories/installed_repository.py

@@ -1,6 +1,10 @@
+import itertools
+
 from typing import Set
+from typing import Union
 
 from poetry.core.packages import Package
+from poetry.core.utils.helpers import module_name
 from poetry.utils._compat import Path
 from poetry.utils._compat import metadata
 from poetry.utils.env import Env
@@ -11,11 +15,17 @@
 _VENDORS = Path(__file__).parent.parent.joinpath("_vendor")
 
 
+try:
+    FileNotFoundError
+except NameError:
+    FileNotFoundError = OSError
+
+
 class InstalledRepository(Repository):
     @classmethod
-    def get_package_paths(cls, sitedir, name):  # type: (Path, str) -> Set[Path]
+    def get_package_paths(cls, env, name):  # type: (Env, str) -> Set[Path]
         """
-        Process a .pth file within the site-packages directory, and return any valid
+        Process a .pth file within the site-packages directories, and return any valid
         paths. We skip executable .pth files as there is no reliable means to do this
         without side-effects to current run-time. Mo check is made that the item refers
         to a directory rather than a file, however, in order to maintain backwards
@@ -24,25 +34,72 @@ def get_package_paths(cls, sitedir, name):  # type: (Path, str) -> Set[Path]
 
         Reference: https://docs.python.org/3.8/library/site.html
 
-        :param sitedir: The site-packages directory to search for .pth file.
+        :param env: The environment to search for the .pth file in.
         :param name: The name of the package to search .pth file for.
         :return: A `Set` of valid `Path` objects.
         """
         paths = set()
 
-        pth_file = sitedir.joinpath("{}.pth".format(name))
-        if pth_file.exists():
+        # we identify the candidate pth files to check, this is done so to handle cases
+        # where the pth file for foo-bar might have been installed as either foo-bar.pth or
+        # foo_bar.pth (expected) in either pure or platform lib directories.
+        candidates = itertools.product(
+            {env.purelib, env.platlib}, {name, module_name(name)},
+        )
+
+        for lib, module in candidates:
+            pth_file = lib.joinpath(module).with_suffix(".pth")
+            if not pth_file.exists():
+                continue
+
             with pth_file.open() as f:
                 for line in f:
                     line = line.strip()
                     if line and not line.startswith(("#", "import ", "import\t")):
                         path = Path(line)
                         if not path.is_absolute():
-                            path = sitedir.joinpath(path)
+                            try:
+                                path = lib.joinpath(path).resolve()
+                            except FileNotFoundError:
+                                # this is required to handle pathlib oddity on win32 python==3.5
+                                path = lib.joinpath(path)
                         paths.add(path)
-
         return paths
 
+    @classmethod
+    def set_package_vcs_properties_from_path(
+        cls, src, package
+    ):  # type: (Path, Package) -> None
+        from poetry.core.vcs.git import Git
+
+        git = Git()
+        revision = git.rev_parse("HEAD", src).strip()
+        url = git.remote_url(src)
+
+        package.source_type = "git"
+        package.source_url = url
+        package.source_reference = revision
+
+    @classmethod
+    def set_package_vcs_properties(cls, package, env):  # type: (Package, Env) -> None
+        src = env.path / "src" / package.name
+        cls.set_package_vcs_properties_from_path(src, package)
+
+    @classmethod
+    def is_vcs_package(cls, package, env):  # type: (Union[Path, Package], Env) -> bool
+        # A VCS dependency should have been installed
+        # in the src directory.
+        src = env.path / "src"
+        if isinstance(package, Package):
+            return src.joinpath(package.name).is_dir()
+
+        try:
+            package.relative_to(env.path / "src")
+        except ValueError:
+            return False
+        else:
+            return True
+
     @classmethod
     def load(cls, env):  # type: (Env) -> InstalledRepository
         """
@@ -75,41 +132,26 @@ def load(cls, env):  # type: (Env) -> InstalledRepository
 
                 repo.add_package(package)
 
-                is_standard_package = True
-                try:
-                    path.relative_to(env.site_packages)
-                except ValueError:
-                    is_standard_package = False
+                is_standard_package = env.is_path_relative_to_lib(path)
 
                 if is_standard_package:
                     if path.name.endswith(".dist-info"):
-                        paths = cls.get_package_paths(
-                            sitedir=env.site_packages, name=package.pretty_name
-                        )
+                        paths = cls.get_package_paths(env=env, name=package.pretty_name)
                         if paths:
-                            # TODO: handle multiple source directories?
-                            package.source_type = "directory"
-                            package.source_url = paths.pop().as_posix()
-
+                            for src in paths:
+                                if cls.is_vcs_package(src, env):
+                                    cls.set_package_vcs_properties(package, env)
+                                    break
+                            else:
+                                # TODO: handle multiple source directories?
+                                package.source_type = "directory"
+                                package.source_url = paths.pop().as_posix()
                     continue
 
-                src_path = env.path / "src"
-
-                # A VCS dependency should have been installed
-                # in the src directory. If not, it's a path dependency
-                try:
-                    path.relative_to(src_path)
-
-                    from poetry.core.vcs.git import Git
-
-                    git = Git()
-                    revision = git.rev_parse("HEAD", src_path / package.name).strip()
-                    url = git.remote_url(src_path / package.name)
-
-                    package.source_type = "git"
-                    package.source_url = url
-                    package.source_reference = revision
-                except ValueError:
+                if cls.is_vcs_package(path, env):
+                    cls.set_package_vcs_properties(package, env)
+                else:
+                    # If not, it's a path dependency
                     package.source_type = "directory"
                     package.source_url = str(path.parent)
 

poetry/utils/env.py

@@ -754,6 +754,8 @@ def __init__(self, path, base=None):  # type: (Path, Optional[Path]) -> None
         self._site_packages = None
         self._paths = None
         self._supported_tags = None
+        self._purelib = None
+        self._platlib = None
 
     @property
     def path(self):  # type: () -> Path
@@ -810,10 +812,37 @@ def pip_version(self):
     @property
     def site_packages(self):  # type: () -> Path
         if self._site_packages is None:
-            self._site_packages = Path(self.paths["purelib"])
+            self._site_packages = Path(self.purelib)
 
         return self._site_packages
 
+    @property
+    def purelib(self):  # type: () -> Path
+        if self._purelib is None:
+            self._purelib = Path(self.paths["purelib"])
+
+        return self._purelib
+
+    @property
+    def platlib(self):  # type: () -> Path
+        if self._platlib is None:
+            if "platlib" in self.paths:
+                self._platlib = Path(self.paths["platlib"])
+            else:
+                self._platlib = self.purelib
+
+        return self._platlib
+
+    def is_path_relative_to_lib(self, path):  # type: (Path) -> bool
+        for lib_path in [self.purelib, self.platlib]:
+            try:
+                path.relative_to(lib_path)
+                return True
+            except ValueError:
+                pass
+
+        return False
+
     @property
     def sys_path(self):  # type: () -> List[str]
         raise NotImplementedError()

tests/fixtures/complete.toml

@@ -22,7 +22,7 @@ toml = "^0.9"
 # Dependencies with extras
 requests = { version = "^2.13", extras = [ "security" ] }
 # Python specific dependencies with prereleases allowed
-pathlib2 = { version = "^2.2", python = "~2.7", allows-prereleases = true }
+pathlib2 = { version = "^2.2", python = "~2.7", allow-prereleases = true }
 # Git dependencies
 cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" }
 

tests/installation/test_installer.py

@@ -890,7 +890,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
     repo.add_package(package_a)
     repo.add_package(package_b)
 
-    package.add_dependency("A", {"version": "*", "allows-prereleases": True})
+    package.add_dependency("A", {"version": "*", "allow-prereleases": True})
     package.add_dependency("B", "^1.1")
 
     installer.update(True)

tests/installation/test_installer_old.py

@@ -861,7 +861,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
     repo.add_package(package_a)
     repo.add_package(package_b)
 
-    package.add_dependency("A", {"version": "*", "allows-prereleases": True})
+    package.add_dependency("A", {"version": "*", "allow-prereleases": True})
     package.add_dependency("B", "^1.1")
 
     installer.update(True)

tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: bender
+Version: 2.0.5
+Summary: Python datetimes made easy
+License: MIT
+Keywords: cli,commands
+Author: Leela
+Author-email: leela@planetexpress.com
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/x-rst

tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender.pth

@@ -0,0 +1 @@
+../../../src/bender

tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA

@@ -0,0 +1,22 @@
+Metadata-Version: 2.1
+Name: lib64
+Version: 2.3.4
+Summary: lib64 description.
+License: MIT
+Keywords: cli,commands
+Author: Foo Bar
+Author-email: foo@bar.com
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/x-rst
+
+lib64
+####

tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: bender
+Version: 2.0.5
+Summary: Python datetimes made easy
+License: MIT
+Keywords: cli,commands
+Author: Leela
+Author-email: leela@planetexpress.com
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/x-rst