Improve editable installs

poetry/console/commands/install.py

@@ -39,7 +39,6 @@ class InstallCommand(EnvCommand):
     _loggers = ["poetry.repositories.pypi_repository"]
 
     def handle(self):
-        from clikit.io import NullIO
         from poetry.installation.installer import Installer
         from poetry.masonry.builders import EditableBuilder
         from poetry.core.masonry.utils.module import ModuleOrPackageNotFound
@@ -69,7 +68,7 @@ def handle(self):
             return 0
 
         try:
-            builder = EditableBuilder(self.poetry, self._env, NullIO())
+            builder = EditableBuilder(self.poetry, self._env, self._io)
         except ModuleOrPackageNotFound:
             # This is likely due to the fact that the project is an application
             # not following the structure expected by Poetry

poetry/installation/pip_installer.py

@@ -175,9 +175,10 @@ def create_temporary_requirement(self, package):
         return name
 
     def install_directory(self, package):
-        from poetry.core.masonry.builder import SdistBuilder
         from poetry.factory import Factory
+        from poetry.io.null_io import NullIO
         from poetry.utils._compat import decode
+        from poetry.masonry.builders.editable import EditableBuilder
         from poetry.utils.toml_file import TomlFile
 
         if package.root_dir:
@@ -197,18 +198,36 @@ def install_directory(self, package):
                 "tool" in pyproject_content and "poetry" in pyproject_content["tool"]
             )
             # Even if there is a build system specified
-            # pip as of right now does not support it fully
-            # TODO: Check for pip version when proper PEP-517 support lands
-            # has_build_system = ("build-system" in pyproject_content)
+            # some versions of pip (< 19.0.0) don't understand it
+            # so we need to check the version of pip to know
+            # if we can rely on the build system
+            pip_version = self._env.pip_version
+            pip_version_with_build_system_support = pip_version.__class__(19, 0, 0)
+            has_build_system = (
+                "build-system" in pyproject_content
+                and pip_version >= pip_version_with_build_system_support
+            )
 
         setup = os.path.join(req, "setup.py")
         has_setup = os.path.exists(setup)
-        if not has_setup and has_poetry and (package.develop or not has_build_system):
-            # We actually need to rely on creating a temporary setup.py
-            # file since pip, as of this comment, does not support
-            # build-system for editable packages
+        if has_poetry and package.develop and not package.build_script:
+            # This is a Poetry package in editable mode
+            # we can use the EditableBuilder without going through pip
+            # to install it, unless it has a build script.
+            builder = EditableBuilder(
+                Factory().create_poetry(pyproject.parent), self._env, NullIO()
+            )
+            builder.build()
+
+            return
+        elif has_poetry and (not has_build_system or package.build_script):
+            from poetry.core.masonry.builders.sdist import SdistBuilder
+
+            # We need to rely on creating a temporary setup.py
+            # file since the version of pip does not support
+            # build-systems
             # We also need it for non-PEP-517 packages
-            builder = SdistBuilder(Factory().create_poetry(pyproject.parent),)
+            builder = SdistBuilder(Factory().create_poetry(pyproject.parent))
 
             with open(setup, "w", encoding="utf-8") as f:
                 f.write(decode(builder.build_setup()))

poetry/masonry/builders/editable.py

@@ -1,16 +1,32 @@
 from __future__ import unicode_literals
 
+import hashlib
 import os
 import shutil
 
-from collections import defaultdict
+from base64 import urlsafe_b64encode
 
 from poetry.core.masonry.builders.builder import Builder
 from poetry.core.masonry.builders.sdist import SdistBuilder
 from poetry.core.semver.version import Version
+from poetry.utils._compat import WINDOWS
+from poetry.utils._compat import Path
 from poetry.utils._compat import decode
 
 
+SCRIPT_TEMPLATE = """\
+#!{python}
+from {module} import {callable_}
+
+if __name__ == '__main__':
+    {callable_}()
+"""
+
+WINDOWS_CMD_TEMPLATE = """\
+@echo off\r\n"{python}" "%~dp0\\{script}" %*\r\n
+"""
+
+
 class EditableBuilder(Builder):
     def __init__(self, poetry, env, io):
         super(EditableBuilder, self).__init__(poetry)
@@ -19,7 +35,22 @@ def __init__(self, poetry, env, io):
         self._io = io
 
     def build(self):
-        return self._setup_build()
+        self._debug(
+            "  - Building package <c1>{}</c1> in <info>editable</info> mode".format(
+                self._package.name
+            )
+        )
+
+        if self._package.build_script:
+            self._debug(
+                "  - <warning>Falling back on using a <b>setup.py</b></warning>"
+            )
+            return self._setup_build()
+
+        added_files = []
+        added_files += self._add_pth()
+        added_files += self._add_scripts()
+        self._add_dist_info(added_files)
 
     def _setup_build(self):
         builder = SdistBuilder(self._poetry)
@@ -36,14 +67,14 @@ def _setup_build(self):
 
         try:
             if self._env.pip_version < Version(19, 0):
-                self._env.run_pip("install", "-e", str(self._path))
+                self._env.run_pip("install", "-e", str(self._path), "--no-deps")
             else:
                 # Temporarily rename pyproject.toml
                 shutil.move(
                     str(self._poetry.file), str(self._poetry.file.with_suffix(".tmp"))
                 )
                 try:
-                    self._env.run_pip("install", "-e", str(self._path))
+                    self._env.run_pip("install", "-e", str(self._path), "--no-deps")
                 finally:
                     shutil.move(
                         str(self._poetry.file.with_suffix(".tmp")),
@@ -53,71 +84,125 @@ def _setup_build(self):
             if not has_setup:
                 os.remove(str(setup))
 
-    def _build_egg_info(self):
-        egg_info = self._path / "{}.egg-info".format(
-            self._package.name.replace("-", "_")
+    def _add_pth(self):
+        pth = self._env.site_packages.joinpath(self._module.name).with_suffix(".pth")
+        self._debug(
+            "  - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
+                pth.name, self._env.site_packages, self._poetry.file.parent
+            )
         )
-        egg_info.mkdir(exist_ok=True)
+        with pth.open("w", encoding="utf-8") as f:
+            f.write(decode(str(self._poetry.file.parent.resolve())))
+
+        return [pth]
+
+    def _add_scripts(self):
+        added = []
+        entry_points = self.convert_entry_points()
+        scripts_path = Path(self._env.paths["scripts"])
+
+        scripts = entry_points.get("console_scripts", [])
+        for script in scripts:
+            name, script = script.split(" = ")
+            module, callable_ = script.split(":")
+            script_file = scripts_path.joinpath(name)
+            self._debug(
+                "  - Adding the <c2>{}</c2> script to <b>{}</b>".format(
+                    name, scripts_path
+                )
+            )
+            with script_file.open("w", encoding="utf-8") as f:
+                f.write(
+                    decode(
+                        SCRIPT_TEMPLATE.format(
+                            python=self._env._bin("python"),
+                            module=module,
+                            callable_=callable_,
+                        )
+                    )
+                )
+
+            script_file.chmod(0o755)
+
+            added.append(script_file)
+
+            if WINDOWS:
+                cmd_script = script_file.with_suffix(".cmd")
+                cmd = WINDOWS_CMD_TEMPLATE.format(
+                    python=self._env._bin("python"), script=name
+                )
+                self._debug(
+                    "  - Adding the <c2>{}</c2> script wrapper to <b>{}</b>".format(
+                        cmd_script.name, scripts_path
+                    )
+                )
+
+                with cmd_script.open("w", encoding="utf-8") as f:
+                    f.write(decode(cmd))
+
+                added.append(cmd_script)
 
-        with egg_info.joinpath("PKG-INFO").open("w", encoding="utf-8") as f:
-            f.write(decode(self.get_metadata_content()))
+        return added
 
-        with egg_info.joinpath("entry_points.txt").open("w", encoding="utf-8") as f:
-            entry_points = self.convert_entry_points()
+    def _add_dist_info(self, added_files):
+        from poetry.core.masonry.builders.wheel import WheelBuilder
 
-            for group_name in sorted(entry_points):
-                f.write("[{}]\n".format(group_name))
-                for ep in sorted(entry_points[group_name]):
-                    f.write(ep.replace(" ", "") + "\n")
+        added_files = added_files[:]
 
-                f.write("\n")
+        builder = WheelBuilder(self._poetry)
+        dist_info = self._env.site_packages.joinpath(builder.dist_info)
+
+        self._debug(
+            "  - Adding the <c2>{}</c2> directory to <b>{}</b>".format(
+                dist_info.name, self._env.site_packages
+            )
+        )
 
-        with egg_info.joinpath("requires.txt").open("w", encoding="utf-8") as f:
-            f.write(self._generate_requires())
+        if dist_info.exists():
+            shutil.rmtree(str(dist_info))
 
-    def _build_egg_link(self):
-        egg_link = self._env.site_packages / "{}.egg-link".format(self._package.name)
-        with egg_link.open("w", encoding="utf-8") as f:
-            f.write(str(self._poetry.file.parent.resolve()) + "\n")
-            f.write(".")
+        dist_info.mkdir()
 
-    def _add_easy_install_entry(self):
-        easy_install_pth = self._env.site_packages / "easy-install.pth"
-        path = str(self._poetry.file.parent.resolve())
-        content = ""
-        if easy_install_pth.exists():
-            with easy_install_pth.open(encoding="utf-8") as f:
-                content = f.read()
+        with dist_info.joinpath("METADATA").open("w", encoding="utf-8") as f:
+            builder._write_metadata_file(f)
 
-        if path in content:
-            return
+        added_files.append(dist_info.joinpath("METADATA"))
 
-        content += "{}\n".format(path)
+        with dist_info.joinpath("INSTALLER").open("w", encoding="utf-8") as f:
+            f.write("poetry")
 
-        with easy_install_pth.open("w", encoding="utf-8") as f:
-            f.write(content)
+        added_files.append(dist_info.joinpath("INSTALLER"))
 
-    def _generate_requires(self):
-        extras = defaultdict(list)
+        if self.convert_entry_points():
+            with dist_info.joinpath("entry_points.txt").open(
+                "w", encoding="utf-8"
+            ) as f:
+                builder._write_entry_points(f)
 
-        requires = ""
-        for dep in sorted(self._package.requires, key=lambda d: d.name):
-            marker = dep.marker
-            if marker.is_any():
-                requires += "{}\n".format(dep.base_pep_508_name)
-                continue
+            added_files.append(dist_info.joinpath("entry_points.txt"))
 
-            extras[str(marker)].append(dep.base_pep_508_name)
+        with dist_info.joinpath("RECORD").open("w", encoding="utf-8") as f:
+            for path in added_files:
+                hash = self._get_file_hash(path)
+                size = path.stat().st_size
+                f.write("{},sha256={},{}\n".format(str(path), hash, size))
 
-        if extras:
-            requires += "\n"
+            # RECORD itself is recorded with no hash or size
+            f.write("{},,\n".format(dist_info.joinpath("RECORD")))
 
-            for marker, deps in sorted(extras.items()):
-                requires += "[:{}]\n".format(marker)
+    def _get_file_hash(self, filepath):
+        hashsum = hashlib.sha256()
+        with filepath.open("rb") as src:
+            while True:
+                buf = src.read(1024 * 8)
+                if not buf:
+                    break
+                hashsum.update(buf)
 
-                for dep in deps:
-                    requires += dep + "\n"
+            src.seek(0)
 
-                requires += "\n"
+        return urlsafe_b64encode(hashsum.digest()).decode("ascii").rstrip("=")
 
-        return requires
+    def _debug(self, msg):
+        if self._io.is_debug():
+            self._io.write_line(msg)

poetry/repositories/installed_repository.py

@@ -14,13 +14,11 @@ class InstalledRepository(Repository):
     def load(cls, env):  # type: (Env) -> InstalledRepository
         """
         Load installed packages.
-
-        For now, it uses the pip "freeze" command.
         """
         repo = cls()
         seen = set()
 
-        for entry in env.sys_path:
+        for entry in reversed(env.sys_path):
             for distribution in sorted(
                 metadata.distributions(path=[entry]), key=lambda d: str(d._path),
             ):