Equivalent behaviour to pip install --find-links

docs/docs/repositories.md

@@ -14,7 +14,8 @@ This represents most cases and will likely be enough for most users.
 ## Using a private repository
 
 However, at times, you may need to keep your package private while still being
-able to share it with your teammates. In this case, you will need to use a private
+able to share it with your teammates. Some public projects also maintain alternative
+package repositories for specialised uses. In either case, you will need to use a private
 repository.
 
 ### Adding a repository
@@ -120,6 +121,24 @@ a custom certificate authority or client certificates, similarly refer to the ex
 `certificates` section. Poetry will use these values to authenticate to your private repository when downloading or
 looking for packages.
 
+### Non-Pep 503 Repositories
+
+A private repository may not follow the structure described in Pep 503. In this case you
+can use the `find_links` keyword. The below provides the equivalent behaviour to `pip
+install --find-links https://foo.bar/index.html`.
+
+
+```toml
+[[tool.poetry.source]]
+name = "foo"
+url = "https://foo.bar/index.html"
+find_links = true
+```
+
+If `find_links` is used the url may return a html page containing links to python
+archives, or may point directly to an installable artifact e.g.
+https://foo.bar/poetry-0.1.0-py3-none-any.whl. The use of `find_links` with local
+paths or file:// urls is not yet supported.
 
 ### Disabling the PyPI repository
 

poetry/factory.py

@@ -125,6 +125,7 @@ def create_legacy_repository(
         self, source, auth_config
     ):  # type: (Dict[str, str], Config) -> LegacyRepository
         from .repositories.auth import Auth
+        from .repositories.find_links_repository import FindLinksRepository
         from .repositories.legacy_repository import LegacyRepository
         from .utils.helpers import get_cert
         from .utils.helpers import get_client_cert
@@ -134,6 +135,10 @@ def create_legacy_repository(
             # PyPI-like repository
             if "name" not in source:
                 raise RuntimeError("Missing [name] in source.")
+            if "find_links" in source and source["find_links"]:
+                repository_class = FindLinksRepository
+            else:
+                repository_class = LegacyRepository
         else:
             raise RuntimeError("Unsupported source specified")
 
@@ -146,7 +151,7 @@ def create_legacy_repository(
         else:
             auth = None
 
-        return LegacyRepository(
+        return repository_class(
             name,
             url,
             auth=auth,

poetry/installation/pip_installer.py

@@ -58,15 +58,17 @@ def install(self, package, update=False):
             if repository.client_cert:
                 args += ["--client-cert", str(repository.client_cert)]
 
-            index_url = repository.authenticated_url
-
-            args += ["--index-url", index_url]
-            if self._pool.has_default():
-                if repository.name != self._pool.repositories[0].name:
-                    args += [
-                        "--extra-index-url",
-                        self._pool.repositories[0].authenticated_url,
-                    ]
+            if package.source_type == "legacy":
+                index_url = repository.authenticated_url
+                args += ["--index-url", index_url]
+                if self._pool.has_default():
+                    if repository.name != self._pool.repositories[0].name:
+                        args += [
+                            "--extra-index-url",
+                            self._pool.repositories[0].authenticated_url,
+                        ]
+            else:
+                args += ["--find-links", repository.url + "/" + repository.file_path]
 
         if update:
             args.append("-U")

poetry/json/schemas/poetry-schema.json

@@ -523,6 +523,10 @@
                 "secondary": {
                     "type": "boolean",
                     "description": "Declare this repository as secondary, i.e. it will only be looked up last for packages."
+                },
+                "find_links": {
+                    "type": "boolean",
+                    "description": "This repository is defined by an url pointing to a webpage hosting links to archives. Equivalent to the pip --find-links flag."
                 }
             }
         }

poetry/repositories/find_links_repository.py

@@ -0,0 +1,124 @@
+import re
+
+from typing import Generator
+from typing import Tuple
+from typing import Union
+
+from poetry.core.packages.utils.link import Link
+
+from .legacy_repository import LegacyRepository
+from .legacy_repository import Page
+
+
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urlparse
+
+try:
+    from html import unescape
+except ImportError:
+    try:
+        from html.parser import HTMLParser
+    except ImportError:
+        from HTMLParser import HTMLParser
+
+    unescape = HTMLParser().unescape
+
+
+def parse_url(url):  # type: (str) -> Tuple[str, str]
+    """Parse an url returning the base url for the repository and
+    the name of any index page that will contain links"""
+    url_parts = urlparse.urlparse(url)
+    path = url_parts.path
+
+    path = path.split("/")
+    if "." in path[-1]:
+        match = re.match(FilteredPage.VERSION_REGEX, path[-1])
+        if match and any(fmt in path[-1] for fmt in FilteredPage.SUPPORTED_FORMATS):
+            index_page = path[-1]
+            single_link = True
+        else:
+            index_page = path[-1]
+            single_link = False
+        return (
+            url_parts._replace(path="/".join(path[:-1])).geturl(),
+            index_page,
+            single_link,
+        )
+    else:
+        return url_parts.geturl().rstrip("/"), "", False
+
+
+class FilteredPage(Page):
+    """A representation of a web page that presents links only for a
+    particular package name."""
+
+    VERSION_REGEX = re.compile(
+        r"(?i)(?P<package_name>[a-z0-9_\-.]+?)-(?=\d)(?P<version>[a-z0-9_.!+-]+)"
+    )
+
+    def __init__(self, url, name, content, headers):
+        self.name = name
+        super(FilteredPage, self).__init__(url, content, headers)
+
+    def _parse_url(self, url):  # type: (str) -> str
+        parsed_url, self.index_page, self.single_link = parse_url(url)
+        return parsed_url
+
+    @property
+    def links(self):  # type: () -> Generator[Link]
+        for anchor in self._parsed.findall(".//a"):
+            if anchor.get("href"):
+                href = anchor.get("href")
+                url = self.clean_link(urlparse.urljoin(self._url + "/", href))
+                pyrequire = anchor.get("data-requires-python")
+                pyrequire = unescape(pyrequire) if pyrequire else None
+
+                url_parts = urlparse.urlparse(url)
+                match = re.search(self.VERSION_REGEX, url_parts.path)
+                if match is None or self.name != match.groupdict()["package_name"]:
+                    continue
+
+                link = Link(url, self, requires_python=pyrequire)
+
+                if link.ext not in self.SUPPORTED_FORMATS:
+                    continue
+
+                yield link
+
+
+class SingleLink(FilteredPage):
+    def __init__(self, url, name):  # type: (str, str) -> None
+        match = re.search(self.VERSION_REGEX, url)
+        if match and name == match.groupdict()["package_name"]:
+            self._link = [Link(url, self, None)]
+        else:
+            self._link = []
+
+    @property
+    def links(self):  # type: (str) -> Generator[Link]
+        for link in self._link:
+            yield link
+
+
+class FindLinksRepository(LegacyRepository):
+    repository_type = "find_links"
+
+    def _parse_url(self, url):  # type: (str) -> str
+        parsed_url, self.file_path, self.single_link = parse_url(url)
+        return parsed_url
+
+    def _get(self, name):  # type: (str) -> Union[Page, None]
+        url = self.full_url
+        if self.single_link:
+            return SingleLink(url, name)
+
+        response = self._session.get(url)
+        if response.status_code == 404:
+            return
+        return FilteredPage(url, name, response.content, response.headers)
+
+    @property
+    def full_url(self):  # type: () -> str
+        return self._url + "/" + self.file_path

poetry/repositories/legacy_repository.py

@@ -21,7 +21,6 @@
 from poetry.core.semver import parse_constraint
 from poetry.locations import REPOSITORY_CACHE_DIR
 from poetry.utils._compat import Path
-from poetry.utils.helpers import canonicalize_name
 from poetry.utils.patterns import wheel_file_re
 
 from ..inspection.info import PackageInfo
@@ -58,7 +57,7 @@
     import html5lib
 
 
-class Page:
+class Page(object):
 
     VERSION_REGEX = re.compile(r"(?i)([a-z0-9_\-.]+?)-(?=\d)([a-z0-9_.!+-]+)")
     SUPPORTED_FORMATS = [
@@ -72,10 +71,7 @@ class Page:
     ]
 
     def __init__(self, url, content, headers):
-        if not url.endswith("/"):
-            url += "/"
-
-        self._url = url
+        self._url = self._parse_url(url)
         encoding = None
         if headers and "Content-Type" in headers:
             content_type, params = cgi.parse_header(headers["Content-Type"])
@@ -156,8 +152,15 @@ def clean_link(self, url):
         % or other characters)."""
         return self._clean_re.sub(lambda match: "%%%2x" % ord(match.group(0)), url)
 
+    def _parse_url(self, url):  # type: (str) -> str
+        if not url.endswith("/"):
+            url += "/"
+        return url
+
 
 class LegacyRepository(PyPiRepository):
+    repository_type = "legacy"
+
     def __init__(
         self, name, url, auth=None, disable_cache=False, cert=None, client_cert=None
     ):  # type: (str, str, Optional[Auth], bool, Optional[Path], Optional[Path]) -> None
@@ -166,7 +169,7 @@ def __init__(
 
         self._packages = []
         self._name = name
-        self._url = url.rstrip("/")
+        self._url = self._parse_url(url)
         self._auth = auth
         self._client_cert = client_cert
         self._cert = cert
@@ -199,6 +202,9 @@ def __init__(
 
         self._disable_cache = disable_cache
 
+    def _parse_url(self, url):  # type: (str) -> str
+        return url.rstrip("/")
+
     @property
     def cert(self):  # type: () -> Optional[Path]
         return self._cert
@@ -251,7 +257,7 @@ def find_packages(
         if self._cache.store("matches").has(key):
             versions = self._cache.store("matches").get(key)
         else:
-            page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
+            page = self._get(name)
             if page is None:
                 return []
 
@@ -271,7 +277,7 @@ def find_packages(
         for package_versions in (versions, ignored_pre_release_versions):
             for version in package_versions:
                 package = Package(name, version)
-                package.source_type = "legacy"
+                package.source_type = self.repository_type
                 package.source_reference = self.name
                 package.source_url = self._url
 
@@ -311,7 +317,8 @@ def package(self, name, version, extras=None):  # type: (...) -> Package
             return self._packages[index]
         except ValueError:
             package = super(LegacyRepository, self).package(name, version, extras)
-            package.source_type = "legacy"
+
+            package.source_type = self.repository_type
             package.source_url = self._url
             package.source_reference = self.name
 
@@ -325,7 +332,7 @@ def find_links_for_package(self, package):
         return list(page.links_for_version(package.version))
 
     def _get_release_info(self, name, version):  # type: (str, str) -> dict
-        page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
+        page = self._get(name)
         if page is None:
             raise PackageNotFound('No package named "{}"'.format(name))
 

tests/installation/test_pip_installer.py

@@ -5,6 +5,7 @@
 from poetry.core.packages.package import Package
 from poetry.installation.pip_installer import PipInstaller
 from poetry.io.null_io import NullIO
+from poetry.repositories.find_links_repository import FindLinksRepository
 from poetry.repositories.legacy_repository import LegacyRepository
 from poetry.repositories.pool import Pool
 from poetry.utils._compat import Path
@@ -75,9 +76,11 @@ def test_requirement_git_develop_false(installer, package_git):
 def test_install_with_non_pypi_default_repository(pool, installer):
     default = LegacyRepository("default", "https://default.com")
     another = LegacyRepository("another", "https://another.com")
+    find_links = FindLinksRepository("find_links", "https://find_links.com/index.html")
 
     pool.add_repository(default, default=True)
     pool.add_repository(another)
+    pool.add_repository(find_links)
 
     foo = Package("foo", "0.0.0")
     foo.source_type = "legacy"
@@ -87,9 +90,18 @@ def test_install_with_non_pypi_default_repository(pool, installer):
     bar.source_type = "legacy"
     bar.source_reference = another._name
     bar.source_url = another._url
+    baz = Package("baz", "0.1.0")
+    baz.source_type = "find_links"
+    baz.source_reference = find_links._name
+    baz.source_url = find_links._url
 
     installer.install(foo)
     installer.install(bar)
+    installer.install(baz)
+
+    find_links_execution = installer._env.executed[-1]
+    assert "--find-links" in find_links_execution
+    assert find_links.full_url in find_links_execution
 
 
 def test_install_with_cert():

tests/repositories/fixtures/find_links/index.html

@@ -0,0 +1,6 @@
+<a href="https://files.pythonhosted.org/packages/41/d8/a945da414f2adc1d9e2f7d6e7445b27f2be42766879062a2e63616ad4199/isort-4.3.4-py2-none-any.whl#sha256=ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497">isort-4.3.4-py2-none-any.whl</a><br/>
+<a href="https://files.pythonhosted.org/packages/1f/2c/22eee714d7199ae0464beda6ad5fedec8fee6a2f7ffd1e8f1840928fe318/isort-4.3.4-py3-none-any.whl#sha256=1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af">isort-4.3.4-py3-none-any.whl</a><br/>
+<a href="https://files.pythonhosted.org/packages/b1/de/a628d16fdba0d38cafb3d7e34d4830f2c9cb3881384ce5c08c44762e1846/isort-4.3.4.tar.gz#sha256=b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8">isort-4.3.4.tar.gz</a><br/>
+<a href="https://files.pythonhosted.org/packages/2d/99/b2c4e9d5a30f6471e410a146232b4118e697fa3ffc06d6a65efde84debd0/futures-3.2.0-py2-none-any.whl#sha256=ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1">futures-3.2.0-py2-none-any.whl</a><br/>
+<a href="https://files.pythonhosted.org/packages/1f/9e/7b2ff7e965fc654592269f2906ade1c7d705f1bf25b7d469fa153f7d19eb/futures-3.2.0.tar.gz#sha256=9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265">futures-3.2.0.tar.gz</a><br/>
+<a href="relative/poetry-0.1.0-py3-none-any.whl">poetry-0.1.0-py3-none-any.whl</a><br/>