Support options in requirements.txt in pip-sync

piptools/scripts/sync.py

@@ -1,14 +1,16 @@
 # coding: utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import itertools
 import os
 import sys
 
 from .. import click, sync
 from .._compat import get_installed_distributions, parse_requirements
 from ..exceptions import PipToolsError
 from ..logging import log
-from ..utils import flat_map
+from ..repositories import PyPIRepository
+from ..utils import create_install_command, flat_map, get_trusted_hosts
 
 DEFAULT_REQUIREMENTS_FILE = "requirements.txt"
 
@@ -104,8 +106,15 @@ def cli(
             log.error("ERROR: " + msg)
             sys.exit(2)
 
+    install_command = create_install_command()
+    options, _ = install_command.parse_args([])
+    session = install_command._build_session(options)
+    finder = install_command._build_package_finder(options=options, session=session)
+
+    # Parse requirements file. Note, all options inside requirements file
+    # will be collected by the finder.
     requirements = flat_map(
-        lambda src: parse_requirements(src, session=True), src_files
+        lambda src: parse_requirements(src, finder=finder, session=session), src_files
     )
 
     try:
@@ -117,26 +126,17 @@ def cli(
     installed_dists = get_installed_distributions(skip=[], user_only=user_only)
     to_install, to_uninstall = sync.diff(requirements, installed_dists)
 
-    install_flags = []
-    for link in find_links or []:
-        install_flags.extend(["-f", link])
-    if no_index:
-        install_flags.append("--no-index")
-    if index_url:
-        install_flags.extend(["-i", index_url])
-    if extra_index_url:
-        for extra_index in extra_index_url:
-            install_flags.extend(["--extra-index-url", extra_index])
-    if trusted_host:
-        for host in trusted_host:
-            install_flags.extend(["--trusted-host", host])
-    if user_only:
-        install_flags.append("--user")
-    if cert:
-        install_flags.extend(["--cert", cert])
-    if client_cert:
-        install_flags.extend(["--client-cert", client_cert])
-
+    install_flags = _compose_install_flags(
+        finder,
+        no_index=no_index,
+        index_url=index_url,
+        extra_index_url=extra_index_url,
+        trusted_host=trusted_host,
+        find_links=find_links,
+        user_only=user_only,
+        cert=cert,
+        client_cert=client_cert,
+    )
     sys.exit(
         sync.sync(
             to_install,
@@ -147,3 +147,65 @@ def cli(
             ask=ask,
         )
     )
+
+
+def _compose_install_flags(
+    finder,
+    no_index=False,
+    index_url=None,
+    extra_index_url=None,
+    trusted_host=None,
+    find_links=None,
+    user_only=False,
+    cert=None,
+    client_cert=None,
+):
+    """
+    Compose install flags with the given finder and CLI options.
+    """
+    result = []
+
+    # Build --index-url/--extra-index-url/--no-index
+    if no_index:
+        result.append("--no-index")
+    elif index_url:
+        result.extend(["--index-url", index_url])
+    elif finder.index_urls:
+        finder_index_url = finder.index_urls[0]
+        if finder_index_url != PyPIRepository.DEFAULT_INDEX_URL:
+            result.extend(["--index-url", finder_index_url])
+        for extra_index in finder.index_urls[1:]:
+            result.extend(["--extra-index-url", extra_index])
+    else:
+        result.append("--no-index")
+
+    for extra_index in extra_index_url or []:
+        result.extend(["--extra-index-url", extra_index])
+
+    # Build --trusted-hosts
+    for host in itertools.chain(trusted_host or [], get_trusted_hosts(finder)):
+        result.extend(["--trusted-host", host])
+
+    # Build --find-links
+    for link in itertools.chain(find_links or [], finder.find_links):
+        result.extend(["--find-links", link])
+
+    # Build format controls --no-binary/--only-binary
+    for format_control in ("no_binary", "only_binary"):
+        formats = getattr(finder.format_control, format_control)
+        if not formats:
+            continue
+        result.extend(
+            ["--" + format_control.replace("_", "-"), ",".join(sorted(formats))]
+        )
+
+    if user_only:
+        result.append("--user")
+
+    if cert:
+        result.extend(["--cert", cert])
+
+    if client_cert:
+        result.extend(["--client-cert", client_cert])
+
+    return result

tests/test_cli_sync.py

@@ -5,7 +5,7 @@
 
 from .utils import invoke
 
-from piptools.scripts.sync import cli
+from piptools.scripts.sync import DEFAULT_REQUIREMENTS_FILE, cli
 
 
 def test_run_as_module_sync():
@@ -111,42 +111,65 @@ def test_merge_error(runner):
 
 
 @pytest.mark.parametrize(
-    ("cli_flags", "expected_install_flags"),
+    "install_flags",
     [
-        (["--find-links", "./libs"], ["-f", "./libs"]),
-        (["--no-index"], ["--no-index"]),
-        (["--index-url", "https://example.com"], ["-i", "https://example.com"]),
-        (
-            ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
-            ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
-        ),
-        (
-            ["--trusted-host", "https://foo", "--trusted-host", "https://bar"],
-            ["--trusted-host", "https://foo", "--trusted-host", "https://bar"],
-        ),
-        (
-            ["--extra-index-url", "https://foo", "--trusted-host", "https://bar"],
-            ["--extra-index-url", "https://foo", "--trusted-host", "https://bar"],
-        ),
-        (["--user"], ["--user"]),
-        (["--cert", "foo.crt"], ["--cert", "foo.crt"]),
-        (["--client-cert", "foo.pem"], ["--client-cert", "foo.pem"]),
+        ["--find-links", "./libs1", "--find-links", "./libs2"],
+        ["--no-index"],
+        ["--index-url", "https://example.com"],
+        ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
+        ["--trusted-host", "foo", "--trusted-host", "bar"],
+        ["--user"],
+        ["--cert", "foo.crt"],
+        ["--client-cert", "foo.pem"],
     ],
 )
 @mock.patch("piptools.sync.check_call")
-def test_pip_install_flags(check_call, cli_flags, expected_install_flags, runner):
+def test_pip_install_flags(check_call, install_flags, runner):
     """
     Test the cli flags have to be passed to the pip install command.
     """
     with open("requirements.txt", "w") as req_in:
         req_in.write("six==1.10.0")
 
-    runner.invoke(cli, cli_flags)
+    runner.invoke(cli, install_flags)
 
     call_args = [call[0][0] for call in check_call.call_args_list]
-    assert [args[6:] for args in call_args if args[3] == "install"] == [
-        expected_install_flags
-    ]
+    called_install_options = [args[6:] for args in call_args if args[3] == "install"]
+    assert called_install_options == [install_flags], "Called args: {}".format(
+        call_args
+    )
+
+
+@pytest.mark.parametrize(
+    "install_flags",
+    [
+        ["--no-index"],
+        ["--index-url", "https://example.com"],
+        ["--extra-index-url", "https://example.com"],
+        ["--find-links", "./libs1"],
+        ["--trusted-host", "example.com"],
+        ["--no-binary", ":all:"],
+        ["--only-binary", ":all:"],
+    ],
+)
+@mock.patch("piptools.sync.check_call")
+def test_pip_install_flags_in_requirements_file(check_call, runner, install_flags):
+    """
+    Test the options from requirements.txt file pass to the pip install command.
+    """
+    with open(DEFAULT_REQUIREMENTS_FILE, "w") as reqs:
+        reqs.write(" ".join(install_flags) + "\n")
+        reqs.write("six==1.10.0")
+
+    out = runner.invoke(cli)
+    assert out.exit_code == 0, out
+
+    # Make sure pip install command has expected options
+    call_args = [call[0][0] for call in check_call.call_args_list]
+    called_install_options = [args[6:] for args in call_args if args[3] == "install"]
+    assert called_install_options == [install_flags], "Called args: {}".format(
+        call_args
+    )
 
 
 @mock.patch("piptools.sync.check_call")