Merge branch 'main' into bugfix/2004

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 24.2.0
+    rev: 24.4.2
     hooks:
       - id: black
         args: [--target-version=py38]
@@ -9,7 +9,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.1
+    rev: v3.15.2
     hooks:
       - id: pyupgrade
         args: [--py38-plus]
@@ -20,7 +20,7 @@ repos:
         additional_dependencies:
           - flake8-pytest-style
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.10.0
     hooks:
       - id: mypy
         # Avoid error: Duplicate module named 'setup'
@@ -37,7 +37,7 @@ repos:
           - pytest==7.4.2
         language_version: python3.8
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.7
+    rev: 1.7.8
     hooks:
       - id: bandit
         args: [--ini, .bandit]

CHANGELOG.md

@@ -1,5 +1,18 @@
+## v7.4.1
+
+05 Mar 2024
+
+Bug Fixes:
+
+- Skip constraint path check ([#2038](https://github.com/jazzband/pip-tools/pull/2038)).
+  Thanks @honnix
+- Fix collecting deps for all extras in multiple input packages
+  ([#1981](https://github.com/jazzband/pip-tools/pull/1981)). Thanks @dragly
+
 ## v7.4.0
 
+16 Feb 2024
+
 Features:
 
 - Allow force-enabling or force-disabling colorized output

README.md

@@ -447,7 +447,7 @@ Sample `.pre-commit-config.yaml`:
 ```yaml
 repos:
   - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.0
+    rev: 7.4.1
     hooks:
       - id: pip-compile
 ```
@@ -457,7 +457,7 @@ You might want to customize `pip-compile` args by configuring `args` and/or `fil
 ```yaml
 repos:
   - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.0
+    rev: 7.4.1
     hooks:
       - id: pip-compile
         files: ^requirements/production\.(in|txt)$
@@ -469,7 +469,7 @@ If you have multiple requirement files make sure you create a hook for each file
 ```yaml
 repos:
   - repo: https://github.com/jazzband/pip-tools
-    rev: 7.4.0
+    rev: 7.4.1
     hooks:
       - id: pip-compile
         name: pip-compile setup.py

docs/requirements.txt

@@ -20,7 +20,7 @@ docutils==0.20.1
     #   sphinx
 furo==2023.8.17
     # via -r requirements.in
-idna==3.4
+idna==3.7
     # via requests
 imagesize==1.4.1
     # via sphinx

piptools/build.py

@@ -14,6 +14,13 @@
 import pyproject_hooks
 from pip._internal.req import InstallRequirement
 from pip._internal.req.constructors import parse_req_from_line
+from pip._vendor.packaging.markers import Marker
+from pip._vendor.packaging.requirements import Requirement
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
 
 from .utils import copy_install_requirement, install_req_from_line
 
@@ -34,23 +41,95 @@ def get_all(self, name: str, failobj: None = None) -> list[Any] | None: ...
         def get_all(self, name: str, failobj: _T) -> list[Any] | _T: ...
 
 
+@dataclass
+class StaticProjectMetadata:
+    extras: tuple[str, ...]
+    requirements: tuple[InstallRequirement, ...]
+
+
 @dataclass
 class ProjectMetadata:
     extras: tuple[str, ...]
     requirements: tuple[InstallRequirement, ...]
     build_requirements: tuple[InstallRequirement, ...]
 
 
+def maybe_statically_parse_project_metadata(
+    src_file: pathlib.Path,
+) -> StaticProjectMetadata | None:
+    """
+    Return the metadata for a project, if it can be statically parsed from ``pyproject.toml``.
+
+    This function is typically significantly faster than invoking a build backend.
+    Returns None if the project metadata cannot be statically parsed.
+    """
+    if src_file.name != PYPROJECT_TOML:
+        return None
+
+    try:
+        with open(src_file, "rb") as f:
+            pyproject_contents = tomllib.load(f)
+    except tomllib.TOMLDecodeError:
+        return None
+
+    # Not valid PEP 621 metadata
+    if (
+        "project" not in pyproject_contents
+        or "name" not in pyproject_contents["project"]
+    ):
+        return None
+
+    project_table = pyproject_contents["project"]
+
+    # Dynamic dependencies require build backend invocation
+    dynamic = project_table.get("dynamic", [])
+    if "dependencies" in dynamic or "optional-dependencies" in dynamic:
+        return None
+
+    package_name = project_table["name"]
+    comes_from = f"{package_name} ({src_file})"
+
+    extras = project_table.get("optional-dependencies", {}).keys()
+    install_requirements = [
+        InstallRequirement(Requirement(req), comes_from)
+        for req in project_table.get("dependencies", [])
+    ]
+    for extra, reqs in (
+        pyproject_contents.get("project", {}).get("optional-dependencies", {}).items()
+    ):
+        for req in reqs:
+            requirement = Requirement(req)
+            if requirement.name == package_name:
+                # Similar to logic for handling self-referential requirements
+                # from _prepare_requirements
+                requirement.url = src_file.parent.as_uri()
+            # Note we don't need to modify `requirement` to include this extra
+            marker = Marker(f"extra == '{extra}'")
+            install_requirements.append(
+                InstallRequirement(requirement, comes_from, markers=marker)
+            )
+
+    return StaticProjectMetadata(
+        extras=tuple(extras),
+        requirements=tuple(install_requirements),
+    )
+
+
 def build_project_metadata(
     src_file: pathlib.Path,
     build_targets: tuple[str, ...],
     *,
+    attempt_static_parse: bool,
     isolated: bool,
     quiet: bool,
-) -> ProjectMetadata:
+) -> ProjectMetadata | StaticProjectMetadata:
     """
     Return the metadata for a project.
 
+    First, optionally attempt to determine the metadata statically from the
+    ``pyproject.toml`` file. This will not work if build_targets are specified,
+    since we cannot determine build requirements statically.
+
     Uses the ``prepare_metadata_for_build_wheel`` hook for the wheel metadata
     if available, otherwise ``build_wheel``.
 
@@ -60,12 +139,25 @@ def build_project_metadata(
     :param src_file: Project source file
     :param build_targets: A tuple of build targets to get the dependencies
                                 of (``sdist`` or ``wheel`` or ``editable``).
+    :param attempt_static_parse: Whether to attempt to statically parse the
+                                 project metadata from ``pyproject.toml``.
+                                 Cannot be used with ``build_targets``.
     :param isolated: Whether to run invoke the backend in the current
                      environment or to create an isolated one and invoke it
                      there.
     :param quiet: Whether to suppress the output of subprocesses.
     """
 
+    if attempt_static_parse:
+        if build_targets:
+            raise ValueError(
+                "Cannot execute the PEP 517 optional get_requires_for_build* "
+                "hooks statically, as build requirements are requested"
+            )
+        project_metadata = maybe_statically_parse_project_metadata(src_file)
+        if project_metadata is not None:
+            return project_metadata
+
     src_dir = src_file.parent
     with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder:
         metadata = _build_project_wheel_metadata(builder)

piptools/exceptions.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import operator
 from typing import Iterable
 
 from pip._internal.index.package_finder import PackageFinder
@@ -27,7 +28,9 @@ def __str__(self) -> str:
         versions = []
         pre_versions = []
 
-        for candidate in sorted(self.candidates_tried):
+        for candidate in sorted(
+            self.candidates_tried, key=operator.attrgetter("version")
+        ):
             version = str(candidate.version)
             if candidate.version.is_prerelease:
                 pre_versions.append(version)

piptools/scripts/compile.py

@@ -15,7 +15,7 @@
 from pip._internal.utils.misc import redact_auth_from_url
 
 from .._compat import parse_requirements
-from ..build import build_project_metadata
+from ..build import ProjectMetadata, build_project_metadata
 from ..cache import DependencyCache
 from ..exceptions import NoCandidateFound, PipToolsError
 from ..logging import log
@@ -370,6 +370,7 @@ def cli(
                 metadata = build_project_metadata(
                     src_file=Path(src_file),
                     build_targets=build_deps_targets,
+                    attempt_static_parse=not bool(build_deps_targets),
                     isolated=build_isolation,
                     quiet=log.verbosity <= 0,
                 )
@@ -383,6 +384,7 @@ def cli(
                 if all_extras:
                     extras += metadata.extras
             if build_deps_targets:
+                assert isinstance(metadata, ProjectMetadata)
                 constraints.extend(metadata.build_requirements)
         else:
             constraints.extend(

piptools/scripts/options.py

@@ -343,14 +343,6 @@ def _get_default_option(option_name: str) -> Any:
 constraint = click.option(
     "-c",
     "--constraint",
-    type=click.Path(
-        exists=True,
-        file_okay=True,
-        dir_okay=False,
-        readable=True,
-        allow_dash=False,
-        path_type=str,
-    ),
     multiple=True,
     help="Constrain versions using the given constraints file; may be used more than once.",
 )

piptools/utils.py

@@ -380,9 +380,14 @@ def get_compile_command(click_ctx: click.Context) -> str:
         # Get the latest option name (usually it'll be a long name)
         option_long_name = option.opts[-1]
 
+        negative_option = None
+        if option.is_flag and option.secondary_opts:
+            # get inverse flag --no-{option_long_name}
+            negative_option = option.secondary_opts[-1]
+
         # Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...)
         # or options that don't change compile behaviour (--verbose/--dry-run/...)
-        if option_long_name in COMPILE_EXCLUDE_OPTIONS:
+        if {option_long_name, negative_option} & COMPILE_EXCLUDE_OPTIONS:
             continue
 
         # Exclude config option if it's the default one

piptools/writer.py

@@ -148,9 +148,27 @@ def write_trusted_hosts(self) -> Iterator[str]:
                 yield f"--trusted-host {trusted_host}"
 
     def write_format_controls(self) -> Iterator[str]:
-        for nb in dedup(sorted(self.format_control.no_binary)):
+        # The ordering of output needs to preserve the behavior of pip's
+        # FormatControl.get_allowed_formats(). The behavior is the following:
+        #
+        #   * Parsing of CLI options happens first to last.
+        #   * --only-binary takes precedence over --no-binary
+        #   * Package names take precedence over :all:
+        #   * We'll never see :all: in both due to mutual exclusion.
+        #
+        # So in summary, we want to emit :all: first and then package names later.
+        no_binary = self.format_control.no_binary.copy()
+        only_binary = self.format_control.only_binary.copy()
+
+        if ":all:" in no_binary:
+            yield "--no-binary :all:"
+            no_binary.remove(":all:")
+        if ":all:" in only_binary:
+            yield "--only-binary :all:"
+            only_binary.remove(":all:")
+        for nb in dedup(sorted(no_binary)):
             yield f"--no-binary {nb}"
-        for ob in dedup(sorted(self.format_control.only_binary)):
+        for ob in dedup(sorted(only_binary)):
             yield f"--only-binary {ob}"
 
     def write_find_links(self) -> Iterator[str]: