diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ed9908d8..148e7e58 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -29,5 +29,5 @@ jobs: - name: Setup run environment run: tox -vv --notest -e docs - - name: Run check for type + - name: Run check for docs run: tox -e docs --skip-pkg-install diff --git a/.readthedocs.yml b/.readthedocs.yml index 60278c43..6faf8d7c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,7 +1,11 @@ version: 2 +build: + os: ubuntu-20.04 + tools: + python: '3.10' + python: - version: 3.8 install: - method: pip path: . diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be92d0b1..5b40e180 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,19 @@ Unreleased - Add schema validation for ``build-system`` table to check conformity with PEP 517 and PEP 518 (`PR #365`_, Fixes `#364`_) +Breaking changes +---------------- + +- The isolated environment interface was redesigned (`PR #361`_) + + - ``IsolatedEnvBuilder`` removed and its functionality taken over by + ``DefaultIsolatedEnv.with_temp_dir`` + - ``ProjectBuiler.from_isolated_env`` added to initialise a builder from + an isolated environment + - ``RunnerType`` callback signature changed to allow modifying the + ``os.environ`` wholesale + +.. _PR #361: https://github.com/pypa/build/pull/361 .. _PR #365: https://github.com/pypa/build/pull/365 .. _PR #392: https://github.com/pypa/build/pull/392 .. _#364: https://github.com/pypa/build/issues/364 diff --git a/docs/api.rst b/docs/api.rst index df3f4b0e..b1436bb7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,8 +1,6 @@ API Documentation ================= -This project exposes 2 modules: - ``build`` module ---------------- @@ -10,6 +8,7 @@ This project exposes 2 modules: :members: :undoc-members: :show-inheritance: + :special-members: __call__ ``build.env`` module -------------------- diff --git a/docs/conf.py b/docs/conf.py index 26e77d08..6dc09e2b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,3 +65,5 @@ # html_static_path = ['_static'] autoclass_content = 'both' +autodoc_member_order = 'bysource' +autodoc_preserve_defaults = True diff --git a/pyproject.toml b/pyproject.toml index a193fb1d..7fdcd3e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ source = [ exclude_lines = [ '\#\s*pragma: no cover', '^\s*raise NotImplementedError\b', + "if TYPE_CHECKING:", + "@overload", ] [tool.coverage.paths] diff --git a/setup.cfg b/setup.cfg index 1621a4a4..d4f89db7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,7 +64,7 @@ test = typing = importlib-metadata>=4.6.4 mypy==0.910 - typing-extensions>=3.7.4.3 + typing-extensions>=3.8;python_version < "3.8" virtualenv = virtualenv>=20.0.35 diff --git a/src/build/__init__.py b/src/build/__init__.py index 41adff72..0f8dc6bf 100644 --- a/src/build/__init__.py +++ b/src/build/__init__.py @@ -3,6 +3,7 @@ """ build - A simple, correct PEP 517 build frontend """ + __version__ = '0.7.0' import contextlib @@ -16,28 +17,21 @@ import warnings import zipfile -from collections import OrderedDict -from typing import ( - AbstractSet, - Any, - Callable, - Dict, - Iterator, - Mapping, - MutableMapping, - Optional, - Sequence, - Set, - Tuple, - Type, - Union, -) +from typing import Any, Dict, Iterator, Mapping, Optional, Set, Tuple, Type, Union import pep517.wrappers - -TOMLDecodeError: Type[Exception] -toml_loads: Callable[[str], MutableMapping[str, Any]] +from . import env +from ._helpers import ( + ConfigSettingsType, + Distribution, + PathType, + RunnerType, + WheelDistribution, + check_dependency, + default_runner, + rewrap_runner_for_pep517_lib, +) try: @@ -47,10 +41,6 @@ from toml import TomlDecodeError as TOMLDecodeError # type: ignore from toml import loads as toml_loads # type: ignore - -RunnerType = Callable[[Sequence[str], Optional[str], Optional[Mapping[str, str]]], None] -ConfigSettingsType = Mapping[str, Union[str, Sequence[str]]] -PathType = Union[str, 'os.PathLike[str]'] _ExcInfoType = Union[Tuple[Type[BaseException], BaseException, types.TracebackType], Tuple[None, None, None]] @@ -67,18 +57,18 @@ } -_logger = logging.getLogger('build') +_logger = logging.getLogger(__name__) class BuildException(Exception): """ - Exception raised by ProjectBuilder + Exception raised by :class:`ProjectBuilder`. """ class BuildBackendException(Exception): """ - Exception raised when the backend fails + Exception raised when a backend operation fails. """ def __init__( @@ -106,7 +96,7 @@ def __str__(self) -> str: class TypoWarning(Warning): """ - Warning raised when a potential typo is found + Warning raised when a possible typo is found. """ @@ -122,56 +112,30 @@ def _working_directory(path: str) -> Iterator[None]: os.chdir(current) -def _validate_source_directory(srcdir: PathType) -> None: - if not os.path.isdir(srcdir): - raise BuildException(f'Source {srcdir} is not a directory') - pyproject_toml = os.path.join(srcdir, 'pyproject.toml') - setup_py = os.path.join(srcdir, 'setup.py') - if not os.path.exists(pyproject_toml) and not os.path.exists(setup_py): - raise BuildException(f'Source {srcdir} does not appear to be a Python project: no pyproject.toml or setup.py') - - -def check_dependency( - req_string: str, ancestral_req_strings: Tuple[str, ...] = (), parent_extras: AbstractSet[str] = frozenset() -) -> Iterator[Tuple[str, ...]]: - """ - Verify that a dependency and all of its dependencies are met. - - :param req_string: Requirement string - :param parent_extras: Extras (eg. "test" in myproject[test]) - :yields: Unmet dependencies - """ - import packaging.requirements +def _parse_source_dir(source_dir: PathType) -> str: + source_dir = os.path.abspath(source_dir) + if not os.path.isdir(source_dir): + raise BuildException(f'Source {source_dir} is not a directory') - if sys.version_info >= (3, 8): - import importlib.metadata as importlib_metadata - else: - import importlib_metadata + pyproject_toml = os.path.join(source_dir, 'pyproject.toml') + setup_py = os.path.join(source_dir, 'setup.py') + if not os.path.exists(pyproject_toml) and not os.path.exists(setup_py): + raise BuildException(f'Source {source_dir} does not appear to be a Python project: no pyproject.toml or setup.py') - req = packaging.requirements.Requirement(req_string) + return source_dir - if req.marker: - extras = frozenset(('',)).union(parent_extras) - # a requirement can have multiple extras but ``evaluate`` can - # only check one at a time. - if all(not req.marker.evaluate(environment={'extra': e}) for e in extras): - # if the marker conditions are not met, we pretend that the - # dependency is satisfied. - return +def _load_pyproject_toml(source_dir: PathType) -> Mapping[str, Any]: + pyproject_toml = os.path.join(source_dir, 'pyproject.toml') try: - dist = importlib_metadata.distribution(req.name) # type: ignore[no-untyped-call] - except importlib_metadata.PackageNotFoundError: - # dependency is not installed in the environment. - yield ancestral_req_strings + (req_string,) - else: - if req.specifier and not req.specifier.contains(dist.version, prereleases=True): - # the installed version is incompatible. - yield ancestral_req_strings + (req_string,) - elif dist.requires: - for other_req_string in dist.requires: - # yields transitive dependencies that are not satisfied. - yield from check_dependency(other_req_string, ancestral_req_strings + (req_string,), req.extras) + with open(pyproject_toml, 'rb') as f: + return toml_loads(f.read().decode()) + except FileNotFoundError: + return {} + except PermissionError as e: + raise BuildException(f"{e.strerror}: '{e.filename}'") + except TOMLDecodeError as e: + raise BuildException(f'Failed to parse {pyproject_toml}: {e}') def _find_typo(dictionary: Mapping[str, str], expected: str) -> None: @@ -223,116 +187,88 @@ def _parse_build_system_table(pyproject_toml: Mapping[str, Any]) -> Dict[str, An class ProjectBuilder: - """ - The PEP 517 consumer API. - """ + """The PEP 517 consumer API.""" def __init__( self, srcdir: PathType, + *, python_executable: str = sys.executable, - scripts_dir: Optional[str] = None, - runner: RunnerType = pep517.wrappers.default_subprocess_runner, + runner: RunnerType = default_runner, + runner_environ: Optional[Dict[str, str]] = None, ) -> None: """ - :param srcdir: The source directory - :param scripts_dir: The location of the scripts dir (defaults to the folder where the python executable lives) - :param python_executable: The python executable where the backend lives - :param runner: An alternative runner for backend subprocesses - - The 'runner', if provided, must accept the following arguments: - - - cmd: a list of strings representing the command and arguments to - execute, as would be passed to e.g. 'subprocess.check_call'. - - cwd: a string representing the working directory that must be - used for the subprocess. Corresponds to the provided srcdir. - - extra_environ: a dict mapping environment variable names to values - which must be set for the subprocess execution. - - The default runner simply calls the backend hooks in a subprocess, writing backend output - to stdout/stderr. + :param srcdir: Project source directory + :param python_executable: Path of Python executable used to invoke PEP 517 hooks + :param runner: Callback for executing PEP 517 hooks in a subprocess + :param runner_environ: Environment variables to be passed to the runner """ - self.srcdir: str = os.path.abspath(srcdir) - _validate_source_directory(srcdir) + self._srcdir = _parse_source_dir(srcdir) - spec_file = os.path.join(srcdir, 'pyproject.toml') + self._python_executable = python_executable + pep517_runner = rewrap_runner_for_pep517_lib(runner, runner_environ) + + self._build_system = _parse_build_system_table(_load_pyproject_toml(self.srcdir)) + self._build_system_requires = set(self._build_system['requires']) + self._build_backend = self._build_system['build-backend'] - try: - with open(spec_file, 'rb') as f: - spec = toml_loads(f.read().decode()) - except FileNotFoundError: - spec = {} - except PermissionError as e: - raise BuildException(f"{e.strerror}: '{e.filename}' ") - except TOMLDecodeError as e: - raise BuildException(f'Failed to parse {spec_file}: {e} ') - - self._build_system = _parse_build_system_table(spec) - self._backend = self._build_system['build-backend'] - self._scripts_dir = scripts_dir - self._hook_runner = runner self._hook = pep517.wrappers.Pep517HookCaller( self.srcdir, - self._backend, + self._build_backend, backend_path=self._build_system.get('backend-path'), - python_executable=python_executable, - runner=self._runner, + runner=pep517_runner, + python_executable=self._python_executable, ) - def _runner( - self, cmd: Sequence[str], cwd: Optional[str] = None, extra_environ: Optional[Mapping[str, str]] = None - ) -> None: - # if script dir is specified must be inserted at the start of PATH (avoid duplicate path while doing so) - if self.scripts_dir is not None: - paths: Dict[str, None] = OrderedDict() - paths[str(self.scripts_dir)] = None - if 'PATH' in os.environ: - paths.update((i, None) for i in os.environ['PATH'].split(os.pathsep)) - extra_environ = {} if extra_environ is None else dict(extra_environ) - extra_environ['PATH'] = os.pathsep.join(paths) - self._hook_runner(cmd, cwd, extra_environ) - - @property - def python_executable(self) -> str: - """ - The Python executable used to invoke the backend. + @classmethod + def from_isolated_env( + cls, + isolated_env: env.IsolatedEnv, + srcdir: PathType, + *, + runner: RunnerType = default_runner, + ) -> 'ProjectBuilder': """ - # make mypy happy - exe: str = self._hook.python_executable - return exe + Instantiate the builder from an isolated environment. - @python_executable.setter - def python_executable(self, value: str) -> None: - self._hook.python_executable = value + :param isolated_env: Isolated environment + :param srcdir: Project source directory + :param runner: Callback for executing PEP 517 hooks in a subprocess + """ + return cls( + srcdir, + python_executable=isolated_env.python_executable, + runner=runner, + runner_environ=isolated_env.environ, + ) @property - def scripts_dir(self) -> Optional[str]: - """ - The folder where the scripts are stored for the python executable. - """ - return self._scripts_dir + def srcdir(self) -> str: + """Project source directory.""" + return self._srcdir - @scripts_dir.setter - def scripts_dir(self, value: Optional[str]) -> None: - self._scripts_dir = value + @property + def python_executable(self) -> str: + """Path of Python executable used to invoke PEP 517 hooks.""" + return self._python_executable @property def build_system_requires(self) -> Set[str]: """ - The dependencies defined in the ``pyproject.toml``'s - ``build-system.requires`` field or the default build dependencies - if ``pyproject.toml`` is missing or ``build-system`` is undefined. + The dependencies specified in the project's ``build-system.requires`` + field or the default build dependencies if unspecified. """ - return set(self._build_system['requires']) + return self._build_system_requires - def get_requires_for_build(self, distribution: str, config_settings: Optional[ConfigSettingsType] = None) -> Set[str]: + def get_requires_for_build( + self, distribution: Distribution, config_settings: Optional[ConfigSettingsType] = None + ) -> Set[str]: """ - Return the dependencies defined by the backend in addition to - :attr:`build_system_requires` for a given distribution. + Get the build dependencies requested by the backend for + a given distribution. - :param distribution: Distribution to get the dependencies of - (``sdist`` or ``wheel``) - :param config_settings: Config settings for the build backend + :param distribution: Distribution to build + :param config_settings: Config settings passed to the backend """ self.log(f'Getting dependencies for {distribution}...') hook_name = f'get_requires_for_build_{distribution}' @@ -342,30 +278,33 @@ def get_requires_for_build(self, distribution: str, config_settings: Optional[Co return set(get_requires(config_settings)) def check_dependencies( - self, distribution: str, config_settings: Optional[ConfigSettingsType] = None + self, distribution: Distribution, config_settings: Optional[ConfigSettingsType] = None ) -> Set[Tuple[str, ...]]: """ - Return the dependencies which are not satisfied from the combined set of - :attr:`build_system_requires` and :meth:`get_requires_for_build` for a given - distribution. + Check that the :attr:`build_system_requires` and :meth:`get_requires_for_build` + dependencies for a given distribution are satisfied and return the dependency + chain of those which aren't. The unmet dependency is the last value in the chain. - :param distribution: Distribution to check (``sdist`` or ``wheel``) - :param config_settings: Config settings for the build backend - :returns: Set of variable-length unmet dependency tuples + :param distribution: Distribution to build + :param config_settings: Config settings passed to the backend + :returns: Unmet dependencies in the PEP 508 format """ dependencies = self.get_requires_for_build(distribution, config_settings).union(self.build_system_requires) return {u for d in dependencies for u in check_dependency(d)} def prepare( - self, distribution: str, output_directory: PathType, config_settings: Optional[ConfigSettingsType] = None + self, + distribution: WheelDistribution, + output_directory: PathType, + config_settings: Optional[ConfigSettingsType] = None, ) -> Optional[str]: """ Prepare metadata for a distribution. - :param distribution: Distribution to build (must be ``wheel``) + :param distribution: Distribution to build :param output_directory: Directory to put the prepared metadata in - :param config_settings: Config settings for the build backend - :returns: The full path to the prepared metadata directory + :param config_settings: Config settings passed to the backend + :returns: The path of the metadata directory """ self.log(f'Getting metadata for {distribution}...') try: @@ -382,7 +321,7 @@ def prepare( def build( self, - distribution: str, + distribution: Distribution, output_directory: PathType, config_settings: Optional[ConfigSettingsType] = None, metadata_directory: Optional[str] = None, @@ -390,12 +329,12 @@ def build( """ Build a distribution. - :param distribution: Distribution to build (``sdist`` or ``wheel``) + :param distribution: Distribution to build :param output_directory: Directory to put the built distribution in - :param config_settings: Config settings for the build backend + :param config_settings: Config settings passed to the backend :param metadata_directory: If provided, should be the return value of a - previous ``prepare`` call on the same ``distribution`` kind - :returns: The full path to the built distribution + previous ``prepare`` call for the same ``distribution`` type + :returns: The path of the built distribution """ self.log(f'Building {distribution}...') kwargs = {} if metadata_directory is None else {'metadata_directory': metadata_directory} @@ -403,12 +342,13 @@ def build( def metadata_path(self, output_directory: PathType) -> str: """ - Generates the metadata directory of a distribution and returns its path. + Generate the metadata directory of a distribution and return its path. If the backend does not support the ``prepare_metadata_for_build_wheel`` - hook, a wheel will be built and the metadata extracted. + hook, a wheel will be built and the metadata will be extracted from it. :param output_directory: Directory to put the metadata distribution in + :returns: The path of the metadata directory """ # prepare_metadata hook metadata = self.prepare('wheel', output_directory) @@ -455,7 +395,7 @@ def _handle_backend(self, hook: str) -> Iterator[None]: except pep517.wrappers.BackendUnavailable as exception: raise BuildBackendException( exception, - f"Backend '{self._backend}' is not available.", + f"Backend '{self._build_backend}' is not available.", sys.exc_info(), ) except subprocess.CalledProcessError as exception: @@ -466,12 +406,12 @@ def _handle_backend(self, hook: str) -> Iterator[None]: @staticmethod def log(message: str) -> None: """ - Prints message + Log a message. The default implementation uses the logging module but this function can be - overwritten by users to have a different implementation. + overridden by users to have a different implementation. - :param msg: Message to output + :param message: Message to output """ if sys.version_info >= (3, 8): _logger.log(logging.INFO, message, stacklevel=2) @@ -479,13 +419,14 @@ def log(message: str) -> None: _logger.log(logging.INFO, message) -__all__ = ( +__all__ = [ '__version__', 'ConfigSettingsType', 'RunnerType', 'BuildException', + 'BuildSystemTableValidationError', 'BuildBackendException', 'TypoWarning', 'check_dependency', 'ProjectBuilder', -) +] diff --git a/src/build/__main__.py b/src/build/__main__.py index d5685cd9..1a5243b2 100644 --- a/src/build/__main__.py +++ b/src/build/__main__.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: MIT - import argparse import contextlib import os @@ -17,11 +16,9 @@ import build -from build import BuildBackendException, BuildException, ConfigSettingsType, PathType, ProjectBuilder -from build.env import IsolatedEnvBuilder - - -__all__ = ['build', 'main', 'main_parser'] +from build import BuildBackendException, BuildException, ProjectBuilder +from build._helpers import ConfigSettingsType, Distribution, PathType +from build.env import DefaultIsolatedEnv _COLORS = { @@ -78,7 +75,7 @@ def _error(msg: str, code: int = 1) -> None: # pragma: no cover :param msg: Error message :param code: Error code """ - print('{red}ERROR{reset} {}'.format(msg, **_STYLES)) + print('{red}ERROR{reset} {}'.format(msg, **_STYLES), file=sys.stderr) exit(code) @@ -88,7 +85,7 @@ def log(message: str) -> None: print('{bold}* {}{reset}'.format(message, **_STYLES)) -class _IsolatedEnvBuilder(IsolatedEnvBuilder): +class _IsolatedEnv(DefaultIsolatedEnv): @staticmethod def log(message: str) -> None: print('{bold}* {}{reset}'.format(message, **_STYLES)) @@ -99,25 +96,25 @@ def _format_dep_chain(dep_chain: Sequence[str]) -> str: def _build_in_isolated_env( - builder: ProjectBuilder, outdir: PathType, distribution: str, config_settings: Optional[ConfigSettingsType] + srcdir: PathType, outdir: PathType, distribution: Distribution, config_settings: Optional[ConfigSettingsType] ) -> str: - with _IsolatedEnvBuilder() as env: - builder.python_executable = env.executable - builder.scripts_dir = env.scripts_dir + with _IsolatedEnv.with_temp_dir() as env: + builder = _ProjectBuilder.from_isolated_env(env, srcdir) # first install the build dependencies - env.install(builder.build_system_requires) + env.install_packages(builder.build_system_requires) # then get the extra required dependencies from the backend (which was installed in the call above :P) - env.install(builder.get_requires_for_build(distribution)) + env.install_packages(builder.get_requires_for_build(distribution)) return builder.build(distribution, outdir, config_settings or {}) def _build_in_current_env( - builder: ProjectBuilder, + srcdir: PathType, outdir: PathType, - distribution: str, + distribution: Distribution, config_settings: Optional[ConfigSettingsType], skip_dependency_check: bool = False, ) -> str: + builder = _ProjectBuilder(srcdir) if not skip_dependency_check: missing = builder.check_dependencies(distribution) if missing: @@ -130,16 +127,16 @@ def _build_in_current_env( def _build( isolation: bool, - builder: ProjectBuilder, + srcdir: PathType, outdir: PathType, - distribution: str, + distribution: Distribution, config_settings: Optional[ConfigSettingsType], skip_dependency_check: bool, ) -> str: if isolation: - return _build_in_isolated_env(builder, outdir, distribution, config_settings) + return _build_in_isolated_env(srcdir, outdir, distribution, config_settings) else: - return _build_in_current_env(builder, outdir, distribution, config_settings, skip_dependency_check) + return _build_in_current_env(srcdir, outdir, distribution, config_settings, skip_dependency_check) @contextlib.contextmanager @@ -162,7 +159,11 @@ def _handle_build_error() -> Iterator[None]: tb = ''.join(tb_lines) else: tb = traceback.format_exc(-1) - print('\n{dim}{}{reset}\n'.format(tb.strip('\n'), **_STYLES)) + print('\n{dim}{}{reset}\n'.format(tb.strip('\n'), **_STYLES), file=sys.stderr) + _error(str(e)) + except Exception as e: # pragma: no cover + tb = traceback.format_exc().strip('\n') + print('\n{dim}{}{reset}\n'.format(tb, **_STYLES), file=sys.stderr) _error(str(e)) @@ -181,7 +182,7 @@ def _natural_language_list(elements: Sequence[str]) -> str: def build_package( srcdir: PathType, outdir: PathType, - distributions: Sequence[str], + distributions: Sequence[Distribution], config_settings: Optional[ConfigSettingsType] = None, isolation: bool = True, skip_dependency_check: bool = False, @@ -197,9 +198,8 @@ def build_package( :param skip_dependency_check: Do not perform the dependency check """ built: List[str] = [] - builder = _ProjectBuilder(srcdir) for distribution in distributions: - out = _build(isolation, builder, outdir, distribution, config_settings, skip_dependency_check) + out = _build(isolation, srcdir, outdir, distribution, config_settings, skip_dependency_check) built.append(os.path.basename(out)) return built @@ -207,7 +207,7 @@ def build_package( def build_package_via_sdist( srcdir: PathType, outdir: PathType, - distributions: Sequence[str], + distributions: Sequence[Distribution], config_settings: Optional[ConfigSettingsType] = None, isolation: bool = True, skip_dependency_check: bool = False, @@ -225,24 +225,23 @@ def build_package_via_sdist( if 'sdist' in distributions: raise ValueError('Only binary distributions are allowed but sdist was specified') - builder = _ProjectBuilder(srcdir) - sdist = _build(isolation, builder, outdir, 'sdist', config_settings, skip_dependency_check) + sdist = _build(isolation, srcdir, outdir, 'sdist', config_settings, skip_dependency_check) sdist_name = os.path.basename(sdist) sdist_out = tempfile.mkdtemp(prefix='build-via-sdist-') built: List[str] = [] - # extract sdist - with tarfile.open(sdist) as t: - t.extractall(sdist_out) - try: - builder = _ProjectBuilder(os.path.join(sdist_out, sdist_name[: -len('.tar.gz')])) - if distributions: - builder.log(f'Building {_natural_language_list(distributions)} from sdist') - for distribution in distributions: - out = _build(isolation, builder, outdir, distribution, config_settings, skip_dependency_check) - built.append(os.path.basename(out)) - finally: - shutil.rmtree(sdist_out, ignore_errors=True) + if distributions: + # extract sdist + with tarfile.open(sdist) as t: + t.extractall(sdist_out) + try: + _ProjectBuilder.log(f'Building {_natural_language_list(distributions)} from sdist') + srcdir = os.path.join(sdist_out, sdist_name[: -len('.tar.gz')]) + for distribution in distributions: + out = _build(isolation, srcdir, outdir, distribution, config_settings, skip_dependency_check) + built.append(os.path.basename(out)) + finally: + shutil.rmtree(sdist_out, ignore_errors=True) return [sdist_name] + built @@ -340,7 +339,7 @@ def main(cli_args: Sequence[str], prog: Optional[str] = None) -> None: # noqa: parser.prog = prog args = parser.parse_args(cli_args) - distributions = [] + distributions: List[Distribution] = [] config_settings = {} if args.config_setting: @@ -367,19 +366,14 @@ def main(cli_args: Sequence[str], prog: Optional[str] = None) -> None: # noqa: else: build_call = build_package_via_sdist distributions = ['wheel'] - try: - with _handle_build_error(): - built = build_call( - args.srcdir, outdir, distributions, config_settings, not args.no_isolation, args.skip_dependency_check - ) - artifact_list = _natural_language_list( - ['{underline}{}{reset}{bold}{green}'.format(artifact, **_STYLES) for artifact in built] - ) - print('{bold}{green}Successfully built {}{reset}'.format(artifact_list, **_STYLES)) - except Exception as e: # pragma: no cover - tb = traceback.format_exc().strip('\n') - print('\n{dim}{}{reset}\n'.format(tb, **_STYLES)) - _error(str(e)) + with _handle_build_error(): + built = build_call( + args.srcdir, outdir, distributions, config_settings, not args.no_isolation, args.skip_dependency_check + ) + artifact_list = _natural_language_list( + ['{underline}{}{reset}{bold}{green}'.format(artifact, **_STYLES) for artifact in built] + ) + print('{bold}{green}Successfully built {}{reset}'.format(artifact_list, **_STYLES)) def entrypoint() -> None: @@ -388,3 +382,9 @@ def entrypoint() -> None: if __name__ == '__main__': # pragma: no cover main(sys.argv[1:], 'python -m build') + + +__all__ = [ + 'main', + 'main_parser', +] diff --git a/src/build/_compat.py b/src/build/_compat.py new file mode 100644 index 00000000..e9541e75 --- /dev/null +++ b/src/build/_compat.py @@ -0,0 +1,51 @@ +import functools +import sys + +from typing import TYPE_CHECKING + + +class _GenericGetitemMeta(type): + # ``__class_getitem__`` was added in 3.7. + # TODO: Merge into ``_GenericGetitem`` when we drop support for Python 3.6. + def __getitem__(self, value: object) -> None: + ... + + +class _GenericGetitem(metaclass=_GenericGetitemMeta): + pass + + +if sys.version_info >= (3, 8): + from typing import Literal, Protocol +else: + if TYPE_CHECKING: + from typing_extensions import Literal, Protocol + else: + from abc import ABC as Protocol + + Literal = _GenericGetitem + + +if sys.version_info >= (3, 8): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + +if sys.version_info >= (3, 9): + cache = functools.cache + +else: + from typing import Callable, TypeVar + + _C = TypeVar('_C', bound=Callable[..., object]) + + def cache(fn: _C) -> _C: + return functools.lru_cache(maxsize=None)(fn) # type: ignore + + +__all__ = [ + 'Literal', + 'Protocol', + 'importlib_metadata', + 'cache', +] diff --git a/src/build/_helpers.py b/src/build/_helpers.py new file mode 100644 index 00000000..aec33b94 --- /dev/null +++ b/src/build/_helpers.py @@ -0,0 +1,93 @@ +import functools +import os +import subprocess + +from typing import AbstractSet, Iterator, Mapping, Optional, Sequence, Tuple, Union + +from ._compat import Literal, Protocol, importlib_metadata + + +ConfigSettingsType = Mapping[str, Union[str, Sequence[str]]] +PathType = Union[str, 'os.PathLike[str]'] + +Distribution = Literal['sdist', 'wheel'] +WheelDistribution = Literal['wheel'] + + +class RunnerType(Protocol): + def __call__(self, cmd: Sequence[str], cwd: Optional[PathType] = None, env: Optional[Mapping[str, str]] = None) -> None: + """ + Run a command in a Python subprocess. + + The parameters mirror those of ``subprocess.run``. + + :param cmd: The command to execute + :param cwd: The working directory + :param env: Variables to be exported to the environment + """ + + +class _Pep517CallbackType(Protocol): + def __call__( + self, cmd: Sequence[str], cwd: Optional[PathType] = None, extra_environ: Optional[Mapping[str, str]] = None + ) -> None: + ... + + +def default_runner(cmd: Sequence[str], cwd: Optional[PathType] = None, env: Optional[Mapping[str, str]] = None) -> None: + subprocess.run(cmd, check=True, cwd=cwd, env=env) + + +def quiet_runner(cmd: Sequence[str], cwd: Optional[PathType] = None, env: Optional[Mapping[str, str]] = None) -> None: + subprocess.run(cmd, check=True, cwd=cwd, env=env, stdout=subprocess.DEVNULL) + + +def rewrap_runner_for_pep517_lib(runner: RunnerType, env: Optional[Mapping[str, str]] = None) -> '_Pep517CallbackType': + @functools.wraps(runner) + def inner(cmd: Sequence[str], cwd: Optional[PathType] = None, extra_environ: Optional[Mapping[str, str]] = None) -> None: + local_env = os.environ.copy() if env is None else dict(env) + if extra_environ: + local_env.update(extra_environ) + + runner(cmd, cwd=cwd, env=local_env) + + return inner + + +def check_dependency( + req_string: str, ancestral_req_strings: Tuple[str, ...] = (), parent_extras: AbstractSet[str] = frozenset() +) -> Iterator[Tuple[str, ...]]: + """ + Verify that a dependency and all of its dependencies are met. + + :param req_string: Requirement string + :param ancestral_req_strings: The dependency chain leading to this ``req_string`` + :param parent_extras: Extras (eg. "test" in ``myproject[test]``) + :yields: Unmet dependencies + """ + import packaging.requirements + + req = packaging.requirements.Requirement(req_string) + + if req.marker: + extras = frozenset(('',)).union(parent_extras) + # a requirement can have multiple extras but ``evaluate`` can + # only check one at a time. + if all(not req.marker.evaluate(environment={'extra': e}) for e in extras): + # if the marker conditions are not met, we pretend that the + # dependency is satisfied. + return + + try: + dist = importlib_metadata.distribution(req.name) # type: ignore[no-untyped-call] + except importlib_metadata.PackageNotFoundError: + # dependency is not installed in the environment. + yield ancestral_req_strings + (req_string,) + else: + if req.specifier and not req.specifier.contains(dist.version, prereleases=True): + # the installed version is incompatible. + yield ancestral_req_strings + (req_string,) + elif dist.requires: + for other_req_string in dist.requires: + # yields transitive dependencies that are not satisfied. + yield from check_dependency(other_req_string, ancestral_req_strings + (req_string,), req.extras) diff --git a/src/build/env.py b/src/build/env.py index 80c42d8e..8001ab83 100644 --- a/src/build/env.py +++ b/src/build/env.py @@ -1,317 +1,253 @@ """ Creates and manages isolated build environments. """ + import abc -import functools +import contextlib import logging import os import platform -import shutil -import subprocess import sys import sysconfig import tempfile -from types import TracebackType -from typing import Callable, Iterable, List, Optional, Tuple, Type - -import build - +from typing import Dict, Iterable, Iterator, Optional, Sequence, Tuple -if sys.version_info < (3, 8): - import importlib_metadata as metadata -else: - from importlib import metadata - -try: - import virtualenv -except ModuleNotFoundError: - virtualenv = None +from ._compat import Protocol, cache, importlib_metadata +from ._helpers import check_dependency, default_runner -_logger = logging.getLogger('build.env') +_logger = logging.getLogger(__name__) -class IsolatedEnv(metaclass=abc.ABCMeta): - """Abstract base of isolated build environments, as required by the build project.""" +class IsolatedEnv(Protocol): + """Protocol for isolated build environments.""" @property @abc.abstractmethod - def executable(self) -> str: - """The executable of the isolated build environment.""" + def environ(self) -> Optional[Dict[str, str]]: + """The isolated environment's environment variables.""" raise NotImplementedError @property @abc.abstractmethod - def scripts_dir(self) -> str: - """The scripts directory of the isolated build environment.""" + def python_executable(self) -> str: + """The isolated environment's Python executable.""" raise NotImplementedError - @abc.abstractmethod - def install(self, requirements: Iterable[str]) -> None: - """ - Install packages from PEP 508 requirements in the isolated build environment. - - :param requirements: PEP 508 requirements - """ - raise NotImplementedError - - -@functools.lru_cache(maxsize=None) -def _should_use_virtualenv() -> bool: - import packaging.requirements - - # virtualenv might be incompatible if it was installed separately - # from build. This verifies that virtualenv and all of its - # dependencies are installed as specified by build. - return virtualenv is not None and not any( - packaging.requirements.Requirement(d[1]).name == 'virtualenv' - for d in build.check_dependency('build[virtualenv]') - if len(d) > 1 - ) +@cache +def _fs_supports_symlinks() -> bool: + """Check if symlinks are supported.""" + # Using definition used by venv.main() + if os.name != 'nt': + return True -def _subprocess(cmd: List[str]) -> None: - """Invoke subprocess and output stdout and stderr if it fails.""" - try: - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - print(e.output.decode(), end='', file=sys.stderr) - raise e - - -class IsolatedEnvBuilder: - """Builder object for isolated environments.""" - - def __init__(self) -> None: - self._path: Optional[str] = None - - def __enter__(self) -> IsolatedEnv: - """ - Create an isolated build environment. - - :return: The isolated build environment - """ - # Call ``realpath`` to prevent spurious warning from being emitted - # that the venv location has changed on Windows. The username is - # DOS-encoded in the output of tempfile - the location is the same - # but the representation of it is different, which confuses venv. - # Ref: https://bugs.python.org/issue46171 - self._path = os.path.realpath(tempfile.mkdtemp(prefix='build-env-')) + # Windows may support symlinks (setting in Windows 10) + with tempfile.TemporaryDirectory(prefix='build-try-symlink-') as temp_dir: try: - # use virtualenv when available (as it's faster than venv) - if _should_use_virtualenv(): - self.log('Creating virtualenv isolated environment...') - executable, scripts_dir = _create_isolated_env_virtualenv(self._path) - else: - self.log('Creating venv isolated environment...') - executable, scripts_dir = _create_isolated_env_venv(self._path) - return _IsolatedEnvVenvPip( - path=self._path, - python_executable=executable, - scripts_dir=scripts_dir, - log=self.log, + os.symlink( + os.path.join(temp_dir, 'foo'), + os.path.join(temp_dir, 'bar'), ) - except Exception: # cleanup folder if creation fails - self.__exit__(*sys.exc_info()) - raise + return True + except (OSError, NotImplementedError, AttributeError): + return False - def __exit__( - self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] - ) -> None: - """ - Delete the created isolated build environment. - :param exc_type: The type of exception raised (if any) - :param exc_val: The value of exception raised (if any) - :param exc_tb: The traceback of exception raised (if any) - """ - if self._path is not None and os.path.exists(self._path): # in case the user already deleted skip remove - shutil.rmtree(self._path) +def _get_isolated_env_executable_and_paths(path: str) -> Tuple[str, Dict[str, str]]: + """ + :param path: venv path on disk + :returns: The Python executable and scripts folder + """ + config_vars = sysconfig.get_config_vars().copy() # globally cached, copy before altering it + config_vars['base'] = path + # The Python that ships with the macOS developer tools varies the + # default scheme depending on whether the ``sys.prefix`` is part of a framework. + # The framework "osx_framework_library" scheme + # can't be used to expand the paths in a venv, which + # can happen if build itself is not installed in a venv. + # If the Apple-custom "osx_framework_library" scheme is available + # we enforce "posix_prefix", the venv scheme, for isolated envs. + if 'osx_framework_library' in sysconfig.get_scheme_names(): + paths = sysconfig.get_paths(scheme='posix_prefix', vars=config_vars) + else: + paths = sysconfig.get_paths(vars=config_vars) + executable = os.path.join(paths['scripts'], 'python.exe' if os.name == 'nt' else 'python') + if not os.path.exists(executable): + raise RuntimeError(f'Virtual environment creation failed, executable {executable} missing') + return executable, paths - @staticmethod - def log(message: str) -> None: - """ - Prints message - The default implementation uses the logging module but this function can be - overwritten by users to have a different implementation. +def _create_isolated_env_venv(path: str) -> Tuple[str, Dict[str, str]]: + """ + :param path: venv path on disk + :returns: The Python executable and scripts folder + """ + import venv - :param msg: Message to output - """ - if sys.version_info >= (3, 8): - _logger.log(logging.INFO, message, stacklevel=2) - else: - _logger.log(logging.INFO, message) + venv.EnvBuilder(symlinks=_fs_supports_symlinks(), with_pip=True).create(path) + return _get_isolated_env_executable_and_paths(path) -class _IsolatedEnvVenvPip(IsolatedEnv): +def _create_isolated_env_virtualenv(path: str) -> Tuple[str, str]: """ - Isolated build environment context manager - - Non-standard paths injected directly to sys.path will still be passed to the environment. + :param path: virtualenv path on disk + :returns: The Python executable and scripts folder """ + import virtualenv - def __init__( - self, - path: str, - python_executable: str, - scripts_dir: str, - log: Callable[[str], None], - ) -> None: - """ - :param path: The path where the environment exists - :param python_executable: The python executable within the environment - :param log: Log function - """ - self._path = path - self._python_executable = python_executable - self._scripts_dir = scripts_dir - self._log = log - - @property - def path(self) -> str: - """The location of the isolated build environment.""" - return self._path + cmd = [path, '--no-setuptools', '--no-wheel', '--activators', ''] + result = virtualenv.cli_run(cmd, setup_logging=False) + executable = str(result.creator.exe) + script_dir = str(result.creator.script_dir) + return executable, script_dir - @property - def executable(self) -> str: - """The python executable of the isolated build environment.""" - return self._python_executable - @property - def scripts_dir(self) -> str: - return self._scripts_dir +@cache +def _should_use_virtualenv() -> bool: + import packaging.requirements - def install(self, requirements: Iterable[str]) -> None: - """ - Install packages from PEP 508 requirements in the isolated build environment. + # virtualenv might be incompatible if it was installed separately + # from build. This verifies that virtualenv and all of its + # dependencies are installed as specified by build. + return not any( + packaging.requirements.Requirement(u).name == 'virtualenv' for d in check_dependency('build[virtualenv]') for u in d + ) - :param requirements: PEP 508 requirement specification to install - :note: Passing non-PEP 508 strings will result in undefined behavior, you *should not* rely on it. It is - merely an implementation detail, it may change any time without warning. - """ - if not requirements: - return +@cache +def _get_min_pip_version() -> str: + if platform.system() == 'Darwin': + release, _, machine = platform.mac_ver() - self._log('Installing packages in isolated environment... ({})'.format(', '.join(sorted(requirements)))) + # Apple Silicon support for wheels shipped with packaging 20.9, + # vendored in pip 21.0.1. + if machine == 'arm64': + return '21.0.1' - # pip does not honour environment markers in command line arguments - # but it does for requirements from a file - with tempfile.NamedTemporaryFile('w+', prefix='build-reqs-', suffix='.txt', delete=False) as req_file: - req_file.write(os.linesep.join(requirements)) - try: - cmd = [ - self.executable, - '-Im', - 'pip', - 'install', - '--use-pep517', - '--no-warn-script-location', - '-r', - os.path.abspath(req_file.name), - ] - _subprocess(cmd) - finally: - os.unlink(req_file.name) + # macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can + # be told to report 10.16 for backwards compatibility; + # but that also fixes earlier versions of pip so this is only needed for 11+. + major_version = int(release[: release.find('.')]) + if major_version >= 11: + return '20.3' + # PEP 517 and manylinux1 were first implemented in 19.1 + return '19.1' -def _create_isolated_env_virtualenv(path: str) -> Tuple[str, str]: - """ - On Python 2 we use the virtualenv package to provision a virtual environment. - :param path: The path where to create the isolated build environment - :return: The Python executable and script folder - """ - cmd = [str(path), '--no-setuptools', '--no-wheel', '--activators', ''] - result = virtualenv.cli_run(cmd, setup_logging=False) - executable = str(result.creator.exe) - script_dir = str(result.creator.script_dir) - return executable, script_dir +class DefaultIsolatedEnv(IsolatedEnv): + """An isolated environment which combines either venv or virtualenv with pip.""" + def _create(self, path: str) -> None: + if _should_use_virtualenv(): + self.log('Creating isolated environment (virtualenv)...') + self._python_executable, scripts_dir = _create_isolated_env_virtualenv(path) + self._environ = self._prepare_environ(scripts_dir) + else: + self.log('Creating isolated environment (venv)...') + # Call ``realpath`` to prevent spurious warning from being emitted + # that the venv location has changed on Windows. The username is + # DOS-encoded in the output of tempfile - the location is the same + # but the representation of it is different, which confuses venv. + # Ref: https://bugs.python.org/issue46171 + path = os.path.realpath(tempfile.mkdtemp(prefix='build-env-')) + self._python_executable, paths = _create_isolated_env_venv(path) + self._environ = self._prepare_environ(paths['scripts']) + self._patch_up_venv(paths['purelib']) -@functools.lru_cache(maxsize=None) -def _fs_supports_symlink() -> bool: - """Return True if symlinks are supported""" - # Using definition used by venv.main() - if os.name != 'nt': - return True + @staticmethod + def _prepare_environ(scripts_dir: str) -> Dict[str, str]: + environ = os.environ.copy() - # Windows may support symlinks (setting in Windows 10) - with tempfile.NamedTemporaryFile(prefix='build-symlink-') as tmp_file: - dest = f'{tmp_file}-b' - try: - os.symlink(tmp_file.name, dest) - os.unlink(dest) - return True - except (OSError, NotImplementedError, AttributeError): - return False + # Make the virtual environment's scripts available to the project's build dependecies. + path = environ.get('PATH') + environ['PATH'] = os.pathsep.join([scripts_dir, path]) if path is not None else scripts_dir + return environ -def _create_isolated_env_venv(path: str) -> Tuple[str, str]: - """ - On Python 3 we use the venv package from the standard library. + def _run(self, cmd: Sequence[str]) -> None: + return default_runner(cmd, env=self.environ) - :param path: The path where to create the isolated build environment - :return: The Python executable and script folder - """ - import venv + def _patch_up_venv(self, venv_purelib: str) -> None: + import packaging.version - import packaging.version + cur_pip_version = next( + d.version + for d in importlib_metadata.distributions(name='pip', path=[venv_purelib]) # type: ignore[no-untyped-call] + ) + min_pip_version = _get_min_pip_version() + if packaging.version.Version(cur_pip_version) < packaging.version.Version(min_pip_version): + self._run([self.python_executable, '-m', 'pip', 'install', f'pip>={min_pip_version}']) - venv.EnvBuilder(with_pip=True, symlinks=_fs_supports_symlink()).create(path) - executable, script_dir, purelib = _find_executable_and_scripts(path) + self._run([self.python_executable, '-m', 'pip', 'uninstall', '-y', 'setuptools']) - # Get the version of pip in the environment - pip_distribution = next(iter(metadata.distributions(name='pip', path=[purelib]))) # type: ignore[no-untyped-call] - current_pip_version = packaging.version.Version(pip_distribution.version) + @classmethod + @contextlib.contextmanager + def with_temp_dir(cls) -> Iterator['DefaultIsolatedEnv']: + with tempfile.TemporaryDirectory(prefix='build-env') as temp_dir: + self = cls() + self._create(temp_dir) + yield self - if platform.system() == 'Darwin' and int(platform.mac_ver()[0].split('.')[0]) >= 11: - # macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be told to report 10.16 for backwards - # compatibility; but that also fixes earlier versions of pip so this is only needed for 11+. - is_apple_silicon_python = platform.machine() != 'x86_64' - minimum_pip_version = '21.0.1' if is_apple_silicon_python else '20.3.0' - else: - # PEP-517 and manylinux1 was first implemented in 19.1 - minimum_pip_version = '19.1.0' + def install_packages(self, requirements: Iterable[str]) -> None: + """ + Install packages in the isolated environment. - if current_pip_version < packaging.version.Version(minimum_pip_version): - _subprocess([executable, '-m', 'pip', 'install', f'pip>={minimum_pip_version}']) + :param requirements: PEP 508-style requirements + """ + req_list = sorted(requirements) + if not req_list: + return - # Avoid the setuptools from ensurepip to break the isolation - _subprocess([executable, '-m', 'pip', 'uninstall', 'setuptools', '-y']) - return executable, script_dir + # pip does not honour environment markers in command line arguments; + # it does for requirements from a file. + with tempfile.NamedTemporaryFile( + 'w', + prefix='build-reqs-', + suffix='.txt', + # On Windows the temp file can't be opened by pip while it is kept open by build + # so we defer deleting it. + delete=False, + ) as req_file: + req_file.write(os.linesep.join(req_list)) + try: + self.log(f'Installing build dependencies... ({", ".join(req_list)})') + self._run( + [ + self.python_executable, + '-m', + 'pip', + 'install', + # Enforce PEP 517 because "legacy" builds won't work for build dependencies + # of a project which does not use setuptools. + '--use-pep517', + '-r', + req_file.name, + ] + ) + finally: + os.unlink(req_file.name) -def _find_executable_and_scripts(path: str) -> Tuple[str, str, str]: - """ - Detect the Python executable and script folder of a virtual environment. + @staticmethod + def log(message: str) -> None: + if sys.version_info >= (3, 8): + _logger.log(logging.INFO, message, stacklevel=2) + else: + _logger.log(logging.INFO, message) - :param path: The location of the virtual environment - :return: The Python executable, script folder, and purelib folder - """ - config_vars = sysconfig.get_config_vars().copy() # globally cached, copy before altering it - config_vars['base'] = path - # The Python that ships with the macOS developer tools varies the - # default scheme depending on whether the ``sys.prefix`` is part of a framework. - # The framework "osx_framework_library" scheme - # can't be used to expand the paths in a venv, which - # can happen if build itself is not installed in a venv. - # If the Apple-custom "osx_framework_library" scheme is available - # we enforce "posix_prefix", the venv scheme, for isolated envs. - if 'osx_framework_library' in sysconfig.get_scheme_names(): - paths = sysconfig.get_paths(scheme='posix_prefix', vars=config_vars) - else: - paths = sysconfig.get_paths(vars=config_vars) - executable = os.path.join(paths['scripts'], 'python.exe' if os.name == 'nt' else 'python') - if not os.path.exists(executable): - raise RuntimeError(f'Virtual environment creation failed, executable {executable} missing') + @property + def environ(self) -> Dict[str, str]: + return self._environ - return executable, paths['scripts'], paths['purelib'] + @property + def python_executable(self) -> str: + return self._python_executable -__all__ = ( - 'IsolatedEnvBuilder', +__all__ = [ 'IsolatedEnv', -) + 'DefaultIsolatedEnv', +] diff --git a/src/build/util.py b/src/build/util.py index be93eee4..81411f40 100644 --- a/src/build/util.py +++ b/src/build/util.py @@ -1,31 +1,23 @@ # SPDX-License-Identifier: MIT -import os import pathlib -import sys import tempfile -import pep517 +from . import ProjectBuilder +from ._compat import importlib_metadata +from ._helpers import PathType, quiet_runner +from .env import DefaultIsolatedEnv -import build -import build.env - -if sys.version_info >= (3, 8): - import importlib.metadata as importlib_metadata -else: - import importlib_metadata - - -def _project_wheel_metadata(builder: build.ProjectBuilder) -> 'importlib_metadata.PackageMetadata': +def _project_wheel_metadata(builder: ProjectBuilder) -> 'importlib_metadata.PackageMetadata': with tempfile.TemporaryDirectory() as tmpdir: path = pathlib.Path(builder.metadata_path(tmpdir)) - # https://github.com/python/importlib_metadata/pull/343 - return importlib_metadata.PathDistribution(path).metadata # type: ignore + # https://github.com/python/importlib_metadata/pull/342 + return importlib_metadata.PathDistribution(path).metadata # type: ignore[arg-type] def project_wheel_metadata( - srcdir: build.PathType, + srcdir: PathType, isolated: bool = True, ) -> 'importlib_metadata.PackageMetadata': """ @@ -35,24 +27,20 @@ def project_wheel_metadata( otherwise ``build_wheel``. :param srcdir: Project source directory - :param isolated: Whether or not to run invoke the backend in the current - environment or to create an isolated one and invoke it - there. + :param isolated: Whether to invoke the backend in the current environment + or create an isolated environment and invoke it there """ - builder = build.ProjectBuilder( - os.fspath(srcdir), - runner=pep517.quiet_subprocess_runner, - ) - if not isolated: + builder = ProjectBuilder(srcdir, runner=quiet_runner) return _project_wheel_metadata(builder) - with build.env.IsolatedEnvBuilder() as env: - builder.python_executable = env.executable - builder.scripts_dir = env.scripts_dir - env.install(builder.build_system_requires) - env.install(builder.get_requires_for_build('wheel')) + with DefaultIsolatedEnv.with_temp_dir() as env: + builder = ProjectBuilder.from_isolated_env(env, srcdir, runner=quiet_runner) + env.install_packages(builder.build_system_requires) + env.install_packages(builder.get_requires_for_build('wheel')) return _project_wheel_metadata(builder) -__all__ = ('project_wheel_metadata',) +__all__ = [ + 'project_wheel_metadata', +] diff --git a/tests/test_env.py b/tests/test_env.py index 831d7254..40a1b621 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: MIT -import collections import logging import os import platform @@ -7,6 +6,7 @@ import subprocess import sys import sysconfig +import types import pytest @@ -21,30 +21,29 @@ @pytest.mark.isolated def test_isolation(): subprocess.check_call([sys.executable, '-c', 'import build.env']) - with build.env.IsolatedEnvBuilder() as env: + with build.env.DefaultIsolatedEnv.with_temp_dir() as env: with pytest.raises(subprocess.CalledProcessError): debug = 'import sys; import os; print(os.linesep.join(sys.path));' - subprocess.check_call([env.executable, '-c', f'{debug} import build.env']) + subprocess.check_call([env.python_executable, '-c', f'{debug} import build.env']) @pytest.mark.isolated def test_isolated_environment_install(mocker): - with build.env.IsolatedEnvBuilder() as env: - mocker.patch('build.env._subprocess') + with build.env.DefaultIsolatedEnv.with_temp_dir() as env: + mocker.patch('build.env.DefaultIsolatedEnv._run') - env.install([]) - build.env._subprocess.assert_not_called() + env.install_packages([]) + build.env.DefaultIsolatedEnv._run.assert_not_called() - env.install(['some', 'requirements']) - build.env._subprocess.assert_called() - args = build.env._subprocess.call_args[0][0][:-1] + env.install_packages(['some', 'requirements']) + build.env.DefaultIsolatedEnv._run.assert_called() + args = build.env.DefaultIsolatedEnv._run.call_args[0][0][:-1] assert args == [ - env.executable, - '-Im', + env.python_executable, + '-m', 'pip', 'install', '--use-pep517', - '--no-warn-script-location', '-r', ] @@ -53,7 +52,7 @@ def test_isolated_environment_install(mocker): @pytest.mark.skipif(sys.platform != 'darwin', reason='workaround for Apple Python') def test_can_get_venv_paths_with_conflicting_default_scheme(mocker): get_scheme_names = mocker.patch('sysconfig.get_scheme_names', return_value=('osx_framework_library',)) - with build.env.IsolatedEnvBuilder(): + with build.env.DefaultIsolatedEnv.with_temp_dir(): pass assert get_scheme_names.call_count == 1 @@ -63,12 +62,13 @@ def test_executable_missing_post_creation(mocker): original_get_paths = sysconfig.get_paths def _get_paths(vars): # noqa - shutil.rmtree(vars['base']) - return original_get_paths(vars=vars) + paths = original_get_paths(vars=vars) + shutil.rmtree(paths['scripts']) + return paths get_paths = mocker.patch('sysconfig.get_paths', side_effect=_get_paths) with pytest.raises(RuntimeError, match='Virtual environment creation failed, executable .* missing'): - with build.env.IsolatedEnvBuilder(): + with build.env.DefaultIsolatedEnv.with_temp_dir(): pass assert get_paths.call_count == 1 @@ -78,75 +78,65 @@ def test_isolated_env_abstract(): build.env.IsolatedEnv() -def test_isolated_env_has_executable_still_abstract(): - class Env(build.env.IsolatedEnv): # noqa - @property - def executable(self): - raise NotImplementedError - - with pytest.raises(TypeError): - Env() - - -def test_isolated_env_has_install_still_abstract(): - class Env(build.env.IsolatedEnv): # noqa - def install(self, requirements): - raise NotImplementedError - - with pytest.raises(TypeError): - Env() - - def test_isolated_env_log(mocker, caplog, package_test_flit): - mocker.patch('build.env._subprocess') + mocker.patch('build.env.DefaultIsolatedEnv._run') caplog.set_level(logging.DEBUG) - builder = build.env.IsolatedEnvBuilder() - builder.log('something') + builder = build.env.DefaultIsolatedEnv.with_temp_dir() with builder as env: - env.install(['something']) + env.log('something') + env.install_packages(['something']) assert [(record.levelname, record.message) for record in caplog.records] == [ + ('INFO', 'Creating isolated environment (venv)...'), ('INFO', 'something'), - ('INFO', 'Creating venv isolated environment...'), - ('INFO', 'Installing packages in isolated environment... (something)'), + ('INFO', 'Installing build dependencies... (something)'), ] if sys.version_info >= (3, 8): # stacklevel - assert [(record.lineno) for record in caplog.records] == [105, 107, 198] + assert [record.lineno for record in caplog.records] == [ + build.env.DefaultIsolatedEnv._create.__code__.co_firstlineno + 6, + test_isolated_env_log.__code__.co_firstlineno + 6, + build.env.DefaultIsolatedEnv.install_packages.__code__.co_firstlineno + 23, + ] @pytest.mark.isolated def test_default_pip_is_never_too_old(): - with build.env.IsolatedEnvBuilder() as env: + with build.env.DefaultIsolatedEnv.with_temp_dir() as env: version = subprocess.check_output( - [env.executable, '-c', 'import pip; print(pip.__version__)'], universal_newlines=True + [env.python_executable, '-c', 'import pip; print(pip.__version__)'], universal_newlines=True ).strip() assert Version(version) >= Version('19.1') @pytest.mark.isolated -@pytest.mark.parametrize('pip_version', ['20.2.0', '20.3.0', '21.0.0', '21.0.1']) @pytest.mark.parametrize('arch', ['x86_64', 'arm64']) -def test_pip_needs_upgrade_mac_os_11(mocker, pip_version, arch): - SimpleNamespace = collections.namedtuple('SimpleNamespace', 'version') - - _subprocess = mocker.patch('build.env._subprocess') +@pytest.mark.parametrize('pip_version', ['20.2.0', '20.3.0', '21.0.0', '21.0.1']) +@pytest.mark.skipif(sys.platform != 'darwin', reason='mac only') +def test_pip_needs_upgrade_mac_os_11(mocker, arch, pip_version): + _subprocess = mocker.patch('build.env.DefaultIsolatedEnv._run') mocker.patch('platform.system', return_value='Darwin') - mocker.patch('platform.machine', return_value=arch) - mocker.patch('platform.mac_ver', return_value=('11.0', ('', '', ''), '')) - mocker.patch('build.env.metadata.distributions', return_value=(SimpleNamespace(version=pip_version),)) + mocker.patch('platform.mac_ver', return_value=('11.0', ('', '', ''), arch)) + + mocker.patch( + 'build._compat.importlib_metadata.distributions', + return_value=iter([types.SimpleNamespace(version=pip_version)]), + ) + + # Cache must be cleared to rerun + build.env._get_min_pip_version.cache_clear() + min_version = build.env._get_min_pip_version() - min_version = Version('20.3' if arch == 'x86_64' else '21.0.1') - with build.env.IsolatedEnvBuilder(): - if Version(pip_version) < min_version: + with build.env.DefaultIsolatedEnv.with_temp_dir(): + if Version(pip_version) < Version(min_version): print(_subprocess.call_args_list) upgrade_call, uninstall_call = _subprocess.call_args_list - answer = 'pip>=20.3.0' if arch == 'x86_64' else 'pip>=21.0.1' + answer = 'pip>=20.3' if arch == 'x86_64' else 'pip>=21.0.1' assert upgrade_call[0][0][1:] == ['-m', 'pip', 'install', answer] - assert uninstall_call[0][0][1:] == ['-m', 'pip', 'uninstall', 'setuptools', '-y'] + assert uninstall_call[0][0][1:] == ['-m', 'pip', 'uninstall', '-y', 'setuptools'] else: (uninstall_call,) = _subprocess.call_args_list - assert uninstall_call[0][0][1:] == ['-m', 'pip', 'uninstall', 'setuptools', '-y'] + assert uninstall_call[0][0][1:] == ['-m', 'pip', 'uninstall', '-y', 'setuptools'] @pytest.mark.isolated @@ -160,8 +150,8 @@ def test_venv_symlink(mocker, has_symlink): mocker.patch('os.symlink', side_effect=OSError()) # Cache must be cleared to rerun - build.env._fs_supports_symlink.cache_clear() - supports_symlink = build.env._fs_supports_symlink() - build.env._fs_supports_symlink.cache_clear() + build.env._fs_supports_symlinks.cache_clear() + supports_symlinks = build.env._fs_supports_symlinks() + build.env._fs_supports_symlinks.cache_clear() - assert supports_symlink is has_symlink + assert supports_symlinks is has_symlink diff --git a/tests/test_main.py b/tests/test_main.py index d34f1bc2..6005b82d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -125,7 +125,7 @@ def test_build_isolated(mocker, package_test_flit): ], ) mocker.patch('build.__main__._error') - install = mocker.patch('build.env._IsolatedEnvVenvPip.install') + install = mocker.patch('build.env.DefaultIsolatedEnv.install_packages') build.__main__.build_package(package_test_flit, '.', ['sdist']) @@ -168,7 +168,7 @@ def test_build_no_isolation_with_check_deps(mocker, package_test_flit, missing_d @pytest.mark.isolated def test_build_raises_build_exception(mocker, package_test_flit): mocker.patch('build.ProjectBuilder.get_requires_for_build', side_effect=build.BuildException) - mocker.patch('build.env._IsolatedEnvVenvPip.install') + mocker.patch('build.env.DefaultIsolatedEnv.install_packages') with pytest.raises(build.BuildException): build.__main__.build_package(package_test_flit, '.', ['sdist']) @@ -177,7 +177,7 @@ def test_build_raises_build_exception(mocker, package_test_flit): @pytest.mark.isolated def test_build_raises_build_backend_exception(mocker, package_test_flit): mocker.patch('build.ProjectBuilder.get_requires_for_build', side_effect=build.BuildBackendException(Exception('a'))) - mocker.patch('build.env._IsolatedEnvVenvPip.install') + mocker.patch('build.env.DefaultIsolatedEnv.install_packages') msg = f"Backend operation failed: Exception('a'{',' if sys.version_info < (3, 7) else ''})" with pytest.raises(build.BuildBackendException, match=re.escape(msg)): @@ -218,15 +218,15 @@ def test_build_package_via_sdist_invalid_distribution(tmp_dir, package_test_setu ( [], [ - '* Creating venv isolated environment...', - '* Installing packages in isolated environment... (setuptools >= 42.0.0, wheel >= 0.36.0)', + '* Creating isolated environment (venv)...', + '* Installing build dependencies... (setuptools >= 42.0.0, wheel >= 0.36.0)', '* Getting dependencies for sdist...', '* Building sdist...', '* Building wheel from sdist', - '* Creating venv isolated environment...', - '* Installing packages in isolated environment... (setuptools >= 42.0.0, wheel >= 0.36.0)', + '* Creating isolated environment (venv)...', + '* Installing build dependencies... (setuptools >= 42.0.0, wheel >= 0.36.0)', '* Getting dependencies for wheel...', - '* Installing packages in isolated environment... (wheel)', + '* Installing build dependencies... (wheel)', '* Building wheel...', 'Successfully built test_setuptools-1.0.0.tar.gz and test_setuptools-1.0.0-py2.py3-none-any.whl', ], @@ -245,10 +245,10 @@ def test_build_package_via_sdist_invalid_distribution(tmp_dir, package_test_setu ( ['--wheel'], [ - '* Creating venv isolated environment...', - '* Installing packages in isolated environment... (setuptools >= 42.0.0, wheel >= 0.36.0)', + '* Creating isolated environment (venv)...', + '* Installing build dependencies... (setuptools >= 42.0.0, wheel >= 0.36.0)', '* Getting dependencies for wheel...', - '* Installing packages in isolated environment... (wheel)', + '* Installing build dependencies... (wheel)', '* Building wheel...', 'Successfully built test_setuptools-1.0.0-py2.py3-none-any.whl', ], @@ -285,7 +285,7 @@ def test_build_package_via_sdist_invalid_distribution(tmp_dir, package_test_setu 'sdist-and-wheel-direct-no-isolation', ], ) -@pytest.mark.flaky(reruns=5) +# @pytest.mark.flaky(reruns=5) def test_output(package_test_setuptools, tmp_dir, capsys, args, output): build.__main__.main([package_test_setuptools, '-o', tmp_dir] + args) stdout, stderr = capsys.readouterr() @@ -301,28 +301,23 @@ def main_reload_styles(): @pytest.mark.parametrize( - ('color', 'stdout_error', 'stdout_body'), + ('color', 'stdout_content', 'stderr_content'), [ ( False, - 'ERROR ', [ - '* Creating venv isolated environment...', - '* Installing packages in isolated environment... (setuptools >= 42.0.0, this is invalid, wheel >= 0.36.0)', - '', - 'Traceback (most recent call last):', + '* Creating isolated environment (venv)...', + '* Installing build dependencies... (setuptools >= 42.0.0, this is invalid, wheel >= 0.36.0)', ], + 'ERROR ', ), ( True, - '\33[91mERROR\33[0m ', [ - '\33[1m* Creating venv isolated environment...\33[0m', - '\33[1m* Installing packages in isolated environment... ' - '(setuptools >= 42.0.0, this is invalid, wheel >= 0.36.0)\33[0m', - '', - '\33[2mTraceback (most recent call last):', + '\33[1m* Creating isolated environment (venv)...\33[0m', + '\33[1m* Installing build dependencies... (setuptools >= 42.0.0, this is invalid, wheel >= 0.36.0)\33[0m', ], + '\33[91mERROR\33[0m ', ), ], ids=['no-color', 'color'], @@ -335,8 +330,8 @@ def test_output_env_subprocess_error( tmp_dir, capsys, color, - stdout_body, - stdout_error, + stdout_content, + stderr_content, ): try: # do not inject hook to have clear output on capsys @@ -354,11 +349,8 @@ def test_output_env_subprocess_error( stdout, stderr = capsys.readouterr() stdout, stderr = stdout.splitlines(), stderr.splitlines() - assert stdout[:4] == stdout_body - assert stdout[-1].startswith(stdout_error) - - assert len(stderr) == 1 - assert stderr[0].startswith('ERROR: Invalid requirement: ') + assert stdout == stdout_content + assert stderr[-1].startswith(stderr_content) @pytest.mark.parametrize( diff --git a/tests/test_projectbuilder.py b/tests/test_projectbuilder.py index 98687f88..d1f13a84 100644 --- a/tests/test_projectbuilder.py +++ b/tests/test_projectbuilder.py @@ -12,6 +12,8 @@ import pytest import build +import build._helpers +import build.env if sys.version_info >= (3, 8): # pragma: no cover @@ -146,30 +148,41 @@ def test_bad_project(package_test_no_project): def test_init(mocker, package_test_flit, package_legacy, test_no_permission, package_test_bad_syntax): - mocker.patch('pep517.wrappers.Pep517HookCaller') + hook = mocker.patch('pep517.wrappers.Pep517HookCaller') + rewrap_runner_for_pep517_lib = mocker.patch('build.rewrap_runner_for_pep517_lib', side_effect=lambda r, e=None: (r, e)) # correct flit pyproject.toml - builder = build.ProjectBuilder(package_test_flit) - pep517.wrappers.Pep517HookCaller.assert_called_with( - package_test_flit, 'flit_core.buildapi', backend_path=None, python_executable=sys.executable, runner=builder._runner + build.ProjectBuilder(package_test_flit) + hook.assert_called_with( + package_test_flit, + 'flit_core.buildapi', + backend_path=None, + runner=(build.default_runner, None), + python_executable=sys.executable, ) - pep517.wrappers.Pep517HookCaller.reset_mock() + hook.reset_mock() + rewrap_runner_for_pep517_lib.reset_mock() # custom python - builder = build.ProjectBuilder(package_test_flit, python_executable='some-python') - pep517.wrappers.Pep517HookCaller.assert_called_with( - package_test_flit, 'flit_core.buildapi', backend_path=None, python_executable='some-python', runner=builder._runner + build.ProjectBuilder(package_test_flit, python_executable='some-python') + hook.assert_called_with( + package_test_flit, + 'flit_core.buildapi', + backend_path=None, + runner=(build.default_runner, None), + python_executable='some-python', ) - pep517.wrappers.Pep517HookCaller.reset_mock() + hook.reset_mock() + rewrap_runner_for_pep517_lib.reset_mock() # FileNotFoundError - builder = build.ProjectBuilder(package_legacy) - pep517.wrappers.Pep517HookCaller.assert_called_with( + build.ProjectBuilder(package_legacy) + hook.assert_called_with( package_legacy, 'setuptools.build_meta:__legacy__', backend_path=None, + runner=(build.default_runner, None), python_executable=sys.executable, - runner=builder._runner, ) # PermissionError @@ -182,13 +195,35 @@ def test_init(mocker, package_test_flit, package_legacy, test_no_permission, pac build.ProjectBuilder(package_test_bad_syntax) -@pytest.mark.parametrize('value', [b'something', 'something_else']) -def test_python_executable(package_test_flit, value): - builder = build.ProjectBuilder(package_test_flit) - - builder.python_executable = value - assert builder.python_executable == value - assert builder._hook.python_executable == value +@pytest.mark.isolated +def test_init_from_isolated_env(mocker, package_test_flit): + hook = mocker.patch('pep517.wrappers.Pep517HookCaller') + rewrap_runner_for_pep517_lib = mocker.patch('build.rewrap_runner_for_pep517_lib', side_effect=lambda r, e=None: (r, e)) + + # default subprocess runner + with build.env.DefaultIsolatedEnv.with_temp_dir() as env: + build.ProjectBuilder.from_isolated_env(env, package_test_flit) + hook.assert_called_with( + package_test_flit, + 'flit_core.buildapi', + backend_path=None, + runner=(build.default_runner, env.environ), + python_executable=env.python_executable, + ) + + hook.reset_mock() + rewrap_runner_for_pep517_lib.reset_mock() + + # custom subprocess runner + with build.env.DefaultIsolatedEnv.with_temp_dir() as env: + build.ProjectBuilder.from_isolated_env(env, package_test_flit, runner=build._helpers.quiet_runner) + hook.assert_called_with( + package_test_flit, + 'flit_core.buildapi', + backend_path=None, + runner=(build._helpers.quiet_runner, env.environ), + python_executable=env.python_executable, + ) @pytest.mark.parametrize('distribution', ['wheel', 'sdist']) @@ -213,7 +248,7 @@ def test_build_missing_backend(packages_path, distribution, tmpdir): builder = build.ProjectBuilder(bad_backend_path) with pytest.raises(build.BuildBackendException): - builder.build(distribution, str(tmpdir)) + builder.build(distribution, tmpdir) def test_check_dependencies(mocker, package_test_flit): @@ -356,9 +391,9 @@ def test_build_not_dir_outdir(mocker, tmp_dir, package_test_flit): def demo_pkg_inline(tmp_path_factory): # builds a wheel without any dependencies and with a console script demo-pkg-inline tmp_path = tmp_path_factory.mktemp('demo-pkg-inline') - builder = build.ProjectBuilder(srcdir=os.path.join(os.path.dirname(__file__), 'packages', 'inline')) + builder = build.ProjectBuilder(os.path.join(os.path.dirname(__file__), 'packages', 'inline')) out = tmp_path / 'dist' - builder.build('wheel', str(out)) + builder.build('wheel', out) return next(out.iterdir()) @@ -570,7 +605,7 @@ def test_log(mocker, caplog, package_test_flit): ('INFO', 'something'), ] if sys.version_info >= (3, 8): # stacklevel - assert caplog.records[-1].lineno == 562 + assert caplog.records[-1].lineno == test_log.__code__.co_firstlineno + 11 @pytest.mark.parametrize( diff --git a/tox.ini b/tox.ini index f5a11d25..de17db5d 100644 --- a/tox.ini +++ b/tox.ini @@ -62,7 +62,7 @@ commands = [testenv:docs] description = build documentations -basepython = python3.8 +basepython = python3.10 extras = docs commands =