diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f3b53aa07..ec2e567a1e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,6 +61,12 @@ jobs: - platform: ubuntu-latest python: "3.10" distutils: stdlib + # Python 3.8, 3.9 are on macos-13 but not macos-latest (macos-14-arm64) + # https://github.com/actions/setup-python/issues/850 + # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 + - {python: "3.8", platform: "macos-13"} + exclude: + - {python: "3.8", platform: "macos-latest"} runs-on: ${{ matrix.platform }} continue-on-error: ${{ matrix.python == '3.13' }} env: @@ -89,7 +95,8 @@ jobs: shell: bash run: | rm -rf dist - pipx run build + # workaround for pypa/setuptools#4333 + pipx run --pip-args 'pyproject-hooks!=1.1' build echo "PRE_BUILT_SETUPTOOLS_SDIST=$(ls dist/*.tar.gz)" >> $GITHUB_ENV echo "PRE_BUILT_SETUPTOOLS_WHEEL=$(ls dist/*.whl)" >> $GITHUB_ENV rm -rf setuptools.egg-info # Avoid interfering with the other tests @@ -122,6 +129,7 @@ jobs: job: - diffcov - docs + - check-extern runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/NEWS.rst b/NEWS.rst index 20c6903a33..73a8148d9c 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -119,7 +119,7 @@ Improved Documentation ---------------------- - Updated documentation referencing obsolete Python 3.7 code. -- by :user:`Avasam` (#4096) -- Changed ``versionadded`` for "Type information included by default" feature from ``v68.3.0`` to ``v69.0.0`` -- by :user:Avasam` (#4182) +- Changed ``versionadded`` for "Type information included by default" feature from ``v68.3.0`` to ``v69.0.0`` -- by :user:`Avasam` (#4182) - Described the auto-generated files -- by :user:`VladimirFokow` (#4198) - Updated "Quickstart" to describe the current status of ``setup.cfg`` and ``pyproject.toml`` -- by :user:`VladimirFokow` (#4200) diff --git a/README.rst b/README.rst index eec6e35531..181c3b2af6 100644 --- a/README.rst +++ b/README.rst @@ -1,32 +1,34 @@ -.. image:: https://img.shields.io/pypi/v/setuptools.svg +.. |pypi-version| image:: https://img.shields.io/pypi/v/setuptools.svg :target: https://pypi.org/project/setuptools -.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg +.. |py-version| image:: https://img.shields.io/pypi/pyversions/setuptools.svg -.. image:: https://github.com/pypa/setuptools/actions/workflows/main.yml/badge.svg +.. |test-badge| image:: https://github.com/pypa/setuptools/actions/workflows/main.yml/badge.svg :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 :alt: tests -.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json - :target: https://github.com/astral-sh/ruff - :alt: Ruff +.. |ruff-badge| image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff -.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg - :target: https://setuptools.pypa.io +.. |docs-badge| image:: https://img.shields.io/readthedocs/setuptools/latest.svg + :target: https://setuptools.pypa.io -.. image:: https://img.shields.io/badge/skeleton-2024-informational +.. |skeleton-badge| image:: https://img.shields.io/badge/skeleton-2024-informational :target: https://blog.jaraco.com/skeleton -.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white +.. |codecov-badge| image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white :target: https://codecov.io/gh/pypa/setuptools -.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat +.. |tidelift-badge| image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme -.. image:: https://img.shields.io/discord/803025117553754132 +.. |discord-badge| image:: https://img.shields.io/discord/803025117553754132 :target: https://discord.com/channels/803025117553754132/815945031150993468 :alt: Discord +|pypi-version| |py-version| |test-badge| |ruff-badge| |docs-badge| |skeleton-badge| |codecov-badge| |discord-badge| + See the `Quickstart `_ and the `User's Guide `_ for instructions on how to use Setuptools. diff --git a/newsfragments/4255.misc.rst b/newsfragments/4255.misc.rst new file mode 100644 index 0000000000..50a0a3d195 --- /dev/null +++ b/newsfragments/4255.misc.rst @@ -0,0 +1 @@ +Treat ``EncodingWarning``s as errors in tests. -- by :user:`Avasam` diff --git a/newsfragments/4309.removal.rst b/newsfragments/4309.removal.rst new file mode 100644 index 0000000000..b69b17d45f --- /dev/null +++ b/newsfragments/4309.removal.rst @@ -0,0 +1,7 @@ +Further adoption of UTF-8 in ``setuptools``. +This change regards mostly files produced and consumed during the build process +(e.g. metadata files, script wrappers, automatically updated config files, etc..) +Although precautions were taken to minimize disruptions, some edge cases might +be subject to backwards incompatibility. + +Support for ``"locale"`` encoding is now **deprecated**. diff --git a/newsfragments/4312.doc.rst b/newsfragments/4312.doc.rst new file mode 100644 index 0000000000..7ada954876 --- /dev/null +++ b/newsfragments/4312.doc.rst @@ -0,0 +1 @@ +Uses RST substitution to put badges in 1 line. diff --git a/newsfragments/4332.feature.rst b/newsfragments/4332.feature.rst new file mode 100644 index 0000000000..9f46298adc --- /dev/null +++ b/newsfragments/4332.feature.rst @@ -0,0 +1 @@ +Modernized and refactored VCS handling in package_index. \ No newline at end of file diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index f8c93fe37b..d32b095a88 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -1524,8 +1524,7 @@ def run_script(self, script_name, namespace): script_filename = self._fn(self.egg_info, script) namespace['__file__'] = script_filename if os.path.exists(script_filename): - with open(script_filename) as fid: - source = fid.read() + source = _read_utf8_with_fallback(script_filename) code = compile(source, script_filename, 'exec') exec(code, namespace, namespace) else: @@ -2175,11 +2174,10 @@ def non_empty_lines(path): """ Yield non-empty lines from file at path """ - with open(path) as f: - for line in f: - line = line.strip() - if line: - yield line + for line in _read_utf8_with_fallback(path).splitlines(): + line = line.strip() + if line: + yield line def resolve_egg_link(path): @@ -3322,3 +3320,35 @@ def _initialize_master_working_set(): # match order list(map(working_set.add_entry, sys.path)) globals().update(locals()) + + +# ---- Ported from ``setuptools`` to avoid introducing an import inter-dependency ---- +LOCALE_ENCODING = "locale" if sys.version_info >= (3, 10) else None + + +def _read_utf8_with_fallback(file: str, fallback_encoding=LOCALE_ENCODING) -> str: + """See setuptools.unicode_utils._read_utf8_with_fallback""" + try: + with open(file, "r", encoding="utf-8") as f: + return f.read() + except UnicodeDecodeError: # pragma: no cover + msg = f"""\ + ******************************************************************************** + `encoding="utf-8"` fails with {file!r}, trying `encoding={fallback_encoding!r}`. + + This fallback behaviour is considered **deprecated** and future versions of + `setuptools/pkg_resources` may not implement it. + + Please encode {file!r} with "utf-8" to ensure future builds will succeed. + + If this file was produced by `setuptools` itself, cleaning up the cached files + and re-building/re-installing the package with a newer version of `setuptools` + (e.g. by updating `build-system.requires` in its `pyproject.toml`) + might solve the problem. + ******************************************************************************** + """ + # TODO: Add a deadline? + # See comment in setuptools.unicode_utils._Utf8EncodingNeeded + warnings.warns(msg, PkgResourcesDeprecationWarning, stacklevel=2) + with open(file, "r", encoding=fallback_encoding) as f: + return f.read() diff --git a/pkg_resources/_vendor/vendored.txt b/pkg_resources/_vendor/vendored.txt index c18a2cc0eb..f8cbfd967e 100644 --- a/pkg_resources/_vendor/vendored.txt +++ b/pkg_resources/_vendor/vendored.txt @@ -9,5 +9,7 @@ jaraco.text==3.7.0 importlib_resources==5.10.2 # required for importlib_resources on older Pythons zipp==3.7.0 +# required for jaraco.functools +more_itertools==10.2.0 # required for jaraco.context on older Pythons backports.tarfile diff --git a/pkg_resources/extern/__init__.py b/pkg_resources/extern/__init__.py index df96f7f26d..7f80b04164 100644 --- a/pkg_resources/extern/__init__.py +++ b/pkg_resources/extern/__init__.py @@ -70,12 +70,21 @@ def install(self): sys.meta_path.append(self) +# [[[cog +# import cog +# from tools.vendored import yield_root_package +# names = "\n".join(f" {x!r}," for x in yield_root_package('pkg_resources')) +# cog.outl(f"names = (\n{names}\n)") +# ]]] names = ( 'packaging', 'platformdirs', + 'typing_extensions', 'jaraco', 'importlib_resources', + 'zipp', 'more_itertools', 'backports', ) +# [[[end]]] VendorImporter(__name__, names).install() diff --git a/pytest.ini b/pytest.ini index e7c96274a3..0c9651d96f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,22 +10,31 @@ filterwarnings= # Fail on warnings error + # Workarounds for pypa/setuptools#3810 + # Can't use EncodingWarning as it doesn't exist on Python 3.9. + # These warnings only appear on Python 3.10+ + ## upstream # Ensure ResourceWarnings are emitted default::ResourceWarning + # python/mypy#17057 + ignore:'encoding' argument not specified::mypy + ignore:'encoding' argument not specified::configparser + # ^-- ConfigParser is called by mypy, + # but ignoring the warning in `mypy` is not enough + # to make it work on PyPy + # realpython/pytest-mypy#152 ignore:'encoding' argument not specified::pytest_mypy - # python/cpython#100750 - ignore:'encoding' argument not specified::platform + # TODO: Set encoding when openning/writing tmpdir files with pytest's LocalPath.open + # see pypa/setuptools#4326 + ignore:'encoding' argument not specified::_pytest - # pypa/build#615 - ignore:'encoding' argument not specified::build.env - - # dateutil/dateutil#1284 - ignore:datetime.datetime.utcfromtimestamp:DeprecationWarning:dateutil.tz.tz + # Already fixed in pypa/distutils, but present in stdlib + ignore:'encoding' argument not specified::distutils ## end upstream @@ -69,11 +78,6 @@ filterwarnings= # https://github.com/pypa/setuptools/issues/3655 ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning - # Workarounds for pypa/setuptools#3810 - # Can't use EncodingWarning as it doesn't exist on Python 3.9 - default:'encoding' argument not specified - default:UTF-8 Mode affects locale.getpreferredencoding(). - # Avoid errors when testing pkg_resources.declare_namespace ignore:.*pkg_resources\.declare_namespace.*:DeprecationWarning diff --git a/setup.cfg b/setup.cfg index c8bb0ed41d..0756fa92ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,6 +76,10 @@ testing = tomli # No Python 3.12 dependencies require importlib_metadata, but needed for type-checking since we import it directly importlib_metadata + pytest-subprocess + + # workaround for pypa/setuptools#4333 + pyproject-hooks!=1.1 docs = # upstream @@ -96,6 +100,9 @@ docs = sphinxcontrib-towncrier sphinx-notfound-page >=1,<2 + # workaround for pypa/setuptools#4333 + pyproject-hooks!=1.1 + ssl = certs = diff --git a/setuptools/_imp.py b/setuptools/_imp.py index 9d4ead0eb0..38b146fc4d 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -6,6 +6,7 @@ import os import importlib.util import importlib.machinery +import tokenize from importlib.util import module_from_spec @@ -60,13 +61,13 @@ def find_module(module, paths=None): if suffix in importlib.machinery.SOURCE_SUFFIXES: kind = PY_SOURCE + file = tokenize.open(path) elif suffix in importlib.machinery.BYTECODE_SUFFIXES: kind = PY_COMPILED + file = open(path, 'rb') elif suffix in importlib.machinery.EXTENSION_SUFFIXES: kind = C_EXTENSION - if kind in {PY_SOURCE, PY_COMPILED}: - file = open(path, mode) else: path = None suffix = mode = '' diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index 2decd2d214..be2742d73d 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -2,7 +2,7 @@ Previously, when a user or a command line tool (let's call it a "frontend") needed to make a request of setuptools to take a certain action, for -example, generating a list of installation requirements, the frontend would +example, generating a list of installation requirements, the frontend would call "setup.py egg_info" or "setup.py bdist_wheel" on the command line. PEP 517 defines a different method of interfacing with setuptools. Rather diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 3687efdf9c..adcb0a1ba1 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -54,7 +54,7 @@ def __bootstrap__(): __bootstrap__() """ ).lstrip() - with open(pyfile, 'w') as f: + with open(pyfile, 'w', encoding="utf-8") as f: f.write(_stub_template % resource) @@ -200,10 +200,9 @@ def run(self): # noqa: C901 # is too complex (14) # FIXME log.info("writing %s", native_libs) if not self.dry_run: ensure_directory(native_libs) - libs_file = open(native_libs, 'wt') - libs_file.write('\n'.join(all_outputs)) - libs_file.write('\n') - libs_file.close() + with open(native_libs, 'wt', encoding="utf-8") as libs_file: + libs_file.write('\n'.join(all_outputs)) + libs_file.write('\n') elif os.path.isfile(native_libs): log.info("removing %s", native_libs) if not self.dry_run: @@ -350,9 +349,8 @@ def write_safety_flag(egg_dir, safe): if safe is None or bool(safe) != flag: os.unlink(fn) elif safe is not None and bool(safe) == flag: - f = open(fn, 'wt') - f.write('\n') - f.close() + with open(fn, 'wt', encoding="utf-8") as f: + f.write('\n') safety_flags = { diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index b5c98c86dc..6056fe9b24 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -342,9 +342,8 @@ def _write_stub_file(self, stub_file: str, ext: Extension, compile=False): if compile and os.path.exists(stub_file): raise BaseError(stub_file + " already exists! Please delete.") if not self.dry_run: - f = open(stub_file, 'w') - f.write( - '\n'.join([ + with open(stub_file, 'w', encoding="utf-8") as f: + content = '\n'.join([ "def __bootstrap__():", " global __bootstrap__, __file__, __loader__", " import sys, os, pkg_resources, importlib.util" + if_dl(", dl"), @@ -368,8 +367,7 @@ def _write_stub_file(self, stub_file: str, ext: Extension, compile=False): "__bootstrap__()", "", # terminal \n ]) - ) - f.close() + f.write(content) if compile: self._compile_and_remove_stub(stub_file) diff --git a/setuptools/command/develop.py b/setuptools/command/develop.py index d8c1b49b3d..d07736a005 100644 --- a/setuptools/command/develop.py +++ b/setuptools/command/develop.py @@ -10,6 +10,8 @@ from setuptools import namespaces import setuptools +from ..unicode_utils import _read_utf8_with_fallback + class develop(namespaces.DevelopInstaller, easy_install): """Set up package for development""" @@ -119,7 +121,7 @@ def install_for_development(self): # create an .egg-link in the installation dir, pointing to our egg log.info("Creating %s (link to %s)", self.egg_link, self.egg_base) if not self.dry_run: - with open(self.egg_link, "w") as f: + with open(self.egg_link, "w", encoding="utf-8") as f: f.write(self.egg_path + "\n" + self.setup_path) # postprocess the installed distro, fixing up .pth, installing scripts, # and handling requirements @@ -128,9 +130,12 @@ def install_for_development(self): def uninstall_link(self): if os.path.exists(self.egg_link): log.info("Removing %s (link to %s)", self.egg_link, self.egg_base) - egg_link_file = open(self.egg_link) - contents = [line.rstrip() for line in egg_link_file] - egg_link_file.close() + + contents = [ + line.rstrip() + for line in _read_utf8_with_fallback(self.egg_link).splitlines() + ] + if contents not in ([self.egg_path], [self.egg_path, self.setup_path]): log.warn("Link points to %s: uninstall aborted", contents) return @@ -156,8 +161,7 @@ def install_egg_scripts(self, dist): for script_name in self.distribution.scripts or []: script_path = os.path.abspath(convert_path(script_name)) script_name = os.path.basename(script_path) - with open(script_path) as strm: - script_text = strm.read() + script_text = _read_utf8_with_fallback(script_path) self.install_script(dist, script_name, script_text, script_path) return None diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 87a68c292a..41ff382fe4 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -873,7 +873,9 @@ def write_script(self, script_name, contents, mode="t", blockers=()): ensure_directory(target) if os.path.exists(target): os.unlink(target) - with open(target, "w" + mode) as f: # TODO: is it safe to use utf-8? + + encoding = None if "b" in mode else "utf-8" + with open(target, "w" + mode, encoding=encoding) as f: f.write(contents) chmod(target, 0o777 - mask) @@ -1017,12 +1019,11 @@ def install_exe(self, dist_filename, tmpdir): # Write EGG-INFO/PKG-INFO if not os.path.exists(pkg_inf): - f = open(pkg_inf, 'w') # TODO: probably it is safe to use utf-8 - f.write('Metadata-Version: 1.0\n') - for k, v in cfg.items('metadata'): - if k != 'target_version': - f.write('%s: %s\n' % (k.replace('_', '-').title(), v)) - f.close() + with open(pkg_inf, 'w', encoding="utf-8") as f: + f.write('Metadata-Version: 1.0\n') + for k, v in cfg.items('metadata'): + if k != 'target_version': + f.write('%s: %s\n' % (k.replace('_', '-').title(), v)) script_dir = os.path.join(_egg_info, 'scripts') # delete entry-point scripts to avoid duping self.delete_blockers([ @@ -1088,9 +1089,8 @@ def process(src, dst): if locals()[name]: txt = os.path.join(egg_tmp, 'EGG-INFO', name + '.txt') if not os.path.exists(txt): - f = open(txt, 'w') # TODO: probably it is safe to use utf-8 - f.write('\n'.join(locals()[name]) + '\n') - f.close() + with open(txt, 'w', encoding="utf-8") as f: + f.write('\n'.join(locals()[name]) + '\n') def install_wheel(self, wheel_path, tmpdir): wheel = Wheel(wheel_path) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py index 1722817f82..b8ed84750a 100644 --- a/setuptools/command/editable_wheel.py +++ b/setuptools/command/editable_wheel.py @@ -565,7 +565,8 @@ def _encode_pth(content: str) -> bytes: This function tries to simulate this behaviour without having to create an actual file, in a way that supports a range of active Python versions. (There seems to be some variety in the way different version of Python handle - ``encoding=None``, not all of them use ``locale.getpreferredencoding(False)``). + ``encoding=None``, not all of them use ``locale.getpreferredencoding(False)`` + or ``locale.getencoding()``). """ with io.BytesIO() as buffer: wrapper = io.TextIOWrapper(buffer, encoding=py39.LOCALE_ENCODING) diff --git a/setuptools/command/install_scripts.py b/setuptools/command/install_scripts.py index 72b2e45cbc..d79a4ab7b0 100644 --- a/setuptools/command/install_scripts.py +++ b/setuptools/command/install_scripts.py @@ -57,10 +57,10 @@ def write_script(self, script_name, contents, mode="t", *ignored): target = os.path.join(self.install_dir, script_name) self.outfiles.append(target) + encoding = None if "b" in mode else "utf-8" mask = current_umask() if not self.dry_run: ensure_directory(target) - f = open(target, "w" + mode) - f.write(contents) - f.close() + with open(target, "w" + mode, encoding=encoding) as f: + f.write(contents) chmod(target, 0o777 - mask) diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index f9a6075128..b78d845e60 100644 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py @@ -5,7 +5,8 @@ import os import configparser -from setuptools import Command +from .. import Command +from ..unicode_utils import _cfg_read_utf8_with_fallback __all__ = ['config_file', 'edit_config', 'option_base', 'setopt'] @@ -36,7 +37,8 @@ def edit_config(filename, settings, dry_run=False): log.debug("Reading configuration from %s", filename) opts = configparser.RawConfigParser() opts.optionxform = lambda x: x - opts.read([filename]) + _cfg_read_utf8_with_fallback(opts, filename) + for section, options in settings.items(): if options is None: log.info("Deleting section [%s] from %s", section, filename) @@ -62,7 +64,7 @@ def edit_config(filename, settings, dry_run=False): log.info("Writing %s", filename) if not dry_run: - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf-8") as f: opts.write(f) diff --git a/setuptools/dist.py b/setuptools/dist.py index 6350e38100..03f6c0398b 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -535,7 +535,8 @@ def warn_dash_deprecation(self, opt, section): def _setuptools_commands(self): try: - return metadata.distribution('setuptools').entry_points.names + entry_points = metadata.distribution('setuptools').entry_points + return {ep.name for ep in entry_points} # Avoid newer API for compatibility except metadata.PackageNotFoundError: # during bootstrapping, distribution doesn't exist return [] @@ -685,7 +686,7 @@ def get_egg_cache_dir(self): os.mkdir(egg_cache_dir) windows_support.hide_file(egg_cache_dir) readme_txt_filename = os.path.join(egg_cache_dir, 'README.txt') - with open(readme_txt_filename, 'w') as f: + with open(readme_txt_filename, 'w', encoding="utf-8") as f: f.write( 'This directory contains eggs that were downloaded ' 'by setuptools to build, test, and run plug-ins.\n\n' diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py index 427b27cb80..16e2c9ea9e 100644 --- a/setuptools/extern/__init__.py +++ b/setuptools/extern/__init__.py @@ -70,16 +70,23 @@ def install(self): sys.meta_path.append(self) +# [[[cog +# import cog +# from tools.vendored import yield_root_package +# names = "\n".join(f" {x!r}," for x in yield_root_package('setuptools')) +# cog.outl(f"names = (\n{names}\n)") +# ]]] names = ( 'packaging', 'ordered_set', 'more_itertools', - 'importlib_metadata', - 'zipp', - 'importlib_resources', 'jaraco', + 'importlib_resources', + 'importlib_metadata', 'typing_extensions', + 'zipp', 'tomli', 'backports', ) +# [[[end]]] VendorImporter(__name__, names, 'setuptools._vendor').install() diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 271aa97f71..c3ffee41a7 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,7 @@ """PyPI and direct package downloading.""" import sys +import subprocess import os import re import io @@ -40,6 +41,8 @@ from setuptools.wheel import Wheel from setuptools.extern.more_itertools import unique_everseen +from .unicode_utils import _read_utf8_with_fallback, _cfg_read_utf8_with_fallback + EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) @@ -419,9 +422,9 @@ def scan_egg_links(self, search_path): list(itertools.starmap(self.scan_egg_link, egg_links)) def scan_egg_link(self, path, entry): - with open(os.path.join(path, entry)) as raw_lines: - # filter non-empty lines - lines = list(filter(None, map(str.strip, raw_lines))) + content = _read_utf8_with_fallback(os.path.join(path, entry)) + # filter non-empty lines + lines = list(filter(None, map(str.strip, content.splitlines()))) if len(lines) != 2: # format is not recognized; punt @@ -585,7 +588,7 @@ def download(self, spec, tmpdir): scheme = URL_SCHEME(spec) if scheme: # It's a url, download it to tmpdir - found = self._download_url(scheme.group(1), spec, tmpdir) + found = self._download_url(spec, tmpdir) base, fragment = egg_info_for_url(spec) if base.endswith('.py'): found = self.gen_setup(found, fragment, tmpdir) @@ -714,7 +717,7 @@ def gen_setup(self, filename, fragment, tmpdir): shutil.copy2(filename, dst) filename = dst - with open(os.path.join(tmpdir, 'setup.py'), 'w') as file: + with open(os.path.join(tmpdir, 'setup.py'), 'w', encoding="utf-8") as file: file.write( "from setuptools import setup\n" "setup(name=%r, version=%r, py_modules=[%r])\n" @@ -814,7 +817,7 @@ def open_url(self, url, warning=None): # noqa: C901 # is too complex (12) else: raise DistutilsError("Download error for %s: %s" % (url, v)) from v - def _download_url(self, scheme, url, tmpdir): + def _download_url(self, url, tmpdir): # Determine download filename # name, fragment = egg_info_for_url(url) @@ -829,19 +832,59 @@ def _download_url(self, scheme, url, tmpdir): filename = os.path.join(tmpdir, name) - # Download the file - # - if scheme == 'svn' or scheme.startswith('svn+'): - return self._download_svn(url, filename) - elif scheme == 'git' or scheme.startswith('git+'): - return self._download_git(url, filename) - elif scheme.startswith('hg+'): - return self._download_hg(url, filename) - elif scheme == 'file': - return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) - else: - self.url_ok(url, True) # raises error if not allowed - return self._attempt_download(url, filename) + return self._download_vcs(url, filename) or self._download_other(url, filename) + + @staticmethod + def _resolve_vcs(url): + """ + >>> rvcs = PackageIndex._resolve_vcs + >>> rvcs('git+http://foo/bar') + 'git' + >>> rvcs('hg+https://foo/bar') + 'hg' + >>> rvcs('git:myhost') + 'git' + >>> rvcs('hg:myhost') + >>> rvcs('http://foo/bar') + """ + scheme = urllib.parse.urlsplit(url).scheme + pre, sep, post = scheme.partition('+') + # svn and git have their own protocol; hg does not + allowed = set(['svn', 'git'] + ['hg'] * bool(sep)) + return next(iter({pre} & allowed), None) + + def _download_vcs(self, url, spec_filename): + vcs = self._resolve_vcs(url) + if not vcs: + return + if vcs == 'svn': + raise DistutilsError( + f"Invalid config, SVN download is not supported: {url}" + ) + + filename, _, _ = spec_filename.partition('#') + url, rev = self._vcs_split_rev_from_url(url) + + self.info(f"Doing {vcs} clone from {url} to {filename}") + subprocess.check_call([vcs, 'clone', '--quiet', url, filename]) + + co_commands = dict( + git=[vcs, '-C', filename, 'checkout', '--quiet', rev], + hg=[vcs, '--cwd', filename, 'up', '-C', '-r', rev, '-q'], + ) + if rev is not None: + self.info(f"Checking out {rev}") + subprocess.check_call(co_commands[vcs]) + + return filename + + def _download_other(self, url, filename): + scheme = urllib.parse.urlsplit(url).scheme + if scheme == 'file': # pragma: no cover + return urllib.request.url2pathname(urllib.parse.urlparse(url).path) + # raise error if not allowed + self.url_ok(url, True) + return self._attempt_download(url, filename) def scan_url(self, url): self.process_url(url, True) @@ -857,64 +900,37 @@ def _invalid_download_html(self, url, headers, filename): os.unlink(filename) raise DistutilsError(f"Unexpected HTML page found at {url}") - def _download_svn(self, url, _filename): - raise DistutilsError(f"Invalid config, SVN download is not supported: {url}") - @staticmethod - def _vcs_split_rev_from_url(url, pop_prefix=False): - scheme, netloc, path, query, frag = urllib.parse.urlsplit(url) + def _vcs_split_rev_from_url(url): + """ + Given a possible VCS URL, return a clean URL and resolved revision if any. + + >>> vsrfu = PackageIndex._vcs_split_rev_from_url + >>> vsrfu('git+https://github.com/pypa/setuptools@v69.0.0#egg-info=setuptools') + ('https://github.com/pypa/setuptools', 'v69.0.0') + >>> vsrfu('git+https://github.com/pypa/setuptools#egg-info=setuptools') + ('https://github.com/pypa/setuptools', None) + >>> vsrfu('http://foo/bar') + ('http://foo/bar', None) + """ + parts = urllib.parse.urlsplit(url) - scheme = scheme.split('+', 1)[-1] + clean_scheme = parts.scheme.split('+', 1)[-1] # Some fragment identification fails - path = path.split('#', 1)[0] - - rev = None - if '@' in path: - path, rev = path.rsplit('@', 1) - - # Also, discard fragment - url = urllib.parse.urlunsplit((scheme, netloc, path, query, '')) + no_fragment_path, _, _ = parts.path.partition('#') - return url, rev + pre, sep, post = no_fragment_path.rpartition('@') + clean_path, rev = (pre, post) if sep else (post, None) - def _download_git(self, url, filename): - filename = filename.split('#', 1)[0] - url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) - - self.info("Doing git clone from %s to %s", url, filename) - os.system("git clone --quiet %s %s" % (url, filename)) - - if rev is not None: - self.info("Checking out %s", rev) - os.system( - "git -C %s checkout --quiet %s" - % ( - filename, - rev, - ) - ) + resolved = parts._replace( + scheme=clean_scheme, + path=clean_path, + # discard the fragment + fragment='', + ).geturl() - return filename - - def _download_hg(self, url, filename): - filename = filename.split('#', 1)[0] - url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) - - self.info("Doing hg clone from %s to %s", url, filename) - os.system("hg clone --quiet %s %s" % (url, filename)) - - if rev is not None: - self.info("Updating to %s", rev) - os.system( - "hg --cwd %s up -C -r %s -q" - % ( - filename, - rev, - ) - ) - - return filename + return resolved, rev def debug(self, msg, *args): log.debug(msg, *args) @@ -1011,7 +1027,7 @@ def __init__(self): rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): - self.read(rc) + _cfg_read_utf8_with_fallback(self, rc) @property def creds_by_repository(self): @@ -1114,8 +1130,7 @@ def local_open(url): for f in os.listdir(filename): filepath = os.path.join(filename, f) if f == 'index.html': - with open(filepath, 'r') as fp: - body = fp.read() + body = _read_utf8_with_fallback(filepath) break elif os.path.isdir(filepath): f += '/' diff --git a/setuptools/tests/__init__.py b/setuptools/tests/__init__.py index 564adf2b0a..738ebf43be 100644 --- a/setuptools/tests/__init__.py +++ b/setuptools/tests/__init__.py @@ -1,10 +1,15 @@ import locale +import sys import pytest __all__ = ['fail_on_ascii'] - -is_ascii = locale.getpreferredencoding() == 'ANSI_X3.4-1968' +locale_encoding = ( + locale.getencoding() + if sys.version_info >= (3, 11) + else locale.getpreferredencoding(False) +) +is_ascii = locale_encoding == 'ANSI_X3.4-1968' fail_on_ascii = pytest.mark.xfail(is_ascii, reason="Test fails in this locale") diff --git a/setuptools/tests/environment.py b/setuptools/tests/environment.py index df2bd37ff6..b9de4fda6b 100644 --- a/setuptools/tests/environment.py +++ b/setuptools/tests/environment.py @@ -17,7 +17,7 @@ class VirtualEnv(jaraco.envs.VirtualEnv): def run(self, cmd, *args, **kwargs): cmd = [self.exe(cmd[0])] + cmd[1:] - kwargs = {"cwd": self.root, **kwargs} # Allow overriding + kwargs = {"cwd": self.root, "encoding": "utf-8", **kwargs} # Allow overriding # In some environments (eg. downstream distro packaging), where: # - tox isn't used to run tests and # - PYTHONPATH is set to point to a specific setuptools codebase and @@ -76,6 +76,7 @@ def run_setup_py(cmd, pypath=None, path=None, data_stream=0, env=None): stderr=_PIPE, shell=shell, env=env, + encoding="utf-8", ) if isinstance(data_stream, tuple): diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index c2a1e6dc75..cc996b4255 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -160,7 +160,7 @@ def run(): # to obtain a distribution object first, and then run the distutils # commands later, because these files will be removed in the meantime. - with open('world.py', 'w') as f: + with open('world.py', 'w', encoding="utf-8") as f: f.write('x = 42') try: @@ -941,14 +941,14 @@ def test_legacy_editable_install(venv, tmpdir, tmpdir_cwd): # First: sanity check cmd = ["pip", "install", "--no-build-isolation", "-e", "."] - output = str(venv.run(cmd, cwd=tmpdir), "utf-8").lower() + output = venv.run(cmd, cwd=tmpdir).lower() assert "running setup.py develop for myproj" not in output assert "created wheel for myproj" in output # Then: real test env = {**os.environ, "SETUPTOOLS_ENABLE_FEATURES": "legacy-editable"} cmd = ["pip", "install", "--no-build-isolation", "-e", "."] - output = str(venv.run(cmd, cwd=tmpdir, env=env), "utf-8").lower() + output = venv.run(cmd, cwd=tmpdir, env=env).lower() assert "running setup.py develop for myproj" in output diff --git a/setuptools/tests/test_build_py.py b/setuptools/tests/test_build_py.py index 4aa1fe68fa..db2052a586 100644 --- a/setuptools/tests/test_build_py.py +++ b/setuptools/tests/test_build_py.py @@ -1,6 +1,7 @@ import os import stat import shutil +import warnings from pathlib import Path from unittest.mock import Mock @@ -162,11 +163,23 @@ def test_excluded_subpackages(tmpdir_cwd): dist.parse_config_files() build_py = dist.get_command_obj("build_py") + msg = r"Python recognizes 'mypkg\.tests' as an importable package" with pytest.warns(SetuptoolsDeprecationWarning, match=msg): # TODO: To fix #3260 we need some transition period to deprecate the # existing behavior of `include_package_data`. After the transition, we # should remove the warning and fix the behaviour. + + if os.getenv("SETUPTOOLS_USE_DISTUTILS") == "stdlib": + # pytest.warns reset the warning filter temporarily + # https://github.com/pytest-dev/pytest/issues/4011#issuecomment-423494810 + warnings.filterwarnings( + "ignore", + "'encoding' argument not specified", + module="distutils.text_file", + # This warning is already fixed in pypa/distutils but not in stdlib + ) + build_py.finalize_options() build_py.run() diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index 950cb23d21..ada4c32285 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -361,7 +361,7 @@ def test_many_pth_distributions_merge_together(self, tmpdir): @pytest.fixture def setup_context(tmpdir): - with (tmpdir / 'setup.py').open('w') as f: + with (tmpdir / 'setup.py').open('w', encoding="utf-8") as f: f.write(SETUP_PY) with tmpdir.as_cwd(): yield tmpdir diff --git a/setuptools/tests/test_editable_install.py b/setuptools/tests/test_editable_install.py index 5da4fccefa..300a02cfb9 100644 --- a/setuptools/tests/test_editable_install.py +++ b/setuptools/tests/test_editable_install.py @@ -118,6 +118,7 @@ def editable_opts(request): SETUP_SCRIPT_STUB = "__import__('setuptools').setup()" +@pytest.mark.xfail(sys.platform == "darwin", reason="pypa/setuptools#4328") @pytest.mark.parametrize( "files", [ @@ -131,7 +132,7 @@ def test_editable_with_pyproject(tmp_path, venv, files, editable_opts): jaraco.path.build(files, prefix=project) cmd = [ - venv.exe(), + "python", "-m", "pip", "install", @@ -140,14 +141,14 @@ def test_editable_with_pyproject(tmp_path, venv, files, editable_opts): str(project), *editable_opts, ] - print(str(subprocess.check_output(cmd), "utf-8")) + print(venv.run(cmd)) - cmd = [venv.exe(), "-m", "mypkg"] - assert subprocess.check_output(cmd).strip() == b"3.14159.post0 Hello World" + cmd = ["python", "-m", "mypkg"] + assert venv.run(cmd).strip() == "3.14159.post0 Hello World" (project / "src/mypkg/data.txt").write_text("foobar", encoding="utf-8") (project / "src/mypkg/mod.py").write_text("x = 42", encoding="utf-8") - assert subprocess.check_output(cmd).strip() == b"3.14159.post0 foobar 42" + assert venv.run(cmd).strip() == "3.14159.post0 foobar 42" def test_editable_with_flat_layout(tmp_path, venv, editable_opts): @@ -176,7 +177,7 @@ def test_editable_with_flat_layout(tmp_path, venv, editable_opts): project = tmp_path / "mypkg" cmd = [ - venv.exe(), + "python", "-m", "pip", "install", @@ -185,9 +186,9 @@ def test_editable_with_flat_layout(tmp_path, venv, editable_opts): str(project), *editable_opts, ] - print(str(subprocess.check_output(cmd), "utf-8")) - cmd = [venv.exe(), "-c", "import pkg, mod; print(pkg.a, mod.b)"] - assert subprocess.check_output(cmd).strip() == b"4 2" + print(venv.run(cmd)) + cmd = ["python", "-c", "import pkg, mod; print(pkg.a, mod.b)"] + assert venv.run(cmd).strip() == "4 2" def test_editable_with_single_module(tmp_path, venv, editable_opts): @@ -214,7 +215,7 @@ def test_editable_with_single_module(tmp_path, venv, editable_opts): project = tmp_path / "mypkg" cmd = [ - venv.exe(), + "python", "-m", "pip", "install", @@ -223,9 +224,9 @@ def test_editable_with_single_module(tmp_path, venv, editable_opts): str(project), *editable_opts, ] - print(str(subprocess.check_output(cmd), "utf-8")) - cmd = [venv.exe(), "-c", "import mod; print(mod.b)"] - assert subprocess.check_output(cmd).strip() == b"2" + print(venv.run(cmd)) + cmd = ["python", "-c", "import mod; print(mod.b)"] + assert venv.run(cmd).strip() == "2" class TestLegacyNamespaces: @@ -384,7 +385,7 @@ def test_namespace_accidental_config_in_lenient_mode(self, venv, tmp_path): opts = ["--no-build-isolation"] # force current version of setuptools venv.run(["python", "-m", "pip", "-v", "install", "-e", str(pkg_A), *opts]) out = venv.run(["python", "-c", "from mypkg.n import pkgA; print(pkgA.a)"]) - assert str(out, "utf-8").strip() == "1" + assert out.strip() == "1" cmd = """\ try: import mypkg.other @@ -392,7 +393,7 @@ def test_namespace_accidental_config_in_lenient_mode(self, venv, tmp_path): print("mypkg.other not defined") """ out = venv.run(["python", "-c", dedent(cmd)]) - assert "mypkg.other not defined" in str(out, "utf-8") + assert "mypkg.other not defined" in out # Moved here from test_develop: @@ -897,6 +898,7 @@ class TestOverallBehaviour: }, } + @pytest.mark.xfail(sys.platform == "darwin", reason="pypa/setuptools#4328") @pytest.mark.parametrize("layout", EXAMPLES.keys()) def test_editable_install(self, tmp_path, venv, layout, editable_opts): project, _ = install_project( @@ -911,7 +913,7 @@ def test_editable_install(self, tmp_path, venv, layout, editable_opts): print(ex) """ out = venv.run(["python", "-c", dedent(cmd_import_error)]) - assert b"No module named 'otherfile'" in out + assert "No module named 'otherfile'" in out # Ensure the modules are importable cmd_get_vars = """\ @@ -919,7 +921,7 @@ def test_editable_install(self, tmp_path, venv, layout, editable_opts): print(mypkg.mod1.var, mypkg.subpackage.mod2.var) """ out = venv.run(["python", "-c", dedent(cmd_get_vars)]) - assert b"42 13" in out + assert "42 13" in out # Ensure resources are reachable cmd_get_resource = """\ @@ -929,7 +931,7 @@ def test_editable_install(self, tmp_path, venv, layout, editable_opts): print(text.read_text(encoding="utf-8")) """ out = venv.run(["python", "-c", dedent(cmd_get_resource)]) - assert b"resource 39" in out + assert "resource 39" in out # Ensure files are editable mod1 = next(project.glob("**/mod1.py")) @@ -941,12 +943,12 @@ def test_editable_install(self, tmp_path, venv, layout, editable_opts): resource_file.write_text("resource 374", encoding="utf-8") out = venv.run(["python", "-c", dedent(cmd_get_vars)]) - assert b"42 13" not in out - assert b"17 781" in out + assert "42 13" not in out + assert "17 781" in out out = venv.run(["python", "-c", dedent(cmd_get_resource)]) - assert b"resource 39" not in out - assert b"resource 374" in out + assert "resource 39" not in out + assert "resource 374" in out class TestLinkTree: @@ -1005,7 +1007,7 @@ def test_strict_install(self, tmp_path, venv): install_project("mypkg", venv, tmp_path, self.FILES, *opts) out = venv.run(["python", "-c", "import mypkg.mod1; print(mypkg.mod1.var)"]) - assert b"42" in out + assert "42" in out # Ensure packages excluded from distribution are not importable cmd_import_error = """\ @@ -1015,7 +1017,7 @@ def test_strict_install(self, tmp_path, venv): print(ex) """ out = venv.run(["python", "-c", dedent(cmd_import_error)]) - assert b"cannot import name 'subpackage'" in out + assert "cannot import name 'subpackage'" in out # Ensure resource files excluded from distribution are not reachable cmd_get_resource = """\ @@ -1028,8 +1030,8 @@ def test_strict_install(self, tmp_path, venv): print(ex) """ out = venv.run(["python", "-c", dedent(cmd_get_resource)]) - assert b"No such file or directory" in out - assert b"resource.not_in_manifest" in out + assert "No such file or directory" in out + assert "resource.not_in_manifest" in out @pytest.mark.filterwarnings("ignore:.*compat.*:setuptools.SetuptoolsDeprecationWarning") @@ -1040,7 +1042,7 @@ def test_compat_install(tmp_path, venv): install_project("mypkg", venv, tmp_path, files, *opts) out = venv.run(["python", "-c", "import mypkg.mod1; print(mypkg.mod1.var)"]) - assert b"42" in out + assert "42" in out expected_path = comparable_path(str(tmp_path)) @@ -1051,7 +1053,7 @@ def test_compat_install(tmp_path, venv): "import other; print(other)", "import mypkg; print(mypkg)", ): - out = comparable_path(str(venv.run(["python", "-c", cmd]), "utf-8")) + out = comparable_path(venv.run(["python", "-c", cmd])) assert expected_path in out # Compatible behaviour will not consider custom mappings @@ -1061,7 +1063,7 @@ def test_compat_install(tmp_path, venv): except ImportError as ex: print(ex) """ - out = str(venv.run(["python", "-c", dedent(cmd)]), "utf-8") + out = venv.run(["python", "-c", dedent(cmd)]) assert "cannot import name 'subpackage'" in out @@ -1105,7 +1107,7 @@ def test_pbr_integration(tmp_path, venv, editable_opts): install_project("mypkg", venv, tmp_path, files, *editable_opts) out = venv.run(["python", "-c", "import mypkg.hello"]) - assert b"Hello world!" in out + assert "Hello world!" in out class TestCustomBuildPy: @@ -1143,11 +1145,11 @@ def test_safeguarded_from_errors(self, tmp_path, venv): """Ensure that errors in custom build_py are reported as warnings""" # Warnings should show up _, out = install_project("mypkg", venv, tmp_path, self.FILES) - assert b"SetuptoolsDeprecationWarning" in out - assert b"ValueError: TEST_RAISE" in out + assert "SetuptoolsDeprecationWarning" in out + assert "ValueError: TEST_RAISE" in out # but installation should be successful out = venv.run(["python", "-c", "import mypkg.mod1; print(mypkg.mod1.var)"]) - assert b"42" in out + assert "42" in out class TestCustomBuildWheel: diff --git a/setuptools/tests/test_egg_info.py b/setuptools/tests/test_egg_info.py index a4b0ecf398..f6b2302d97 100644 --- a/setuptools/tests/test_egg_info.py +++ b/setuptools/tests/test_egg_info.py @@ -213,13 +213,9 @@ def test_license_is_a_string(self, tmpdir_cwd, env): with pytest.raises(AssertionError) as exc: self._run_egg_info_command(tmpdir_cwd, env) - # Hopefully this is not too fragile: the only argument to the - # assertion error should be a traceback, ending with: - # ValueError: .... - # - # assert not 1 - tb = exc.value.args[0].split('\n') - assert tb[-3].lstrip().startswith('ValueError') + # The only argument to the assertion error should be a traceback + # containing a ValueError + assert 'ValueError' in exc.value.args[0] def test_rebuilt(self, tmpdir_cwd, env): """Ensure timestamps are updated when the command is re-run.""" diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 1aa16172b5..71f10d9a6e 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -99,6 +99,11 @@ def test_pbr(install_context): @pytest.mark.xfail +@pytest.mark.filterwarnings("ignore:'encoding' argument not specified") +# ^-- Dependency chain: `python-novaclient` < `oslo-utils` < `netifaces==0.11.0` +# netifaces' setup.py uses `open` without `encoding="utf-8"` which is hijacked by +# `setuptools.sandbox._open` and triggers the EncodingWarning. +# Can't use EncodingWarning in the filter, as it does not exist on Python < 3.10. def test_python_novaclient(install_context): _install_one('python-novaclient', install_context, 'novaclient', 'base.py') diff --git a/setuptools/tests/test_packageindex.py b/setuptools/tests/test_packageindex.py index 93474ae5af..f5f37e0563 100644 --- a/setuptools/tests/test_packageindex.py +++ b/setuptools/tests/test_packageindex.py @@ -3,7 +3,6 @@ import urllib.error import http.client from inspect import cleandoc -from unittest import mock import pytest @@ -171,49 +170,46 @@ def test_egg_fragment(self): assert dists[0].version == '' assert dists[1].version == vc - def test_download_git_with_rev(self, tmpdir): + def test_download_git_with_rev(self, tmp_path, fp): url = 'git+https://github.example/group/project@master#egg=foo' index = setuptools.package_index.PackageIndex() - with mock.patch("os.system") as os_system_mock: - result = index.download(url, str(tmpdir)) + expected_dir = tmp_path / 'project@master' + fp.register([ + 'git', + 'clone', + '--quiet', + 'https://github.example/group/project', + expected_dir, + ]) + fp.register(['git', '-C', expected_dir, 'checkout', '--quiet', 'master']) - os_system_mock.assert_called() + result = index.download(url, tmp_path) - expected_dir = str(tmpdir / 'project@master') - expected = ( - 'git clone --quiet ' 'https://github.example/group/project {expected_dir}' - ).format(**locals()) - first_call_args = os_system_mock.call_args_list[0][0] - assert first_call_args == (expected,) + assert result == str(expected_dir) + assert len(fp.calls) == 2 - tmpl = 'git -C {expected_dir} checkout --quiet master' - expected = tmpl.format(**locals()) - assert os_system_mock.call_args_list[1][0] == (expected,) - assert result == expected_dir - - def test_download_git_no_rev(self, tmpdir): + def test_download_git_no_rev(self, tmp_path, fp): url = 'git+https://github.example/group/project#egg=foo' index = setuptools.package_index.PackageIndex() - with mock.patch("os.system") as os_system_mock: - result = index.download(url, str(tmpdir)) - - os_system_mock.assert_called() - - expected_dir = str(tmpdir / 'project') - expected = ( - 'git clone --quiet ' 'https://github.example/group/project {expected_dir}' - ).format(**locals()) - os_system_mock.assert_called_once_with(expected) - - def test_download_svn(self, tmpdir): + expected_dir = tmp_path / 'project' + fp.register([ + 'git', + 'clone', + '--quiet', + 'https://github.example/group/project', + expected_dir, + ]) + index.download(url, tmp_path) + + def test_download_svn(self, tmp_path): url = 'svn+https://svn.example/project#egg=foo' index = setuptools.package_index.PackageIndex() msg = r".*SVN download is not supported.*" with pytest.raises(distutils.errors.DistutilsError, match=msg): - index.download(url, str(tmpdir)) + index.download(url, tmp_path) class TestContentCheckers: diff --git a/setuptools/tests/test_windows_wrappers.py b/setuptools/tests/test_windows_wrappers.py index 3f321386f1..b272689351 100644 --- a/setuptools/tests/test_windows_wrappers.py +++ b/setuptools/tests/test_windows_wrappers.py @@ -110,7 +110,11 @@ def test_basic(self, tmpdir): 'arg5 a\\\\b', ] proc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, text=True + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + text=True, + encoding="utf-8", ) stdout, stderr = proc.communicate('hello\nworld\n') actual = stdout.replace('\r\n', '\n') @@ -143,7 +147,11 @@ def test_symlink(self, tmpdir): 'arg5 a\\\\b', ] proc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, text=True + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + text=True, + encoding="utf-8", ) stdout, stderr = proc.communicate('hello\nworld\n') actual = stdout.replace('\r\n', '\n') @@ -191,6 +199,7 @@ def test_with_options(self, tmpdir): stdin=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + encoding="utf-8", ) stdout, stderr = proc.communicate() actual = stdout.replace('\r\n', '\n') @@ -240,6 +249,7 @@ def test_basic(self, tmpdir): stdin=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + encoding="utf-8", ) stdout, stderr = proc.communicate() assert not stdout diff --git a/setuptools/unicode_utils.py b/setuptools/unicode_utils.py index d43dcc11f9..696b34c46a 100644 --- a/setuptools/unicode_utils.py +++ b/setuptools/unicode_utils.py @@ -1,5 +1,9 @@ import unicodedata import sys +from configparser import ConfigParser + +from .compat import py39 +from .warnings import SetuptoolsDeprecationWarning # HFS Plus uses decomposed UTF-8 @@ -42,3 +46,57 @@ def try_encode(string, enc): return string.encode(enc) except UnicodeEncodeError: return None + + +def _read_utf8_with_fallback(file: str, fallback_encoding=py39.LOCALE_ENCODING) -> str: + """ + First try to read the file with UTF-8, if there is an error fallback to a + different encoding ("locale" by default). Returns the content of the file. + Also useful when reading files that might have been produced by an older version of + setuptools. + """ + try: + with open(file, "r", encoding="utf-8") as f: + return f.read() + except UnicodeDecodeError: # pragma: no cover + _Utf8EncodingNeeded.emit(file=file, fallback_encoding=fallback_encoding) + with open(file, "r", encoding=fallback_encoding) as f: + return f.read() + + +def _cfg_read_utf8_with_fallback( + cfg: ConfigParser, file: str, fallback_encoding=py39.LOCALE_ENCODING +) -> None: + """Same idea as :func:`_read_utf8_with_fallback`, but for the + :meth:`ConfigParser.read` method. + + This method may call ``cfg.clear()``. + """ + try: + cfg.read(file, encoding="utf-8") + except UnicodeDecodeError: # pragma: no cover + _Utf8EncodingNeeded.emit(file=file, fallback_encoding=fallback_encoding) + cfg.clear() + cfg.read(file, encoding=fallback_encoding) + + +class _Utf8EncodingNeeded(SetuptoolsDeprecationWarning): + _SUMMARY = """ + `encoding="utf-8"` fails with {file!r}, trying `encoding={fallback_encoding!r}`. + """ + + _DETAILS = """ + Fallback behaviour for UTF-8 is considered **deprecated** and future versions of + `setuptools` may not implement it. + + Please encode {file!r} with "utf-8" to ensure future builds will succeed. + + If this file was produced by `setuptools` itself, cleaning up the cached files + and re-building/re-installing the package with a newer version of `setuptools` + (e.g. by updating `build-system.requires` in its `pyproject.toml`) + might solve the problem. + """ + # TODO: Add a deadline? + # Will we be able to remove this? + # The question comes to mind mainly because of sdists that have been produced + # by old versions of setuptools and published to PyPI... diff --git a/setuptools/wheel.py b/setuptools/wheel.py index 9861b5cf1c..e06daec4d0 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -18,6 +18,8 @@ from setuptools.command.egg_info import write_requirements, _egg_basename from setuptools.archive_util import _unpack_zipfile_obj +from .unicode_utils import _read_utf8_with_fallback + WHEEL_NAME = re.compile( r"""^(?P.+?)-(?P\d.*?) @@ -222,13 +224,13 @@ def _move_data_entries(destination_eggdir, dist_data): def _fix_namespace_packages(egg_info, destination_eggdir): namespace_packages = os.path.join(egg_info, 'namespace_packages.txt') if os.path.exists(namespace_packages): - with open(namespace_packages) as fp: - namespace_packages = fp.read().split() + namespace_packages = _read_utf8_with_fallback(namespace_packages).split() + for mod in namespace_packages: mod_dir = os.path.join(destination_eggdir, *mod.split('.')) mod_init = os.path.join(mod_dir, '__init__.py') if not os.path.exists(mod_dir): os.mkdir(mod_dir) if not os.path.exists(mod_init): - with open(mod_init, 'w') as fp: + with open(mod_init, 'w', encoding="utf-8") as fp: fp.write(NAMESPACE_PACKAGE_INIT) diff --git a/tools/vendored.py b/tools/vendored.py index 232e9625d2..63797ea24a 100644 --- a/tools/vendored.py +++ b/tools/vendored.py @@ -166,4 +166,18 @@ def update_setuptools(): rewrite_more_itertools(vendor / "more_itertools") +def yield_root_package(name): + """Useful when defining the MetaPathFinder + >>> examples = set(yield_root_package("setuptools")) & {"jaraco", "backports"} + >>> list(sorted(examples)) + ['backports', 'jaraco'] + """ + vendored = Path(f"{name}/_vendor/vendored.txt") + yield from ( + line.partition("=")[0].partition(".")[0].replace("-", "_") + for line in vendored.read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + ) + + __name__ == '__main__' and update_vendored() diff --git a/tox.ini b/tox.ini index 7412730008..22dd7af8da 100644 --- a/tox.ini +++ b/tox.ini @@ -69,12 +69,16 @@ pass_env = * commands = python tools/finalize.py -[testenv:vendor] +[testenv:{vendor,check-extern}] skip_install = True +allowlist_externals = sh deps = path + cogapp commands = - python -m tools.vendored + vendor: python -m tools.vendored + vendor: sh -c "git grep -l -F '\[\[\[cog' | xargs cog -I {toxinidir} -r" # update `*.extern` + check-extern: sh -c "git grep -l -F '\[\[\[cog' | xargs cog -I {toxinidir} --check" [testenv:generate-validation-code] skip_install = True