From 4f47cac1925a2232892ceae438e2c62f81517714 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Jan 2024 17:00:47 -0800 Subject: [PATCH] Add --unstable flag (#4096) --- CHANGES.md | 17 ++ docs/faq.md | 7 +- docs/the_black_code_style/current_style.md | 6 + docs/the_black_code_style/future_style.md | 217 +++++++++--------- docs/the_black_code_style/index.md | 8 +- .../black_as_a_server.md | 6 + docs/usage_and_configuration/the_basics.md | 31 ++- pyproject.toml | 9 +- src/black/__init__.py | 49 +++- src/black/mode.py | 39 ++-- src/blackd/__init__.py | 104 +++++---- tests/data/cases/preview_cantfit.py | 14 -- tests/data/cases/preview_cantfit_string.py | 18 ++ tests/data/cases/preview_comments7.py | 2 +- tests/data/cases/preview_long_dict_values.py | 2 +- tests/data/cases/preview_long_strings.py | 2 +- .../preview_long_strings__east_asian_width.py | 2 +- .../cases/preview_long_strings__edge_case.py | 2 +- .../cases/preview_long_strings__regression.py | 2 +- tests/data/cases/preview_multiline_strings.py | 2 +- ...eview_return_annotation_brackets_string.py | 2 +- tests/test_black.py | 63 ++--- tests/util.py | 16 +- 23 files changed, 368 insertions(+), 252 deletions(-) create mode 100644 tests/data/cases/preview_cantfit_string.py diff --git a/CHANGES.md b/CHANGES.md index 45cb967e74a..0496603e2c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,14 @@ changes: - Fix incorrect formatting of certain async statements (#3609) - Allow combining `# fmt: skip` with other comments (#3959) +There are already a few improvements in the `--preview` style, which are slated for the +2025 stable style. Try them out and +[share your feedback](https://github.com/psf/black/issues). In the past, the preview +style has included some features that we were not able to stabilize. This year, we're +adding a separate `--unstable` style for features with known problems. Now, the +`--preview` style only includes features that we actually expect to make it into next +year's stable style. + ### Stable style @@ -53,6 +61,12 @@ release: +- Add `--unstable` style, covering preview features that have known problems that would + block them from going into the stable style. Also add the `--enable-unstable-feature` + flag; for example, use + `--enable-unstable-feature hug_parens_with_braces_and_square_brackets` to apply this + preview style throughout 2024, even if a later Black release downgrades the feature to + unstable (#4096) - Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) @@ -66,6 +80,9 @@ release: - Fix symlink handling, properly catch and ignore symlinks that point outside of root (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) +- Remove the long-deprecated `--experimental-string-processing` flag. This feature can + currently be enabled with `--preview --enable-unstable-feature string_processing`. + (#4096) ### Packaging diff --git a/docs/faq.md b/docs/faq.md index c62e1b504b5..124a096efac 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -41,9 +41,10 @@ other tools, such as `# noqa`, may be moved by _Black_. See below for more detai Stable. _Black_ aims to enforce one style and one style only, with some room for pragmatism. See [The Black Code Style](the_black_code_style/index.md) for more details. -Starting in 2022, the formatting output will be stable for the releases made in the same -year (other than unintentional bugs). It is possible to opt-in to the latest formatting -styles, using the `--preview` flag. +Starting in 2022, the formatting output is stable for the releases made in the same year +(other than unintentional bugs). At the beginning of every year, the first release will +make changes to the stable style. It is possible to opt in to the latest formatting +styles using the `--preview` flag. ## Why is my file not formatted? diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 00bd81416dc..ca5d1d4a701 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -449,6 +449,12 @@ file that are not enforced yet but might be in a future version of the formatter _Black_ will normalize line endings (`\n` or `\r\n`) based on the first line ending of the file. +### Form feed characters + +_Black_ will retain form feed characters on otherwise empty lines at the module level. +Only one form feed is retained for a group of consecutive empty lines. Where there are +two empty lines in a row, the form feed is placed on the second line. + ## Pragmatism Early versions of _Black_ used to be absolutist in some respects. They took after its diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f55ea5f60a9..1cdd25fdb7c 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -1,54 +1,5 @@ # The (future of the) Black code style -```{warning} -Changes to this document often aren't tied and don't relate to releases of -_Black_. It's recommended that you read the latest version available. -``` - -## Using backslashes for with statements - -[Backslashes are bad and should be never be used](labels/why-no-backslashes) however -there is one exception: `with` statements using multiple context managers. Before Python -3.9 Python's grammar does not allow organizing parentheses around the series of context -managers. - -We don't want formatting like: - -```py3 -with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: - ... # nothing to split on - line too long -``` - -So _Black_ will, when we implement this, format it like this: - -```py3 -with \ - make_context_manager1() as cm1, \ - make_context_manager2() as cm2, \ - make_context_manager3() as cm3, \ - make_context_manager4() as cm4 \ -: - ... # backslashes and an ugly stranded colon -``` - -Although when the target version is Python 3.9 or higher, _Black_ uses parentheses -instead in `--preview` mode (see below) since they're allowed in Python 3.9 and higher. - -An alternative to consider if the backslashes in the above formatting are undesirable is -to use {external:py:obj}`contextlib.ExitStack` to combine context managers in the -following way: - -```python -with contextlib.ExitStack() as exit_stack: - cm1 = exit_stack.enter_context(make_context_manager1()) - cm2 = exit_stack.enter_context(make_context_manager2()) - cm3 = exit_stack.enter_context(make_context_manager3()) - cm4 = exit_stack.enter_context(make_context_manager4()) - ... -``` - -(labels/preview-style)= - ## Preview style Experimental, potentially disruptive style changes are gathered under the `--preview` @@ -56,62 +7,38 @@ CLI flag. At the end of each year, these changes may be adopted into the default as described in [The Black Code Style](index.md). Because the functionality is experimental, feedback and issue reports are highly encouraged! -### Improved string processing - -_Black_ will split long string literals and merge short ones. Parentheses are used where -appropriate. When split, parts of f-strings that don't need formatting are converted to -plain strings. User-made splits are respected when they do not exceed the line length -limit. Line continuation backslashes are converted into parenthesized strings. -Unnecessary parentheses are stripped. The stability and status of this feature is -tracked in [this issue](https://github.com/psf/black/issues/2188). - -### Improved line breaks - -For assignment expressions, _Black_ now prefers to split and wrap the right side of the -assignment instead of left side. For example: - -```python -some_dict[ - "with_a_long_key" -] = some_looooooooong_module.some_looooooooooooooong_function_name( - first_argument, second_argument, third_argument -) -``` - -will be changed to: +In the past, the preview style included some features with known bugs, so that we were +unable to move these features to the stable style. Therefore, such features are now +moved to the `--unstable` style. All features in the `--preview` style are expected to +make it to next year's stable style; features in the `--unstable` style will be +stabilized only if issues with them are fixed. If bugs are discovered in a `--preview` +feature, it is demoted to the `--unstable` style. To avoid thrash when a feature is +demoted from the `--preview` to the `--unstable` style, users can use the +`--enable-unstable-feature` flag to enable specific unstable features. -```python -some_dict["with_a_long_key"] = ( - some_looooooooong_module.some_looooooooooooooong_function_name( - first_argument, second_argument, third_argument - ) -) -``` +Currently, the following features are included in the preview style: -### Improved parentheses management +- `hex_codes_in_unicode_sequences`: normalize casing of Unicode escape characters in + strings +- `unify_docstring_detection`: fix inconsistencies in whether certain strings are + detected as docstrings +- `hug_parens_with_braces_and_square_brackets`: more compact formatting of nested + brackets ([see below](labels/hug-parens)) +- `no_normalize_fmt_skip_whitespace`: whitespace before `# fmt: skip` comments is no + longer normalized -For dict literals with long values, they are now wrapped in parentheses. Unnecessary -parentheses are now removed. For example: +(labels/unstable-features)= -```python -my_dict = { - "a key in my dict": a_very_long_variable - * and_a_very_long_function_call() - / 100000.0, - "another key": (short_value), -} -``` +The unstable style additionally includes the following features: -will be changed to: +- `string_processing`: split long string literals and related changes + ([see below](labels/string-processing)) +- `wrap_long_dict_values_in_parens`: add parentheses to long values in dictionaries + ([see below](labels/wrap-long-dict-values)) +- `multiline_string_handling`: more compact formatting of expressions involving + multiline strings ([see below](labels/multiline-string-handling)) -```python -my_dict = { - "a key in my dict": ( - a_very_long_variable * and_a_very_long_function_call() / 100000.0 - ), - "another key": short_value, -} -``` +(labels/hug-parens)= ### Improved multiline dictionary and list indentation for sole function parameter @@ -185,6 +112,46 @@ foo( ) ``` +(labels/string-processing)= + +### Improved string processing + +_Black_ will split long string literals and merge short ones. Parentheses are used where +appropriate. When split, parts of f-strings that don't need formatting are converted to +plain strings. User-made splits are respected when they do not exceed the line length +limit. Line continuation backslashes are converted into parenthesized strings. +Unnecessary parentheses are stripped. The stability and status of this feature is +tracked in [this issue](https://github.com/psf/black/issues/2188). + +(labels/wrap-long-dict-values)= + +### Improved parentheses management in dicts + +For dict literals with long values, they are now wrapped in parentheses. Unnecessary +parentheses are now removed. For example: + +```python +my_dict = { + "a key in my dict": a_very_long_variable + * and_a_very_long_function_call() + / 100000.0, + "another key": (short_value), +} +``` + +will be changed to: + +```python +my_dict = { + "a key in my dict": ( + a_very_long_variable * and_a_very_long_function_call() / 100000.0 + ), + "another key": short_value, +} +``` + +(labels/multiline-string-handling)= + ### Improved multiline string handling _Black_ is smarter when formatting multiline strings, especially in function arguments, @@ -297,13 +264,51 @@ s = ( # Top comment ) ``` -======= +## Potential future changes + +This section lists changes that we may want to make in the future, but that aren't +implemented yet. + +### Using backslashes for with statements + +[Backslashes are bad and should be never be used](labels/why-no-backslashes) however +there is one exception: `with` statements using multiple context managers. Before Python +3.9 Python's grammar does not allow organizing parentheses around the series of context +managers. + +We don't want formatting like: + +```py3 +with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: + ... # nothing to split on - line too long +``` + +So _Black_ will, when we implement this, format it like this: + +```py3 +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + ... # backslashes and an ugly stranded colon +``` + +Although when the target version is Python 3.9 or higher, _Black_ uses parentheses +instead in `--preview` mode (see below) since they're allowed in Python 3.9 and higher. -### Form feed characters +An alternative to consider if the backslashes in the above formatting are undesirable is +to use {external:py:obj}`contextlib.ExitStack` to combine context managers in the +following way: -_Black_ will now retain form feed characters on otherwise empty lines at the module -level. Only one form feed is retained for a group of consecutive empty lines. Where -there are two empty lines in a row, the form feed will be placed on the second line. +```python +with contextlib.ExitStack() as exit_stack: + cm1 = exit_stack.enter_context(make_context_manager1()) + cm2 = exit_stack.enter_context(make_context_manager2()) + cm3 = exit_stack.enter_context(make_context_manager3()) + cm4 = exit_stack.enter_context(make_context_manager4()) + ... +``` -_Black_ already retained form feed literals inside a comment or inside a string. This -remains the case. +(labels/preview-style)= diff --git a/docs/the_black_code_style/index.md b/docs/the_black_code_style/index.md index 1719347eec8..58f28673022 100644 --- a/docs/the_black_code_style/index.md +++ b/docs/the_black_code_style/index.md @@ -42,9 +42,11 @@ _Black_: enabled by newer Python language syntax as well as due to improvements in the formatting logic. -- The `--preview` flag is exempt from this policy. There are no guarantees around the - stability of the output with that flag passed into _Black_. This flag is intended for - allowing experimentation with the proposed changes to the _Black_ code style. +- The `--preview` and `--unstable` flags are exempt from this policy. There are no + guarantees around the stability of the output with these flags passed into _Black_. + They are intended for allowing experimentation with proposed changes to the _Black_ + code style. The `--preview` style at the end of a year should closely match the stable + style for the next year, but we may always make changes. Documentation for both the current and future styles can be found: diff --git a/docs/usage_and_configuration/black_as_a_server.md b/docs/usage_and_configuration/black_as_a_server.md index f24fb34d915..0a7edb57fd7 100644 --- a/docs/usage_and_configuration/black_as_a_server.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -62,6 +62,12 @@ The headers controlling how source code is formatted are: - `X-Preview`: corresponds to the `--preview` command line flag. If present and its value is not an empty string, experimental and potentially disruptive style changes will be used. +- `X-Unstable`: corresponds to the `--unstable` command line flag. If present and its + value is not an empty string, experimental style changes that are known to be buggy + will be used. +- `X-Enable-Unstable-Feature`: corresponds to the `--enable-unstable-feature` flag. The + contents of the flag must be a comma-separated list of unstable features to be + enabled. Example: `X-Enable-Unstable-Feature: feature1, feature2`. - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as _Black_ does when passed the `--fast` command line flag. - `X-Python-Variant`: if set to `pyi`, `blackd` will act as _Black_ does when passed the diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index b541f07907c..a42e093155b 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -144,9 +144,34 @@ magic trailing comma is ignored. #### `--preview` -Enable potentially disruptive style changes that may be added to Black's main -functionality in the next major release. Read more about -[our preview style](labels/preview-style). +Enable potentially disruptive style changes that we expect to add to Black's main +functionality in the next major release. Use this if you want a taste of what next +year's style will look like. + +Read more about [our preview style](labels/preview-style). + +There is no guarantee on the code style produced by this flag across releases. + +#### `--unstable` + +Enable all style changes in `--preview`, plus additional changes that we would like to +make eventually, but that have known issues that need to be fixed before they can move +back to the `--preview` style. Use this if you want to experiment with these changes and +help fix issues with them. + +There is no guarantee on the code style produced by this flag across releases. + +#### `--enable-unstable-feature` + +Enable specific features from the `--unstable` style. See +[the preview style documentation](labels/unstable-features) for the list of supported +features. This flag can only be used when `--preview` is enabled. Users are encouraged +to use this flag if they use `--preview` style and a feature that affects their code is +moved from the `--preview` to the `--unstable` style, but they want to avoid the thrash +from undoing this change. + +There are no guarantees on the behavior of these features, or even their existence, +across releases. (labels/exit-code)= diff --git a/pyproject.toml b/pyproject.toml index 24b9c07674d..fa3654b8d67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,11 @@ extend-exclude = ''' | profiling )/ ''' -# We use preview style for formatting Black itself. If you -# want stable formatting across releases, you should keep -# this off. -preview = true +# We use the unstable style for formatting Black itself. If you +# want bug-free formatting, you should keep this off. If you want +# stable formatting across releases, you should also keep `preview = true` +# (which is implied by this flag) off. +unstable = true # Build system information and other project-specific configuration below. # NOTE: You don't need this in your own Black configuration. diff --git a/src/black/__init__.py b/src/black/__init__.py index 961ed9479a8..ebc7ac8eda5 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -68,7 +68,7 @@ from black.lines import EmptyLineTracker, LinesBlock from black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature from black.mode import Mode as Mode # re-exported -from black.mode import TargetVersion, supports_feature +from black.mode import Preview, TargetVersion, supports_feature from black.nodes import ( STARS, is_number_token, @@ -209,6 +209,13 @@ def target_version_option_callback( return [TargetVersion[val.upper()] for val in v] +def enable_unstable_feature_callback( + c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...] +) -> List[Preview]: + """Compute the features from an --enable-unstable-feature flag.""" + return [Preview[val] for val in v] + + def re_compile_maybe_verbose(regex: str) -> Pattern[str]: """Compile a regular expression string in `regex`. @@ -303,12 +310,6 @@ def validate_regex( is_flag=True, help="Don't use trailing commas as a reason to split lines.", ) -@click.option( - "--experimental-string-processing", - is_flag=True, - hidden=True, - help="(DEPRECATED and now included in --preview) Normalize string literals.", -) @click.option( "--preview", is_flag=True, @@ -317,6 +318,26 @@ def validate_regex( " functionality in the next major release." ), ) +@click.option( + "--unstable", + is_flag=True, + help=( + "Enable potentially disruptive style changes that have known bugs or are not" + " currently expected to make it into the stable style Black's next major" + " release. Implies --preview." + ), +) +@click.option( + "--enable-unstable-feature", + type=click.Choice([v.name for v in Preview]), + callback=enable_unstable_feature_callback, + multiple=True, + help=( + "Enable specific features included in the `--unstable` style. Requires" + " `--preview`. No compatibility guarantees are provided on the behavior" + " or existence of any unstable features." + ), +) @click.option( "--check", is_flag=True, @@ -507,8 +528,9 @@ def main( # noqa: C901 skip_source_first_line: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, - experimental_string_processing: bool, preview: bool, + unstable: bool, + enable_unstable_feature: List[Preview], quiet: bool, verbose: bool, required_version: Optional[str], @@ -534,6 +556,14 @@ def main( # noqa: C901 out(main.get_usage(ctx) + "\n\nOne of 'SRC' or 'code' is required.") ctx.exit(1) + # It doesn't do anything if --unstable is also passed, so just allow it. + if enable_unstable_feature and not (preview or unstable): + out( + main.get_usage(ctx) + + "\n\n'--enable-unstable-feature' requires '--preview'." + ) + ctx.exit(1) + root, method = ( find_project_root(src, stdin_filename) if code is None else (None, None) ) @@ -595,9 +625,10 @@ def main( # noqa: C901 skip_source_first_line=skip_source_first_line, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, - experimental_string_processing=experimental_string_processing, preview=preview, + unstable=unstable, python_cell_magics=set(python_cell_magics), + enabled_features=set(enable_unstable_feature), ) lines: List[Tuple[int, int]] = [] diff --git a/src/black/mode.py b/src/black/mode.py index a1519f17bcc..68919fb4901 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -9,7 +9,6 @@ from hashlib import sha256 from operator import attrgetter from typing import Dict, Final, Set -from warnings import warn from black.const import DEFAULT_LINE_LENGTH @@ -179,6 +178,16 @@ class Preview(Enum): multiline_string_handling = auto() +UNSTABLE_FEATURES: Set[Preview] = { + # Many issues, see summary in https://github.com/psf/black/issues/4042 + Preview.string_processing, + # See issues #3452 and #4158 + Preview.wrap_long_dict_values_in_parens, + # See issue #4159 + Preview.multiline_string_handling, +} + + class Deprecated(UserWarning): """Visible deprecation warning.""" @@ -192,28 +201,24 @@ class Mode: is_ipynb: bool = False skip_source_first_line: bool = False magic_trailing_comma: bool = True - experimental_string_processing: bool = False python_cell_magics: Set[str] = field(default_factory=set) preview: bool = False - - def __post_init__(self) -> None: - if self.experimental_string_processing: - warn( - "`experimental string processing` has been included in `preview`" - " and deprecated. Use `preview` instead.", - Deprecated, - ) + unstable: bool = False + enabled_features: Set[Preview] = field(default_factory=set) def __contains__(self, feature: Preview) -> bool: """ Provide `Preview.FEATURE in Mode` syntax that mirrors the ``preview`` flag. - The argument is not checked and features are not differentiated. - They only exist to make development easier by clarifying intent. + In unstable mode, all features are enabled. In preview mode, all features + except those in UNSTABLE_FEATURES are enabled. Any features in + `self.enabled_features` are also enabled. """ - if feature is Preview.string_processing: - return self.preview or self.experimental_string_processing - return self.preview + if self.unstable: + return True + if feature in self.enabled_features: + return True + return self.preview and feature not in UNSTABLE_FEATURES def get_cache_key(self) -> str: if self.target_versions: @@ -231,7 +236,9 @@ def get_cache_key(self) -> str: str(int(self.is_ipynb)), str(int(self.skip_source_first_line)), str(int(self.magic_trailing_comma)), - str(int(self.experimental_string_processing)), + sha256( + (",".join(sorted(f.name for f in self.enabled_features))).encode() + ).hexdigest(), str(int(self.preview)), sha256((",".join(sorted(self.python_cell_magics))).encode()).hexdigest(), ] diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 6b0f3d33295..7041671f596 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -8,6 +8,7 @@ try: from aiohttp import web + from multidict import MultiMapping from .middlewares import cors except ImportError as ie: @@ -34,6 +35,8 @@ SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization" SKIP_MAGIC_TRAILING_COMMA = "X-Skip-Magic-Trailing-Comma" PREVIEW = "X-Preview" +UNSTABLE = "X-Unstable" +ENABLE_UNSTABLE_FEATURE = "X-Enable-Unstable-Feature" FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe" DIFF_HEADER = "X-Diff" @@ -45,6 +48,8 @@ SKIP_STRING_NORMALIZATION_HEADER, SKIP_MAGIC_TRAILING_COMMA, PREVIEW, + UNSTABLE, + ENABLE_UNSTABLE_FEATURE, FAST_OR_SAFE_HEADER, DIFF_HEADER, ] @@ -53,6 +58,10 @@ BLACK_VERSION_HEADER = "X-Black-Version" +class HeaderError(Exception): + pass + + class InvalidVariantHeader(Exception): pass @@ -93,55 +102,21 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: return web.Response( status=501, text="This server only supports protocol version 1" ) - try: - line_length = int( - request.headers.get(LINE_LENGTH_HEADER, black.DEFAULT_LINE_LENGTH) - ) - except ValueError: - return web.Response(status=400, text="Invalid line length header value") - if PYTHON_VARIANT_HEADER in request.headers: - value = request.headers[PYTHON_VARIANT_HEADER] - try: - pyi, versions = parse_python_variant_header(value) - except InvalidVariantHeader as e: - return web.Response( - status=400, - text=f"Invalid value for {PYTHON_VARIANT_HEADER}: {e.args[0]}", - ) - else: - pyi = False - versions = set() - - skip_string_normalization = bool( - request.headers.get(SKIP_STRING_NORMALIZATION_HEADER, False) - ) - skip_magic_trailing_comma = bool( - request.headers.get(SKIP_MAGIC_TRAILING_COMMA, False) - ) - skip_source_first_line = bool( - request.headers.get(SKIP_SOURCE_FIRST_LINE, False) - ) - preview = bool(request.headers.get(PREVIEW, False)) fast = False if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast": fast = True - mode = black.FileMode( - target_versions=versions, - is_pyi=pyi, - line_length=line_length, - skip_source_first_line=skip_source_first_line, - string_normalization=not skip_string_normalization, - magic_trailing_comma=not skip_magic_trailing_comma, - preview=preview, - ) + try: + mode = parse_mode(request.headers) + except HeaderError as e: + return web.Response(status=400, text=e.args[0]) req_bytes = await request.content.read() charset = request.charset if request.charset is not None else "utf8" req_str = req_bytes.decode(charset) then = datetime.now(timezone.utc) header = "" - if skip_source_first_line: + if mode.skip_source_first_line: first_newline_position: int = req_str.find("\n") + 1 header = req_str[:first_newline_position] req_str = req_str[first_newline_position:] @@ -190,6 +165,57 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: return web.Response(status=500, headers=headers, text=str(e)) +def parse_mode(headers: MultiMapping[str]) -> black.Mode: + try: + line_length = int(headers.get(LINE_LENGTH_HEADER, black.DEFAULT_LINE_LENGTH)) + except ValueError: + raise HeaderError("Invalid line length header value") from None + + if PYTHON_VARIANT_HEADER in headers: + value = headers[PYTHON_VARIANT_HEADER] + try: + pyi, versions = parse_python_variant_header(value) + except InvalidVariantHeader as e: + raise HeaderError( + f"Invalid value for {PYTHON_VARIANT_HEADER}: {e.args[0]}", + ) from None + else: + pyi = False + versions = set() + + skip_string_normalization = bool( + headers.get(SKIP_STRING_NORMALIZATION_HEADER, False) + ) + skip_magic_trailing_comma = bool(headers.get(SKIP_MAGIC_TRAILING_COMMA, False)) + skip_source_first_line = bool(headers.get(SKIP_SOURCE_FIRST_LINE, False)) + + preview = bool(headers.get(PREVIEW, False)) + unstable = bool(headers.get(UNSTABLE, False)) + enable_features: Set[black.Preview] = set() + enable_unstable_features = headers.get(ENABLE_UNSTABLE_FEATURE, "").split(",") + for piece in enable_unstable_features: + piece = piece.strip() + if piece: + try: + enable_features.add(black.Preview[piece]) + except KeyError: + raise HeaderError( + f"Invalid value for {ENABLE_UNSTABLE_FEATURE}: {piece}", + ) from None + + return black.FileMode( + target_versions=versions, + is_pyi=pyi, + line_length=line_length, + skip_source_first_line=skip_source_first_line, + string_normalization=not skip_string_normalization, + magic_trailing_comma=not skip_magic_trailing_comma, + preview=preview, + unstable=unstable, + enabled_features=enable_features, + ) + + def parse_python_variant_header(value: str) -> Tuple[bool, Set[black.TargetVersion]]: if value == "pyi": return True, set() diff --git a/tests/data/cases/preview_cantfit.py b/tests/data/cases/preview_cantfit.py index d5da6654f0c..29789c7e653 100644 --- a/tests/data/cases/preview_cantfit.py +++ b/tests/data/cases/preview_cantfit.py @@ -20,12 +20,6 @@ normal_name = but_the_function_name_is_now_ridiculously_long_and_it_is_still_super_annoying( [1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3 ) -# long arguments -normal_name = normal_function_name( - "but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit", - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs", - this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, -) string_variable_name = ( "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa ) @@ -78,14 +72,6 @@ [1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3 ) ) -# long arguments -normal_name = normal_function_name( - "but with super long string arguments that on their own exceed the line limit so" - " there's no way it can ever fit", - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" - " with spam and eggs and spam with eggs", - this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, -) string_variable_name = "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa for key in """ hostname diff --git a/tests/data/cases/preview_cantfit_string.py b/tests/data/cases/preview_cantfit_string.py new file mode 100644 index 00000000000..3b48e318ade --- /dev/null +++ b/tests/data/cases/preview_cantfit_string.py @@ -0,0 +1,18 @@ +# flags: --unstable +# long arguments +normal_name = normal_function_name( + "but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs", + this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, +) + +# output + +# long arguments +normal_name = normal_function_name( + "but with super long string arguments that on their own exceed the line limit so" + " there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" + " with spam and eggs and spam with eggs", + this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, +) diff --git a/tests/data/cases/preview_comments7.py b/tests/data/cases/preview_comments7.py index 006d4f7266f..e4d547138db 100644 --- a/tests/data/cases/preview_comments7.py +++ b/tests/data/cases/preview_comments7.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable from .config import ( Any, Bool, diff --git a/tests/data/cases/preview_long_dict_values.py b/tests/data/cases/preview_long_dict_values.py index 54da76038dc..a19210605f6 100644 --- a/tests/data/cases/preview_long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable my_dict = { "something_something": r"Lorem ipsum dolor sit amet, an sed convenire eloquentiam \t" diff --git a/tests/data/cases/preview_long_strings.py b/tests/data/cases/preview_long_strings.py index 19ac47a7032..86fa1b0c7e1 100644 --- a/tests/data/cases/preview_long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable x = "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." x += "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." diff --git a/tests/data/cases/preview_long_strings__east_asian_width.py b/tests/data/cases/preview_long_strings__east_asian_width.py index d190f422a60..022b0452522 100644 --- a/tests/data/cases/preview_long_strings__east_asian_width.py +++ b/tests/data/cases/preview_long_strings__east_asian_width.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable # The following strings do not have not-so-many chars, but are long enough # when these are rendered in a monospace font (if the renderer respects # Unicode East Asian Width properties). diff --git a/tests/data/cases/preview_long_strings__edge_case.py b/tests/data/cases/preview_long_strings__edge_case.py index a8e8971968c..28497e731bc 100644 --- a/tests/data/cases/preview_long_strings__edge_case.py +++ b/tests/data/cases/preview_long_strings__edge_case.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable some_variable = "This string is long but not so long that it needs to be split just yet" some_variable = 'This string is long but not so long that it needs to be split just yet' some_variable = "This string is long, just long enough that it needs to be split, u get?" diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index 5e76a8cf61c..afe2b311cf4 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable class A: def foo(): result = type(message)("") diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/preview_multiline_strings.py index 3ff643610b7..9288f6991bd 100644 --- a/tests/data/cases/preview_multiline_strings.py +++ b/tests/data/cases/preview_multiline_strings.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable """cow say""", call(3, "dogsay", textwrap.dedent("""dove diff --git a/tests/data/cases/preview_return_annotation_brackets_string.py b/tests/data/cases/preview_return_annotation_brackets_string.py index fea0ea6839a..2f937cf54ef 100644 --- a/tests/data/cases/preview_return_annotation_brackets_string.py +++ b/tests/data/cases/preview_return_annotation_brackets_string.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable # Long string example def frobnicate() -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": pass diff --git a/tests/test_black.py b/tests/test_black.py index a979a95b674..6dbe25a90b6 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -158,13 +158,11 @@ def test_empty_ff(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_one_empty_line(self) -> None: - mode = black.Mode(preview=True) for nl in ["\n", "\r\n"]: source = expected = nl - assert_format(source, expected, mode=mode) + assert_format(source, expected) def test_one_empty_line_ff(self) -> None: - mode = black.Mode(preview=True) for nl in ["\n", "\r\n"]: expected = nl tmp_file = Path(black.dump_to_file(nl)) @@ -175,20 +173,13 @@ def test_one_empty_line_ff(self) -> None: with open(tmp_file, "wb") as f: f.write(nl.encode("utf-8")) try: - self.assertFalse( - ff(tmp_file, mode=mode, write_back=black.WriteBack.YES) - ) + self.assertFalse(ff(tmp_file, write_back=black.WriteBack.YES)) with open(tmp_file, "rb") as f: actual = f.read().decode("utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) - def test_experimental_string_processing_warns(self) -> None: - self.assertWarns( - black.mode.Deprecated, black.Mode, experimental_string_processing=True - ) - def test_piping(self) -> None: _, source, expected = read_data_from_file( PROJECT_ROOT / "src/black/__init__.py" @@ -252,21 +243,6 @@ def test_piping_diff_with_color(self) -> None: self.assertIn("\033[31m", actual) self.assertIn("\033[0m", actual) - @patch("black.dump_to_file", dump_to_stderr) - def _test_wip(self) -> None: - source, expected = read_data("miscellaneous", "wip") - sys.settrace(tracefunc) - mode = replace( - DEFAULT_MODE, - experimental_string_processing=False, - target_versions={black.TargetVersion.PY38}, - ) - actual = fs(source, mode=mode) - sys.settrace(None) - self.assertFormatEqual(expected, actual) - black.assert_equivalent(source, actual) - black.assert_stable(source, actual, black.FileMode()) - def test_pep_572_version_detection(self) -> None: source, _ = read_data("cases", "pep_572") root = black.lib2to3_parse(source) @@ -374,7 +350,7 @@ def test_detect_debug_f_strings(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_string_quotes(self) -> None: source, expected = read_data("miscellaneous", "string_quotes") - mode = black.Mode(preview=True) + mode = black.Mode(unstable=True) assert_format(source, expected, mode) mode = replace(mode, string_normalization=False) not_normalized = fs(source, mode=mode) @@ -1052,7 +1028,6 @@ def test_format_file_contents(self) -> None: black.format_file_contents(invalid, mode=mode, fast=False) self.assertEqual(str(e.exception), "Cannot parse: 1:7: return if you can") - mode = black.Mode(preview=True) just_crlf = "\r\n" with self.assertRaises(black.NothingChanged): black.format_file_contents(just_crlf, mode=mode, fast=False) @@ -1396,7 +1371,6 @@ def get_output(*args: Any, **kwargs: Any) -> io.TextIOWrapper: return get_output - mode = black.Mode(preview=True) for content, expected in cases: output = io.StringIO() io_TextIOWrapper = io.TextIOWrapper @@ -1407,26 +1381,27 @@ def get_output(*args: Any, **kwargs: Any) -> io.TextIOWrapper: fast=True, content=content, write_back=black.WriteBack.YES, - mode=mode, + mode=DEFAULT_MODE, ) except io.UnsupportedOperation: pass # StringIO does not support detach assert output.getvalue() == expected - # An empty string is the only test case for `preview=False` - output = io.StringIO() - io_TextIOWrapper = io.TextIOWrapper - with patch("io.TextIOWrapper", _new_wrapper(output, io_TextIOWrapper)): - try: - black.format_stdin_to_stdout( - fast=True, - content="", - write_back=black.WriteBack.YES, - mode=DEFAULT_MODE, - ) - except io.UnsupportedOperation: - pass # StringIO does not support detach - assert output.getvalue() == "" + def test_cli_unstable(self) -> None: + self.invokeBlack(["--unstable", "-c", "0"], exit_code=0) + self.invokeBlack(["--preview", "-c", "0"], exit_code=0) + # Must also pass --preview + self.invokeBlack( + ["--enable-unstable-feature", "string_processing", "-c", "0"], exit_code=1 + ) + self.invokeBlack( + ["--preview", "--enable-unstable-feature", "string_processing", "-c", "0"], + exit_code=0, + ) + self.invokeBlack( + ["--unstable", "--enable-unstable-feature", "string_processing", "-c", "0"], + exit_code=0, + ) def test_invalid_cli_regex(self) -> None: for option in ["--include", "--exclude", "--extend-exclude", "--force-exclude"]: diff --git a/tests/util.py b/tests/util.py index 9ea30e62fe3..d5425f1f743 100644 --- a/tests/util.py +++ b/tests/util.py @@ -112,16 +112,24 @@ def assert_format( # For both preview and non-preview tests, ensure that Black doesn't crash on # this code, but don't pass "expected" because the precise output may differ. try: + if mode.unstable: + new_mode = replace(mode, unstable=False, preview=False) + else: + new_mode = replace(mode, preview=not mode.preview) _assert_format_inner( source, None, - replace(mode, preview=not mode.preview), + new_mode, fast=fast, minimum_version=minimum_version, lines=lines, ) except Exception as e: - text = "non-preview" if mode.preview else "preview" + text = ( + "unstable" + if mode.unstable + else "non-preview" if mode.preview else "preview" + ) raise FormatFailure( f"Black crashed formatting this case in {text} mode." ) from e @@ -138,7 +146,7 @@ def assert_format( _assert_format_inner( source, None, - replace(mode, preview=preview_mode, line_length=1), + replace(mode, preview=preview_mode, line_length=1, unstable=False), fast=fast, minimum_version=minimum_version, lines=lines, @@ -241,6 +249,7 @@ def get_flags_parser() -> argparse.ArgumentParser: "--skip-magic-trailing-comma", default=False, action="store_true" ) parser.add_argument("--preview", default=False, action="store_true") + parser.add_argument("--unstable", default=False, action="store_true") parser.add_argument("--fast", default=False, action="store_true") parser.add_argument( "--minimum-version", @@ -278,6 +287,7 @@ def parse_mode(flags_line: str) -> TestCaseArgs: is_ipynb=args.ipynb, magic_trailing_comma=not args.skip_magic_trailing_comma, preview=args.preview, + unstable=args.unstable, ) if args.line_ranges: lines = parse_line_ranges(args.line_ranges)