Format subscriptions in a PEP-8 compliant way

.flake8

@@ -2,7 +2,7 @@
 # Keep in sync with setup.cfg which is used for source packages.
 
 [flake8]
-ignore = E266, E501, W503
+ignore = E203, E266, E501, W503
 max-line-length = 80
 max-complexity = 15
 select = B,C,E,F,W,T4,B9

README.md

@@ -302,6 +302,19 @@ This behaviour may raise ``W503 line break before binary operator`` warnings in
 style guide enforcement tools like Flake8. Since ``W503`` is not PEP 8 compliant,
 you should tell Flake8 to ignore these warnings.
 
+### Whiespace In Subscripts and ``:``
+
+PEP 8 [recommends](https://www.python.org/dev/peps/pep-0008/#whitespace-in-expressions-and-statements)
+to treat ``:`` in slices as a binary operator with the lowest priority, and to
+leave an equal amount of space on either side, except if a parameter is omitted
+(e.g. ``ham[1 + 1:]``). It also states that for extended slices, both ``:``
+operators have to have the same amount of spacing, except if a parameter is
+omitted (``ham[1 + 1 ::]``). *Black* enforces these rules consistently.
+
+This behaviour may raise ``E203 whitespace before ':'`` warnings in style guide
+enforcement tools like Flake8. Since ``E203`` is not PEP 8 compliant, you should
+tell Flake8 to ignore these warnings.
+
 ### Parentheses
 
 Some parentheses are optional in the Python grammar.  Any expression can

black.py

@@ -89,11 +89,11 @@ def __init__(self, consumed: int) -> None:
         self.consumed = consumed
 
     def trim_prefix(self, leaf: Leaf) -> None:
-        leaf.prefix = leaf.prefix[self.consumed:]
+        leaf.prefix = leaf.prefix[self.consumed :]
 
     def leaf_from_consumed(self, leaf: Leaf) -> Leaf:
         """Returns a new Leaf from the consumed part of the prefix."""
-        unformatted_prefix = leaf.prefix[:self.consumed]
+        unformatted_prefix = leaf.prefix[: self.consumed]
         return Leaf(token.NEWLINE, unformatted_prefix)
 
 
@@ -582,6 +582,16 @@ def show(cls, code: str) -> None:
     syms.listmaker,
     syms.testlist_gexp,
 }
+EXPRS = {
+    syms.expr,
+    syms.xor_expr,
+    syms.and_expr,
+    syms.shift_expr,
+    syms.arith_expr,
+    syms.trailer,
+    syms.term,
+    syms.power,
+}
 COMPREHENSION_PRIORITY = 20
 COMMA_PRIORITY = 10
 TERNARY_PRIORITY = 7
@@ -726,7 +736,26 @@ def append(self, leaf: Leaf, preformatted: bool = False) -> None:
         if self.leaves and not preformatted:
             # Note: at this point leaf.prefix should be empty except for
             # imports, for which we only preserve newlines.
-            leaf.prefix += whitespace(leaf)
+            lsqb_leaf: Optional[Leaf]
+            if leaf.type == token.LSQB:
+                lsqb_leaf = leaf
+            else:
+                lsqb_leaf = self.bracket_tracker.bracket_match.get(
+                    (self.bracket_tracker.depth - 1, token.RSQB)
+                )
+            complex_subscript = False
+            if lsqb_leaf is not None:
+                subscript_start = lsqb_leaf.next_sibling
+                assert subscript_start is not None, "LSQB always has a right sibling"
+                if (
+                    isinstance(subscript_start, Node)
+                    and subscript_start.type == syms.subscriptlist
+                ):
+                    subscript_start = child_towards(subscript_start, leaf)
+                complex_subscript = subscript_start is not None and any(
+                    map(lambda n: n.type in EXPRS, subscript_start.pre_order())
+                )
+            leaf.prefix += whitespace(leaf, complex_subscript=complex_subscript)
         if self.inside_brackets or not preformatted:
             self.bracket_tracker.mark(leaf)
             self.maybe_remove_trailing_comma(leaf)
@@ -859,7 +888,7 @@ def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
         else:
             return False
 
-        for leaf in self.leaves[_opening_index + 1:]:
+        for leaf in self.leaves[_opening_index + 1 :]:
             if leaf is closing:
                 break
 
@@ -1303,8 +1332,12 @@ def __attrs_post_init__(self) -> None:
 ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
 
 
-def whitespace(leaf: Leaf) -> str:  # noqa C901
-    """Return whitespace prefix if needed for the given `leaf`."""
+def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str:  # noqa C901
+    """Return whitespace prefix if needed for the given `leaf`.
+
+    `complex_subscript` signals whether the given leaf is part of a subscription
+    which has non-trivial arguments, like arithmetic expressions or function calls.
+    """
     NO = ""
     SPACE = " "
     DOUBLESPACE = "  "
@@ -1318,7 +1351,10 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
         return DOUBLESPACE
 
     assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}"
-    if t == token.COLON and p.type not in {syms.subscript, syms.subscriptlist}:
+    if (
+        t == token.COLON
+        and p.type not in {syms.subscript, syms.subscriptlist, syms.sliceop}
+    ):
         return NO
 
     prev = leaf.prev_sibling
@@ -1328,7 +1364,10 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
             return NO
 
         if t == token.COLON:
-            return SPACE if prevp.type == token.COMMA else NO
+            if prevp.type == token.COLON:
+                return NO
+
+            return SPACE if prevp.type == token.COMMA or complex_subscript else NO
 
         if prevp.type == token.EQUAL:
             if prevp.parent:
@@ -1349,7 +1388,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
 
         elif prevp.type == token.COLON:
             if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:
-                return NO
+                return SPACE if complex_subscript else NO
 
         elif (
             prevp.parent
@@ -1455,7 +1494,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
         if prev and prev.type == token.LPAR:
             return NO
 
-    elif p.type == syms.subscript:
+    elif p.type in {syms.subscript, syms.sliceop}:
         # indexing
         if not prev:
             assert p.parent is not None, "subscripts are always parented"
@@ -1464,8 +1503,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
 
             return NO
 
-        else:
-            return NO
+        return SPACE if complex_subscript else NO
 
     elif p.type == syms.atom:
         if prev and t == token.DOT:
@@ -1534,6 +1572,14 @@ def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
     return None
 
 
+def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:
+    """Return the child of `ancestor` that contains `descendant`."""
+    node: Optional[LN] = descendant
+    while node and node.parent != ancestor:
+        node = node.parent
+    return node
+
+
 def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
     """Return the priority of the `leaf` delimiter, given a line break after it.
 
@@ -1994,6 +2040,7 @@ def explode_split(
 
     try:
         yield from delimiter_split(new_lines[1], py36)
+
     except CannotSplit:
         yield new_lines[1]
 
@@ -2061,7 +2108,7 @@ def normalize_string_quotes(leaf: Leaf) -> None:
     unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
     escaped_new_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{new_quote}")
     escaped_orig_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{orig_quote}")
-    body = leaf.value[first_quote_pos + len(orig_quote):-len(orig_quote)]
+    body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)]
     if "r" in prefix.casefold():
         if unescaped_new_quote.search(body):
             # There's at least one unescaped new_quote in this raw string

tests/expression.diff

@@ -130,7 +130,7 @@
  slice[0]
  slice[0:1]
 @@ -123,88 +145,114 @@
- numpy[-(c + 1):, d]
+ numpy[-(c + 1) :, d]
  numpy[:, l[-2]]
  numpy[:, ::-1]
  numpy[np.newaxis, :]

tests/expression.py

@@ -105,7 +105,7 @@
 slice[:-1]
 slice[1:]
 slice[::-1]
-slice[d::d + 1]
+slice[d :: d + 1]
 slice[:c, c - 1]
 numpy[:, 0:1]
 numpy[:, :-1]
@@ -119,8 +119,8 @@
 numpy[:, (0, 1, 2, 5)]
 numpy[0, [0]]
 numpy[:, [i]]
-numpy[1:c + 1, c]
-numpy[-(c + 1):, d]
+numpy[1 : c + 1, c]
+numpy[-(c + 1) :, d]
 numpy[:, l[-2]]
 numpy[:, ::-1]
 numpy[np.newaxis, :]
@@ -341,7 +341,7 @@ async def f():
 slice[:-1]
 slice[1:]
 slice[::-1]
-slice[d::d + 1]
+slice[d :: d + 1]
 slice[:c, c - 1]
 numpy[:, 0:1]
 numpy[:, :-1]
@@ -355,8 +355,8 @@ async def f():
 numpy[:, (0, 1, 2, 5)]
 numpy[0, [0]]
 numpy[:, [i]]
-numpy[1:c + 1, c]
-numpy[-(c + 1):, d]
+numpy[1 : c + 1, c]
+numpy[-(c + 1) :, d]
 numpy[:, l[-2]]
 numpy[:, ::-1]
 numpy[np.newaxis, :]

tests/fmtonoff.py

@@ -121,7 +121,7 @@ def function_signature_stress_test(number:int,no_annotation=None,text:str='defau
 # fmt: on
 def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r""):
     offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200000)))
-    assert task._cancel_stack[:len(old_stack)] == old_stack
+    assert task._cancel_stack[: len(old_stack)] == old_stack
 
 
 def spaces_types(

tests/function.py

@@ -133,7 +133,7 @@ def function_signature_stress_test(
 
 def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r""):
     offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200000)))
-    assert task._cancel_stack[:len(old_stack)] == old_stack
+    assert task._cancel_stack[: len(old_stack)] == old_stack
 
 
 def spaces_types(

tests/slices.py

@@ -0,0 +1,20 @@
+slice[a.b : c.d]
+slice[d :: d + 1]
+slice[d + 1 :: d]
+slice[d::d]
+slice[0]
+slice[-1]
+slice[:-1]
+slice[::-1]
+slice[:c, c - 1]
+slice[c, c + 1, d::]
+slice[ham[c::d] :: 1]
+slice[ham[cheese ** 2 : -1] : 1 : 1, ham[1:2]]
+slice[:-1:]
+
+# These are from PEP-8:
+ham[1:9], ham[1:9:3], ham[:9:3], ham[1::3], ham[1:9:]
+ham[lower:upper], ham[lower:upper:], ham[lower::step]
+# ham[lower+offset : upper+offset]
+ham[: upper_fn(x) : step_fn(x)], ham[:: step_fn(x)]
+ham[lower + offset : upper + offset]

tests/test_black.py

@@ -229,6 +229,14 @@ def test_string_quotes(self) -> None:
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, line_length=ll)
 
+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_slices(self) -> None:
+        source, expected = read_data("slices")
+        actual = fs(source)
+        self.assertFormatEqual(expected, actual)
+        black.assert_equivalent(source, actual)
+        black.assert_stable(source, actual, line_length=ll)
+
     @patch("black.dump_to_file", dump_to_stderr)
     def test_comments(self) -> None:
         source, expected = read_data("comments")