Fix various things with Literal

CHANGELOG.md

@@ -3,6 +3,16 @@
 - Add `typing_extensions.Buffer`, a marker class for buffer types, as proposed
   by PEP 688. Equivalent to `collections.abc.Buffer` in Python 3.12. Patch by
   Jelle Zijlstra.
+- Backport two CPython PRs fixing various issues with `typing.Literal`:
+  https://github.com/python/cpython/pull/23294 and
+  https://github.com/python/cpython/pull/23383. Both CPython PRs were
+  originally by Yurii Karabas, and both were backported to Python >=3.9.1, but
+  no earlier. Patch by Alex Waygood.
+
+  A side effect of one of the changes is that equality comparisons of `Literal`
+  objects will now raise a `TypeError` if one of the `Literal` objects being
+  compared has a mutable parameter. (Using mutable parameters with `Literal` is
+  not supported by PEP 586 or by any major static type checkers.)
 - Backport [CPython PR 26067](https://github.com/python/cpython/pull/26067)
   (originally by Yurii Karabas), ensuring that `isinstance()` calls on
   protocols raise `TypeError` when the protocol is not decorated with

README.md

@@ -143,6 +143,9 @@ Certain objects were changed after they were added to `typing`, and
 - `TypeVar` gains two additional parameters, `default=` and `infer_variance=`,
   in the draft PEPs [695](https://peps.python.org/pep-0695/) and [696](https://peps.python.org/pep-0696/), which are being considered for inclusion
   in Python 3.12.
+- `Literal` does not flatten or deduplicate parameters on Python <3.9.1. The
+  `typing_extensions` version flattens and deduplicates parameters on all
+  Python versions.
 
 There are a few types whose interface was modified between different
 versions of typing. For example, `typing.Sequence` was modified to

src/test_typing_extensions.py

@@ -615,7 +615,8 @@ def test_literals_inside_other_types(self):
         List[Literal[("foo", "bar", "baz")]]
 
     def test_repr(self):
-        if hasattr(typing, 'Literal'):
+        # we backport various bugfixes that were added in 3.9.1
+        if sys.version_info >= (3, 9, 1):
             mod_name = 'typing'
         else:
             mod_name = 'typing_extensions'
@@ -624,6 +625,7 @@ def test_repr(self):
         self.assertEqual(repr(Literal[int]), mod_name + ".Literal[int]")
         self.assertEqual(repr(Literal), mod_name + ".Literal")
         self.assertEqual(repr(Literal[None]), mod_name + ".Literal[None]")
+        self.assertEqual(repr(Literal[1, 2, 3, 3]), mod_name + ".Literal[1, 2, 3]")
 
     def test_cannot_init(self):
         with self.assertRaises(TypeError):
@@ -655,6 +657,39 @@ def test_no_multiple_subscripts(self):
         with self.assertRaises(TypeError):
             Literal[1][1]
 
+    def test_equal(self):
+        self.assertNotEqual(Literal[0], Literal[False])
+        self.assertNotEqual(Literal[True], Literal[1])
+        self.assertNotEqual(Literal[1], Literal[2])
+        self.assertNotEqual(Literal[1, True], Literal[1])
+        self.assertEqual(Literal[1], Literal[1])
+        self.assertEqual(Literal[1, 2], Literal[2, 1])
+        self.assertEqual(Literal[1, 2, 3], Literal[1, 2, 3, 3])
+
+    def test_hash(self):
+        self.assertEqual(hash(Literal[1]), hash(Literal[1]))
+        self.assertEqual(hash(Literal[1, 2]), hash(Literal[2, 1]))
+        self.assertEqual(hash(Literal[1, 2, 3]), hash(Literal[1, 2, 3, 3]))
+
+    def test_args(self):
+        self.assertEqual(Literal[1, 2, 3].__args__, (1, 2, 3))
+        self.assertEqual(Literal[1, 2, 3, 3].__args__, (1, 2, 3))
+        self.assertEqual(Literal[1, Literal[2], Literal[3, 4]].__args__, (1, 2, 3, 4))
+        # Mutable arguments will not be deduplicated
+        self.assertEqual(Literal[[], []].__args__, ([], []))
+
+    def test_flatten(self):
+        l1 = Literal[Literal[1], Literal[2], Literal[3]]
+        l2 = Literal[Literal[1, 2], 3]
+        l3 = Literal[Literal[1, 2, 3]]
+        for lit in l1, l2, l3:
+            self.assertEqual(lit, Literal[1, 2, 3])
+            self.assertEqual(lit.__args__, (1, 2, 3))
+
+    def test_caching_of_Literal_respects_type(self):
+        self.assertIs(type(Literal[1].__args__[0]), int)
+        self.assertIs(type(Literal[True].__args__[0]), bool)
+
 
 class MethodHolder:
     @classmethod
@@ -3566,6 +3601,8 @@ def test_typing_extensions_defers_when_possible(self):
             'get_type_hints',
             'is_typeddict',
         }
+        if sys.version_info < (3, 9, 1):
+            exclude |= {"Literal"}
         if sys.version_info < (3, 10):
             exclude |= {'get_args', 'get_origin'}
         if sys.version_info < (3, 11):

src/typing_extensions.py

@@ -261,21 +261,70 @@ def IntVar(name):
     return typing.TypeVar(name)
 
 
-# 3.8+:
-if hasattr(typing, 'Literal'):
+# Various Literal bugs were fixed in 3.9.1, but not backported earlier than that
+if sys.version_info >= (3, 9, 1):
     Literal = typing.Literal
-# 3.7:
 else:
+    def _flatten_literal_params(parameters):
+        """An internal helper for Literal creation: flatten Literals among parameters"""
+        params = []
+        for p in parameters:
+            if isinstance(p, _LiteralGenericAlias):
+                params.extend(p.__args__)
+            else:
+                params.append(p)
+        return tuple(params)
+
+    def _value_and_type_iter(params):
+        for p in params:
+            yield p, type(p)
+
+    class _LiteralGenericAlias(typing._GenericAlias, _root=True):
+        def __eq__(self, other):
+            if not isinstance(other, _LiteralGenericAlias):
+                return NotImplemented
+            these_args_deduped = set(_value_and_type_iter(self.__args__))
+            other_args_deduped = set(_value_and_type_iter(other.__args__))
+            return these_args_deduped == other_args_deduped
+
+        def __hash__(self):
+            return hash(frozenset(_value_and_type_iter(self.__args__)))
+
     class _LiteralForm(typing._SpecialForm, _root=True):
+        def __init__(self, doc: str):
+            self._name = 'Literal'
+            self._doc = self.__doc__ = doc
 
         def __repr__(self):
             return 'typing_extensions.' + self._name
 
         def __getitem__(self, parameters):
-            return typing._GenericAlias(self, parameters)
+            if not isinstance(parameters, tuple):
+                parameters = (parameters,)
+
+            parameters = _flatten_literal_params(parameters)
 
-    Literal = _LiteralForm('Literal',
-                           doc="""A type that can be used to indicate to type checkers
+            val_type_pairs = list(_value_and_type_iter(parameters))
+            try:
+                deduped_pairs = set(val_type_pairs)
+            except TypeError:
+                # unhashable parameters
+                pass
+            else:
+                # similar logic to typing._deduplicate on Python 3.9+
+                if len(deduped_pairs) < len(val_type_pairs):
+                    new_parameters = []
+                    for pair in val_type_pairs:
+                        if pair in deduped_pairs:
+                            new_parameters.append(pair[0])
+                            deduped_pairs.remove(pair)
+                    assert not deduped_pairs, deduped_pairs
+                    parameters = tuple(new_parameters)
+
+            return _LiteralGenericAlias(self, parameters)
+
+    Literal = _LiteralForm(doc="""\
+                           A type that can be used to indicate to type checkers
                            that the corresponding value has a value literally equivalent
                            to the provided parameter. For example: