Minimal changes for Python 3.12

- No opcode implementations, yet. - Python 3.12 added pseudo and instrumentation opcodes to `dis` collections. pytype doesn't need those, so we ignore them in the generate_opcode_diff.py script. - pycnite changes are submitted and this bumps the min pycnite version number. - The YIELD_VALUE opcode changed from having no arguments to having an argument. Looks like that hasn't happened before in pytype's history. I solved that by adding a `for_python_version` function, which returns a nested class. The nested class has the same name as the top-level class, which is important because pytype relies on `__class__.__name__`. - Starting with Python 3.12, pylint complains about too long lines. This is arguably a formatter concern. Disabling the rule. PiperOrigin-RevId: 642962053
google · Jun 13, 2024 · 4155664 · 4155664
1 parent 46acb94
commit 4155664
Show file tree

Hide file tree

Showing 9 changed files with 363 additions and 81 deletions.
diff --git a/docs/developers/python_version_upgrades.md b/docs/developers/python_version_upgrades.md
@@ -31,10 +31,10 @@ updated to accept the new version.
 
 #### opcode changes
 
-If the new version adds or removes any opcodes, then an updated opcode mapping
-should be added to
-[pytype/pyc/opcodes.py][pyc.opcodes.python_3_7_mapping] and new opcodes
-implemented in [pytype/vm.py][vm.VirtualMachine.byte_LOAD_METHOD].
+If the new version adds or removes any opcodes, then updated opcodes should be
+added to [pytype/pyc/opcodes.py][pyc.opcodes], an opcode mapping to
+[pycnite/mapping.py][pycnite.mapping] and new opcodes implemented in
+[pytype/vm.py][vm.VirtualMachine.byte_LOAD_METHOD].
 
 TIP: [pytype/pyc/generate_opcode_diffs.py][pyc.generate_opcode_diffs] will
 generate the changes you need to make to opcodes.py, as well as
@@ -65,7 +65,7 @@ https://github.com/python/cpython/blob/master/Python/ceval.c).
 #### magic numbers
 
 Magic numbers for the new version should be copied from the
-[CPython source code][cpython-source] to [pytype/pyc/magic.py][pyc.magic].
+[CPython source code][cpython-source] to [pycnite/magic.py][pycnite.magic].
 
 ### regression tests
 
@@ -105,14 +105,17 @@ and by searching for "New in version 3.x" in the
 [cpython-opcode]: https://github.com/python/cpython/blob/master/Lib/opcode.py
 
 <!-- References with different internal and external versions -->
+<!-- mdformat off(mdformat adds/removes newlines, which make these harder to read) -->
 
 [cpython-source]: https://github.com/python/cpython/blob/beba1a808000d5fc445cb28eab96bdb4cdb7c959/Lib/importlib/_bootstrap_external.py#L245
 
 [pyc.generate_opcode_diffs]: https://github.com/google/pytype/blob/main/pytype/pyc/generate_opcode_diffs.py
 
-[pyc.magic]: https://github.com/google/pytype/blob/ee51995a1c5937cb4ebee291acb2e049fb0f81cc/pytype/pyc/magic.py#L97
+[pyc.opcodes]: https://github.com/google/pytype/blob/6516ebd5def4ac507a5449b0c57297a53b7e9a9f/pytype/pyc/opcodes.py#L201-L1018
 
-[pyc.opcodes.python_3_7_mapping]: https://github.com/google/pytype/blob/ee51995a1c5937cb4ebee291acb2e049fb0f81cc/pytype/pyc/opcodes.py#L1101
+[pycnite.mapping]: https://github.com/google/pycnite/blob/25326a096278a8372e03bbefab8fa4b725f96245/pycnite/mapping.py#L196
+
+[pycnite.magic]: https://github.com/google/pycnite/blob/25326a096278a8372e03bbefab8fa4b725f96245/pycnite/magic.py#L20
 
 [test_data.simple]: https://github.com/google/pytype/blob/main/pytype/test_data/simple.py
 

diff --git a/pylintrc b/pylintrc
@@ -40,6 +40,7 @@ disable=
   import-outside-toplevel,
   inconsistent-return-statements,
   invalid-name,
+  line-too-long,
   method-hidden,
   missing-docstring,
   modified-iterating-dict,

diff --git a/pytype/pyc/generate_opcode_diffs.py b/pytype/pyc/generate_opcode_diffs.py
@@ -27,12 +27,20 @@
 """
 
 import json
+import opcode
 import subprocess
 import sys
 import tempfile
 import textwrap
 
 
+# Starting with Python 3.12, `dis` collections contain pseudo instructions and
+# instrumented instructions. These are opcodes with values >= MIN_PSEUDO_OPCODE
+# and >= MIN_INSTRUMENTED_OPCODE.
+# Pytype doesn't care about those, so we ignore them here.
+_MIN_INSTRUMENTED_OPCODE = getattr(opcode, 'MIN_INSTRUMENTED_OPCODE', 237)
+
+
 def generate_diffs(argv):
   """Generate diffs."""
   version1, version2 = argv
@@ -57,11 +65,8 @@ def generate_diffs(argv):
         'HAS_JABS': dis.hasjabs,
         'HAS_LOCAL': dis.haslocal,
         'HAS_FREE': dis.hasfree,
-        'HAS_NARGS': dis.hasnargs,
+        'HAS_NARGS': getattr(dis, 'hasnargs', []), # Removed in Python 3.12
       }
-      for attr in dis.__all__:
-        if attr.startswith('has'):
-          output[attr] = getattr(dis, attr)
       print(json.dumps(output))
     """))
     f.flush()
@@ -77,16 +82,14 @@ def generate_diffs(argv):
   #   index: ('CHANGE', old opcode at this index, new opcode at this index)
   #   index: ('FLAG_CHANGE', opcode)
   #   index: ('ADD', new opcode)
-  num_opcodes = len(dis1['opname'])
-  assert num_opcodes == len(dis2['opname'])
   changed = {}
   impl_changed = set()
   name_unchanged = set()
 
   def is_unset(opname_entry):
     return opname_entry == f'<{i}>'
 
-  for i in range(num_opcodes):
+  for i in range(_MIN_INSTRUMENTED_OPCODE - 1):
     opname1 = dis1['opname'][i]
     opname2 = dis2['opname'][i]
     if opname1 == opname2:
@@ -133,55 +136,83 @@ def is_unset(opname_entry):
         continue
       flags.append(k)
     if flags:
-      cls.append('  FLAGS = ' + ' | '.join(flags))
+      cls.append('  _FLAGS = ' + ' | '.join(flags))
     cls.append('  __slots__ = ()')
     classes.append(cls)
 
+  # Generate stub implementations.
+  stubs = []
+  for _, diff in sorted(changed.items()):
+    if diff[0] == 'DELETE':
+      continue
+    name = diff[-1]
+    if name in impl_changed:
+      continue
+    stubs.append(
+        [f'def byte_{name}(self, state, op):', '  del op', '  return state']
+    )
+
   # Generate a mapping diff.
-  diffs = []
+  mapping = []
   for op, diff in sorted(changed.items()):
     if diff[0] == 'DELETE':
       name = diff[1]
-      diffs.append(f'{op}: None,  # was {name} in {version1}')
+      mapping.append(f'{op}: None,  # was {name} in {version1}')
     elif diff[0] == 'CHANGE':
       old_name, new_name = diff[1:]  # pytype: disable=bad-unpacking
-      diffs.append(f'{op}: {new_name},  # was {old_name} in {version1}')
+      mapping.append(f'{op}: "{new_name}",  # was {old_name} in {version1}')
     elif diff[0] == 'ADD':
       name = diff[1]
-      diffs.append(f'{op}: {name},')
+      mapping.append(f'{op}: "{name}",')
     else:
       assert diff[0] == 'FLAG_CHANGE'
 
-  # Generate stub implementations.
-  stubs = []
-  for op, diff in sorted(changed.items()):
+  # Generate arg type diff
+  arg_types = []
+  for _, diff in sorted(changed.items()):
     if diff[0] == 'DELETE':
       continue
     name = diff[-1]
-    if name in impl_changed:
-      continue
-    stubs.append([f'def byte_{name}(self, state, op):',
-                  '  del op',
-                  '  return state'])
+    old_type = _get_arg_type(dis1, name)
+    new_type = _get_arg_type(dis2, name)
+    if new_type != old_type:
+      arg_types.append(f'"{name}": {new_type},')
+
+  return classes, stubs, sorted(impl_changed), mapping, arg_types
 
-  return classes, diffs, stubs, sorted(impl_changed)
+
+def _get_arg_type(dis, opname):
+  all_types = ['CONST', 'NAME', 'JREL', 'JABS', 'LOCAL', 'FREE', 'NARGS']
+  for t in all_types:
+    k = f'HAS_{t}'
+    if k in dis and opname in dis['opmap'] and dis['opmap'][opname] in dis[k]:
+      return t
+  return None
 
 
 def main(argv):
-  classes, diff, stubs, impl_changed = generate_diffs(argv)
+  classes, stubs, impl_changed, mapping, arg_types = generate_diffs(argv)
+  print('==== PYTYPE CHANGES ====\n')
   print('---- NEW OPCODES (pyc/opcodes.py) ----\n')
   print('\n\n\n'.join('\n'.join(cls) for cls in classes))
   if impl_changed:
-    print('\nNOTE: Delete the old class definitions for the following '
-          'modified opcodes: ' + ', '.join(impl_changed))
-  print('\n---- OPCODE MAPPING DIFF (pyc/opcodes.py) ----\n')
-  print('    ' + '\n    '.join(diff))
+    print(
+        '\nNOTE: Delete the old class definitions for the following '
+        'modified opcodes: '
+        + ', '.join(impl_changed)
+    )
   print('\n---- OPCODE STUB IMPLEMENTATIONS (vm.py) ----\n')
   print('\n\n'.join('  ' + '\n  '.join(stub) for stub in stubs))
   if impl_changed:
     print('\nNOTE: The implementations of the following modified opcodes may '
           'need to be updated: ' + ', '.join(impl_changed))
 
+  print('\n\n==== PYCNITE CHANGES ====\n')
+  print('---- OPCODE MAPPING DIFF (mapping.py) ----\n')
+  print('    ' + '\n    '.join(mapping))
+  print('\n---- OPCODE ARG TYPE DIFF (mapping.py) ----\n')
+  print('    ' + '\n    '.join(arg_types))
+
 
 if __name__ == '__main__':
   main(sys.argv[1:])
diff --git a/pytype/pyc/generate_opcode_diffs_test.py b/pytype/pyc/generate_opcode_diffs_test.py
@@ -13,20 +13,47 @@ class GenerateOpcodeDiffsTest(unittest.TestCase):
 
   def _generate_diffs(self):
     with mock.patch.object(subprocess, 'run') as mock_run:
+      more_opcode_names = [
+          f'<{i}>'
+          for i in range(9, generate_opcode_diffs._MIN_INSTRUMENTED_OPCODE)
+      ]
       mapping_38 = json.dumps({
           'opmap': {'DO_THIS': 1, 'I_MOVE': 2, 'DO_EIGHT': 5, 'JUMP': 8},
-          'opname': ['<0>', 'DO_THIS', 'I_MOVE', '<3>', '<4>', 'DO_EIGHT',
-                     '<6>', '<7>', 'JUMP'],
+          'opname': [
+              '<0>',
+              'DO_THIS',
+              'I_MOVE',
+              '<3>',
+              '<4>',
+              'DO_EIGHT',
+              '<6>',
+              '<7>',
+              'JUMP',
+          ] + more_opcode_names,
           'HAVE_ARGUMENT': 3,
           'HAS_CONST': [],
           'HAS_NAME': [],
           'HAS_JREL': [],
       })
       mapping_39 = json.dumps({
-          'opmap': {'I_MOVE': 3, 'DO_THAT': 4, 'DO_THAT_TOO': 5, 'DO_NINE': 7,
-                    'JUMP': 8},
-          'opname': ['<0>', '<1>', '<2>', 'I_MOVE', 'DO_THAT', 'DO_THAT_TOO',
-                     '<6>', 'DO_NINE', 'JUMP'],
+          'opmap': {
+              'I_MOVE': 3,
+              'DO_THAT': 4,
+              'DO_THAT_TOO': 5,
+              'DO_NINE': 7,
+              'JUMP': 8,
+          },
+          'opname': [
+              '<0>',
+              '<1>',
+              '<2>',
+              'I_MOVE',
+              'DO_THAT',
+              'DO_THAT_TOO',
+              '<6>',
+              'DO_NINE',
+              'JUMP',
+          ] + more_opcode_names,
           'HAVE_ARGUMENT': 6,
           'HAS_CONST': [7],
           'HAS_NAME': [5, 7],
@@ -37,7 +64,7 @@ def _generate_diffs(self):
       return generate_opcode_diffs.generate_diffs(['3.8', '3.9'])
 
   def test_classes(self):
-    classes, _, _, _ = self._generate_diffs()
+    classes, _, _, _, _ = self._generate_diffs()
     i_move, do_that, do_that_too, do_nine, jump = classes
     self.assertMultiLineEqual('\n'.join(i_move), textwrap.dedent("""
       class I_MOVE(Opcode):
@@ -47,35 +74,33 @@ class I_MOVE(Opcode):
       class DO_THAT(Opcode):
         __slots__ = ()
     """).strip())
-    self.assertMultiLineEqual('\n'.join(do_that_too), textwrap.dedent("""
-      class DO_THAT_TOO(Opcode):
-        FLAGS = HAS_NAME
-        __slots__ = ()
-    """).strip())
-    self.assertMultiLineEqual('\n'.join(do_nine), textwrap.dedent("""
-      class DO_NINE(OpcodeWithArg):
-        FLAGS = HAS_ARGUMENT | HAS_CONST | HAS_NAME
-        __slots__ = ()
-    """).strip())
-    self.assertMultiLineEqual('\n'.join(jump), textwrap.dedent("""
-      class JUMP(OpcodeWithArg):
-        FLAGS = HAS_ARGUMENT | HAS_JREL
-        __slots__ = ()
-    """).strip())
-
-  def test_diff(self):
-    _, diff, _, _ = self._generate_diffs()
-    self.assertMultiLineEqual('\n'.join(diff), textwrap.dedent("""
-      1: None,  # was DO_THIS in 3.8
-      2: None,  # was I_MOVE in 3.8
-      3: I_MOVE,
-      4: DO_THAT,
-      5: DO_THAT_TOO,  # was DO_EIGHT in 3.8
-      7: DO_NINE,
-    """).strip())
+    self.assertMultiLineEqual(
+        '\n'.join(do_that_too),
+        textwrap.dedent("""
+          class DO_THAT_TOO(Opcode):
+            _FLAGS = HAS_NAME
+            __slots__ = ()
+        """).strip(),
+    )
+    self.assertMultiLineEqual(
+        '\n'.join(do_nine),
+        textwrap.dedent("""
+          class DO_NINE(OpcodeWithArg):
+            _FLAGS = HAS_ARGUMENT | HAS_CONST | HAS_NAME
+            __slots__ = ()
+        """).strip(),
+    )
+    self.assertMultiLineEqual(
+        '\n'.join(jump),
+        textwrap.dedent("""
+          class JUMP(OpcodeWithArg):
+            _FLAGS = HAS_ARGUMENT | HAS_JREL
+            __slots__ = ()
+        """).strip(),
+    )
 
   def test_stubs(self):
-    _, _, stubs, _ = self._generate_diffs()
+    _, stubs, _, _, _ = self._generate_diffs()
     do_that, do_that_too, do_nine = stubs
     self.assertMultiLineEqual('\n'.join(do_that), textwrap.dedent("""
       def byte_DO_THAT(self, state, op):
@@ -94,9 +119,34 @@ def byte_DO_NINE(self, state, op):
     """).strip())
 
   def test_impl_changed(self):
-    _, _, _, impl_changed = self._generate_diffs()
+    _, _, impl_changed, _, _ = self._generate_diffs()
     self.assertEqual(impl_changed, ['I_MOVE', 'JUMP'])
 
+  def test_mapping(self):
+    _, _, _, mapping, _ = self._generate_diffs()
+    self.assertMultiLineEqual(
+        '\n'.join(mapping),
+        textwrap.dedent("""
+          1: None,  # was DO_THIS in 3.8
+          2: None,  # was I_MOVE in 3.8
+          3: "I_MOVE",
+          4: "DO_THAT",
+          5: "DO_THAT_TOO",  # was DO_EIGHT in 3.8
+          7: "DO_NINE",
+        """).strip(),
+    )
+
+  def test_arg_types(self):
+    _, _, _, _, arg_types = self._generate_diffs()
+    self.assertMultiLineEqual(
+        '\n'.join(arg_types),
+        textwrap.dedent("""
+          "DO_THAT_TOO": NAME,
+          "DO_NINE": CONST,
+          "JUMP": JREL,
+        """).strip(),
+    )
+
 
 if __name__ == '__main__':
   unittest.main()