Merge branch 'main' into typewatch

* main:
  bpo-35540 dataclasses.asdict now supports defaultdict fields (pythongh-32056)
  pythonGH-91052: Add C API for watching dictionaries (pythonGH-31787)
  bpo-38693: Use f-strings instead of str.format() within importlib (python#17058)

Doc/c-api/dict.rst

@@ -238,3 +238,54 @@ Dictionary Objects
           for key, value in seq2:
               if override or key not in a:
                   a[key] = value
+
+.. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback)
+
+   Register *callback* as a dictionary watcher. Return a non-negative integer
+   id which must be passed to future calls to :c:func:`PyDict_Watch`. In case
+   of error (e.g. no more watcher IDs available), return ``-1`` and set an
+   exception.
+
+.. c:function:: int PyDict_ClearWatcher(int watcher_id)
+
+   Clear watcher identified by *watcher_id* previously returned from
+   :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g.
+   if the given *watcher_id* was never registered.)
+
+.. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict)
+
+   Mark dictionary *dict* as watched. The callback granted *watcher_id* by
+   :c:func:`PyDict_AddWatcher` will be called when *dict* is modified or
+   deallocated.
+
+.. c:type:: PyDict_WatchEvent
+
+   Enumeration of possible dictionary watcher events: ``PyDict_EVENT_ADDED``,
+   ``PyDict_EVENT_MODIFIED``, ``PyDict_EVENT_DELETED``, ``PyDict_EVENT_CLONED``,
+   ``PyDict_EVENT_CLEARED``, or ``PyDict_EVENT_DEALLOCATED``.
+
+.. c:type:: int (*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value)
+
+   Type of a dict watcher callback function.
+
+   If *event* is ``PyDict_EVENT_CLEARED`` or ``PyDict_EVENT_DEALLOCATED``, both
+   *key* and *new_value* will be ``NULL``. If *event* is ``PyDict_EVENT_ADDED``
+   or ``PyDict_EVENT_MODIFIED``, *new_value* will be the new value for *key*.
+   If *event* is ``PyDict_EVENT_DELETED``, *key* is being deleted from the
+   dictionary and *new_value* will be ``NULL``.
+
+   ``PyDict_EVENT_CLONED`` occurs when *dict* was previously empty and another
+   dict is merged into it. To maintain efficiency of this operation, per-key
+   ``PyDict_EVENT_ADDED`` events are not issued in this case; instead a
+   single ``PyDict_EVENT_CLONED`` is issued, and *key* will be the source
+   dictionary.
+
+   The callback may inspect but must not modify *dict*; doing so could have
+   unpredictable effects, including infinite recursion.
+
+   Callbacks occur before the notified modification to *dict* takes place, so
+   the prior state of *dict* can be inspected.
+
+   If the callback returns with an exception set, it must return ``-1``; this
+   exception will be printed as an unraisable exception using
+   :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.

Include/cpython/dictobject.h

@@ -83,3 +83,26 @@ typedef struct {
 
 PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *);
 PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other);
+
+/* Dictionary watchers */
+
+typedef enum {
+    PyDict_EVENT_ADDED,
+    PyDict_EVENT_MODIFIED,
+    PyDict_EVENT_DELETED,
+    PyDict_EVENT_CLONED,
+    PyDict_EVENT_CLEARED,
+    PyDict_EVENT_DEALLOCATED,
+} PyDict_WatchEvent;
+
+// Callback to be invoked when a watched dict is cleared, dealloced, or modified.
+// In clear/dealloc case, key and new_value will be NULL. Otherwise, new_value will be the
+// new value for key, NULL if key is being deleted.
+typedef int(*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value);
+
+// Register/unregister a dict-watcher callback
+PyAPI_FUNC(int) PyDict_AddWatcher(PyDict_WatchCallback callback);
+PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id);
+
+// Mark given dictionary as "watched" (callback will be called if it is modified)
+PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict);

Include/internal/pycore_dict.h

@@ -154,7 +154,32 @@ struct _dictvalues {
 
 extern uint64_t _pydict_global_version;
 
-#define DICT_NEXT_VERSION() (++_pydict_global_version)
+#define DICT_MAX_WATCHERS 8
+#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS)
+#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1)
+
+#define DICT_NEXT_VERSION() (_pydict_global_version += DICT_VERSION_INCREMENT)
+
+void
+_PyDict_SendEvent(int watcher_bits,
+                  PyDict_WatchEvent event,
+                  PyDictObject *mp,
+                  PyObject *key,
+                  PyObject *value);
+
+static inline uint64_t
+_PyDict_NotifyEvent(PyDict_WatchEvent event,
+                    PyDictObject *mp,
+                    PyObject *key,
+                    PyObject *value)
+{
+    int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK;
+    if (watcher_bits) {
+        _PyDict_SendEvent(watcher_bits, event, mp, key, value);
+        return DICT_NEXT_VERSION() | watcher_bits;
+    }
+    return DICT_NEXT_VERSION();
+}
 
 extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
 extern PyObject *_PyDict_FromItems(

Include/internal/pycore_interp.h

@@ -144,6 +144,8 @@ struct _is {
     // Initialized to _PyEval_EvalFrameDefault().
     _PyFrameEvalFunction eval_frame;
 
+    PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS];
+
     Py_ssize_t co_extra_user_count;
     freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
 

Lib/dataclasses.py

@@ -1325,6 +1325,14 @@ def _asdict_inner(obj, dict_factory):
         # generator (which is not true for namedtuples, handled
         # above).
         return type(obj)(_asdict_inner(v, dict_factory) for v in obj)
+    elif isinstance(obj, dict) and hasattr(type(obj), 'default_factory'):
+        # obj is a defaultdict, which has a different constructor from
+        # dict as it requires the default_factory as its first arg.
+        # https://bugs.python.org/issue35540
+        result = type(obj)(getattr(obj, 'default_factory'))
+        for k, v in obj.items():
+            result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory)
+        return result
     elif isinstance(obj, dict):
         return type(obj)((_asdict_inner(k, dict_factory),
                           _asdict_inner(v, dict_factory))

Lib/importlib/_bootstrap.py

@@ -136,7 +136,7 @@ def release(self):
                     self.wakeup.release()
 
     def __repr__(self):
-        return '_ModuleLock({!r}) at {}'.format(self.name, id(self))
+        return f'_ModuleLock({self.name!r}) at {id(self)}'
 
 
 class _DummyModuleLock:
@@ -157,7 +157,7 @@ def release(self):
         self.count -= 1
 
     def __repr__(self):
-        return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self))
+        return f'_DummyModuleLock({self.name!r}) at {id(self)}'
 
 
 class _ModuleLockManager:
@@ -253,7 +253,7 @@ def _requires_builtin(fxn):
     """Decorator to verify the named module is built-in."""
     def _requires_builtin_wrapper(self, fullname):
         if fullname not in sys.builtin_module_names:
-            raise ImportError('{!r} is not a built-in module'.format(fullname),
+            raise ImportError(f'{fullname!r} is not a built-in module',
                               name=fullname)
         return fxn(self, fullname)
     _wrap(_requires_builtin_wrapper, fxn)
@@ -264,7 +264,7 @@ def _requires_frozen(fxn):
     """Decorator to verify the named module is frozen."""
     def _requires_frozen_wrapper(self, fullname):
         if not _imp.is_frozen(fullname):
-            raise ImportError('{!r} is not a frozen module'.format(fullname),
+            raise ImportError(f'{fullname!r} is not a frozen module',
                               name=fullname)
         return fxn(self, fullname)
     _wrap(_requires_frozen_wrapper, fxn)
@@ -305,11 +305,11 @@ def _module_repr(module):
         filename = module.__file__
     except AttributeError:
         if loader is None:
-            return '<module {!r}>'.format(name)
+            return f'<module {name!r}>'
         else:
-            return '<module {!r} ({!r})>'.format(name, loader)
+            return f'<module {name!r} ({loader!r})>'
     else:
-        return '<module {!r} from {!r}>'.format(name, filename)
+        return f'<module {name!r} from {filename!r}>'
 
 
 class ModuleSpec:
@@ -363,14 +363,12 @@ def __init__(self, name, loader, *, origin=None, loader_state=None,
         self._cached = None
 
     def __repr__(self):
-        args = ['name={!r}'.format(self.name),
-                'loader={!r}'.format(self.loader)]
+        args = [f'name={self.name!r}', f'loader={self.loader!r}']
         if self.origin is not None:
-            args.append('origin={!r}'.format(self.origin))
+            args.append(f'origin={self.origin!r}')
         if self.submodule_search_locations is not None:
-            args.append('submodule_search_locations={}'
-                        .format(self.submodule_search_locations))
-        return '{}({})'.format(self.__class__.__name__, ', '.join(args))
+            args.append(f'submodule_search_locations={self.submodule_search_locations}')
+        return f'{self.__class__.__name__}({", ".join(args)})'
 
     def __eq__(self, other):
         smsl = self.submodule_search_locations
@@ -580,14 +578,14 @@ def _module_repr_from_spec(spec):
     name = '?' if spec.name is None else spec.name
     if spec.origin is None:
         if spec.loader is None:
-            return '<module {!r}>'.format(name)
+            return f'<module {name!r}>'
         else:
-            return '<module {!r} ({!r})>'.format(name, spec.loader)
+            return f'<module {name!r} ({spec.loader!r})>'
     else:
         if spec.has_location:
-            return '<module {!r} from {!r}>'.format(name, spec.origin)
+            return f'<module {name!r} from {spec.origin!r}>'
         else:
-            return '<module {!r} ({})>'.format(spec.name, spec.origin)
+            return f'<module {spec.name!r} ({spec.origin})>'
 
 
 # Used by importlib.reload() and _load_module_shim().
@@ -596,7 +594,7 @@ def _exec(spec, module):
     name = spec.name
     with _ModuleLockManager(name):
         if sys.modules.get(name) is not module:
-            msg = 'module {!r} not in sys.modules'.format(name)
+            msg = f'module {name!r} not in sys.modules'
             raise ImportError(msg, name=name)
         try:
             if spec.loader is None:
@@ -756,7 +754,7 @@ def find_module(cls, fullname, path=None):
     def create_module(spec):
         """Create a built-in module"""
         if spec.name not in sys.builtin_module_names:
-            raise ImportError('{!r} is not a built-in module'.format(spec.name),
+            raise ImportError(f'{spec.name!r} is not a built-in module',
                               name=spec.name)
         return _call_with_frames_removed(_imp.create_builtin, spec)
 
@@ -1012,7 +1010,7 @@ def _resolve_name(name, package, level):
     if len(bits) < level:
         raise ImportError('attempted relative import beyond top-level package')
     base = bits[0]
-    return '{}.{}'.format(base, name) if name else base
+    return f'{base}.{name}' if name else base
 
 
 def _find_spec_legacy(finder, name, path):
@@ -1075,7 +1073,7 @@ def _find_spec(name, path, target=None):
 def _sanity_check(name, package, level):
     """Verify arguments are "sane"."""
     if not isinstance(name, str):
-        raise TypeError('module name must be str, not {}'.format(type(name)))
+        raise TypeError(f'module name must be str, not {type(name)}')
     if level < 0:
         raise ValueError('level must be >= 0')
     if level > 0:
@@ -1105,13 +1103,13 @@ def _find_and_load_unlocked(name, import_):
         try:
             path = parent_module.__path__
         except AttributeError:
-            msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent)
+            msg = f'{_ERR_MSG_PREFIX} {name!r}; {parent!r} is not a package'
             raise ModuleNotFoundError(msg, name=name) from None
         parent_spec = parent_module.__spec__
         child = name.rpartition('.')[2]
     spec = _find_spec(name, path)
     if spec is None:
-        raise ModuleNotFoundError(_ERR_MSG.format(name), name=name)
+        raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name)
     else:
         if parent_spec:
             # Temporarily add child we are currently importing to parent's
@@ -1156,8 +1154,7 @@ def _find_and_load(name, import_):
         _lock_unlock_module(name)
 
     if module is None:
-        message = ('import of {} halted; '
-                   'None in sys.modules'.format(name))
+        message = f'import of {name} halted; None in sys.modules'
         raise ModuleNotFoundError(message, name=name)
 
     return module
@@ -1201,7 +1198,7 @@ def _handle_fromlist(module, fromlist, import_, *, recursive=False):
                 _handle_fromlist(module, module.__all__, import_,
                                  recursive=True)
         elif not hasattr(module, x):
-            from_name = '{}.{}'.format(module.__name__, x)
+            from_name = f'{module.__name__}.{x}'
             try:
                 _call_with_frames_removed(import_, from_name)
             except ModuleNotFoundError as exc: