gh-101892: Fix SystemError when a callable iterator call exhausts the iterator

Lib/test/test_iter.py

@@ -267,6 +267,23 @@ def spam(state=[0]):
             return i
         self.check_iterator(iter(spam, 20), list(range(10)), pickle=False)
 
+    # Test two-argument iter() with function that empties its associated
+    # iterator via list() (or anything else but next()) but returns
+    # a non-sentinel value thus claiming that the iterator can yield more.
+    def test_iter_function_concealing_reentrant_exhaustion(self):
+        HAS_MORE = 1
+        NO_MORE = 2
+        def spam():
+            if spam.need_reentrance:
+                spam.need_reentrance = False
+                list(spam.iterator) # Implicitly call ourselves
+                return HAS_MORE
+            return NO_MORE
+        spam.need_reentrance = True
+        spam.iterator = iter(spam, NO_MORE)
+        with self.assertRaises(StopIteration):
+            next(spam.iterator)
+
     # Test exception propagation through function iterator
     def test_exception_function(self):
         def spam(state=[0]):

Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst

@@ -0,0 +1 @@
+Callable iterators no longer raise :class:`SystemError` by exhausting themselves.

Objects/iterobject.c

@@ -214,15 +214,14 @@ calliter_iternext(calliterobject *it)
     }
 
     result = _PyObject_CallNoArgs(it->it_callable);
-    if (result != NULL) {
+    if (result != NULL && it->it_sentinel != NULL){
         int ok;
 
         ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ);
         if (ok == 0) {
             return result; /* Common case, fast path */
         }
 
-        Py_DECREF(result);
         if (ok > 0) {
             Py_CLEAR(it->it_callable);
             Py_CLEAR(it->it_sentinel);
@@ -233,6 +232,7 @@ calliter_iternext(calliterobject *it)
         Py_CLEAR(it->it_callable);
         Py_CLEAR(it->it_sentinel);
     }
+    Py_XDECREF(result);
     return NULL;
 }