Fix issue with re-entrant clearing.

Author: ZeroIntensity

Programs/_testembed.c

@@ -2878,6 +2878,63 @@ test_concurrent_finalization_stress(void)
 
 #undef NUM_THREADS
 
+/* A capsule destructor that calls Ensure/Release while the tstate is being
+ * cleared by PyThreadState_Release. */
+static void
+capsule_destructor(PyObject *capsule)
+{
+    assert(capsule != NULL);
+    PyInterpreterGuard *guard = PyCapsule_GetPointer(capsule, "x");
+    PyThreadStateToken *token = PyThreadState_Ensure(guard);
+    assert(token != NULL);
+    PyThreadState_Release(token);
+}
+
+static void
+thread_with_tstate_destructor(void *arg)
+{
+    PyInterpreterGuard *guard = (PyInterpreterGuard *)arg;
+
+    /* Triggers fresh tstate path */
+    PyThreadStateToken *token = PyThreadState_Ensure(guard);
+    assert(token != NULL);
+
+    /* Stash a capsule whose destructor will run during PyThreadState_Clear. */
+    PyObject *capsule = PyCapsule_New(guard, "x", capsule_destructor);
+    assert(capsule != NULL);
+
+    /* We need to put it somewhere it gets cleaned up at PyThreadState_Clear.
+     * tstate->dict is cleared during PyThreadState_Clear. */
+    PyObject *dict = PyThreadState_GetDict();
+    assert(dict != NULL);
+    int res = PyDict_SetItemString(dict, "key", capsule);
+    assert(res == 0);
+    Py_DECREF(capsule);
+
+    PyThreadState_Release(token);
+    PyInterpreterGuard_Close(guard);
+}
+
+static int
+test_thread_state_release_with_destructor(void)
+{
+    _testembed_initialize();
+    PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent();
+    assert(guard != NULL);
+
+    PyThread_handle_t handle;
+    PyThread_ident_t ident;
+    if (PyThread_start_joinable_thread(thread_with_tstate_destructor,
+                                       guard, &ident, &handle) < 0) {
+        PyInterpreterGuard_Close(guard);
+        return -1;
+    }
+
+    PyThread_join_thread(handle);
+    Py_Finalize();
+    return 0;
+}
+
 /* *********************************************************
  * List of test cases and the function that implements it.
  *
@@ -2976,6 +3033,7 @@ static struct TestCase TestCases[] = {
     {"test_main_interpreter_view", test_main_interpreter_view},
     {"test_thread_state_ensure_from_view", test_thread_state_ensure_from_view},
     {"test_concurrent_finalization_stress", test_concurrent_finalization_stress},
+    {"test_thread_state_release_with_destructor", test_thread_state_release_with_destructor},
     {NULL, NULL}
 };
 

Python/pystate.c

@@ -3567,7 +3567,9 @@ PyThreadState_Release(PyThreadStateToken *token)
     PyInterpreterGuard *owned_guard = tstate->ensure.owned_guard;
     assert(tstate->ensure.delete_on_release == 1 || tstate->ensure.delete_on_release == 0);
     if (tstate->ensure.delete_on_release) {
+        ++tstate->ensure.counter;
         PyThreadState_Clear(tstate);
+        --tstate->ensure.counter;
     }
 
     PyThreadState *check_tstate = PyThreadState_Swap(to_restore);