From a8dadd39f1f9e87adedc12a2f33bbff80ebdfca5 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 21:43:28 -0500 Subject: [PATCH 01/31] Simpler initial implementation. Thanks for the advice, Yury! --- Include/internal/pycore_pystate.h | 4 +- Include/internal/pycore_tstate.h | 16 + Lib/concurrent/interpreters/__init__.py | 18 +- Modules/_interpretersmodule.c | 870 +++++++++++++++++++++++- Modules/clinic/_interpretersmodule.c.h | 14 +- Python/pystate.c | 36 + 6 files changed, 951 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 189a8dde9f09ed..cd06f8e3589b95 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -135,13 +135,13 @@ _PyThreadState_IsAttached(PyThreadState *tstate) // // High-level code should generally call PyEval_RestoreThread() instead, which // calls this function. -extern void _PyThreadState_Attach(PyThreadState *tstate); +PyAPI_FUNC(void) _PyThreadState_Attach(PyThreadState *tstate); // Detaches the current thread from the interpreter. // // High-level code should generally call PyEval_SaveThread() instead, which // calls this function. -extern void _PyThreadState_Detach(PyThreadState *tstate); +PyAPI_FUNC(void) _PyThreadState_Detach(PyThreadState *tstate); // Detaches the current thread to the "suspended" state if a stop-the-world // pause is in progress. diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 50048801b2e4ee..a9ecb487c86a98 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -54,6 +54,20 @@ typedef struct _PyJitTracerState { } _PyJitTracerState; #endif +typedef void (*_PyThreadState_ClearCallback)(PyThreadState *, void *); + +struct _PyThreadState_ClearNode { + struct _PyThreadState_ClearNode *next; + _PyThreadState_ClearCallback callback; + void *arg; +}; + +// Export for '_interpreters' shared extension. +PyAPI_FUNC(int) +_PyThreadState_AddClearCallback(PyThreadState *tstate, + _PyThreadState_ClearCallback callback, + void *arg); + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -121,6 +135,8 @@ typedef struct _PyThreadStateImpl { #if _Py_TIER2 _PyJitTracerState jit_tracer_state; #endif + + struct _PyThreadState_ClearNode *clear_callbacks; } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index ea4147ee9a25da..87e56e621ee377 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -7,7 +7,7 @@ # aliases: from _interpreters import ( InterpreterError, InterpreterNotFoundError, NotShareableError, - is_shareable, + is_shareable, SharedObjectProxy ) from ._queues import ( create as create_queue, @@ -245,3 +245,19 @@ def call_in_thread(self, callable, /, *args, **kwargs): t = threading.Thread(target=self._call, args=(callable, args, kwargs)) t.start() return t + + +def _can_natively_share(obj): + if isinstance(obj, SharedObjectProxy): + return False + + return _interpreters.is_shareable(obj) + + +def share(obj): + """Wrap the object in a shareable object proxy that allows cross-interpreter + access. + """ + if _can_natively_share(obj): + return obj + return _interpreters.share(obj) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 2aee8b07891c91..402902a4613b17 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -6,15 +6,19 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _PyEval_GetANext() #include "pycore_code.h" // _PyCode_HAS_EXECUTORS() #include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_interp.h" // _PyInterpreterState_IDIncref() +#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_IsRunningMain() +#include "pycore_runtime_structs.h" +#include "pycore_pyatomic_ft_wrappers.h" #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -317,8 +321,6 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) return 0; } - - /* module state *************************************************************/ typedef struct { @@ -326,6 +328,9 @@ typedef struct { /* heap types */ PyTypeObject *XIBufferViewType; + + /* Linked list of shared object proxies available for use. */ + PyObject *available_proxies; } module_state; static inline module_state * @@ -370,6 +375,820 @@ clear_module_state(module_state *state) return 0; } +/* Shared object proxies. */ + +typedef struct _Py_shared_object_proxy { + PyObject_HEAD + PyInterpreterState *interp; + PyObject *object; +#ifdef Py_GIL_DISABLED + struct { + _Py_hashtable_t *table; + PyMutex mutex; + } thread_states; +#else + _Py_hashtable_t *thread_states; +#endif +} SharedObjectProxy; + +#define SharedObjectProxy_CAST(op) ((SharedObjectProxy *)op) +#define SharedObjectProxy_OBJECT(op) FT_ATOMIC_LOAD_PTR_RELAXED(SharedObjectProxy_CAST(op)->object) +#define SharedObjectProxy_CheckExact(op) (Py_TYPE(_PyObject_CAST(op)) == &SharedObjectProxy_Type) + +#ifdef Py_GIL_DISABLED +#define SharedObjectProxy_TSTATES(op) ((op)->thread_states.table) +#define SharedObjectProxy_LOCK_TSTATES(op) PyMutex_Lock(&(op)->thread_states.mutex) +#define SharedObjectProxy_UNLOCK_TSTATES(op) PyMutex_Unlock(&(op)->thread_states.mutex) +#else +#define SharedObjectProxy_TSTATES(op) ((op)->thread_states) +#define SharedObjectProxy_LOCK_TSTATES(op) +#define SharedObjectProxy_UNLOCK_TSTATES(op) +#endif + +static int +sharedobjectproxy_clear(PyObject *op) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + // Don't clear from another interpreter + if (self->interp != _PyInterpreterState_GET()) { + return 0; + } + + Py_CLEAR(self->object); + return 0; +} + +static int +sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + // Don't traverse from another interpreter + if (self->interp != _PyInterpreterState_GET()) { + return 0; + } + + Py_VISIT(self->object); + return 0; +} + +static void +sharedobjectproxy_dealloc(PyObject *op) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + assert(_PyInterpreterState_GET() == self->interp); + PyTypeObject *tp = Py_TYPE(self); + (void)sharedobjectproxy_clear(op); + PyObject_GC_UnTrack(self); + tp->tp_free(self); +} + +static PyObject * +sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(type->tp_alloc(type, 0)); + if (self == NULL) { + return NULL; + } + + _Py_hashtable_allocator_t alloc = { + .malloc = PyMem_RawMalloc, + .free = PyMem_RawFree, + }; + _Py_hashtable_t *tstates = _Py_hashtable_new_full(_Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, + NULL, + &alloc); + if (tstates == NULL) { + Py_DECREF(self); + return PyErr_NoMemory(); + } + + SharedObjectProxy_TSTATES(self) = tstates; + self->object = Py_None; + self->interp = _PyInterpreterState_GET(); + + return (PyObject *)self; +} + +typedef struct { + PyThreadState *to_restore; + PyThreadState *for_call; +} _PyXI_proxy_state; + +static void +_sharedobjectproxy_destroy_tstate(PyThreadState *tstate, void *arg) +{ + assert(tstate != NULL); + assert(arg != NULL); + SharedObjectProxy *self = SharedObjectProxy_CAST(arg); + SharedObjectProxy_LOCK_TSTATES(self); + _Py_hashtable_t *table = SharedObjectProxy_TSTATES(self); + PyThreadState *to_destroy = _Py_hashtable_steal(table, tstate); + assert(to_destroy != NULL); + PyThreadState_Swap(to_destroy); + PyThreadState_Clear(to_destroy); + PyThreadState_DeleteCurrent(); + _PyThreadState_Attach(tstate); + SharedObjectProxy_UNLOCK_TSTATES(self); +} + +static int +_sharedobjectproxy_enter_lock_held(SharedObjectProxy *self, _PyXI_proxy_state *state, + PyThreadState *tstate) +{ + _Py_hashtable_t *table = SharedObjectProxy_TSTATES(self); + PyThreadState *cached_tstate = _Py_hashtable_get(table, tstate); + + if (cached_tstate != NULL) { + _PyThreadState_Detach(tstate); + state->for_call = cached_tstate; + return 0; + } + + PyThreadState *for_call = _PyThreadState_NewBound(self->interp, + _PyThreadState_WHENCE_EXEC); + state->for_call = for_call; + if (for_call == NULL) { + PyErr_NoMemory(); + return -1; + } + if (_Py_hashtable_set(table, tstate, for_call) < 0) { + PyErr_NoMemory(); + return -1; + } + if (_PyThreadState_AddClearCallback(tstate, + _sharedobjectproxy_destroy_tstate, + self) < 0) { + + PyErr_NoMemory(); + return -1; + } + _PyThreadState_Detach(tstate); + return 0; +} + +static int +_sharedobjectproxy_enter(SharedObjectProxy *self, _PyXI_proxy_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + assert(self != NULL); + assert(tstate != NULL); + if (tstate->interp == self->interp) { + // No need to switch; already in the correct interpreter + state->to_restore = NULL; + state->for_call = NULL; + return 0; + } + state->to_restore = tstate; + SharedObjectProxy_LOCK_TSTATES(self); + int res = _sharedobjectproxy_enter_lock_held(self, state, tstate); + SharedObjectProxy_UNLOCK_TSTATES(self); + if (res < 0) { + return -1; + } + _PyThreadState_Attach(state->for_call); + assert(_PyInterpreterState_GET() == self->interp); + return 0; +} + +static int +_sharedobjectproxy_exit(SharedObjectProxy *self, _PyXI_proxy_state *state) +{ + assert(_PyInterpreterState_GET() == self->interp); + if (state->to_restore == NULL) { + // Nothing to do. We were already in the correct interpreter. + return PyErr_Occurred() == NULL ? 0 : -1; + } + + PyThreadState *tstate = state->for_call; + int should_throw = 0; + if (_PyErr_Occurred(tstate)) { + // TODO: Serialize and transfer the exception to the calling + // interpreter. + PyErr_FormatUnraisable("Exception occured in interpreter"); + should_throw = 1; + } + + assert(state->for_call == _PyThreadState_GET()); + PyThreadState_Swap(state->to_restore); + + if (should_throw) { + _PyErr_SetString(state->to_restore, PyExc_RuntimeError, "exception in interpreter"); + return -1; + } + return 0; +} + +PyObject * +_sharedobjectproxy_create(PyObject *object); + +typedef struct { + _PyXIData_t *xidata; + PyObject *object; +} _PyXI_proxy_share; + +static PyTypeObject SharedObjectProxy_Type; + +/* Use this in the calling interpreter. */ +static int +_sharedobjectproxy_init_share(_PyXI_proxy_share *share, + SharedObjectProxy *self, PyObject *op) +{ + assert(op != NULL); + assert(share != NULL); + if (Py_TYPE(op) == &SharedObjectProxy_Type) { + // Already an object proxy; nothing to do + share->object = op; + share->xidata = NULL; + return 0; + } + + _PyXIData_t *xidata = _PyXIData_New(); + if (xidata == NULL) { + return -1; + } + + if (_PyObject_GetXIData(_PyThreadState_GET(), op, + _PyXIDATA_XIDATA_ONLY, xidata) < 0) { + PyErr_Clear(); + share->object = _sharedobjectproxy_create(op); + share->xidata = NULL; + _PyXIData_Free(xidata); + if (share->object == NULL) { + return -1; + } + } else { + share->object = NULL; + share->xidata = xidata; + } + + return 0; +} + +/* Use this in the switched interpreter. */ +static PyObject * +_sharedobjectproxy_as_shared(_PyXI_proxy_share *share) +{ + assert(share != NULL); + _PyXIData_t *xidata = share->xidata; + if (xidata == NULL) { + // Not shareable; use the object proxy + return share->object; + } else { + PyObject *result = _PyXIData_NewObject(xidata); + assert(share->object == NULL); + return result; + } +} + +static void +_sharedobjectproxy_finish_share(_PyXI_proxy_share *share) +{ + if (share->xidata != NULL) { + _PyXIData_Free(share->xidata); + } +#ifdef Py_DEBUG + share->xidata = NULL; + if (share->object != NULL) { + share->object = NULL; + } +#endif +} + +static PyObject * +_sharedobjectproxy_wrap_result(SharedObjectProxy *self, PyObject *result, + _PyXI_proxy_state *state) +{ + if (result == NULL) { + (void)_sharedobjectproxy_exit(self, state); + return NULL; + } + + assert(result != NULL); + _PyXI_proxy_share shared_result; + if (_sharedobjectproxy_init_share(&shared_result, self, result) < 0) { + Py_DECREF(result); + (void)_sharedobjectproxy_exit(self, state); + return NULL; + } + + Py_DECREF(result); + if (_sharedobjectproxy_exit(self, state) < 0) { + _sharedobjectproxy_finish_share(&shared_result); + return NULL; + } + + PyObject *ret = _sharedobjectproxy_as_shared(&shared_result); + //_sharedobjectproxy_finish_share(&shared_result); + return ret; +} + +static PyObject * +sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + Py_ssize_t size = PyTuple_Size(args); + _PyXI_proxy_share *shared_args_state = PyMem_RawMalloc(size * sizeof(_PyXI_proxy_share)); + + for (Py_ssize_t i = 0; i < size; ++i) { + PyObject *arg = PyTuple_GetItem(args, i); + if (arg == NULL) { + PyMem_RawFree(shared_args_state); + return NULL; + } + if (_sharedobjectproxy_init_share(&shared_args_state[i], self, arg) < 0) { + // TODO: Fix leaks from prior iterations + PyMem_RawFree(shared_args_state); + return NULL; + } + } + + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(self, &state) < 0) { + PyMem_RawFree(shared_args_state); + return NULL; + } + PyObject *shared_args = PyTuple_New(size); + if (shared_args == NULL) { + (void)_sharedobjectproxy_exit(self, &state); + PyMem_RawFree(shared_args_state); + return NULL; + } + + for (Py_ssize_t i = 0; i < size; ++i) { + PyObject *shared = _sharedobjectproxy_as_shared(&shared_args_state[i]); + if (shared == NULL) { + (void)_sharedobjectproxy_exit(self, &state); + PyMem_RawFree(shared_args_state); + return NULL; + } + PyTuple_SET_ITEM(shared_args, i, shared); + } + + // kwargs aren't supported yet + PyObject *res = PyObject_Call(SharedObjectProxy_OBJECT(self), + shared_args, NULL); + Py_DECREF(shared_args); + + return _sharedobjectproxy_wrap_result(self, res, &state); +} + +static PyObject * +_sharedobjectproxy_no_arg(PyObject *op, unaryfunc call) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(self, &state) < 0) { + return NULL; + } + + PyObject *result = call(SharedObjectProxy_OBJECT(op)); + return _sharedobjectproxy_wrap_result(self, result, &state); +} + +static PyObject * +_sharedobjectproxy_single_share_common(SharedObjectProxy *self, PyObject *to_share, + _PyXI_proxy_state *state, + _PyXI_proxy_share *shared_arg) +{ + if (_sharedobjectproxy_init_share(shared_arg, self, to_share) < 0) { + return NULL; + } + if (_sharedobjectproxy_enter(self, state) < 0) { + _sharedobjectproxy_finish_share(shared_arg); + return NULL; + } + PyObject *shared_obj = _sharedobjectproxy_as_shared(shared_arg); + if (shared_obj == NULL) { + (void)_sharedobjectproxy_exit(self, state); + _sharedobjectproxy_finish_share(shared_arg); + return NULL; + } + return shared_obj; +} + +static PyObject * +_sharedobjectproxy_single_share(PyObject *op, PyObject *other, binaryfunc call) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + _PyXI_proxy_share shared_arg; + PyObject *shared_obj = _sharedobjectproxy_single_share_common(self, other, + &state, &shared_arg); + if (shared_obj == NULL) { + return NULL; + } + PyObject *result = call(SharedObjectProxy_OBJECT(op), shared_obj); + Py_DECREF(shared_obj); + PyObject *ret = _sharedobjectproxy_wrap_result(self, result, &state); + _sharedobjectproxy_finish_share(&shared_arg); + return ret; +} + +static int +_sharedobjectproxy_double_share_common(SharedObjectProxy *self, + _PyXI_proxy_state *state, + _PyXI_proxy_share *shared_first, + PyObject *first, + PyObject **first_ptr, + _PyXI_proxy_share *shared_second, + PyObject *second, + PyObject **second_ptr) +{ + if (_sharedobjectproxy_init_share(shared_first, self, first) < 0) { + return -1; + } + if (_sharedobjectproxy_init_share(shared_second, self, second) < 0) { + return -1; + } + if (_sharedobjectproxy_enter(self, state) < 0) { + _sharedobjectproxy_finish_share(shared_first); + _sharedobjectproxy_finish_share(shared_second); + return -1; + } + PyObject *first_obj = _sharedobjectproxy_as_shared(shared_first); + if (first_obj == NULL) { + (void)_sharedobjectproxy_exit(self, state); + _sharedobjectproxy_finish_share(shared_first); + _sharedobjectproxy_finish_share(shared_second); + return -1; + } + PyObject *second_obj = _sharedobjectproxy_as_shared(shared_second); + if (second_obj == NULL) { + Py_DECREF(first_obj); + (void)_sharedobjectproxy_exit(self, state); + _sharedobjectproxy_finish_share(shared_first); + _sharedobjectproxy_finish_share(shared_second); + return -1; + } + + *first_ptr = first_obj; + *second_ptr = second_obj; + return 0; +} + +static PyObject * +_sharedobjectproxy_double_share(PyObject *op, PyObject *first, + PyObject *second, ternaryfunc call) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + _PyXI_proxy_share shared_first; + _PyXI_proxy_share shared_second; + PyObject *first_obj; + PyObject *second_obj; + if (_sharedobjectproxy_double_share_common(self, &state, &shared_first, + first, &first_obj, &shared_second, + second, &second_obj) < 0) { + return NULL; + } + PyObject *result = call(SharedObjectProxy_OBJECT(op), first_obj, second_obj); + Py_DECREF(first_obj); + Py_DECREF(second_obj); + PyObject *ret = _sharedobjectproxy_wrap_result(self, result, &state); + _sharedobjectproxy_finish_share(&shared_first); + _sharedobjectproxy_finish_share(&shared_second); + return ret; +} + +static int +_sharedobjectproxy_double_share_int(PyObject *op, PyObject *first, + PyObject *second, objobjargproc call) +{ + + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + _PyXI_proxy_share shared_first; + _PyXI_proxy_share shared_second; + PyObject *first_obj; + PyObject *second_obj; + if (_sharedobjectproxy_double_share_common(self, &state, &shared_first, + first, &first_obj, &shared_second, + second, &second_obj) < 0) { + return -1; + } + int result = call(SharedObjectProxy_OBJECT(op), first_obj, second_obj); + Py_DECREF(first_obj); + Py_DECREF(second_obj); + if (_sharedobjectproxy_exit(self, &state) < 0) { + _sharedobjectproxy_finish_share(&shared_first); + _sharedobjectproxy_finish_share(&shared_second); + return -1; + } + _sharedobjectproxy_finish_share(&shared_first); + _sharedobjectproxy_finish_share(&shared_second); + return result; +} + + +static PyObject * +_sharedobjectproxy_ssize_arg(PyObject *op, Py_ssize_t count, ssizeargfunc call) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(self, &state) < 0) { + return NULL; + } + PyObject *result = call(SharedObjectProxy_OBJECT(op), count); + return _sharedobjectproxy_wrap_result(self, result, &state); +} + +static Py_ssize_t +_sharedobjectproxy_ssize_result(PyObject *op, lenfunc call) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(self, &state) < 0) { + return -1; + } + Py_ssize_t result = call(SharedObjectProxy_OBJECT(op)); + if (_sharedobjectproxy_exit(self, &state) < 0) { + return -1; + } + + return result; +} + + +#define _SharedObjectProxy_ONE_ARG(name, func) \ +static PyObject * \ +sharedobjectproxy_ ##name(PyObject *op, PyObject *other) \ +{ \ + return _sharedobjectproxy_single_share(op, other, func);\ +} \ + +#define _SharedObjectProxy_TWO_ARG(name, func) \ +static PyObject * \ +sharedobjectproxy_ ##name(PyObject *op, PyObject *first, PyObject *second) \ +{ \ + return _sharedobjectproxy_double_share(op, first, second, func); \ +} \ + +#define _SharedObjectProxy_TWO_ARG_INT(name, func) \ +static int \ +sharedobjectproxy_ ##name(PyObject *op, PyObject *first, PyObject *second) \ +{ \ + return _sharedobjectproxy_double_share_int(op, first, second, func); \ +} \ + +#define _SharedObjectProxy_NO_ARG(name, func) \ +static PyObject * \ +sharedobjectproxy_ ##name(PyObject *op) \ +{ \ + return _sharedobjectproxy_no_arg(op, func); \ +} \ + +#define _SharedObjectProxy_SSIZE_ARG(name, func) \ +static PyObject * \ +sharedobjectproxy_ ##name(PyObject *op, Py_ssize_t count) \ +{ \ + return _sharedobjectproxy_ssize_arg(op, count, func); \ +} + +#define _SharedObjectProxy_SSIZE_RETURN(name, func) \ +static Py_ssize_t \ +sharedobjectproxy_ ##name(PyObject *op) \ +{ \ + return _sharedobjectproxy_ssize_result(op, func); \ +} + +#define _SharedObjectProxy_FIELD(name) .name = sharedobjectproxy_ ##name + +_SharedObjectProxy_NO_ARG(tp_iter, PyObject_GetIter); +_SharedObjectProxy_NO_ARG(tp_iternext, PyIter_Next); +_SharedObjectProxy_NO_ARG(tp_str, PyObject_Str); +_SharedObjectProxy_NO_ARG(tp_repr, PyObject_Repr); +_SharedObjectProxy_ONE_ARG(tp_getattro, PyObject_GetAttr); +_SharedObjectProxy_TWO_ARG_INT(tp_setattro, PyObject_SetAttr); + +static Py_hash_t +sharedobjectproxy_tp_hash(PyObject *op) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(self, &state) < 0) { + return -1; + } + + Py_hash_t result = PyObject_Hash(SharedObjectProxy_OBJECT(op)); + + if (_sharedobjectproxy_exit(self, &state) < 0) { + return -1; + } + + return result; +} + +_SharedObjectProxy_ONE_ARG(nb_add, PyNumber_Add); +_SharedObjectProxy_ONE_ARG(nb_subtract, PyNumber_Subtract); +_SharedObjectProxy_ONE_ARG(nb_multiply, PyNumber_Multiply); +_SharedObjectProxy_ONE_ARG(nb_remainder, PyNumber_Remainder); +_SharedObjectProxy_ONE_ARG(nb_divmod, PyNumber_Divmod); +_SharedObjectProxy_TWO_ARG(nb_power, PyNumber_Power); +_SharedObjectProxy_NO_ARG(nb_negative, PyNumber_Negative); +_SharedObjectProxy_NO_ARG(nb_positive, PyNumber_Positive); +_SharedObjectProxy_NO_ARG(nb_absolute, PyNumber_Absolute); +_SharedObjectProxy_NO_ARG(nb_invert, PyNumber_Invert); +_SharedObjectProxy_ONE_ARG(nb_lshift, PyNumber_Lshift); +_SharedObjectProxy_ONE_ARG(nb_rshift, PyNumber_Rshift); +_SharedObjectProxy_ONE_ARG(nb_and, PyNumber_And); +_SharedObjectProxy_ONE_ARG(nb_xor, PyNumber_Xor); +_SharedObjectProxy_ONE_ARG(nb_or, PyNumber_Or); +_SharedObjectProxy_NO_ARG(nb_int, PyNumber_Long); +_SharedObjectProxy_NO_ARG(nb_float, PyNumber_Float); +_SharedObjectProxy_ONE_ARG(nb_inplace_add, PyNumber_InPlaceAdd); +_SharedObjectProxy_ONE_ARG(nb_inplace_subtract, PyNumber_InPlaceSubtract); +_SharedObjectProxy_ONE_ARG(nb_inplace_multiply, PyNumber_InPlaceMultiply); +_SharedObjectProxy_ONE_ARG(nb_inplace_remainder, PyNumber_InPlaceRemainder); +_SharedObjectProxy_TWO_ARG(nb_inplace_power, PyNumber_InPlacePower); +_SharedObjectProxy_ONE_ARG(nb_inplace_lshift, PyNumber_InPlaceLshift); +_SharedObjectProxy_ONE_ARG(nb_inplace_rshift, PyNumber_InPlaceRshift); +_SharedObjectProxy_ONE_ARG(nb_inplace_and, PyNumber_InPlaceAnd); +_SharedObjectProxy_ONE_ARG(nb_inplace_xor, PyNumber_InPlaceXor); +_SharedObjectProxy_ONE_ARG(nb_inplace_or, PyNumber_InPlaceOr); +_SharedObjectProxy_ONE_ARG(nb_floor_divide, PyNumber_FloorDivide); +_SharedObjectProxy_ONE_ARG(nb_true_divide, PyNumber_TrueDivide); +_SharedObjectProxy_ONE_ARG(nb_inplace_floor_divide, PyNumber_InPlaceFloorDivide); +_SharedObjectProxy_ONE_ARG(nb_inplace_true_divide, PyNumber_InPlaceTrueDivide); +_SharedObjectProxy_NO_ARG(nb_index, PyNumber_Index); +_SharedObjectProxy_ONE_ARG(nb_matrix_multiply, PyNumber_MatrixMultiply); +_SharedObjectProxy_ONE_ARG(nb_inplace_matrix_multiply, PyNumber_InPlaceMatrixMultiply); + +static PyNumberMethods SharedObjectProxy_NumberMethods = { + _SharedObjectProxy_FIELD(nb_add), + _SharedObjectProxy_FIELD(nb_subtract), + _SharedObjectProxy_FIELD(nb_multiply), + _SharedObjectProxy_FIELD(nb_remainder), + _SharedObjectProxy_FIELD(nb_power), + _SharedObjectProxy_FIELD(nb_divmod), + _SharedObjectProxy_FIELD(nb_negative), + _SharedObjectProxy_FIELD(nb_positive), + _SharedObjectProxy_FIELD(nb_absolute), + _SharedObjectProxy_FIELD(nb_invert), + _SharedObjectProxy_FIELD(nb_lshift), + _SharedObjectProxy_FIELD(nb_rshift), + _SharedObjectProxy_FIELD(nb_and), + _SharedObjectProxy_FIELD(nb_xor), + _SharedObjectProxy_FIELD(nb_or), + _SharedObjectProxy_FIELD(nb_int), + _SharedObjectProxy_FIELD(nb_float), + _SharedObjectProxy_FIELD(nb_inplace_add), + _SharedObjectProxy_FIELD(nb_inplace_subtract), + _SharedObjectProxy_FIELD(nb_inplace_multiply), + _SharedObjectProxy_FIELD(nb_inplace_remainder), + _SharedObjectProxy_FIELD(nb_inplace_power), + _SharedObjectProxy_FIELD(nb_inplace_lshift), + _SharedObjectProxy_FIELD(nb_inplace_rshift), + _SharedObjectProxy_FIELD(nb_inplace_and), + _SharedObjectProxy_FIELD(nb_inplace_xor), + _SharedObjectProxy_FIELD(nb_inplace_or), + _SharedObjectProxy_FIELD(nb_floor_divide), + _SharedObjectProxy_FIELD(nb_true_divide), + _SharedObjectProxy_FIELD(nb_inplace_floor_divide), + _SharedObjectProxy_FIELD(nb_inplace_true_divide), + _SharedObjectProxy_FIELD(nb_index), + _SharedObjectProxy_FIELD(nb_matrix_multiply), + _SharedObjectProxy_FIELD(nb_inplace_matrix_multiply) +}; + +_SharedObjectProxy_NO_ARG(am_await, _PyCoro_GetAwaitableIter); +_SharedObjectProxy_NO_ARG(am_aiter, PyObject_GetAIter); +_SharedObjectProxy_NO_ARG(am_anext, _PyEval_GetANext); + +static PyAsyncMethods SharedObjectProxy_AsyncMethods = { + _SharedObjectProxy_FIELD(am_await), + _SharedObjectProxy_FIELD(am_aiter), + _SharedObjectProxy_FIELD(am_anext), +}; + +_SharedObjectProxy_SSIZE_RETURN(sq_length, PySequence_Size); +_SharedObjectProxy_ONE_ARG(sq_concat, PySequence_Concat); +_SharedObjectProxy_SSIZE_ARG(sq_repeat, PySequence_Repeat) +_SharedObjectProxy_SSIZE_ARG(sq_item, PySequence_GetItem); +_SharedObjectProxy_ONE_ARG(sq_inplace_concat, PySequence_InPlaceConcat); +_SharedObjectProxy_SSIZE_ARG(sq_inplace_repeat, PySequence_InPlaceRepeat); + +static int +sharedobjectproxy_sq_ass_item(PyObject *op, Py_ssize_t index, PyObject *item) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + _PyXI_proxy_share shared_arg; + PyObject *shared_obj = _sharedobjectproxy_single_share_common(self, item, + &state, &shared_arg); + if (shared_obj == NULL) { + Py_DECREF(shared_obj); + (void)_sharedobjectproxy_exit(self, &state); + _sharedobjectproxy_finish_share(&shared_arg); + return -1; + } + int result = PySequence_SetItem(SharedObjectProxy_OBJECT(op), index, shared_obj); + Py_DECREF(shared_obj); + if (_sharedobjectproxy_exit(self, &state) < 0) { + _sharedobjectproxy_finish_share(&shared_arg); + return -1; + } + _sharedobjectproxy_finish_share(&shared_arg); + return result; +} + +static int +sharedobjectproxy_sq_contains(PyObject *op, PyObject *item) +{ + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_state state; + _PyXI_proxy_share shared_arg; + PyObject *shared_obj = _sharedobjectproxy_single_share_common(self, item, + &state, &shared_arg); + if (shared_obj == NULL) { + Py_DECREF(shared_obj); + (void)_sharedobjectproxy_exit(self, &state); + _sharedobjectproxy_finish_share(&shared_arg); + return -1; + } + int result = PySequence_Contains(SharedObjectProxy_OBJECT(op), shared_obj); + Py_DECREF(shared_obj); + if (_sharedobjectproxy_exit(self, &state) < 0) { + _sharedobjectproxy_finish_share(&shared_arg); + return -1; + } + _sharedobjectproxy_finish_share(&shared_arg); + return result; +} + +static PySequenceMethods SharedObjectProxy_SequenceMethods = { + _SharedObjectProxy_FIELD(sq_concat), + _SharedObjectProxy_FIELD(sq_length), + _SharedObjectProxy_FIELD(sq_repeat), + _SharedObjectProxy_FIELD(sq_item), + _SharedObjectProxy_FIELD(sq_inplace_concat), + _SharedObjectProxy_FIELD(sq_inplace_repeat), + _SharedObjectProxy_FIELD(sq_ass_item), + _SharedObjectProxy_FIELD(sq_contains) +}; + + +_SharedObjectProxy_SSIZE_RETURN(mp_length, PyMapping_Length); +_SharedObjectProxy_ONE_ARG(mp_subscript, PyObject_GetItem); +_SharedObjectProxy_TWO_ARG_INT(mp_ass_subscript, PyObject_SetItem); + +static PyMappingMethods SharedObjectProxy_MappingMethods = { + _SharedObjectProxy_FIELD(mp_length), + _SharedObjectProxy_FIELD(mp_subscript), + _SharedObjectProxy_FIELD(mp_ass_subscript) +}; + +/* This has to be a static type as it can be referenced from any interpreter + * through a Py_TYPE() on a proxy instance. */ +static PyTypeObject SharedObjectProxy_Type = { + .tp_name = MODULE_NAME_STR ".SharedObjectProxy", + .tp_basicsize = sizeof(SharedObjectProxy), + .tp_flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .tp_new = sharedobjectproxy_new, + .tp_traverse = sharedobjectproxy_traverse, + .tp_clear = sharedobjectproxy_clear, + .tp_dealloc = sharedobjectproxy_dealloc, + _SharedObjectProxy_FIELD(tp_getattro), + _SharedObjectProxy_FIELD(tp_setattro), + _SharedObjectProxy_FIELD(tp_call), + _SharedObjectProxy_FIELD(tp_repr), + _SharedObjectProxy_FIELD(tp_str), + _SharedObjectProxy_FIELD(tp_iter), + _SharedObjectProxy_FIELD(tp_iternext), + _SharedObjectProxy_FIELD(tp_hash), + .tp_as_number = &SharedObjectProxy_NumberMethods, + .tp_as_async = &SharedObjectProxy_AsyncMethods, + .tp_as_sequence = &SharedObjectProxy_SequenceMethods, + .tp_as_mapping = &SharedObjectProxy_MappingMethods +}; + +static PyObject * +sharedobjectproxy_xid(_PyXIData_t *data) +{ + return _sharedobjectproxy_create(data->obj); +} + +static int +sharedobjectproxy_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) +{ + _PyXIData_Init(data, tstate->interp, NULL, obj, sharedobjectproxy_xid); + return 0; +} + +static int +register_sharedobjectproxy(PyObject *mod) +{ + if (PyModule_AddType(mod, &SharedObjectProxy_Type) < 0) { + return -1; + } + + if (ensure_xid_class(&SharedObjectProxy_Type, GETDATA(sharedobjectproxy_shared)) < 0) { + return -1; + } + + return 0; +} static PyTypeObject * _get_current_xibufferview_type(void) @@ -381,6 +1200,23 @@ _get_current_xibufferview_type(void) return state->XIBufferViewType; } +PyObject * +_sharedobjectproxy_create(PyObject *object) +{ + assert(object != NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp != NULL); + + SharedObjectProxy *proxy = SharedObjectProxy_CAST(sharedobjectproxy_new(&SharedObjectProxy_Type, + NULL, NULL)); + if (proxy == NULL) { + return NULL; + } + + proxy->object = Py_NewRef(object); + proxy->interp = _PyInterpreterState_GET(); + return (PyObject *)proxy; +} /* interpreter-specific code ************************************************/ @@ -1553,6 +2389,30 @@ _interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) return captured; } +/*[clinic input] +_interpreters.share + op: object, + / + + +Wrap an object in a shareable proxy that allows cross-interpreter access. + +The proxy will be assigned a context and may have its references cleared by +_interpreters.close_proxy(). +[clinic start generated code]*/ + +static PyObject * +_interpreters_share(PyObject *module, PyObject *op) +/*[clinic end generated code: output=e2ce861ae3b58508 input=d333c93f128faf93]*/ +{ + if (Py_IsNone(op)) { + PyErr_SetString(PyExc_ValueError, + "None is a reserved value for dead object proxies, and " + "does not need to be shared"); + return NULL; + } + return _sharedobjectproxy_create(op); +} static PyMethodDef module_functions[] = { {"new_config", _PyCFunction_CAST(interp_new_config), @@ -1578,9 +2438,9 @@ static PyMethodDef module_functions[] = { _INTERPRETERS_DECREF_METHODDEF _INTERPRETERS_IS_SHAREABLE_METHODDEF - _INTERPRETERS_CAPTURE_EXCEPTION_METHODDEF + _INTERPRETERS_SHARE_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1627,6 +2487,10 @@ module_exec(PyObject *mod) goto error; } + if (register_sharedobjectproxy(mod) < 0) { + goto error; + } + return 0; error: diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h index d70ffcea527895..47c7b255fcd31c 100644 --- a/Modules/clinic/_interpretersmodule.c.h +++ b/Modules/clinic/_interpretersmodule.c.h @@ -1198,4 +1198,16 @@ _interpreters_capture_exception(PyObject *module, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=c80f73761f860f6c input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_interpreters_share__doc__, +"share($module, op, /)\n" +"--\n" +"\n" +"Wrap an object in a shareable proxy that allows cross-interpreter access.\n" +"\n" +"The proxy will be assigned a context and may have its references cleared by\n" +"_interpreters.close_proxy()."); + +#define _INTERPRETERS_SHARE_METHODDEF \ + {"share", (PyCFunction)_interpreters_share, METH_O, _interpreters_share__doc__}, +/*[clinic end generated code: output=c1a117cea9045d1c input=a9049054013a1b77]*/ diff --git a/Python/pystate.c b/Python/pystate.c index c12a1418e74309..17e499e2947e9c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1638,11 +1638,47 @@ clear_datastack(PyThreadState *tstate) } } +int +_PyThreadState_AddClearCallback(PyThreadState *tstate, + _PyThreadState_ClearCallback callback, + void *arg) +{ + assert(tstate != NULL); + assert(_PyThreadState_IsAttached(tstate)); + assert(callback != NULL); + _PyThreadStateImpl *impl = (_PyThreadStateImpl *)tstate; + struct _PyThreadState_ClearNode *node = PyMem_Malloc(sizeof(struct _PyThreadState_ClearNode)); + if (node == NULL) { + PyErr_NoMemory(); + return -1; + } + node->callback = callback; + node->next = impl->clear_callbacks; + node->arg = arg; + impl->clear_callbacks = node; + return 0; +} + +void +call_clear_callbacks(PyThreadState *tstate) +{ + assert(tstate != NULL); + assert(tstate == current_fast_get()); + _PyThreadStateImpl *impl = (_PyThreadStateImpl *)tstate; + struct _PyThreadState_ClearNode *head = impl->clear_callbacks; + while (head != NULL) { + head->callback(tstate, head->arg); + head = head->next; + } +} + void PyThreadState_Clear(PyThreadState *tstate) { + assert(tstate != NULL); assert(tstate->_status.initialized && !tstate->_status.cleared); assert(current_fast_get()->interp == tstate->interp); + call_clear_callbacks(tstate); // GH-126016: In the _interpreters module, KeyboardInterrupt exceptions // during PyEval_EvalCode() are sent to finalization, which doesn't let us // mark threads as "not running main". So, for now this assertion is From 35cab484be7efd641c38d447847bab0998aea8d0 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 21:46:44 -0500 Subject: [PATCH 02/31] Remove special case. --- Modules/_interpretersmodule.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 402902a4613b17..4a1623c884e46d 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -2405,12 +2405,6 @@ static PyObject * _interpreters_share(PyObject *module, PyObject *op) /*[clinic end generated code: output=e2ce861ae3b58508 input=d333c93f128faf93]*/ { - if (Py_IsNone(op)) { - PyErr_SetString(PyExc_ValueError, - "None is a reserved value for dead object proxies, and " - "does not need to be shared"); - return NULL; - } return _sharedobjectproxy_create(op); } From db0c2ff41bf7a07d615771272e358f3d55ccad4b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 22:08:02 -0500 Subject: [PATCH 03/31] Add some basic tests that probably don't work. --- .../test_interpreters/test_object_proxy.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 Lib/test/test_interpreters/test_object_proxy.py diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py new file mode 100644 index 00000000000000..e9e773339f0a59 --- /dev/null +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -0,0 +1,48 @@ +import unittest + +from test.support import import_helper +from test.support import threading_helper +# Raise SkipTest if subinterpreters not supported. +import_helper.import_module('_interpreters') +from concurrent.interpreters import share, SharedObjectProxy +from .utils import TestBase + + +class SharedObjectProxyTests(TestBase): + def unshareable(self): + class Test: + def __init__(self): + pass + + return Test() + + def test_create(self): + proxy = share(self.unshareable()) + self.assertIsInstance(proxy, SharedObjectProxy) + + # Shareable objects should pass through + for shareable in (None, True, False, 100, 10000, "hello", b"world", memoryview(b"test")): + self.assertTrue(shareable) + with self.subTest(shareable=shareable): + not_a_proxy = share(shareable) + self.assertNotIsInstance(not_a_proxy, SharedObjectProxy) + self.assertIs(not_a_proxy, shareable) + + @threading_helper.requires_working_threading() + def test_create_concurrently(self): + def thread(): + for iteration in range(100): + with self.subTest(iteration=iteration): + self.test_create() + + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads((thread for _ in range(4))): + pass + + if cm.exc_value is not None: + raise cm.exc_value + + +if __name__ == '__main__': + # Test needs to be a package, so we can do relative imports. + unittest.main() From 10903669140ee8285a2fb3c0df6e4166980f783c Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 22:11:54 -0500 Subject: [PATCH 04/31] Fix the test. --- .../test_interpreters/test_object_proxy.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index e9e773339f0a59..6d5f04d8fbb7fc 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -5,7 +5,9 @@ # Raise SkipTest if subinterpreters not supported. import_helper.import_module('_interpreters') from concurrent.interpreters import share, SharedObjectProxy -from .utils import TestBase +from test.test_interpreters.utils import TestBase +from threading import Barrier, Thread +from concurrent import interpreters class SharedObjectProxyTests(TestBase): @@ -22,7 +24,7 @@ def test_create(self): # Shareable objects should pass through for shareable in (None, True, False, 100, 10000, "hello", b"world", memoryview(b"test")): - self.assertTrue(shareable) + self.assertTrue(interpreters.is_shareable(shareable)) with self.subTest(shareable=shareable): not_a_proxy = share(shareable) self.assertNotIsInstance(not_a_proxy, SharedObjectProxy) @@ -30,13 +32,22 @@ def test_create(self): @threading_helper.requires_working_threading() def test_create_concurrently(self): + barrier = Barrier(4) def thread(): + interp = interpreters.create() + barrier.wait() for iteration in range(100): with self.subTest(iteration=iteration): - self.test_create() + interp.exec("""if True: + from concurrent.interpreters import share + import os + + unshareable = open(os.devnull) + proxy = share(unshareable) + unshareable.close()""") with threading_helper.catch_threading_exception() as cm: - with threading_helper.start_threads((thread for _ in range(4))): + with threading_helper.start_threads((Thread(target=thread) for _ in range(4))): pass if cm.exc_value is not None: @@ -44,5 +55,4 @@ def thread(): if __name__ == '__main__': - # Test needs to be a package, so we can do relative imports. unittest.main() From e0c6e59eb35ae3c72043742aa1fd843f1cadb8e5 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 22:31:44 -0500 Subject: [PATCH 05/31] Add a test for access in another interpreter. It crashes right now though. --- .../test_interpreters/test_object_proxy.py | 12 +++++- Modules/_interpretersmodule.c | 39 ++++++++++++++----- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 6d5f04d8fbb7fc..a8a1a939368263 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -16,7 +16,9 @@ class Test: def __init__(self): pass - return Test() + instance = Test() + self.assertFalse(interpreters.is_shareable(instance)) + return instance def test_create(self): proxy = share(self.unshareable()) @@ -53,6 +55,14 @@ def thread(): if cm.exc_value is not None: raise cm.exc_value + def test_access_proxy(self): + interp = interpreters.create() + obj = self.unshareable() + proxy = share(obj) + obj.test = "silly" + interp.prepare_main(proxy=proxy) + interp.exec("print(proxy.test, proxy.test == 'silly', type(proxy.test))") + if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 4a1623c884e46d..973c2af405a103 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -391,20 +391,34 @@ typedef struct _Py_shared_object_proxy { #endif } SharedObjectProxy; -#define SharedObjectProxy_CAST(op) ((SharedObjectProxy *)op) +static inline SharedObjectProxy * +SharedObjectProxy_CAST(PyObject *op) +{ + assert(op != NULL); + SharedObjectProxy *proxy = (SharedObjectProxy *)op; + assert(proxy->object != NULL); + assert(Py_REFCNT(proxy->object) > 0); + assert(!_PyMem_IsPtrFreed(proxy->object)); + assert(proxy->interp != NULL); + assert(!_PyMem_IsPtrFreed(proxy->interp)); + return proxy; +} +#define SharedObjectProxy_CAST(op) SharedObjectProxy_CAST(_PyObject_CAST(op)) #define SharedObjectProxy_OBJECT(op) FT_ATOMIC_LOAD_PTR_RELAXED(SharedObjectProxy_CAST(op)->object) #define SharedObjectProxy_CheckExact(op) (Py_TYPE(_PyObject_CAST(op)) == &SharedObjectProxy_Type) #ifdef Py_GIL_DISABLED -#define SharedObjectProxy_TSTATES(op) ((op)->thread_states.table) -#define SharedObjectProxy_LOCK_TSTATES(op) PyMutex_Lock(&(op)->thread_states.mutex) -#define SharedObjectProxy_UNLOCK_TSTATES(op) PyMutex_Unlock(&(op)->thread_states.mutex) +#define SharedObjectProxy_TSTATES(op) ((SharedObjectProxy_CAST(op))->thread_states.table) +#define SharedObjectProxy_LOCK_TSTATES(op) PyMutex_Lock(&(SharedObjectProxy_CAST(op))->thread_states.mutex) +#define SharedObjectProxy_UNLOCK_TSTATES(op) PyMutex_Unlock(&(SharedObjectProxy_CAST(op))->thread_states.mutex) #else #define SharedObjectProxy_TSTATES(op) ((op)->thread_states) #define SharedObjectProxy_LOCK_TSTATES(op) #define SharedObjectProxy_UNLOCK_TSTATES(op) #endif +static PyTypeObject SharedObjectProxy_Type; + static int sharedobjectproxy_clear(PyObject *op) { @@ -414,6 +428,7 @@ sharedobjectproxy_clear(PyObject *op) return 0; } + assert(!SharedObjectProxy_CheckExact(self->object)); Py_CLEAR(self->object); return 0; } @@ -427,6 +442,7 @@ sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) return 0; } + assert(!SharedObjectProxy_CheckExact(self->object)); Py_VISIT(self->object); return 0; } @@ -435,7 +451,6 @@ static void sharedobjectproxy_dealloc(PyObject *op) { SharedObjectProxy *self = SharedObjectProxy_CAST(op); - assert(_PyInterpreterState_GET() == self->interp); PyTypeObject *tp = Py_TYPE(self); (void)sharedobjectproxy_clear(op); PyObject_GC_UnTrack(self); @@ -445,7 +460,7 @@ sharedobjectproxy_dealloc(PyObject *op) static PyObject * sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - SharedObjectProxy *self = SharedObjectProxy_CAST(type->tp_alloc(type, 0)); + SharedObjectProxy *self = (SharedObjectProxy *)type->tp_alloc(type, 0); if (self == NULL) { return NULL; } @@ -506,6 +521,7 @@ _sharedobjectproxy_enter_lock_held(SharedObjectProxy *self, _PyXI_proxy_state *s return 0; } + assert(self->interp != NULL); PyThreadState *for_call = _PyThreadState_NewBound(self->interp, _PyThreadState_WHENCE_EXEC); state->for_call = for_call; @@ -588,8 +604,6 @@ typedef struct { PyObject *object; } _PyXI_proxy_share; -static PyTypeObject SharedObjectProxy_Type; - /* Use this in the calling interpreter. */ static int _sharedobjectproxy_init_share(_PyXI_proxy_share *share, @@ -1207,14 +1221,21 @@ _sharedobjectproxy_create(PyObject *object) PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp != NULL); + if (SharedObjectProxy_CheckExact(object)) { + object = SharedObjectProxy_CAST(object)->object; + interp = SharedObjectProxy_CAST(object)->interp; + } + SharedObjectProxy *proxy = SharedObjectProxy_CAST(sharedobjectproxy_new(&SharedObjectProxy_Type, NULL, NULL)); if (proxy == NULL) { return NULL; } + assert(!SharedObjectProxy_CheckExact(object)); + assert(interp != NULL); proxy->object = Py_NewRef(object); - proxy->interp = _PyInterpreterState_GET(); + proxy->interp = interp; return (PyObject *)proxy; } From 0867c095f2e150f29530ae7b1314c758356b1597 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 22:40:36 -0500 Subject: [PATCH 06/31] Ensure that copied proxies have the correct interpreters. --- Lib/test/test_interpreters/test_object_proxy.py | 14 +++++++++++++- Modules/_interpretersmodule.c | 15 +++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index a8a1a939368263..ffe6677e5fba87 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -16,6 +16,9 @@ class Test: def __init__(self): pass + def silly(self): + return "silly" + instance = Test() self.assertFalse(interpreters.is_shareable(instance)) return instance @@ -61,7 +64,16 @@ def test_access_proxy(self): proxy = share(obj) obj.test = "silly" interp.prepare_main(proxy=proxy) - interp.exec("print(proxy.test, proxy.test == 'silly', type(proxy.test))") + interp.exec("assert proxy.test == 'silly'") + interp.exec("assert isinstance(proxy.test, str)") + interp.exec("""if True: + from concurrent.interpreters import SharedObjectProxy + method = proxy.silly + assert isinstance(method, SharedObjectProxy) + assert method() == 'silly' + assert isinstance(method(), str) + """) + #interp.exec("proxy.noexist") if __name__ == '__main__': diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 973c2af405a103..ed2d25db10db4a 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -391,10 +391,14 @@ typedef struct _Py_shared_object_proxy { #endif } SharedObjectProxy; +static PyTypeObject SharedObjectProxy_Type; +#define SharedObjectProxy_CheckExact(op) (Py_TYPE(_PyObject_CAST(op)) == &SharedObjectProxy_Type) + static inline SharedObjectProxy * SharedObjectProxy_CAST(PyObject *op) { assert(op != NULL); + assert(SharedObjectProxy_CheckExact(op)); SharedObjectProxy *proxy = (SharedObjectProxy *)op; assert(proxy->object != NULL); assert(Py_REFCNT(proxy->object) > 0); @@ -405,7 +409,6 @@ SharedObjectProxy_CAST(PyObject *op) } #define SharedObjectProxy_CAST(op) SharedObjectProxy_CAST(_PyObject_CAST(op)) #define SharedObjectProxy_OBJECT(op) FT_ATOMIC_LOAD_PTR_RELAXED(SharedObjectProxy_CAST(op)->object) -#define SharedObjectProxy_CheckExact(op) (Py_TYPE(_PyObject_CAST(op)) == &SharedObjectProxy_Type) #ifdef Py_GIL_DISABLED #define SharedObjectProxy_TSTATES(op) ((SharedObjectProxy_CAST(op))->thread_states.table) @@ -417,8 +420,6 @@ SharedObjectProxy_CAST(PyObject *op) #define SharedObjectProxy_UNLOCK_TSTATES(op) #endif -static PyTypeObject SharedObjectProxy_Type; - static int sharedobjectproxy_clear(PyObject *op) { @@ -1180,7 +1181,13 @@ static PyTypeObject SharedObjectProxy_Type = { static PyObject * sharedobjectproxy_xid(_PyXIData_t *data) { - return _sharedobjectproxy_create(data->obj); + SharedObjectProxy *proxy = SharedObjectProxy_CAST(data->obj); + SharedObjectProxy *new_proxy = SharedObjectProxy_CAST(_sharedobjectproxy_create(proxy->object)); + if (new_proxy == NULL) { + return NULL; + } + new_proxy->interp = proxy->interp; + return (PyObject *)new_proxy; } static int From cd1d139289534cc7f13fc72c3643e9402bce16b3 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 23:08:11 -0500 Subject: [PATCH 07/31] Create proxies in the switched interpreter, not the calling interpreter. --- Modules/_interpretersmodule.c | 158 +++++++++++++++------------------- 1 file changed, 69 insertions(+), 89 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ed2d25db10db4a..9032a3c22dfbee 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -420,6 +420,54 @@ SharedObjectProxy_CAST(PyObject *op) #define SharedObjectProxy_UNLOCK_TSTATES(op) #endif +static PyObject * +sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + SharedObjectProxy *self = (SharedObjectProxy *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + _Py_hashtable_allocator_t alloc = { + .malloc = PyMem_RawMalloc, + .free = PyMem_RawFree, + }; + _Py_hashtable_t *tstates = _Py_hashtable_new_full(_Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, + NULL, + &alloc); + if (tstates == NULL) { + Py_DECREF(self); + return PyErr_NoMemory(); + } + + SharedObjectProxy_TSTATES(self) = tstates; + self->object = Py_None; + self->interp = _PyInterpreterState_GET(); + + return (PyObject *)self; +} + +PyObject * +_sharedobjectproxy_create(PyObject *object, PyInterpreterState *owning_interp) +{ + assert(object != NULL); + assert(owning_interp != NULL); + assert(!SharedObjectProxy_CheckExact(object)); + + SharedObjectProxy *proxy = SharedObjectProxy_CAST(sharedobjectproxy_new(&SharedObjectProxy_Type, + NULL, NULL)); + if (proxy == NULL) { + return NULL; + } + + proxy->object = Py_NewRef(object); + proxy->interp = owning_interp; + return (PyObject *)proxy; +} + + static int sharedobjectproxy_clear(PyObject *op) { @@ -458,35 +506,6 @@ sharedobjectproxy_dealloc(PyObject *op) tp->tp_free(self); } -static PyObject * -sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - SharedObjectProxy *self = (SharedObjectProxy *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - - _Py_hashtable_allocator_t alloc = { - .malloc = PyMem_RawMalloc, - .free = PyMem_RawFree, - }; - _Py_hashtable_t *tstates = _Py_hashtable_new_full(_Py_hashtable_hash_ptr, - _Py_hashtable_compare_direct, - NULL, - NULL, - &alloc); - if (tstates == NULL) { - Py_DECREF(self); - return PyErr_NoMemory(); - } - - SharedObjectProxy_TSTATES(self) = tstates; - self->object = Py_None; - self->interp = _PyInterpreterState_GET(); - - return (PyObject *)self; -} - typedef struct { PyThreadState *to_restore; PyThreadState *for_call; @@ -597,12 +616,10 @@ _sharedobjectproxy_exit(SharedObjectProxy *self, _PyXI_proxy_state *state) return 0; } -PyObject * -_sharedobjectproxy_create(PyObject *object); - typedef struct { _PyXIData_t *xidata; - PyObject *object; + PyObject *object_to_wrap; + PyInterpreterState *owner; } _PyXI_proxy_share; /* Use this in the calling interpreter. */ @@ -612,29 +629,20 @@ _sharedobjectproxy_init_share(_PyXI_proxy_share *share, { assert(op != NULL); assert(share != NULL); - if (Py_TYPE(op) == &SharedObjectProxy_Type) { - // Already an object proxy; nothing to do - share->object = op; - share->xidata = NULL; - return 0; - } - _PyXIData_t *xidata = _PyXIData_New(); if (xidata == NULL) { return -1; } + share->owner = _PyInterpreterState_GET(); if (_PyObject_GetXIData(_PyThreadState_GET(), op, _PyXIDATA_XIDATA_ONLY, xidata) < 0) { PyErr_Clear(); - share->object = _sharedobjectproxy_create(op); + share->object_to_wrap = Py_NewRef(op); share->xidata = NULL; _PyXIData_Free(xidata); - if (share->object == NULL) { - return -1; - } } else { - share->object = NULL; + share->object_to_wrap = NULL; share->xidata = xidata; } @@ -643,16 +651,16 @@ _sharedobjectproxy_init_share(_PyXI_proxy_share *share, /* Use this in the switched interpreter. */ static PyObject * -_sharedobjectproxy_as_shared(_PyXI_proxy_share *share) +_sharedobjectproxy_copy_for_interp(_PyXI_proxy_share *share) { assert(share != NULL); _PyXIData_t *xidata = share->xidata; if (xidata == NULL) { - // Not shareable; use the object proxy - return share->object; + // Not shareable; use an object proxy + return _sharedobjectproxy_create(share->object_to_wrap, share->owner); } else { + assert(share->object_to_wrap == NULL); PyObject *result = _PyXIData_NewObject(xidata); - assert(share->object == NULL); return result; } } @@ -661,13 +669,15 @@ static void _sharedobjectproxy_finish_share(_PyXI_proxy_share *share) { if (share->xidata != NULL) { + assert(share->object_to_wrap == NULL); _PyXIData_Free(share->xidata); + } else { + assert(share->object_to_wrap != NULL); + Py_DECREF(share->object_to_wrap); } #ifdef Py_DEBUG share->xidata = NULL; - if (share->object != NULL) { - share->object = NULL; - } + share->object_to_wrap = NULL; #endif } @@ -694,7 +704,7 @@ _sharedobjectproxy_wrap_result(SharedObjectProxy *self, PyObject *result, return NULL; } - PyObject *ret = _sharedobjectproxy_as_shared(&shared_result); + PyObject *ret = _sharedobjectproxy_copy_for_interp(&shared_result); //_sharedobjectproxy_finish_share(&shared_result); return ret; } @@ -731,7 +741,7 @@ sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { } for (Py_ssize_t i = 0; i < size; ++i) { - PyObject *shared = _sharedobjectproxy_as_shared(&shared_args_state[i]); + PyObject *shared = _sharedobjectproxy_copy_for_interp(&shared_args_state[i]); if (shared == NULL) { (void)_sharedobjectproxy_exit(self, &state); PyMem_RawFree(shared_args_state); @@ -773,7 +783,7 @@ _sharedobjectproxy_single_share_common(SharedObjectProxy *self, PyObject *to_sha _sharedobjectproxy_finish_share(shared_arg); return NULL; } - PyObject *shared_obj = _sharedobjectproxy_as_shared(shared_arg); + PyObject *shared_obj = _sharedobjectproxy_copy_for_interp(shared_arg); if (shared_obj == NULL) { (void)_sharedobjectproxy_exit(self, state); _sharedobjectproxy_finish_share(shared_arg); @@ -821,14 +831,14 @@ _sharedobjectproxy_double_share_common(SharedObjectProxy *self, _sharedobjectproxy_finish_share(shared_second); return -1; } - PyObject *first_obj = _sharedobjectproxy_as_shared(shared_first); + PyObject *first_obj = _sharedobjectproxy_copy_for_interp(shared_first); if (first_obj == NULL) { (void)_sharedobjectproxy_exit(self, state); _sharedobjectproxy_finish_share(shared_first); _sharedobjectproxy_finish_share(shared_second); return -1; } - PyObject *second_obj = _sharedobjectproxy_as_shared(shared_second); + PyObject *second_obj = _sharedobjectproxy_copy_for_interp(shared_second); if (second_obj == NULL) { Py_DECREF(first_obj); (void)_sharedobjectproxy_exit(self, state); @@ -1182,12 +1192,7 @@ static PyObject * sharedobjectproxy_xid(_PyXIData_t *data) { SharedObjectProxy *proxy = SharedObjectProxy_CAST(data->obj); - SharedObjectProxy *new_proxy = SharedObjectProxy_CAST(_sharedobjectproxy_create(proxy->object)); - if (new_proxy == NULL) { - return NULL; - } - new_proxy->interp = proxy->interp; - return (PyObject *)new_proxy; + return _sharedobjectproxy_create(proxy->object, proxy->interp); } static int @@ -1221,31 +1226,6 @@ _get_current_xibufferview_type(void) return state->XIBufferViewType; } -PyObject * -_sharedobjectproxy_create(PyObject *object) -{ - assert(object != NULL); - PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp != NULL); - - if (SharedObjectProxy_CheckExact(object)) { - object = SharedObjectProxy_CAST(object)->object; - interp = SharedObjectProxy_CAST(object)->interp; - } - - SharedObjectProxy *proxy = SharedObjectProxy_CAST(sharedobjectproxy_new(&SharedObjectProxy_Type, - NULL, NULL)); - if (proxy == NULL) { - return NULL; - } - - assert(!SharedObjectProxy_CheckExact(object)); - assert(interp != NULL); - proxy->object = Py_NewRef(object); - proxy->interp = interp; - return (PyObject *)proxy; -} - /* interpreter-specific code ************************************************/ static int @@ -2433,7 +2413,7 @@ static PyObject * _interpreters_share(PyObject *module, PyObject *op) /*[clinic end generated code: output=e2ce861ae3b58508 input=d333c93f128faf93]*/ { - return _sharedobjectproxy_create(op); + return _sharedobjectproxy_create(op, _PyInterpreterState_GET()); } static PyMethodDef module_functions[] = { From 6fa7e0ac0ead240e5c935028ce20ca6beb0aa319 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 23:10:36 -0500 Subject: [PATCH 08/31] Some general test improvements. --- .../test_interpreters/test_object_proxy.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index ffe6677e5fba87..43ab2ce40354b7 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -11,20 +11,8 @@ class SharedObjectProxyTests(TestBase): - def unshareable(self): - class Test: - def __init__(self): - pass - - def silly(self): - return "silly" - - instance = Test() - self.assertFalse(interpreters.is_shareable(instance)) - return instance - def test_create(self): - proxy = share(self.unshareable()) + proxy = share(object()) self.assertIsInstance(proxy, SharedObjectProxy) # Shareable objects should pass through @@ -59,8 +47,12 @@ def thread(): raise cm.exc_value def test_access_proxy(self): + class Test: + def silly(self): + return "silly" + interp = interpreters.create() - obj = self.unshareable() + obj = Test() proxy = share(obj) obj.test = "silly" interp.prepare_main(proxy=proxy) @@ -73,7 +65,8 @@ def test_access_proxy(self): assert method() == 'silly' assert isinstance(method(), str) """) - #interp.exec("proxy.noexist") + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec("proxy.noexist") if __name__ == '__main__': From 6b64744f0116377e90c94ba7aa5a44f90786cc84 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 23:20:46 -0500 Subject: [PATCH 09/31] Hold a reference to object proxies in clear callbacks. --- .../test_interpreters/test_object_proxy.py | 58 ++++++++++++++----- Modules/_interpretersmodule.c | 5 +- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 43ab2ce40354b7..3eaf7e29355d6a 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -6,11 +6,25 @@ import_helper.import_module('_interpreters') from concurrent.interpreters import share, SharedObjectProxy from test.test_interpreters.utils import TestBase -from threading import Barrier, Thread +from threading import Barrier, Thread, Lock from concurrent import interpreters class SharedObjectProxyTests(TestBase): + def run_concurrently(self, func, num_threads=4): + barrier = Barrier(num_threads) + def thread(): + interp = interpreters.create() + barrier.wait() + func(interp) + + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads((Thread(target=thread) for _ in range(num_threads))): + pass + + if cm.exc_value is not None: + raise cm.exc_value + def test_create(self): proxy = share(object()) self.assertIsInstance(proxy, SharedObjectProxy) @@ -25,26 +39,15 @@ def test_create(self): @threading_helper.requires_working_threading() def test_create_concurrently(self): - barrier = Barrier(4) - def thread(): - interp = interpreters.create() - barrier.wait() + def thread(interp): for iteration in range(100): with self.subTest(iteration=iteration): interp.exec("""if True: from concurrent.interpreters import share - import os - unshareable = open(os.devnull) - proxy = share(unshareable) - unshareable.close()""") + share(object())""") - with threading_helper.catch_threading_exception() as cm: - with threading_helper.start_threads((Thread(target=thread) for _ in range(4))): - pass - - if cm.exc_value is not None: - raise cm.exc_value + self.run_concurrently(thread) def test_access_proxy(self): class Test: @@ -68,6 +71,31 @@ def silly(self): with self.assertRaises(interpreters.ExecutionFailed): interp.exec("proxy.noexist") + @threading_helper.requires_working_threading() + def test_access_proxy_concurrently(self): + class Test: + def __init__(self): + self.lock = Lock() + self.value = 0 + + def increment(self): + with self.lock: + self.value += 1 + + test = Test() + proxy = share(test) + + def thread(interp): + interp.prepare_main(proxy=proxy) + for _ in range(100): + interp.exec("proxy.increment()") + interp.exec("assert isinstance(proxy.value, int)") + + self.run_concurrently(thread) + self.assertEqual(test.value, 400) + + + if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 9032a3c22dfbee..db32c4d9322b02 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -398,7 +398,7 @@ static inline SharedObjectProxy * SharedObjectProxy_CAST(PyObject *op) { assert(op != NULL); - assert(SharedObjectProxy_CheckExact(op)); + _PyObject_ASSERT(op, SharedObjectProxy_CheckExact(op)); SharedObjectProxy *proxy = (SharedObjectProxy *)op; assert(proxy->object != NULL); assert(Py_REFCNT(proxy->object) > 0); @@ -526,6 +526,7 @@ _sharedobjectproxy_destroy_tstate(PyThreadState *tstate, void *arg) PyThreadState_DeleteCurrent(); _PyThreadState_Attach(tstate); SharedObjectProxy_UNLOCK_TSTATES(self); + Py_DECREF(arg); } static int @@ -555,7 +556,7 @@ _sharedobjectproxy_enter_lock_held(SharedObjectProxy *self, _PyXI_proxy_state *s } if (_PyThreadState_AddClearCallback(tstate, _sharedobjectproxy_destroy_tstate, - self) < 0) { + Py_NewRef(self)) < 0) { PyErr_NoMemory(); return -1; From 5d0b3088c61b573eec916738591f330e6203bf2c Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 23:30:22 -0500 Subject: [PATCH 10/31] Remove the per-object thread state cache. This should be done at the interpreter level, not the object level. --- Modules/_interpretersmodule.c | 92 ++++------------------------------- 1 file changed, 9 insertions(+), 83 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index db32c4d9322b02..b1283f08a87904 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -410,16 +410,6 @@ SharedObjectProxy_CAST(PyObject *op) #define SharedObjectProxy_CAST(op) SharedObjectProxy_CAST(_PyObject_CAST(op)) #define SharedObjectProxy_OBJECT(op) FT_ATOMIC_LOAD_PTR_RELAXED(SharedObjectProxy_CAST(op)->object) -#ifdef Py_GIL_DISABLED -#define SharedObjectProxy_TSTATES(op) ((SharedObjectProxy_CAST(op))->thread_states.table) -#define SharedObjectProxy_LOCK_TSTATES(op) PyMutex_Lock(&(SharedObjectProxy_CAST(op))->thread_states.mutex) -#define SharedObjectProxy_UNLOCK_TSTATES(op) PyMutex_Unlock(&(SharedObjectProxy_CAST(op))->thread_states.mutex) -#else -#define SharedObjectProxy_TSTATES(op) ((op)->thread_states) -#define SharedObjectProxy_LOCK_TSTATES(op) -#define SharedObjectProxy_UNLOCK_TSTATES(op) -#endif - static PyObject * sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -428,21 +418,6 @@ sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - _Py_hashtable_allocator_t alloc = { - .malloc = PyMem_RawMalloc, - .free = PyMem_RawFree, - }; - _Py_hashtable_t *tstates = _Py_hashtable_new_full(_Py_hashtable_hash_ptr, - _Py_hashtable_compare_direct, - NULL, - NULL, - &alloc); - if (tstates == NULL) { - Py_DECREF(self); - return PyErr_NoMemory(); - } - - SharedObjectProxy_TSTATES(self) = tstates; self->object = Py_None; self->interp = _PyInterpreterState_GET(); @@ -511,60 +486,6 @@ typedef struct { PyThreadState *for_call; } _PyXI_proxy_state; -static void -_sharedobjectproxy_destroy_tstate(PyThreadState *tstate, void *arg) -{ - assert(tstate != NULL); - assert(arg != NULL); - SharedObjectProxy *self = SharedObjectProxy_CAST(arg); - SharedObjectProxy_LOCK_TSTATES(self); - _Py_hashtable_t *table = SharedObjectProxy_TSTATES(self); - PyThreadState *to_destroy = _Py_hashtable_steal(table, tstate); - assert(to_destroy != NULL); - PyThreadState_Swap(to_destroy); - PyThreadState_Clear(to_destroy); - PyThreadState_DeleteCurrent(); - _PyThreadState_Attach(tstate); - SharedObjectProxy_UNLOCK_TSTATES(self); - Py_DECREF(arg); -} - -static int -_sharedobjectproxy_enter_lock_held(SharedObjectProxy *self, _PyXI_proxy_state *state, - PyThreadState *tstate) -{ - _Py_hashtable_t *table = SharedObjectProxy_TSTATES(self); - PyThreadState *cached_tstate = _Py_hashtable_get(table, tstate); - - if (cached_tstate != NULL) { - _PyThreadState_Detach(tstate); - state->for_call = cached_tstate; - return 0; - } - - assert(self->interp != NULL); - PyThreadState *for_call = _PyThreadState_NewBound(self->interp, - _PyThreadState_WHENCE_EXEC); - state->for_call = for_call; - if (for_call == NULL) { - PyErr_NoMemory(); - return -1; - } - if (_Py_hashtable_set(table, tstate, for_call) < 0) { - PyErr_NoMemory(); - return -1; - } - if (_PyThreadState_AddClearCallback(tstate, - _sharedobjectproxy_destroy_tstate, - Py_NewRef(self)) < 0) { - - PyErr_NoMemory(); - return -1; - } - _PyThreadState_Detach(tstate); - return 0; -} - static int _sharedobjectproxy_enter(SharedObjectProxy *self, _PyXI_proxy_state *state) { @@ -578,12 +499,14 @@ _sharedobjectproxy_enter(SharedObjectProxy *self, _PyXI_proxy_state *state) return 0; } state->to_restore = tstate; - SharedObjectProxy_LOCK_TSTATES(self); - int res = _sharedobjectproxy_enter_lock_held(self, state, tstate); - SharedObjectProxy_UNLOCK_TSTATES(self); - if (res < 0) { + PyThreadState *for_call = _PyThreadState_NewBound(self->interp, + _PyThreadState_WHENCE_EXEC); + state->for_call = for_call; + if (for_call == NULL) { + PyErr_NoMemory(); return -1; } + _PyThreadState_Detach(tstate); _PyThreadState_Attach(state->for_call); assert(_PyInterpreterState_GET() == self->interp); return 0; @@ -608,12 +531,15 @@ _sharedobjectproxy_exit(SharedObjectProxy *self, _PyXI_proxy_state *state) } assert(state->for_call == _PyThreadState_GET()); + PyThreadState_Clear(state->for_call); PyThreadState_Swap(state->to_restore); + PyThreadState_Delete(state->for_call); if (should_throw) { _PyErr_SetString(state->to_restore, PyExc_RuntimeError, "exception in interpreter"); return -1; } + return 0; } From 84a286ba322a71290382211b9918e07760ca3b92 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 26 Nov 2025 23:42:55 -0500 Subject: [PATCH 11/31] Fix assertion failures when GC clears the proxy before reference counting does. --- .../test_interpreters/test_object_proxy.py | 19 +++++++++++++++++++ Modules/_interpretersmodule.c | 11 ++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 3eaf7e29355d6a..d9693d7f36ab65 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -94,6 +94,25 @@ def thread(interp): self.run_concurrently(thread) self.assertEqual(test.value, 400) + def test_proxy_call(self): + constant = 67 # Hilarious + def my_function(arg=1, /, *, arg2=2): + # We need the constant here to make this function unshareable. + return constant + arg + arg2 + + proxy = share(my_function) + self.assertIsInstance(proxy, SharedObjectProxy) + self.assertEqual(proxy(), 70) + self.assertEqual(proxy(0, arg2=1), 68) + self.assertEqual(proxy(2), 71) + + interp = interpreters.create() + interp.exec("""if True: + assert isinstance(proxy(), int) + assert proxy() == 70 + assert proxy(0, arg2=1) == 68 + assert proxy(2) == 71""") + diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index b1283f08a87904..a75908e3577758 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -393,6 +393,7 @@ typedef struct _Py_shared_object_proxy { static PyTypeObject SharedObjectProxy_Type; #define SharedObjectProxy_CheckExact(op) (Py_TYPE(_PyObject_CAST(op)) == &SharedObjectProxy_Type) +#define SharedObjectProxy_RAW_CAST(op) ((SharedObjectProxy *)op) static inline SharedObjectProxy * SharedObjectProxy_CAST(PyObject *op) @@ -446,13 +447,13 @@ _sharedobjectproxy_create(PyObject *object, PyInterpreterState *owning_interp) static int sharedobjectproxy_clear(PyObject *op) { - SharedObjectProxy *self = SharedObjectProxy_CAST(op); + SharedObjectProxy *self = SharedObjectProxy_RAW_CAST(op); // Don't clear from another interpreter if (self->interp != _PyInterpreterState_GET()) { return 0; } - assert(!SharedObjectProxy_CheckExact(self->object)); + assert(self->object == NULL || !SharedObjectProxy_CheckExact(self->object)); Py_CLEAR(self->object); return 0; } @@ -460,13 +461,13 @@ sharedobjectproxy_clear(PyObject *op) static int sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) { - SharedObjectProxy *self = SharedObjectProxy_CAST(op); + SharedObjectProxy *self = SharedObjectProxy_RAW_CAST(op); // Don't traverse from another interpreter if (self->interp != _PyInterpreterState_GET()) { return 0; } - assert(!SharedObjectProxy_CheckExact(self->object)); + assert(self->object == NULL || !SharedObjectProxy_CheckExact(self->object)); Py_VISIT(self->object); return 0; } @@ -474,7 +475,7 @@ sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) static void sharedobjectproxy_dealloc(PyObject *op) { - SharedObjectProxy *self = SharedObjectProxy_CAST(op); + SharedObjectProxy *self = SharedObjectProxy_RAW_CAST(op); PyTypeObject *tp = Py_TYPE(self); (void)sharedobjectproxy_clear(op); PyObject_GC_UnTrack(self); From 516f3fc60c3aa9537f4ebeab8ef4be29964b03c8 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 00:09:06 -0500 Subject: [PATCH 12/31] Add support for keyword arguments in proxy calls. It's still pretty leaky right now though. --- .../test_interpreters/test_object_proxy.py | 1 + Modules/_interpretersmodule.c | 107 ++++++++++++++++-- 2 files changed, 101 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index d9693d7f36ab65..7b237b55cc6318 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -107,6 +107,7 @@ def my_function(arg=1, /, *, arg2=2): self.assertEqual(proxy(2), 71) interp = interpreters.create() + interp.prepare_main(proxy=proxy) interp.exec("""if True: assert isinstance(proxy(), int) assert proxy() == 70 diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a75908e3577758..1c1d63b0fd1eb4 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -640,10 +640,17 @@ _sharedobjectproxy_wrap_result(SharedObjectProxy *self, PyObject *result, static PyObject * sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { SharedObjectProxy *self = SharedObjectProxy_CAST(op); - Py_ssize_t size = PyTuple_Size(args); - _PyXI_proxy_share *shared_args_state = PyMem_RawMalloc(size * sizeof(_PyXI_proxy_share)); + assert(PyTuple_Check(args)); + Py_ssize_t args_size = PyTuple_GET_SIZE(args); + assert(kwargs == NULL || PyDict_Check(kwargs)); + Py_ssize_t kwarg_size = kwargs == NULL ? -1 : PyDict_GET_SIZE(kwargs); + _PyXI_proxy_share *shared_args_state = PyMem_RawMalloc(args_size * sizeof(_PyXI_proxy_share)); + if (shared_args_state == NULL) { + PyErr_NoMemory(); + return NULL; + } - for (Py_ssize_t i = 0; i < size; ++i) { + for (Py_ssize_t i = 0; i < args_size; ++i) { PyObject *arg = PyTuple_GetItem(args, i); if (arg == NULL) { PyMem_RawFree(shared_args_state); @@ -656,19 +663,62 @@ sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { } } + struct _dict_pair { + const char *key; + Py_ssize_t key_length; + _PyXI_proxy_share value; + }; + struct _dict_pair *kwarg_pairs = NULL; + if (kwargs != NULL) { + kwarg_pairs = PyMem_RawCalloc(kwarg_size, sizeof(struct _dict_pair)); + if (kwarg_pairs == NULL) { + PyErr_NoMemory(); + PyMem_RawFree(shared_args_state); + return NULL; + } + + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(kwargs, &pos, &key, &value)) { + // XXX Can kwarg keys be dictionary subclasses? + assert(PyUnicode_Check(key)); + Py_ssize_t index = pos - 1; + assert(index >= 0); + assert(index < kwarg_size); + struct _dict_pair *pair = &kwarg_pairs[index]; + assert(pair->key == NULL); + assert(pair->key_length == 0); + const char *key_str = PyUnicode_AsUTF8AndSize(key, &pair->key_length); + if (key_str == NULL) { + PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); + return NULL; + + } + pair->key = key_str; + if (_sharedobjectproxy_init_share(&pair->value, self, value) < 0) { + PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); + return NULL; + } + } + } + _PyXI_proxy_state state; if (_sharedobjectproxy_enter(self, &state) < 0) { PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); return NULL; } - PyObject *shared_args = PyTuple_New(size); + PyObject *shared_args = PyTuple_New(args_size); if (shared_args == NULL) { (void)_sharedobjectproxy_exit(self, &state); PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); return NULL; } - for (Py_ssize_t i = 0; i < size; ++i) { + for (Py_ssize_t i = 0; i < args_size; ++i) { PyObject *shared = _sharedobjectproxy_copy_for_interp(&shared_args_state[i]); if (shared == NULL) { (void)_sharedobjectproxy_exit(self, &state); @@ -678,10 +728,53 @@ sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { PyTuple_SET_ITEM(shared_args, i, shared); } - // kwargs aren't supported yet + PyObject *shared_kwargs = NULL; + if (kwargs != NULL) { + shared_kwargs = PyDict_New(); + if (shared_kwargs == NULL) { + Py_DECREF(shared_args); + (void)_sharedobjectproxy_exit(self, &state); + PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); + return NULL; + } + for (Py_ssize_t i = 0; i < kwarg_size; ++i) { + struct _dict_pair *pair = &kwarg_pairs[i]; + assert(pair->key != NULL); + PyObject *key = PyUnicode_FromStringAndSize(pair->key, pair->key_length); + if (key == NULL) { + Py_DECREF(shared_args); + Py_DECREF(shared_kwargs); + (void)_sharedobjectproxy_exit(self, &state); + PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); + return NULL; + } + PyObject *shared_kwarg = _sharedobjectproxy_copy_for_interp(&pair->value); + if (shared_kwarg == NULL) { + Py_DECREF(shared_args); + Py_DECREF(shared_kwargs); + (void)_sharedobjectproxy_exit(self, &state); + PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); + return NULL; + } + if (PyDict_SetItem(shared_kwargs, key, shared_kwarg) < 0) { + Py_DECREF(shared_args); + Py_DECREF(shared_kwargs); + (void)_sharedobjectproxy_exit(self, &state); + PyMem_RawFree(shared_args_state); + PyMem_RawFree(kwarg_pairs); + return NULL; + } + } + } + + PyObject *res = PyObject_Call(SharedObjectProxy_OBJECT(self), - shared_args, NULL); + shared_args, shared_kwargs); Py_DECREF(shared_args); + Py_XDECREF(shared_kwargs); return _sharedobjectproxy_wrap_result(self, res, &state); } From 47242c8d2beccbef4868c36c0820e4fb3ddcf648 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 02:59:41 -0500 Subject: [PATCH 13/31] Fix leaks in object proxy call slots. --- Modules/_interpretersmodule.c | 260 +++++++++++++++++++++------------- 1 file changed, 161 insertions(+), 99 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 1c1d63b0fd1eb4..cf8bcdfd57d51b 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -637,13 +637,11 @@ _sharedobjectproxy_wrap_result(SharedObjectProxy *self, PyObject *result, return ret; } -static PyObject * -sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { - SharedObjectProxy *self = SharedObjectProxy_CAST(op); +static _PyXI_proxy_share * +_sharedobjectproxy_init_shared_args(PyObject *args, SharedObjectProxy *self) +{ assert(PyTuple_Check(args)); Py_ssize_t args_size = PyTuple_GET_SIZE(args); - assert(kwargs == NULL || PyDict_Check(kwargs)); - Py_ssize_t kwarg_size = kwargs == NULL ? -1 : PyDict_GET_SIZE(kwargs); _PyXI_proxy_share *shared_args_state = PyMem_RawMalloc(args_size * sizeof(_PyXI_proxy_share)); if (shared_args_state == NULL) { PyErr_NoMemory(); @@ -651,132 +649,196 @@ sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { } for (Py_ssize_t i = 0; i < args_size; ++i) { - PyObject *arg = PyTuple_GetItem(args, i); - if (arg == NULL) { - PyMem_RawFree(shared_args_state); - return NULL; - } + PyObject *arg = PyTuple_GET_ITEM(args, i); if (_sharedobjectproxy_init_share(&shared_args_state[i], self, arg) < 0) { - // TODO: Fix leaks from prior iterations PyMem_RawFree(shared_args_state); + for (int x = 0; x < i; ++x) { + _sharedobjectproxy_finish_share(&shared_args_state[i]); + } return NULL; } } + return shared_args_state; +} - struct _dict_pair { - const char *key; - Py_ssize_t key_length; - _PyXI_proxy_share value; - }; - struct _dict_pair *kwarg_pairs = NULL; - if (kwargs != NULL) { - kwarg_pairs = PyMem_RawCalloc(kwarg_size, sizeof(struct _dict_pair)); - if (kwarg_pairs == NULL) { - PyErr_NoMemory(); - PyMem_RawFree(shared_args_state); - return NULL; - } - - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(kwargs, &pos, &key, &value)) { - // XXX Can kwarg keys be dictionary subclasses? - assert(PyUnicode_Check(key)); - Py_ssize_t index = pos - 1; - assert(index >= 0); - assert(index < kwarg_size); - struct _dict_pair *pair = &kwarg_pairs[index]; - assert(pair->key == NULL); - assert(pair->key_length == 0); - const char *key_str = PyUnicode_AsUTF8AndSize(key, &pair->key_length); - if (key_str == NULL) { - PyMem_RawFree(shared_args_state); - PyMem_RawFree(kwarg_pairs); - return NULL; - - } - pair->key = key_str; - if (_sharedobjectproxy_init_share(&pair->value, self, value) < 0) { - PyMem_RawFree(shared_args_state); - PyMem_RawFree(kwarg_pairs); - return NULL; - } - } +static void +_sharedobjectproxy_close_shared_args(PyObject *args, _PyXI_proxy_share *shared_args_state) +{ + Py_ssize_t args_size = PyTuple_GET_SIZE(args); + for (Py_ssize_t i = 0; i < args_size; ++i) { + _sharedobjectproxy_finish_share(&shared_args_state[i]); } + PyMem_RawFree(shared_args_state); +} - _PyXI_proxy_state state; - if (_sharedobjectproxy_enter(self, &state) < 0) { - PyMem_RawFree(shared_args_state); - PyMem_RawFree(kwarg_pairs); - return NULL; - } +static PyObject * +_sharedobjectproxy_construct_shared_args(PyObject *args, _PyXI_proxy_share *shared_args_state) +{ + Py_ssize_t args_size = PyTuple_GET_SIZE(args); PyObject *shared_args = PyTuple_New(args_size); if (shared_args == NULL) { - (void)_sharedobjectproxy_exit(self, &state); - PyMem_RawFree(shared_args_state); - PyMem_RawFree(kwarg_pairs); return NULL; } - for (Py_ssize_t i = 0; i < args_size; ++i) { PyObject *shared = _sharedobjectproxy_copy_for_interp(&shared_args_state[i]); if (shared == NULL) { - (void)_sharedobjectproxy_exit(self, &state); - PyMem_RawFree(shared_args_state); + Py_DECREF(shared_args); return NULL; } PyTuple_SET_ITEM(shared_args, i, shared); } - PyObject *shared_kwargs = NULL; - if (kwargs != NULL) { - shared_kwargs = PyDict_New(); - if (shared_kwargs == NULL) { - Py_DECREF(shared_args); - (void)_sharedobjectproxy_exit(self, &state); - PyMem_RawFree(shared_args_state); + return shared_args; +} + +typedef struct { + const char *key; + Py_ssize_t key_length; + _PyXI_proxy_share value; +} _SharedObjectProxy_dict_pair; + +static int +_sharedobjectproxy_init_shared_kwargs(PyObject *kwargs, SharedObjectProxy *self, + _SharedObjectProxy_dict_pair **kwarg_pairs) +{ + if (kwargs == NULL) { + return 0; + } + Py_ssize_t kwarg_size = PyDict_GET_SIZE(kwargs); + *kwarg_pairs = PyMem_RawCalloc(kwarg_size, sizeof(_SharedObjectProxy_dict_pair)); + if (*kwarg_pairs == NULL) { + PyErr_NoMemory(); + return -1; + } + + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(kwargs, &pos, &key, &value)) { + // XXX Can kwarg keys be dictionary subclasses? + assert(PyUnicode_Check(key)); + Py_ssize_t index = pos - 1; + assert(index >= 0); + assert(index < kwarg_size); + _SharedObjectProxy_dict_pair *pair = kwarg_pairs[index]; + assert(pair->key == NULL); + assert(pair->key_length == 0); + const char *key_str = PyUnicode_AsUTF8AndSize(key, &pair->key_length); + if (key_str == NULL) { + for (Py_ssize_t i = 0; i < pos; ++i) { + _sharedobjectproxy_finish_share(&kwarg_pairs[i]->value); + } PyMem_RawFree(kwarg_pairs); - return NULL; + return -1; + } - for (Py_ssize_t i = 0; i < kwarg_size; ++i) { - struct _dict_pair *pair = &kwarg_pairs[i]; - assert(pair->key != NULL); - PyObject *key = PyUnicode_FromStringAndSize(pair->key, pair->key_length); - if (key == NULL) { - Py_DECREF(shared_args); - Py_DECREF(shared_kwargs); - (void)_sharedobjectproxy_exit(self, &state); - PyMem_RawFree(shared_args_state); - PyMem_RawFree(kwarg_pairs); - return NULL; - } - PyObject *shared_kwarg = _sharedobjectproxy_copy_for_interp(&pair->value); - if (shared_kwarg == NULL) { - Py_DECREF(shared_args); - Py_DECREF(shared_kwargs); - (void)_sharedobjectproxy_exit(self, &state); - PyMem_RawFree(shared_args_state); - PyMem_RawFree(kwarg_pairs); - return NULL; - } - if (PyDict_SetItem(shared_kwargs, key, shared_kwarg) < 0) { - Py_DECREF(shared_args); - Py_DECREF(shared_kwargs); - (void)_sharedobjectproxy_exit(self, &state); - PyMem_RawFree(shared_args_state); - PyMem_RawFree(kwarg_pairs); - return NULL; + pair->key = key_str; + if (_sharedobjectproxy_init_share(&pair->value, self, value) < 0) { + for (Py_ssize_t i = 0; i < pos; ++i) { + _sharedobjectproxy_finish_share(&kwarg_pairs[i]->value); } + PyMem_RawFree(kwarg_pairs); + return -1; + } + } + + return 0; +} + +static PyObject * +_sharedobjectproxy_construct_shared_kwargs(PyObject *kwargs, _SharedObjectProxy_dict_pair *pairs) +{ + if (kwargs == NULL) { + return NULL; + } + PyObject *shared_kwargs = PyDict_New(); + if (shared_kwargs == NULL) { + return NULL; + } + for (Py_ssize_t i = 0; i < PyDict_GET_SIZE(kwargs); ++i) { + _SharedObjectProxy_dict_pair *pair = &pairs[i]; + assert(pair->key != NULL); + PyObject *key = PyUnicode_FromStringAndSize(pair->key, pair->key_length); + if (key == NULL) { + Py_DECREF(shared_kwargs); + return NULL; + } + PyObject *shared_kwarg = _sharedobjectproxy_copy_for_interp(&pair->value); + if (shared_kwarg == NULL) { + Py_DECREF(shared_kwargs); + Py_DECREF(key); + return NULL; + } + int res = PyDict_SetItem(shared_kwargs, key, shared_kwarg); + Py_DECREF(key); + Py_DECREF(shared_kwarg); + if (res < 0) { + Py_DECREF(shared_kwargs); + return NULL; } } + return shared_kwargs; +} + +static void +_sharedobjectproxy_close_shared_kwargs(PyObject *kwargs, _SharedObjectProxy_dict_pair *pairs) +{ + if (kwargs == NULL) { + return; + } + Py_ssize_t size = PyDict_GET_SIZE(kwargs); + for (Py_ssize_t i = 0; i < size; ++i) { + _sharedobjectproxy_finish_share(&pairs[i].value); + } + PyMem_RawFree(pairs); +} + +static PyObject * +sharedobjectproxy_tp_call(PyObject *op, PyObject *args, PyObject *kwargs) { + SharedObjectProxy *self = SharedObjectProxy_CAST(op); + _PyXI_proxy_share *args_state = _sharedobjectproxy_init_shared_args(args, self); + if (args_state == NULL) { + return NULL; + } + + _SharedObjectProxy_dict_pair *kwarg_pairs; + if (_sharedobjectproxy_init_shared_kwargs(kwargs, self, &kwarg_pairs) < 0) { + _sharedobjectproxy_close_shared_args(args, args_state); + return NULL; + } + + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(self, &state) < 0) { + return NULL; + } + + PyObject *shared_args = _sharedobjectproxy_construct_shared_args(args, args_state); + if (shared_args == NULL) { + (void)_sharedobjectproxy_exit(self, &state); + _sharedobjectproxy_close_shared_args(args, args_state); + _sharedobjectproxy_close_shared_kwargs(kwargs, kwarg_pairs); + return NULL; + } + + + PyObject *shared_kwargs = _sharedobjectproxy_construct_shared_kwargs(kwargs, kwarg_pairs); + if (shared_kwargs == NULL && PyErr_Occurred()) { + Py_DECREF(shared_args); + (void)_sharedobjectproxy_exit(self, &state); + _sharedobjectproxy_close_shared_args(args, args_state); + _sharedobjectproxy_close_shared_kwargs(kwargs, kwarg_pairs); + return NULL; + } PyObject *res = PyObject_Call(SharedObjectProxy_OBJECT(self), shared_args, shared_kwargs); Py_DECREF(shared_args); Py_XDECREF(shared_kwargs); - return _sharedobjectproxy_wrap_result(self, res, &state); + PyObject *ret = _sharedobjectproxy_wrap_result(self, res, &state); + _sharedobjectproxy_close_shared_args(args, args_state); + _sharedobjectproxy_close_shared_kwargs(kwargs, kwarg_pairs); + return ret; } static PyObject * From 87348d85a9213a6ced53c038846045a7974a2bd0 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 03:07:17 -0500 Subject: [PATCH 14/31] Fix a small leak. --- Modules/_interpretersmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index cf8bcdfd57d51b..16adabefa8794f 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -907,6 +907,7 @@ _sharedobjectproxy_double_share_common(SharedObjectProxy *self, return -1; } if (_sharedobjectproxy_init_share(shared_second, self, second) < 0) { + _sharedobjectproxy_finish_share(shared_first); return -1; } if (_sharedobjectproxy_enter(self, state) < 0) { From 355b37e6f40b14d190fe19c7bac4f4a0e00476a9 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 12:12:44 -0500 Subject: [PATCH 15/31] Fix leak at the end of _sharedobjectproxy_wrap_result(). --- Modules/_interpretersmodule.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 16adabefa8794f..7fa49b2f0c23de 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -627,13 +627,20 @@ _sharedobjectproxy_wrap_result(SharedObjectProxy *self, PyObject *result, } Py_DECREF(result); + PyObject *ret; + if (state->to_restore != NULL) { + PyThreadState *save = PyThreadState_Swap(state->to_restore); + ret = _sharedobjectproxy_copy_for_interp(&shared_result); + PyThreadState_Swap(save); + } else { + ret = _sharedobjectproxy_copy_for_interp(&shared_result); + } + + _sharedobjectproxy_finish_share(&shared_result); if (_sharedobjectproxy_exit(self, state) < 0) { - _sharedobjectproxy_finish_share(&shared_result); return NULL; } - PyObject *ret = _sharedobjectproxy_copy_for_interp(&shared_result); - //_sharedobjectproxy_finish_share(&shared_result); return ret; } From 323b96a09b6aabd2a8f0777492715d0a55c0a3d3 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 13:03:43 -0500 Subject: [PATCH 16/31] Turn the SharedObjectProxy type into a heap type, and fix related leaks. --- Modules/_interpretersmodule.c | 208 +++++++++++++++++++--------------- 1 file changed, 117 insertions(+), 91 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 7fa49b2f0c23de..2da068fc7fd8b6 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -328,9 +328,7 @@ typedef struct { /* heap types */ PyTypeObject *XIBufferViewType; - - /* Linked list of shared object proxies available for use. */ - PyObject *available_proxies; + PyTypeObject *SharedObjectProxyType; } module_state; static inline module_state * @@ -362,6 +360,7 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) { /* heap types */ Py_VISIT(state->XIBufferViewType); + Py_VISIT(state->SharedObjectProxyType); return 0; } @@ -371,6 +370,7 @@ clear_module_state(module_state *state) { /* heap types */ Py_CLEAR(state->XIBufferViewType); + Py_CLEAR(state->SharedObjectProxyType); return 0; } @@ -391,15 +391,14 @@ typedef struct _Py_shared_object_proxy { #endif } SharedObjectProxy; -static PyTypeObject SharedObjectProxy_Type; -#define SharedObjectProxy_CheckExact(op) (Py_TYPE(_PyObject_CAST(op)) == &SharedObjectProxy_Type) +static PyTypeObject * +_get_current_sharedobjectproxy_type(void); #define SharedObjectProxy_RAW_CAST(op) ((SharedObjectProxy *)op) static inline SharedObjectProxy * SharedObjectProxy_CAST(PyObject *op) { assert(op != NULL); - _PyObject_ASSERT(op, SharedObjectProxy_CheckExact(op)); SharedObjectProxy *proxy = (SharedObjectProxy *)op; assert(proxy->object != NULL); assert(Py_REFCNT(proxy->object) > 0); @@ -430,15 +429,23 @@ _sharedobjectproxy_create(PyObject *object, PyInterpreterState *owning_interp) { assert(object != NULL); assert(owning_interp != NULL); - assert(!SharedObjectProxy_CheckExact(object)); - SharedObjectProxy *proxy = SharedObjectProxy_CAST(sharedobjectproxy_new(&SharedObjectProxy_Type, + PyTypeObject *type = _get_current_sharedobjectproxy_type(); + if (type == NULL) { + return NULL; + } + assert(Py_TYPE(object) != type); + SharedObjectProxy *proxy = SharedObjectProxy_CAST(sharedobjectproxy_new(type, NULL, NULL)); if (proxy == NULL) { return NULL; } - proxy->object = Py_NewRef(object); + assert(PyObject_GC_IsTracked((PyObject *)proxy)); + if (_PyInterpreterState_GET() == owning_interp) { + Py_INCREF(object); + } + proxy->object = object; proxy->interp = owning_interp; return (PyObject *)proxy; } @@ -453,7 +460,6 @@ sharedobjectproxy_clear(PyObject *op) return 0; } - assert(self->object == NULL || !SharedObjectProxy_CheckExact(self->object)); Py_CLEAR(self->object); return 0; } @@ -467,7 +473,6 @@ sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) return 0; } - assert(self->object == NULL || !SharedObjectProxy_CheckExact(self->object)); Py_VISIT(self->object); return 0; } @@ -480,6 +485,7 @@ sharedobjectproxy_dealloc(PyObject *op) (void)sharedobjectproxy_clear(op); PyObject_GC_UnTrack(self); tp->tp_free(self); + Py_DECREF(tp); } typedef struct { @@ -1068,8 +1074,6 @@ sharedobjectproxy_ ##name(PyObject *op) \ return _sharedobjectproxy_ssize_result(op, func); \ } -#define _SharedObjectProxy_FIELD(name) .name = sharedobjectproxy_ ##name - _SharedObjectProxy_NO_ARG(tp_iter, PyObject_GetIter); _SharedObjectProxy_NO_ARG(tp_iternext, PyIter_Next); _SharedObjectProxy_NO_ARG(tp_str, PyObject_Str); @@ -1095,6 +1099,7 @@ sharedobjectproxy_tp_hash(PyObject *op) return result; } +/* Number wrappers */ _SharedObjectProxy_ONE_ARG(nb_add, PyNumber_Add); _SharedObjectProxy_ONE_ARG(nb_subtract, PyNumber_Subtract); _SharedObjectProxy_ONE_ARG(nb_multiply, PyNumber_Multiply); @@ -1130,53 +1135,12 @@ _SharedObjectProxy_NO_ARG(nb_index, PyNumber_Index); _SharedObjectProxy_ONE_ARG(nb_matrix_multiply, PyNumber_MatrixMultiply); _SharedObjectProxy_ONE_ARG(nb_inplace_matrix_multiply, PyNumber_InPlaceMatrixMultiply); -static PyNumberMethods SharedObjectProxy_NumberMethods = { - _SharedObjectProxy_FIELD(nb_add), - _SharedObjectProxy_FIELD(nb_subtract), - _SharedObjectProxy_FIELD(nb_multiply), - _SharedObjectProxy_FIELD(nb_remainder), - _SharedObjectProxy_FIELD(nb_power), - _SharedObjectProxy_FIELD(nb_divmod), - _SharedObjectProxy_FIELD(nb_negative), - _SharedObjectProxy_FIELD(nb_positive), - _SharedObjectProxy_FIELD(nb_absolute), - _SharedObjectProxy_FIELD(nb_invert), - _SharedObjectProxy_FIELD(nb_lshift), - _SharedObjectProxy_FIELD(nb_rshift), - _SharedObjectProxy_FIELD(nb_and), - _SharedObjectProxy_FIELD(nb_xor), - _SharedObjectProxy_FIELD(nb_or), - _SharedObjectProxy_FIELD(nb_int), - _SharedObjectProxy_FIELD(nb_float), - _SharedObjectProxy_FIELD(nb_inplace_add), - _SharedObjectProxy_FIELD(nb_inplace_subtract), - _SharedObjectProxy_FIELD(nb_inplace_multiply), - _SharedObjectProxy_FIELD(nb_inplace_remainder), - _SharedObjectProxy_FIELD(nb_inplace_power), - _SharedObjectProxy_FIELD(nb_inplace_lshift), - _SharedObjectProxy_FIELD(nb_inplace_rshift), - _SharedObjectProxy_FIELD(nb_inplace_and), - _SharedObjectProxy_FIELD(nb_inplace_xor), - _SharedObjectProxy_FIELD(nb_inplace_or), - _SharedObjectProxy_FIELD(nb_floor_divide), - _SharedObjectProxy_FIELD(nb_true_divide), - _SharedObjectProxy_FIELD(nb_inplace_floor_divide), - _SharedObjectProxy_FIELD(nb_inplace_true_divide), - _SharedObjectProxy_FIELD(nb_index), - _SharedObjectProxy_FIELD(nb_matrix_multiply), - _SharedObjectProxy_FIELD(nb_inplace_matrix_multiply) -}; - +/* Async wrappers */ _SharedObjectProxy_NO_ARG(am_await, _PyCoro_GetAwaitableIter); _SharedObjectProxy_NO_ARG(am_aiter, PyObject_GetAIter); _SharedObjectProxy_NO_ARG(am_anext, _PyEval_GetANext); -static PyAsyncMethods SharedObjectProxy_AsyncMethods = { - _SharedObjectProxy_FIELD(am_await), - _SharedObjectProxy_FIELD(am_aiter), - _SharedObjectProxy_FIELD(am_anext), -}; - +/* Sequence wrappers */ _SharedObjectProxy_SSIZE_RETURN(sq_length, PySequence_Size); _SharedObjectProxy_ONE_ARG(sq_concat, PySequence_Concat); _SharedObjectProxy_SSIZE_ARG(sq_repeat, PySequence_Repeat) @@ -1232,39 +1196,19 @@ sharedobjectproxy_sq_contains(PyObject *op, PyObject *item) return result; } -static PySequenceMethods SharedObjectProxy_SequenceMethods = { - _SharedObjectProxy_FIELD(sq_concat), - _SharedObjectProxy_FIELD(sq_length), - _SharedObjectProxy_FIELD(sq_repeat), - _SharedObjectProxy_FIELD(sq_item), - _SharedObjectProxy_FIELD(sq_inplace_concat), - _SharedObjectProxy_FIELD(sq_inplace_repeat), - _SharedObjectProxy_FIELD(sq_ass_item), - _SharedObjectProxy_FIELD(sq_contains) -}; - +/* Mapping wrappers */ _SharedObjectProxy_SSIZE_RETURN(mp_length, PyMapping_Length); _SharedObjectProxy_ONE_ARG(mp_subscript, PyObject_GetItem); _SharedObjectProxy_TWO_ARG_INT(mp_ass_subscript, PyObject_SetItem); -static PyMappingMethods SharedObjectProxy_MappingMethods = { - _SharedObjectProxy_FIELD(mp_length), - _SharedObjectProxy_FIELD(mp_subscript), - _SharedObjectProxy_FIELD(mp_ass_subscript) -}; +#define _SharedObjectProxy_FIELD(name) {Py_ ##name, sharedobjectproxy_ ##name} -/* This has to be a static type as it can be referenced from any interpreter - * through a Py_TYPE() on a proxy instance. */ -static PyTypeObject SharedObjectProxy_Type = { - .tp_name = MODULE_NAME_STR ".SharedObjectProxy", - .tp_basicsize = sizeof(SharedObjectProxy), - .tp_flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), - .tp_new = sharedobjectproxy_new, - .tp_traverse = sharedobjectproxy_traverse, - .tp_clear = sharedobjectproxy_clear, - .tp_dealloc = sharedobjectproxy_dealloc, +static PyType_Slot SharedObjectProxy_slots[] = { + {Py_tp_new, sharedobjectproxy_new}, + {Py_tp_traverse, sharedobjectproxy_traverse}, + {Py_tp_clear, sharedobjectproxy_clear}, + {Py_tp_dealloc, sharedobjectproxy_dealloc}, _SharedObjectProxy_FIELD(tp_getattro), _SharedObjectProxy_FIELD(tp_setattro), _SharedObjectProxy_FIELD(tp_call), @@ -1273,12 +1217,67 @@ static PyTypeObject SharedObjectProxy_Type = { _SharedObjectProxy_FIELD(tp_iter), _SharedObjectProxy_FIELD(tp_iternext), _SharedObjectProxy_FIELD(tp_hash), - .tp_as_number = &SharedObjectProxy_NumberMethods, - .tp_as_async = &SharedObjectProxy_AsyncMethods, - .tp_as_sequence = &SharedObjectProxy_SequenceMethods, - .tp_as_mapping = &SharedObjectProxy_MappingMethods + _SharedObjectProxy_FIELD(mp_length), + _SharedObjectProxy_FIELD(mp_subscript), + _SharedObjectProxy_FIELD(mp_ass_subscript), + _SharedObjectProxy_FIELD(sq_concat), + _SharedObjectProxy_FIELD(sq_length), + _SharedObjectProxy_FIELD(sq_repeat), + _SharedObjectProxy_FIELD(sq_item), + _SharedObjectProxy_FIELD(sq_inplace_concat), + _SharedObjectProxy_FIELD(sq_inplace_repeat), + _SharedObjectProxy_FIELD(sq_ass_item), + _SharedObjectProxy_FIELD(sq_contains), + _SharedObjectProxy_FIELD(am_await), + _SharedObjectProxy_FIELD(am_aiter), + _SharedObjectProxy_FIELD(am_anext), + _SharedObjectProxy_FIELD(nb_add), + _SharedObjectProxy_FIELD(nb_subtract), + _SharedObjectProxy_FIELD(nb_multiply), + _SharedObjectProxy_FIELD(nb_remainder), + _SharedObjectProxy_FIELD(nb_power), + _SharedObjectProxy_FIELD(nb_divmod), + _SharedObjectProxy_FIELD(nb_negative), + _SharedObjectProxy_FIELD(nb_positive), + _SharedObjectProxy_FIELD(nb_absolute), + _SharedObjectProxy_FIELD(nb_invert), + _SharedObjectProxy_FIELD(nb_lshift), + _SharedObjectProxy_FIELD(nb_rshift), + _SharedObjectProxy_FIELD(nb_and), + _SharedObjectProxy_FIELD(nb_xor), + _SharedObjectProxy_FIELD(nb_or), + _SharedObjectProxy_FIELD(nb_int), + _SharedObjectProxy_FIELD(nb_float), + _SharedObjectProxy_FIELD(nb_inplace_add), + _SharedObjectProxy_FIELD(nb_inplace_subtract), + _SharedObjectProxy_FIELD(nb_inplace_multiply), + _SharedObjectProxy_FIELD(nb_inplace_remainder), + _SharedObjectProxy_FIELD(nb_inplace_power), + _SharedObjectProxy_FIELD(nb_inplace_lshift), + _SharedObjectProxy_FIELD(nb_inplace_rshift), + _SharedObjectProxy_FIELD(nb_inplace_and), + _SharedObjectProxy_FIELD(nb_inplace_xor), + _SharedObjectProxy_FIELD(nb_inplace_or), + _SharedObjectProxy_FIELD(nb_floor_divide), + _SharedObjectProxy_FIELD(nb_true_divide), + _SharedObjectProxy_FIELD(nb_inplace_floor_divide), + _SharedObjectProxy_FIELD(nb_inplace_true_divide), + _SharedObjectProxy_FIELD(nb_index), + _SharedObjectProxy_FIELD(nb_matrix_multiply), + _SharedObjectProxy_FIELD(nb_inplace_matrix_multiply), + {0, NULL}, +}; + +static PyType_Spec SharedObjectProxy_spec = { + .name = MODULE_NAME_STR ".SharedObjectProxy", + .basicsize = sizeof(SharedObjectProxy), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC), + .slots = SharedObjectProxy_slots, }; + static PyObject * sharedobjectproxy_xid(_PyXIData_t *data) { @@ -1286,21 +1285,37 @@ sharedobjectproxy_xid(_PyXIData_t *data) return _sharedobjectproxy_create(proxy->object, proxy->interp); } +static void +sharedobjectproxy_shared_free(void *data) +{ + SharedObjectProxy *proxy = SharedObjectProxy_CAST(data); + Py_DECREF(proxy); +} + static int sharedobjectproxy_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { _PyXIData_Init(data, tstate->interp, NULL, obj, sharedobjectproxy_xid); + data->free = sharedobjectproxy_shared_free; return 0; } static int -register_sharedobjectproxy(PyObject *mod) +register_sharedobjectproxy(PyObject *mod, PyTypeObject **p_state) { - if (PyModule_AddType(mod, &SharedObjectProxy_Type) < 0) { + assert(*p_state == NULL); + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &SharedObjectProxy_spec, NULL); + if (cls == NULL) { + return -1; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); return -1; } + *p_state = cls; - if (ensure_xid_class(&SharedObjectProxy_Type, GETDATA(sharedobjectproxy_shared)) < 0) { + if (ensure_xid_class(cls, GETDATA(sharedobjectproxy_shared)) < 0) { return -1; } @@ -1317,6 +1332,17 @@ _get_current_xibufferview_type(void) return state->XIBufferViewType; } +static PyTypeObject * +_get_current_sharedobjectproxy_type(void) +{ + module_state *state = _get_current_module_state(); + if (state == NULL) { + return NULL; + } + + return state->SharedObjectProxyType; +} + /* interpreter-specific code ************************************************/ static int @@ -2580,7 +2606,7 @@ module_exec(PyObject *mod) goto error; } - if (register_sharedobjectproxy(mod) < 0) { + if (register_sharedobjectproxy(mod, &state->SharedObjectProxyType) < 0) { goto error; } From b414dbb4b5a60fc2fbb29166b47d2fc5ef2b9391 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 13:05:23 -0500 Subject: [PATCH 17/31] Remove some problematic assertions. --- Modules/_interpretersmodule.c | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 2da068fc7fd8b6..0133624d692092 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -393,21 +393,7 @@ typedef struct _Py_shared_object_proxy { static PyTypeObject * _get_current_sharedobjectproxy_type(void); -#define SharedObjectProxy_RAW_CAST(op) ((SharedObjectProxy *)op) - -static inline SharedObjectProxy * -SharedObjectProxy_CAST(PyObject *op) -{ - assert(op != NULL); - SharedObjectProxy *proxy = (SharedObjectProxy *)op; - assert(proxy->object != NULL); - assert(Py_REFCNT(proxy->object) > 0); - assert(!_PyMem_IsPtrFreed(proxy->object)); - assert(proxy->interp != NULL); - assert(!_PyMem_IsPtrFreed(proxy->interp)); - return proxy; -} -#define SharedObjectProxy_CAST(op) SharedObjectProxy_CAST(_PyObject_CAST(op)) +#define SharedObjectProxy_CAST(op) ((SharedObjectProxy *)op) #define SharedObjectProxy_OBJECT(op) FT_ATOMIC_LOAD_PTR_RELAXED(SharedObjectProxy_CAST(op)->object) static PyObject * @@ -454,7 +440,7 @@ _sharedobjectproxy_create(PyObject *object, PyInterpreterState *owning_interp) static int sharedobjectproxy_clear(PyObject *op) { - SharedObjectProxy *self = SharedObjectProxy_RAW_CAST(op); + SharedObjectProxy *self = SharedObjectProxy_CAST(op); // Don't clear from another interpreter if (self->interp != _PyInterpreterState_GET()) { return 0; @@ -467,7 +453,7 @@ sharedobjectproxy_clear(PyObject *op) static int sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) { - SharedObjectProxy *self = SharedObjectProxy_RAW_CAST(op); + SharedObjectProxy *self = SharedObjectProxy_CAST(op); // Don't traverse from another interpreter if (self->interp != _PyInterpreterState_GET()) { return 0; @@ -480,7 +466,7 @@ sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) static void sharedobjectproxy_dealloc(PyObject *op) { - SharedObjectProxy *self = SharedObjectProxy_RAW_CAST(op); + SharedObjectProxy *self = SharedObjectProxy_CAST(op); PyTypeObject *tp = Py_TYPE(self); (void)sharedobjectproxy_clear(op); PyObject_GC_UnTrack(self); From 1effb7ad4753da2e1036d921983bbb2adc1efc15 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 13:12:57 -0500 Subject: [PATCH 18/31] Ensure that all interpreters are closed in the tests. --- .../test_interpreters/test_object_proxy.py | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 7b237b55cc6318..633b95c841894d 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -8,15 +8,26 @@ from test.test_interpreters.utils import TestBase from threading import Barrier, Thread, Lock from concurrent import interpreters +from contextlib import contextmanager class SharedObjectProxyTests(TestBase): - def run_concurrently(self, func, num_threads=4): + @contextmanager + def create_interp(self, **to_prepare): + interp = interpreters.create() + try: + if to_prepare != {}: + interp.prepare_main(**to_prepare) + yield interp + finally: + interp.close() + + def run_concurrently(self, func, num_threads=4, **to_prepare): barrier = Barrier(num_threads) def thread(): - interp = interpreters.create() - barrier.wait() - func(interp) + with self.create_interp(**to_prepare) as interp: + barrier.wait() + func(interp) with threading_helper.catch_threading_exception() as cm: with threading_helper.start_threads((Thread(target=thread) for _ in range(num_threads))): @@ -54,22 +65,21 @@ class Test: def silly(self): return "silly" - interp = interpreters.create() obj = Test() - proxy = share(obj) obj.test = "silly" - interp.prepare_main(proxy=proxy) - interp.exec("assert proxy.test == 'silly'") - interp.exec("assert isinstance(proxy.test, str)") - interp.exec("""if True: - from concurrent.interpreters import SharedObjectProxy - method = proxy.silly - assert isinstance(method, SharedObjectProxy) - assert method() == 'silly' - assert isinstance(method(), str) - """) - with self.assertRaises(interpreters.ExecutionFailed): - interp.exec("proxy.noexist") + proxy = share(obj) + with self.create_interp(proxy=proxy) as interp: + interp.exec("assert proxy.test == 'silly'") + interp.exec("assert isinstance(proxy.test, str)") + interp.exec("""if True: + from concurrent.interpreters import SharedObjectProxy + method = proxy.silly + assert isinstance(method, SharedObjectProxy) + assert method() == 'silly' + assert isinstance(method(), str) + """) + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec("proxy.noexist") @threading_helper.requires_working_threading() def test_access_proxy_concurrently(self): @@ -86,12 +96,11 @@ def increment(self): proxy = share(test) def thread(interp): - interp.prepare_main(proxy=proxy) for _ in range(100): interp.exec("proxy.increment()") interp.exec("assert isinstance(proxy.value, int)") - self.run_concurrently(thread) + self.run_concurrently(thread, proxy=proxy) self.assertEqual(test.value, 400) def test_proxy_call(self): @@ -106,13 +115,12 @@ def my_function(arg=1, /, *, arg2=2): self.assertEqual(proxy(0, arg2=1), 68) self.assertEqual(proxy(2), 71) - interp = interpreters.create() - interp.prepare_main(proxy=proxy) - interp.exec("""if True: - assert isinstance(proxy(), int) - assert proxy() == 70 - assert proxy(0, arg2=1) == 68 - assert proxy(2) == 71""") + with self.create_interp(proxy=proxy) as interp: + interp.exec("""if True: + assert isinstance(proxy(), int) + assert proxy() == 70 + assert proxy(0, arg2=1) == 68 + assert proxy(2) == 71""") From c4ac44202dc60ae9704abef7060ecb1a28d3a63f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 14:49:13 -0500 Subject: [PATCH 19/31] Fix assertion failure when creating a new proxy. --- Lib/test/test_interpreters/test_object_proxy.py | 14 ++++++++++++++ Modules/_interpretersmodule.c | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 633b95c841894d..25a3fc03153c10 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -122,6 +122,20 @@ def my_function(arg=1, /, *, arg2=2): assert proxy(0, arg2=1) == 68 assert proxy(2) == 71""") + def test_proxy_call_args(self): + def shared(arg): + return type(arg).__name__ + + proxy = share(shared) + self.assertEqual(proxy(1), "int") + self.assertEqual(proxy('test'), "str") + self.assertEqual(proxy(object()), "SharedObjectProxy") + + with self.create_interp(proxy=proxy) as interp: + interp.exec("assert proxy(1) == 'int'") + interp.exec("assert proxy('test') == 'str'") + interp.exec("assert proxy(object()) == 'SharedObjectProxy'") + diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 0133624d692092..5fd32ea3853bbc 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -560,7 +560,7 @@ _sharedobjectproxy_init_share(_PyXI_proxy_share *share, PyErr_Clear(); share->object_to_wrap = Py_NewRef(op); share->xidata = NULL; - _PyXIData_Free(xidata); + PyMem_RawFree(xidata); } else { share->object_to_wrap = NULL; share->xidata = xidata; @@ -641,7 +641,7 @@ _sharedobjectproxy_init_shared_args(PyObject *args, SharedObjectProxy *self) { assert(PyTuple_Check(args)); Py_ssize_t args_size = PyTuple_GET_SIZE(args); - _PyXI_proxy_share *shared_args_state = PyMem_RawMalloc(args_size * sizeof(_PyXI_proxy_share)); + _PyXI_proxy_share *shared_args_state = PyMem_RawCalloc(args_size, sizeof(_PyXI_proxy_share)); if (shared_args_state == NULL) { PyErr_NoMemory(); return NULL; From 14122007308f1210b041edfd682e991af5fea908 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 15:11:01 -0500 Subject: [PATCH 20/31] Give each proxy a dedicated reference. --- .../test_interpreters/test_object_proxy.py | 17 ++ Modules/_interpretersmodule.c | 163 ++++++++++-------- 2 files changed, 109 insertions(+), 71 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 25a3fc03153c10..119cf784cb8f20 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -136,7 +136,24 @@ def shared(arg): interp.exec("assert proxy('test') == 'str'") interp.exec("assert proxy(object()) == 'SharedObjectProxy'") + def test_proxy_call_return(self): + class Test: + def __init__(self, silly): + self.silly = silly + + def shared(): + return Test("silly") + + proxy = share(shared) + res = proxy() + self.assertIsInstance(res, SharedObjectProxy) + self.assertEqual(res.silly, "silly") + with self.create_interp(proxy=proxy) as interp: + interp.exec("""if True: + obj = proxy() + assert obj.silly == 'silly' + assert type(obj).__name__ == 'SharedObjectProxy'""") if __name__ == '__main__': diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 5fd32ea3853bbc..ba980c99f16023 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -396,6 +396,68 @@ _get_current_sharedobjectproxy_type(void); #define SharedObjectProxy_CAST(op) ((SharedObjectProxy *)op) #define SharedObjectProxy_OBJECT(op) FT_ATOMIC_LOAD_PTR_RELAXED(SharedObjectProxy_CAST(op)->object) +typedef struct { + PyThreadState *to_restore; + PyThreadState *for_call; +} _PyXI_proxy_state; + +static int +_sharedobjectproxy_enter(SharedObjectProxy *self, _PyXI_proxy_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + assert(self != NULL); + assert(tstate != NULL); + if (tstate->interp == self->interp) { + // No need to switch; already in the correct interpreter + state->to_restore = NULL; + state->for_call = NULL; + return 0; + } + state->to_restore = tstate; + PyThreadState *for_call = _PyThreadState_NewBound(self->interp, + _PyThreadState_WHENCE_EXEC); + state->for_call = for_call; + if (for_call == NULL) { + PyErr_NoMemory(); + return -1; + } + _PyThreadState_Detach(tstate); + _PyThreadState_Attach(state->for_call); + assert(_PyInterpreterState_GET() == self->interp); + return 0; +} + +static int +_sharedobjectproxy_exit(SharedObjectProxy *self, _PyXI_proxy_state *state) +{ + assert(_PyInterpreterState_GET() == self->interp); + if (state->to_restore == NULL) { + // Nothing to do. We were already in the correct interpreter. + return PyErr_Occurred() == NULL ? 0 : -1; + } + + PyThreadState *tstate = state->for_call; + int should_throw = 0; + if (_PyErr_Occurred(tstate)) { + // TODO: Serialize and transfer the exception to the calling + // interpreter. + PyErr_FormatUnraisable("Exception occured in interpreter"); + should_throw = 1; + } + + assert(state->for_call == _PyThreadState_GET()); + PyThreadState_Clear(state->for_call); + PyThreadState_Swap(state->to_restore); + PyThreadState_Delete(state->for_call); + + if (should_throw) { + _PyErr_SetString(state->to_restore, PyExc_RuntimeError, "exception in interpreter"); + return -1; + } + + return 0; +} + static PyObject * sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -428,11 +490,24 @@ _sharedobjectproxy_create(PyObject *object, PyInterpreterState *owning_interp) } assert(PyObject_GC_IsTracked((PyObject *)proxy)); - if (_PyInterpreterState_GET() == owning_interp) { - Py_INCREF(object); - } - proxy->object = object; + proxy->object = NULL; proxy->interp = owning_interp; + + // We have to be in the correct interpreter to increment the object's + // reference count. + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(proxy, &state) < 0) { + Py_DECREF(proxy); + return NULL; + } + + proxy->object = Py_NewRef(object); + + if (_sharedobjectproxy_exit(proxy, &state)) { + Py_DECREF(proxy); + return NULL; + } + return (PyObject *)proxy; } @@ -441,21 +516,25 @@ static int sharedobjectproxy_clear(PyObject *op) { SharedObjectProxy *self = SharedObjectProxy_CAST(op); - // Don't clear from another interpreter - if (self->interp != _PyInterpreterState_GET()) { + if (self->object == NULL) { return 0; } + _PyXI_proxy_state state; + if (_sharedobjectproxy_enter(self, &state) < 0) { + // The object leaks :( + return -1; + } Py_CLEAR(self->object); - return 0; + return _sharedobjectproxy_exit(self, &state); } static int sharedobjectproxy_traverse(PyObject *op, visitproc visit, void *arg) { SharedObjectProxy *self = SharedObjectProxy_CAST(op); - // Don't traverse from another interpreter if (self->interp != _PyInterpreterState_GET()) { + // Don't traverse from another interpreter return 0; } @@ -468,72 +547,14 @@ sharedobjectproxy_dealloc(PyObject *op) { SharedObjectProxy *self = SharedObjectProxy_CAST(op); PyTypeObject *tp = Py_TYPE(self); - (void)sharedobjectproxy_clear(op); + PyObject *err = PyErr_GetRaisedException(); + if (sharedobjectproxy_clear(op) < 0) { + PyErr_FormatUnraisable("Exception in proxy destructor"); + }; PyObject_GC_UnTrack(self); tp->tp_free(self); Py_DECREF(tp); -} - -typedef struct { - PyThreadState *to_restore; - PyThreadState *for_call; -} _PyXI_proxy_state; - -static int -_sharedobjectproxy_enter(SharedObjectProxy *self, _PyXI_proxy_state *state) -{ - PyThreadState *tstate = _PyThreadState_GET(); - assert(self != NULL); - assert(tstate != NULL); - if (tstate->interp == self->interp) { - // No need to switch; already in the correct interpreter - state->to_restore = NULL; - state->for_call = NULL; - return 0; - } - state->to_restore = tstate; - PyThreadState *for_call = _PyThreadState_NewBound(self->interp, - _PyThreadState_WHENCE_EXEC); - state->for_call = for_call; - if (for_call == NULL) { - PyErr_NoMemory(); - return -1; - } - _PyThreadState_Detach(tstate); - _PyThreadState_Attach(state->for_call); - assert(_PyInterpreterState_GET() == self->interp); - return 0; -} - -static int -_sharedobjectproxy_exit(SharedObjectProxy *self, _PyXI_proxy_state *state) -{ - assert(_PyInterpreterState_GET() == self->interp); - if (state->to_restore == NULL) { - // Nothing to do. We were already in the correct interpreter. - return PyErr_Occurred() == NULL ? 0 : -1; - } - - PyThreadState *tstate = state->for_call; - int should_throw = 0; - if (_PyErr_Occurred(tstate)) { - // TODO: Serialize and transfer the exception to the calling - // interpreter. - PyErr_FormatUnraisable("Exception occured in interpreter"); - should_throw = 1; - } - - assert(state->for_call == _PyThreadState_GET()); - PyThreadState_Clear(state->for_call); - PyThreadState_Swap(state->to_restore); - PyThreadState_Delete(state->for_call); - - if (should_throw) { - _PyErr_SetString(state->to_restore, PyExc_RuntimeError, "exception in interpreter"); - return -1; - } - - return 0; + PyErr_SetRaisedException(err); } typedef struct { From 8f82d4643908ef41bb8e1fbd61c6ac8ee5674aad Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 15:15:19 -0500 Subject: [PATCH 21/31] Add a test for calling concurrently. --- .../test_interpreters/test_object_proxy.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 119cf784cb8f20..79426977e941b9 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -155,6 +155,31 @@ def shared(): assert obj.silly == 'silly' assert type(obj).__name__ == 'SharedObjectProxy'""") + def test_proxy_call_concurrently(self): + def shared(arg, *, kwarg): + return arg + kwarg + + class Weird: + def __add__(_self, other): + self.assertIsInstance(_self, Weird) + self.assertIsInstance(other, int) + if other == 24: + ob = Weird() + ob.silly = "test" + return ob + return 42 + + def thread(interp): + interp.exec("assert proxy(1, kwarg=2) == 3") + interp.exec("assert proxy(2, kwarg=5) == 7") + interp.exec("assert proxy(weird, kwarg=5) == 42") + interp.exec("assert proxy(weird, kwarg=24).silly == 'test'") + + + proxy = share(shared) + weird = share(Weird()) + self.run_concurrently(thread, proxy=proxy, weird=weird) + if __name__ == '__main__': unittest.main() From d206dcdc66af80d182bfd65099cd3756847792be Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 15:20:55 -0500 Subject: [PATCH 22/31] Add a test for reference cycles. --- .../test_interpreters/test_object_proxy.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 79426977e941b9..5f0e944df312f1 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -180,6 +180,31 @@ def thread(interp): weird = share(Weird()) self.run_concurrently(thread, proxy=proxy, weird=weird) + def test_proxy_reference_cycle(self): + import gc + + called = 0 + + class Cycle: + def __init__(self, other): + self.cycle = self + self.other = other + + def __del__(self): + nonlocal called + called += 1 + + cycle_type = share(Cycle) + interp_a = cycle_type(0) + with self.create_interp(cycle_type=cycle_type, interp_a=interp_a) as interp: + interp.exec("interp_b = cycle_type(interp_a)") + + del interp_a + for _ in range(3): + gc.collect() + + self.assertEqual(called, 2) + if __name__ == '__main__': unittest.main() From b37c36a82e1ca59da16edb05e96ef0f2face0851 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 15:21:23 -0500 Subject: [PATCH 23/31] Add an extra assertion. --- Lib/test/test_interpreters/test_object_proxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 5f0e944df312f1..3cf7f62e3b42b2 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -199,6 +199,7 @@ def __del__(self): with self.create_interp(cycle_type=cycle_type, interp_a=interp_a) as interp: interp.exec("interp_b = cycle_type(interp_a)") + self.assertEqual(called, 0) del interp_a for _ in range(3): gc.collect() From 7d26ac0e5f6b534604f7c6ca5ce3b64d26ddc7c2 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 15:29:07 -0500 Subject: [PATCH 24/31] Add a test for concurrent attribute access. --- Lib/test/test_interpreters/test_object_proxy.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 3cf7f62e3b42b2..851214595a83bd 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -206,6 +206,20 @@ def __del__(self): self.assertEqual(called, 2) + def test_proxy_attribute_concurrently(self): + class Test: + def __init__(self): + self.value = 0 + + proxy = share(Test()) + def thread(interp): + for _ in range(1000): + interp.exec("proxy.value = 42") + interp.exec("proxy.value = 0") + interp.exec("assert proxy.value in (0, 42)") + + self.run_concurrently(thread, proxy=proxy) + if __name__ == '__main__': unittest.main() From 5cc541079eab82b1553d6d5f9182ddcbdb3d38e1 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 15:31:23 -0500 Subject: [PATCH 25/31] Run formatter on the tests. My file my rules :D --- .../test_interpreters/test_object_proxy.py | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 851214595a83bd..286268bf764157 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -2,8 +2,9 @@ from test.support import import_helper from test.support import threading_helper + # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_interpreters') +import_helper.import_module("_interpreters") from concurrent.interpreters import share, SharedObjectProxy from test.test_interpreters.utils import TestBase from threading import Barrier, Thread, Lock @@ -24,13 +25,16 @@ def create_interp(self, **to_prepare): def run_concurrently(self, func, num_threads=4, **to_prepare): barrier = Barrier(num_threads) + def thread(): with self.create_interp(**to_prepare) as interp: barrier.wait() func(interp) with threading_helper.catch_threading_exception() as cm: - with threading_helper.start_threads((Thread(target=thread) for _ in range(num_threads))): + with threading_helper.start_threads( + (Thread(target=thread) for _ in range(num_threads)) + ): pass if cm.exc_value is not None: @@ -41,7 +45,16 @@ def test_create(self): self.assertIsInstance(proxy, SharedObjectProxy) # Shareable objects should pass through - for shareable in (None, True, False, 100, 10000, "hello", b"world", memoryview(b"test")): + for shareable in ( + None, + True, + False, + 100, + 10000, + "hello", + b"world", + memoryview(b"test"), + ): self.assertTrue(interpreters.is_shareable(shareable)) with self.subTest(shareable=shareable): not_a_proxy = share(shareable) @@ -53,10 +66,12 @@ def test_create_concurrently(self): def thread(interp): for iteration in range(100): with self.subTest(iteration=iteration): - interp.exec("""if True: + interp.exec( + """if True: from concurrent.interpreters import share - share(object())""") + share(object())""" + ) self.run_concurrently(thread) @@ -71,13 +86,15 @@ def silly(self): with self.create_interp(proxy=proxy) as interp: interp.exec("assert proxy.test == 'silly'") interp.exec("assert isinstance(proxy.test, str)") - interp.exec("""if True: + interp.exec( + """if True: from concurrent.interpreters import SharedObjectProxy method = proxy.silly assert isinstance(method, SharedObjectProxy) assert method() == 'silly' assert isinstance(method(), str) - """) + """ + ) with self.assertRaises(interpreters.ExecutionFailed): interp.exec("proxy.noexist") @@ -105,6 +122,7 @@ def thread(interp): def test_proxy_call(self): constant = 67 # Hilarious + def my_function(arg=1, /, *, arg2=2): # We need the constant here to make this function unshareable. return constant + arg + arg2 @@ -116,11 +134,13 @@ def my_function(arg=1, /, *, arg2=2): self.assertEqual(proxy(2), 71) with self.create_interp(proxy=proxy) as interp: - interp.exec("""if True: + interp.exec( + """if True: assert isinstance(proxy(), int) assert proxy() == 70 assert proxy(0, arg2=1) == 68 - assert proxy(2) == 71""") + assert proxy(2) == 71""" + ) def test_proxy_call_args(self): def shared(arg): @@ -128,7 +148,7 @@ def shared(arg): proxy = share(shared) self.assertEqual(proxy(1), "int") - self.assertEqual(proxy('test'), "str") + self.assertEqual(proxy("test"), "str") self.assertEqual(proxy(object()), "SharedObjectProxy") with self.create_interp(proxy=proxy) as interp: @@ -150,10 +170,12 @@ def shared(): self.assertEqual(res.silly, "silly") with self.create_interp(proxy=proxy) as interp: - interp.exec("""if True: + interp.exec( + """if True: obj = proxy() assert obj.silly == 'silly' - assert type(obj).__name__ == 'SharedObjectProxy'""") + assert type(obj).__name__ == 'SharedObjectProxy'""" + ) def test_proxy_call_concurrently(self): def shared(arg, *, kwarg): @@ -175,7 +197,6 @@ def thread(interp): interp.exec("assert proxy(weird, kwarg=5) == 42") interp.exec("assert proxy(weird, kwarg=24).silly == 'test'") - proxy = share(shared) weird = share(Weird()) self.run_concurrently(thread, proxy=proxy, weird=weird) @@ -212,6 +233,7 @@ def __init__(self): self.value = 0 proxy = share(Test()) + def thread(interp): for _ in range(1000): interp.exec("proxy.value = 42") @@ -221,5 +243,5 @@ def thread(interp): self.run_concurrently(thread, proxy=proxy) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 04e3778cc28881203391bee15c8dcbeb90a6fd5e Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 15:34:42 -0500 Subject: [PATCH 26/31] Goodbye silly clear callback mechanism. --- Include/internal/pycore_tstate.h | 16 -------------- Python/pystate.c | 36 -------------------------------- 2 files changed, 52 deletions(-) diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index a9ecb487c86a98..50048801b2e4ee 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -54,20 +54,6 @@ typedef struct _PyJitTracerState { } _PyJitTracerState; #endif -typedef void (*_PyThreadState_ClearCallback)(PyThreadState *, void *); - -struct _PyThreadState_ClearNode { - struct _PyThreadState_ClearNode *next; - _PyThreadState_ClearCallback callback; - void *arg; -}; - -// Export for '_interpreters' shared extension. -PyAPI_FUNC(int) -_PyThreadState_AddClearCallback(PyThreadState *tstate, - _PyThreadState_ClearCallback callback, - void *arg); - // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -135,8 +121,6 @@ typedef struct _PyThreadStateImpl { #if _Py_TIER2 _PyJitTracerState jit_tracer_state; #endif - - struct _PyThreadState_ClearNode *clear_callbacks; } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Python/pystate.c b/Python/pystate.c index 17e499e2947e9c..c12a1418e74309 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1638,47 +1638,11 @@ clear_datastack(PyThreadState *tstate) } } -int -_PyThreadState_AddClearCallback(PyThreadState *tstate, - _PyThreadState_ClearCallback callback, - void *arg) -{ - assert(tstate != NULL); - assert(_PyThreadState_IsAttached(tstate)); - assert(callback != NULL); - _PyThreadStateImpl *impl = (_PyThreadStateImpl *)tstate; - struct _PyThreadState_ClearNode *node = PyMem_Malloc(sizeof(struct _PyThreadState_ClearNode)); - if (node == NULL) { - PyErr_NoMemory(); - return -1; - } - node->callback = callback; - node->next = impl->clear_callbacks; - node->arg = arg; - impl->clear_callbacks = node; - return 0; -} - -void -call_clear_callbacks(PyThreadState *tstate) -{ - assert(tstate != NULL); - assert(tstate == current_fast_get()); - _PyThreadStateImpl *impl = (_PyThreadStateImpl *)tstate; - struct _PyThreadState_ClearNode *head = impl->clear_callbacks; - while (head != NULL) { - head->callback(tstate, head->arg); - head = head->next; - } -} - void PyThreadState_Clear(PyThreadState *tstate) { - assert(tstate != NULL); assert(tstate->_status.initialized && !tstate->_status.cleared); assert(current_fast_get()->interp == tstate->interp); - call_clear_callbacks(tstate); // GH-126016: In the _interpreters module, KeyboardInterrupt exceptions // during PyEval_EvalCode() are sent to finalization, which doesn't let us // mark threads as "not running main". So, for now this assertion is From d279720959da98445b00bbe68b00c8ca15c85907 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 16:12:42 -0500 Subject: [PATCH 27/31] Use a thread state cache instead of creating a new one for each access. --- Include/internal/pycore_pystate.h | 3 + .../test_interpreters/test_object_proxy.py | 16 ++++- Modules/_interpretersmodule.c | 8 +-- Python/pystate.c | 69 ++++++++++++++++++- 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index cd06f8e3589b95..2cba98dc321615 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -231,6 +231,9 @@ extern PyThreadState * _PyThreadState_RemoveExcept(PyThreadState *tstate); extern void _PyThreadState_DeleteList(PyThreadState *list, int is_after_fork); extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); +// Export for '_interpreters' shared extension +PyAPI_FUNC(PyThreadState *) _PyThreadState_NewForExec(PyInterpreterState *interp); + // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 286268bf764157..274c94173a0a09 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -7,7 +7,7 @@ import_helper.import_module("_interpreters") from concurrent.interpreters import share, SharedObjectProxy from test.test_interpreters.utils import TestBase -from threading import Barrier, Thread, Lock +from threading import Barrier, Thread, Lock, local from concurrent import interpreters from contextlib import contextmanager @@ -242,6 +242,20 @@ def thread(interp): self.run_concurrently(thread, proxy=proxy) + def test_retain_thread_local_variables(self): + thread_local = local() + thread_local.value = 42 + + def test(): + old = thread_local.value + thread_local.value = 24 + return old + + proxy = share(test) + with self.create_interp(proxy=proxy) as interp: + interp.exec("assert proxy() == 42") + self.assertEqual(thread_local.value, 24) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ba980c99f16023..469e1b4a80b3a9 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -414,8 +414,7 @@ _sharedobjectproxy_enter(SharedObjectProxy *self, _PyXI_proxy_state *state) return 0; } state->to_restore = tstate; - PyThreadState *for_call = _PyThreadState_NewBound(self->interp, - _PyThreadState_WHENCE_EXEC); + PyThreadState *for_call = _PyThreadState_NewForExec(self->interp); state->for_call = for_call; if (for_call == NULL) { PyErr_NoMemory(); @@ -446,9 +445,10 @@ _sharedobjectproxy_exit(SharedObjectProxy *self, _PyXI_proxy_state *state) } assert(state->for_call == _PyThreadState_GET()); - PyThreadState_Clear(state->for_call); PyThreadState_Swap(state->to_restore); - PyThreadState_Delete(state->for_call); + // If we created a new thread state, we don't want to delete it. + // It's likely that it will be used again, but if not, the interpreter + // will clean it up at the end anyway. if (should_throw) { _PyErr_SetString(state->to_restore, PyExc_RuntimeError, "exception in interpreter"); diff --git a/Python/pystate.c b/Python/pystate.c index c12a1418e74309..513145b9cbe860 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -80,6 +80,56 @@ _Py_thread_local PyThreadState *_Py_tss_gilstate = NULL; and is same as tstate->interp. */ _Py_thread_local PyInterpreterState *_Py_tss_interp = NULL; +/* The last thread state used for each interpreter by this thread. */ +_Py_thread_local _Py_hashtable_t *_Py_tss_tstate_map = NULL; + +// TODO: Let's add a way to use _Py_hashtable_t statically to avoid the +// extra heap allocation. + +static void +mark_thread_state_used(PyThreadState *tstate) +{ + assert(tstate != NULL); + if (_Py_tss_tstate_map == NULL) { + _Py_hashtable_allocator_t alloc = { + .malloc = PyMem_RawMalloc, + .free = PyMem_RawFree + }; + _Py_tss_tstate_map = _Py_hashtable_new_full(_Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, NULL, &alloc); + if (_Py_tss_tstate_map == NULL) { + return; + } + } + + (void)_Py_hashtable_steal(_Py_tss_tstate_map, tstate->interp); + (void)_Py_hashtable_set(_Py_tss_tstate_map, tstate->interp, tstate); +} + +static PyThreadState * +last_thread_state_for_interp(PyInterpreterState *interp) +{ + assert(interp != NULL); + if (_Py_tss_tstate_map == NULL) { + return NULL; + } + + return _Py_hashtable_get(_Py_tss_tstate_map, interp); +} + +static void +mark_thread_state_dead(PyThreadState *tstate) +{ + if (_Py_tss_tstate_map == NULL) { + return; + } + + if (tstate == _Py_hashtable_get(_Py_tss_tstate_map, tstate->interp)) { + (void)_Py_hashtable_steal(_Py_tss_tstate_map, tstate->interp); + } +} + static inline PyThreadState * current_fast_get(void) { @@ -1603,6 +1653,21 @@ _PyThreadState_NewBound(PyInterpreterState *interp, int whence) return tstate; } +/* Get the last thread state used for this interpreter, or create a new + * one if none exists. + * The thread state returned by this may or may not be attached. */ +PyThreadState * +_PyThreadState_NewForExec(PyInterpreterState *interp) +{ + assert(interp != NULL); + PyThreadState *cached = last_thread_state_for_interp(interp); + if (cached != NULL) { + return cached; + } + + return _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); +} + // This must be followed by a call to _PyThreadState_Bind(); PyThreadState * _PyThreadState_New(PyInterpreterState *interp, int whence) @@ -1649,6 +1714,7 @@ PyThreadState_Clear(PyThreadState *tstate) // disabled. // XXX assert(!_PyThreadState_IsRunningMain(tstate)); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); + mark_thread_state_dead(tstate); tstate->_status.finalizing = 1; // just in case /* XXX Conditions we need to enforce: @@ -1961,7 +2027,6 @@ _PyThreadState_DeleteList(PyThreadState *list, int is_after_fork) } } - //---------- // accessors //---------- @@ -2168,6 +2233,8 @@ _PyThreadState_Attach(PyThreadState *tstate) #if defined(Py_DEBUG) errno = err; #endif + + mark_thread_state_used(tstate); } static void From 1b23e2d56f450d9b6458cd45ac3e2e39d7a7cd1e Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 16:42:48 -0500 Subject: [PATCH 28/31] Add a test for destruction in another interpreter. --- Lib/test/test_interpreters/test_object_proxy.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 274c94173a0a09..70abef40efd49a 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -256,6 +256,23 @@ def test(): interp.exec("assert proxy() == 42") self.assertEqual(thread_local.value, 24) + def test_destruct_object_in_subinterp(self): + called = False + + class Test: + def __del__(self): + nonlocal called + called = True + + foo = Test() + proxy = share(foo) + with self.create_interp(proxy=proxy): + self.assertFalse(called) + del foo, proxy + self.assertFalse(called) + + self.assertTrue(called) + if __name__ == "__main__": unittest.main() From 7cd14690101c008262f40461c3ad355e255c0a12 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 27 Nov 2025 18:25:09 -0500 Subject: [PATCH 29/31] Add a test for ensuring the switched interpreter is correct. --- Lib/test/test_interpreters/test_object_proxy.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index 70abef40efd49a..c09c62cd3b9df0 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -273,6 +273,20 @@ def __del__(self): self.assertTrue(called) + def test_called_in_correct_interpreter(self): + called = False + + def foo(): + nonlocal called + self.assertEqual(interpreters.get_current(), interpreters.get_main()) + called = True + + proxy = share(foo) + with self.create_interp(proxy=proxy) as interp: + interp.exec("proxy()") + + self.assertTrue(called) + if __name__ == "__main__": unittest.main() From 37e4556908758e766908acad5913696d9af3ffe4 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 23 Feb 2026 11:30:38 -0500 Subject: [PATCH 30/31] Implement the __share__() method. --- Modules/_interpretersmodule.c | 83 ++++++++++++++++++++++---- Modules/clinic/_interpretersmodule.c.h | 39 ++++++++++-- 2 files changed, 103 insertions(+), 19 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 469e1b4a80b3a9..d5ae12c65d79f5 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -33,8 +33,9 @@ /*[clinic input] module _interpreters +class _interpreters.SharedObjectProxy "SharedObjectProxy *" "&PyType_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=bfd967980a0de892]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4bb543de3f19aa0b]*/ static PyInterpreterState * _get_current_interp(void) @@ -458,17 +459,44 @@ _sharedobjectproxy_exit(SharedObjectProxy *self, _PyXI_proxy_state *state) return 0; } -static PyObject * -sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static SharedObjectProxy * +sharedobjectproxy_alloc(PyTypeObject *type) { - SharedObjectProxy *self = (SharedObjectProxy *)type->tp_alloc(type, 0); + assert(type != NULL); + assert(PyType_Check(type)); + SharedObjectProxy *self = SharedObjectProxy_CAST(type->tp_alloc(type, 0)); if (self == NULL) { return NULL; } - self->object = Py_None; self->interp = _PyInterpreterState_GET(); +#ifndef NDEBUG + self->object = NULL; +#endif + + return self; +} + +/*[clinic input] +@classmethod +_interpreters.SharedObjectProxy.__new__ as sharedobjectproxy_new + + obj: object, + / + +Create a new cross-interpreter proxy. +[clinic start generated code]*/ +static PyObject * +sharedobjectproxy_new_impl(PyTypeObject *type, PyObject *obj) +/*[clinic end generated code: output=42ed0a0bc47ecedf input=fce004d93517c6df]*/ +{ + SharedObjectProxy *self = sharedobjectproxy_alloc(type); + if (self == NULL) { + return NULL; + } + + self->object = Py_NewRef(obj); return (PyObject *)self; } @@ -483,8 +511,7 @@ _sharedobjectproxy_create(PyObject *object, PyInterpreterState *owning_interp) return NULL; } assert(Py_TYPE(object) != type); - SharedObjectProxy *proxy = SharedObjectProxy_CAST(sharedobjectproxy_new(type, - NULL, NULL)); + SharedObjectProxy *proxy = sharedobjectproxy_alloc(type); if (proxy == NULL) { return NULL; } @@ -1278,8 +1305,7 @@ static PyType_Slot SharedObjectProxy_slots[] = { static PyType_Spec SharedObjectProxy_spec = { .name = MODULE_NAME_STR ".SharedObjectProxy", .basicsize = sizeof(SharedObjectProxy), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), .slots = SharedObjectProxy_slots, }; @@ -2521,6 +2547,32 @@ _interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) return captured; } +static PyObject * +call_share_method_steal(PyObject *method) +{ + assert(method != NULL); + PyObject *result = PyObject_CallNoArgs(method); + Py_DECREF(method); + if (result == NULL) { + return NULL; + } + + PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate != NULL); + + if (_PyObject_CheckXIData(tstate, result) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyXIData_FormatNotShareableError(tstate, "__share__() returned unshareable object: %R", result); + PyObject *new_exc = _PyErr_GetRaisedException(tstate); + PyException_SetCause(new_exc, exc); + PyErr_SetRaisedException(new_exc); + Py_DECREF(result); + return NULL; + } + + return result; +} + /*[clinic input] _interpreters.share op: object, @@ -2528,15 +2580,20 @@ _interpreters.share Wrap an object in a shareable proxy that allows cross-interpreter access. - -The proxy will be assigned a context and may have its references cleared by -_interpreters.close_proxy(). [clinic start generated code]*/ static PyObject * _interpreters_share(PyObject *module, PyObject *op) -/*[clinic end generated code: output=e2ce861ae3b58508 input=d333c93f128faf93]*/ +/*[clinic end generated code: output=e2ce861ae3b58508 input=5fb300b5598bb7d2]*/ { + PyObject *share_method; + if (PyObject_GetOptionalAttrString(op, "__share__", &share_method) < 0) { + return NULL; + } + if (share_method != NULL) { + return call_share_method_steal(share_method /* stolen */); + } + return _sharedobjectproxy_create(op, _PyInterpreterState_GET()); } diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h index 47c7b255fcd31c..767dae31d08d1b 100644 --- a/Modules/clinic/_interpretersmodule.c.h +++ b/Modules/clinic/_interpretersmodule.c.h @@ -6,7 +6,37 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(sharedobjectproxy_new__doc__, +"SharedObjectProxy(obj, /)\n" +"--\n" +"\n" +"Create a new cross-interpreter proxy."); + +static PyObject * +sharedobjectproxy_new_impl(PyTypeObject *type, PyObject *obj); + +static PyObject * +sharedobjectproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyType_Type; + PyObject *obj; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("SharedObjectProxy", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("SharedObjectProxy", PyTuple_GET_SIZE(args), 1, 1)) { + goto exit; + } + obj = PyTuple_GET_ITEM(args, 0); + return_value = sharedobjectproxy_new_impl(type, obj); + +exit: + return return_value; +} PyDoc_STRVAR(_interpreters_create__doc__, "create($module, /, config=\'isolated\', *, reqrefs=False)\n" @@ -1203,11 +1233,8 @@ PyDoc_STRVAR(_interpreters_share__doc__, "share($module, op, /)\n" "--\n" "\n" -"Wrap an object in a shareable proxy that allows cross-interpreter access.\n" -"\n" -"The proxy will be assigned a context and may have its references cleared by\n" -"_interpreters.close_proxy()."); +"Wrap an object in a shareable proxy that allows cross-interpreter access."); #define _INTERPRETERS_SHARE_METHODDEF \ {"share", (PyCFunction)_interpreters_share, METH_O, _interpreters_share__doc__}, -/*[clinic end generated code: output=c1a117cea9045d1c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=24102c5dcbc26a72 input=a9049054013a1b77]*/ From 58c255ba8b02bb1847011767e719dade9424abfb Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 23 Feb 2026 11:41:57 -0500 Subject: [PATCH 31/31] Add some more tests. --- Lib/concurrent/interpreters/__init__.py | 18 +------ .../test_interpreters/test_object_proxy.py | 52 ++++++++++++++++++- Modules/_interpretersmodule.c | 14 +++-- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index 87e56e621ee377..cdc91059712c87 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -7,7 +7,7 @@ # aliases: from _interpreters import ( InterpreterError, InterpreterNotFoundError, NotShareableError, - is_shareable, SharedObjectProxy + is_shareable, SharedObjectProxy, share ) from ._queues import ( create as create_queue, @@ -245,19 +245,3 @@ def call_in_thread(self, callable, /, *args, **kwargs): t = threading.Thread(target=self._call, args=(callable, args, kwargs)) t.start() return t - - -def _can_natively_share(obj): - if isinstance(obj, SharedObjectProxy): - return False - - return _interpreters.is_shareable(obj) - - -def share(obj): - """Wrap the object in a shareable object proxy that allows cross-interpreter - access. - """ - if _can_natively_share(obj): - return obj - return _interpreters.share(obj) diff --git a/Lib/test/test_interpreters/test_object_proxy.py b/Lib/test/test_interpreters/test_object_proxy.py index c09c62cd3b9df0..f9d084cb2a677d 100644 --- a/Lib/test/test_interpreters/test_object_proxy.py +++ b/Lib/test/test_interpreters/test_object_proxy.py @@ -5,7 +5,7 @@ # Raise SkipTest if subinterpreters not supported. import_helper.import_module("_interpreters") -from concurrent.interpreters import share, SharedObjectProxy +from concurrent.interpreters import NotShareableError, share, SharedObjectProxy from test.test_interpreters.utils import TestBase from threading import Barrier, Thread, Lock, local from concurrent import interpreters @@ -287,6 +287,56 @@ def foo(): self.assertTrue(called) + def test_proxy_reshare_does_not_copy(self): + class Test: + pass + + proxy = share(Test()) + reproxy = share(proxy) + self.assertIs(proxy, reproxy) + + def test_object_share_method(self): + class Test: + def __share__(self): + return 42 + + shared = share(Test()) + self.assertEqual(shared, 42) + + def test_object_share_method_failure(self): + class Test: + def __share__(self): + return self + + exception = RuntimeError("ouch") + class Evil: + def __share__(self): + raise exception + + with self.assertRaises(NotShareableError): + share(Test()) + + with self.assertRaises(RuntimeError) as exc: + share(Evil()) + + self.assertIs(exc.exception, exception) + + def test_proxy_manual_construction(self): + called = False + + class Test: + def __init__(self): + self.attr = 24 + + def __share__(self): + nonlocal called + called = True + return 42 + + proxy = SharedObjectProxy(Test()) + self.assertIsInstance(proxy, SharedObjectProxy) + self.assertFalse(called) + self.assertEqual(proxy.attr, 24) if __name__ == "__main__": unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index d5ae12c65d79f5..1c4f05b855be94 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -2548,8 +2548,9 @@ _interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) } static PyObject * -call_share_method_steal(PyObject *method) +call_share_method_steal(PyThreadState *tstate, PyObject *method) { + assert(tstate != NULL); assert(method != NULL); PyObject *result = PyObject_CallNoArgs(method); Py_DECREF(method); @@ -2557,9 +2558,6 @@ call_share_method_steal(PyObject *method) return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); - assert(tstate != NULL); - if (_PyObject_CheckXIData(tstate, result) < 0) { PyObject *exc = _PyErr_GetRaisedException(tstate); _PyXIData_FormatNotShareableError(tstate, "__share__() returned unshareable object: %R", result); @@ -2586,12 +2584,18 @@ static PyObject * _interpreters_share(PyObject *module, PyObject *op) /*[clinic end generated code: output=e2ce861ae3b58508 input=5fb300b5598bb7d2]*/ { + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyObject_CheckXIData(tstate, op) == 0) { + return Py_NewRef(op); + } + PyErr_Clear(); + PyObject *share_method; if (PyObject_GetOptionalAttrString(op, "__share__", &share_method) < 0) { return NULL; } if (share_method != NULL) { - return call_share_method_steal(share_method /* stolen */); + return call_share_method_steal(tstate, share_method /* stolen */); } return _sharedobjectproxy_create(op, _PyInterpreterState_GET());