Fix INTERPRETER_EXIT tracing

Author: Fidget-Spinner

Include/internal/pycore_ceval.h

@@ -391,6 +391,9 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame
 #define SPECIAL___AEXIT__   3
 #define SPECIAL_MAX   3
 
+struct _PyCode12 _PyCode_DEF(12);
+PyAPI_DATA(const struct _PyCode12) _PyEntryFrameCode;
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_code.h

@@ -672,6 +672,12 @@ PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *);
  * compare bytes and str which can raise a BytesWarning exception. */
 extern PyObject* _PyCode_ConstantKey(PyObject *obj);
 
+#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3)
+
+static const PyBytesObject no_location = {
+    PyVarObject_HEAD_INIT(&PyBytes_Type, 1)
+    .ob_sval = { NO_LOC_4 }
+};
 
 #ifdef __cplusplus
 }

Include/internal/pycore_global_objects_fini_generated.h

@@ -67,6 +67,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(TextIOWrapper)
         STRUCT_FOR_ID(True)
         STRUCT_FOR_ID(WarningMessage)
+        STRUCT_FOR_ID(_PyEval_EvalFrameDefault)
         STRUCT_FOR_ID(_WindowsConsoleIO)
         STRUCT_FOR_ID(__IOBase_closed)
         STRUCT_FOR_ID(__abc_tpflags__)

Include/internal/pycore_global_strings.h

@@ -170,7 +170,9 @@ dummy_func(
 
         op(_QUICKEN_RESUME, (--)) {
             #if ENABLE_SPECIALIZATION_FT
-            if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
+            PyCodeObject *code = _PyFrame_GetCode(frame);
+            if (tstate->tracing == 0 && this_instr->op.code == RESUME &&
+                code != (PyCodeObject *)&_Py_InitCleanup && code != (PyCodeObject *)&_PyEntryFrameCode) {
                 FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
             }
             #endif  /* ENABLE_SPECIALIZATION_FT */
@@ -1218,19 +1220,33 @@ dummy_func(
             tstate->current_frame = frame->previous;
             assert(!_PyErr_Occurred(tstate));
             PyObject *result = PyStackRef_AsPyObjectSteal(retval);
+            if (IS_JIT_TRACING()) {
+#if _Py_TIER2
+                _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
+                LEAVE_TRACING();
+                int err = bail_tracing_and_jit(tstate, frame);
+                if (err < 0) {
+                    Py_DECREF(result);
+                    ERROR_IF(true);
+                }
+                return result;
+#endif
+            }
+            else {
 #if !_Py_TAIL_CALL_INTERP
-            assert(frame == &entry.frame);
+                assert(frame == &entry.frame);
 #endif
 #ifdef _Py_TIER2
-            _PyStackRef executor = frame->localsplus[0];
-            assert(tstate->current_executor == NULL);
-            if (!PyStackRef_IsNull(executor)) {
-                tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
-                PyStackRef_CLOSE(executor);
-            }
+                _PyStackRef executor = frame->localsplus[0];
+                assert(tstate->current_executor == NULL);
+                if (!PyStackRef_IsNull(executor)) {
+                    tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
+                    PyStackRef_CLOSE(executor);
+                }
 #endif
-            LLTRACE_RESUME_FRAME();
-            return result;
+                LLTRACE_RESUME_FRAME();
+                return result;
+            }
         }
 
         // The stack effect here is a bit misleading.
@@ -3020,15 +3036,6 @@ dummy_func(
 
         tier1 inst(ENTER_EXECUTOR, (--)) {
             #ifdef _Py_TIER2
-            // We want to end any current trace here, before we possibly need
-            // to start tracing new ones due to recursive traces in any inner C functions
-            // in tier2 code.
-            if (IS_JIT_TRACING()) {
-                _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
-                LEAVE_TRACING();
-                int err = bail_tracing_and_jit(tstate, frame);
-                ERROR_IF(err < 0);
-            }
             PyCodeObject *code = _PyFrame_GetCode(frame);
             _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
             assert(executor->vm_data.index == INSTR_OFFSET() - 1);
@@ -3038,14 +3045,19 @@ dummy_func(
             /* If the eval breaker is set then stay in tier 1.
              * This avoids any potentially infinite loops
              * involving _RESUME_CHECK */
-            if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+            if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
                 opcode = executor->vm_data.opcode;
                 oparg = (oparg & ~255) | executor->vm_data.oparg;
                 next_instr = this_instr;
                 if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
                     PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
                 }
-                DISPATCH_GOTO();
+                if (IS_JIT_TRACING()) {
+                    DISPATCH_GOTO_NON_TRACING();
+                }
+                else {
+                    DISPATCH_GOTO();
+                }
             }
             assert(executor != tstate->interp->cold_executor);
             tstate->jit_exit = NULL;

Include/internal/pycore_opcode_metadata.h

@@ -931,14 +931,39 @@ int _Py_CheckRecursiveCallPy(
     return 0;
 }
 
-static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
-    /* Put a NOP at the start, so that the IP points into
-    * the code, rather than before it */
-    { .op.code = NOP, .op.arg = 0 },
-    { .op.code = INTERPRETER_EXIT, .op.arg = 0 },  /* reached on return */
-    { .op.code = NOP, .op.arg = 0 },
-    { .op.code = INTERPRETER_EXIT, .op.arg = 0 },  /* reached on yield */
-    { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START }
+
+#ifdef Py_GIL_DISABLED
+static _PyCodeArray emtry_cleanup_tlbc = {
+    .size = 1,
+    .entries = {(char*) &_PyEntryFrameCode.co_code_adaptive},
+};
+#endif
+
+const struct _PyCode12 _PyEntryFrameCode = {
+    _PyVarObject_HEAD_INIT(&PyCode_Type, 5),
+    .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
+    .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty),
+    .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
+    .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS,
+    .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
+    .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
+    .co_filename = &_Py_ID(_PyEval_EvalFrameDefault),
+    .co_name = &_Py_ID(_PyEval_EvalFrameDefault),
+    .co_qualname = &_Py_ID(_PyEval_EvalFrameDefault),
+    .co_linetable = (PyObject *)&no_location,
+    ._co_firsttraceable = 4,
+    .co_stacksize = 2,
+    .co_framesize = 2 + FRAME_SPECIALS_SIZE,
+#ifdef Py_GIL_DISABLED
+    .co_tlbc = &emtry_cleanup_tlbc,
+#endif
+    .co_code_adaptive = {
+        NOP, 0,
+        INTERPRETER_EXIT, 0, /* reached on return */
+        NOP, 0,
+        INTERPRETER_EXIT, 0, /* reached on yield */
+        RESUME, RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START
+    }
 };
 
 #ifdef Py_DEBUG
@@ -1014,7 +1039,13 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame)
         // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
         // to be valid to access.
         if (err <= 0) {
-            exit->temperature = restart_backoff_counter(exit->temperature);
+            // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again.
+            if (tstate->interp->jit_state.prev_instr->op.code == INTERPRETER_EXIT) {
+                exit->temperature = initial_unreachable_backoff_counter();
+            }
+            else {
+                exit->temperature = restart_backoff_counter(exit->temperature);
+            }
         }
         else {
             exit->temperature = initial_temperature_backoff_counter();
@@ -1105,8 +1136,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     entry.frame.f_globals = (PyObject*)0xaaa3;
     entry.frame.f_builtins = (PyObject*)0xaaa4;
 #endif
-    entry.frame.f_executable = PyStackRef_None;
-    entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1;
+    entry.frame.f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)&_PyEntryFrameCode);
+    entry.frame.instr_ptr = ((_Py_CODEUNIT *)_PyEntryFrameCode.co_code_adaptive) + 1;
     entry.frame.stackpointer = entry.stack;
     entry.frame.owner = FRAME_OWNED_BY_INTERPRETER;
     entry.frame.visited = 0;