Merge branch 'main' into gh-148874/with-signal-exit-skip

Author: NekoAsakura

Doc/glossary.rst

@@ -39,10 +39,11 @@ Glossary
       ABCs with the :mod:`abc` module.
 
    annotate function
-      A function that can be called to retrieve the :term:`annotations <annotation>`
-      of an object. This function is accessible as the :attr:`~object.__annotate__`
-      attribute of functions, classes, and modules. Annotate functions are a
-      subset of :term:`evaluate functions <evaluate function>`.
+      A callable that can be called to retrieve the :term:`annotations <annotation>` of
+      an object. Annotate functions are usually :term:`functions <function>`,
+      automatically generated as the :attr:`~object.__annotate__` attribute of functions,
+      classes, and modules. Annotate functions are a subset of
+      :term:`evaluate functions <evaluate function>`.
 
    annotation
       A label associated with a variable, a class

Doc/library/annotationlib.rst

@@ -510,6 +510,81 @@ annotations from the class and puts them in a separate attribute:
          return typ
 
 
+Creating a custom callable annotate function
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Custom :term:`annotate functions <annotate function>` may be literal functions like those
+automatically generated for functions, classes, and modules. Or, they may wish to utilise
+the encapsulation provided by classes, in which case any :term:`callable` can be used as
+an :term:`annotate function`.
+
+To provide the :attr:`~Format.VALUE`, :attr:`~Format.STRING`, or
+:attr:`~Format.FORWARDREF` formats directly, an :term:`annotate function` must provide
+the following attribute:
+
+* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not
+  raise a :exc:`NotImplementedError` when called with a supported format.
+
+To provide the :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` format, which is used to
+automatically generate :attr:`~Format.STRING` or :attr:`~Format.FORWARDREF` if they are
+not supported directly, :term:`annotate functions <annotate function>` must provide the
+following attributes:
+
+* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not
+  raise a :exc:`NotImplementedError` when called with
+  :attr:`~Format.VALUE_WITH_FAKE_GLOBALS`.
+* A :ref:`code object <code-objects>` ``__code__`` containing the compiled code for the
+  annotate function.
+* Optional: A tuple of the function's positional defaults ``__kwdefaults__``, if the
+  function represented by ``__code__`` uses any positional defaults.
+* Optional: A dict of the function's keyword defaults ``__defaults__``, if the function
+  represented by ``__code__`` uses any keyword defaults.
+* Optional: All other :ref:`function attributes <inspect-types>`.
+
+.. code-block:: python
+
+   class Annotate:
+       called_formats = []
+
+       def __call__(self, format=None, /, *, _self=None):
+           # When called with fake globals, `_self` will be the
+           # actual self value, and `self` will be the format.
+           if _self is not None:
+               self, format = _self, self
+
+           self.called_formats.append(format)
+           if format <= 2:  # VALUE or VALUE_WITH_FAKE_GLOBALS
+               return {"x": MyType}
+           raise NotImplementedError
+
+       __code__ = __call__.__code__
+       __defaults__ = (None,)
+       __kwdefaults__ = property(lambda self: dict(_self=self))
+
+       __globals__ = {}
+       __builtins__ = {}
+       __closure__ = None
+
+This can then be called with:
+
+.. code-block:: pycon
+
+   >>> from annotationlib import call_annotate_function, Format
+   >>> call_annotate_function(Annotate(), format=Format.STRING)
+   {'x': 'MyType'}
+
+Or used as the annotate function for an object:
+
+.. code-block:: pycon
+
+   >>> from annotationlib import get_annotations, Format
+   >>> class C:
+   ...   pass
+   >>> C.__annotate__ = Annotate()
+   >>> get_annotations(Annotate(), format=Format.STRING)
+   {'x': 'MyType'}
+
+
 Limitations of the ``STRING`` format
 ------------------------------------
 

Doc/library/typing.rst

@@ -2361,6 +2361,12 @@ without the dedicated syntax, as documented below.
          >>> Alias.__module__
          '__main__'
 
+      This attribute is writable.
+
+      .. versionchanged:: 3.15
+
+         The attribute is now writable.
+
    .. attribute:: __type_params__
 
       The type parameters of the type alias, or an empty tuple if the alias is

Include/internal/pycore_opcode_utils.h

@@ -75,7 +75,12 @@ extern "C" {
 #define CONSTANT_BUILTIN_ANY 4
 #define CONSTANT_BUILTIN_LIST 5
 #define CONSTANT_BUILTIN_SET 6
-#define NUM_COMMON_CONSTANTS 7
+#define CONSTANT_NONE 7
+#define CONSTANT_EMPTY_STR 8
+#define CONSTANT_TRUE 9
+#define CONSTANT_FALSE 10
+#define CONSTANT_MINUS_ONE 11
+#define NUM_COMMON_CONSTANTS 12
 
 /* Values used in the oparg for RESUME */
 #define RESUME_AT_FUNC_START 0

Lib/_pyrepl/reader.py

@@ -38,12 +38,13 @@
 )
 from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines
 from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay
-from .utils import ANSI_ESCAPE_SEQUENCE, THEME, StyleRef, wlen, gen_colors
+from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, wlen, gen_colors
 from .trace import trace
 
 
 # types
 Command = commands.Command
+from collections.abc import Callable, Iterator
 from .types import (
     Callback,
     CommandName,
@@ -304,6 +305,7 @@ class Reader:
     lxy: CursorXY = field(init=False)
     scheduled_commands: list[CommandName] = field(default_factory=list)
     can_colorize: bool = False
+    gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors
     threading_hook: Callback | None = None
     invalidation: RefreshInvalidation = field(init=False)
 
@@ -534,7 +536,7 @@ def _build_content_lines(
         prompt_from_cache: bool,
     ) -> tuple[ContentLine, ...]:
         if self.can_colorize:
-            colors = list(gen_colors(self.get_unicode()))
+            colors = list(self.gen_colors(self.get_unicode()))
         else:
             colors = None
         trace("colors = {colors}", colors=colors)

Lib/dis.py

@@ -651,6 +651,7 @@ def get_argval_argrepr(self, op, arg, offset):
                     argrepr = str(argval)
             elif deop == LOAD_COMMON_CONSTANT:
                 obj = _common_constants[arg]
+                argval = obj
                 if isinstance(obj, type):
                     argrepr = obj.__name__
                 else:
@@ -700,10 +701,15 @@ def _get_const_value(op, arg, co_consts):
        Otherwise (if it is a LOAD_CONST and co_consts is not
        provided) returns the dis.UNKNOWN sentinel.
     """
-    assert op in hasconst or op == LOAD_SMALL_INT
+    assert op in hasconst or op == LOAD_SMALL_INT or op == LOAD_COMMON_CONSTANT
 
     if op == LOAD_SMALL_INT:
         return arg
+    if op == LOAD_COMMON_CONSTANT:
+        # Opargs 0-6 are callables; 7-11 are literal values.
+        if 7 <= arg <= 11:
+            return _common_constants[arg]
+        return UNKNOWN
     argval = UNKNOWN
     if co_consts is not None:
         argval = co_consts[arg]
@@ -1023,8 +1029,9 @@ def _find_imports(co):
         if op == IMPORT_NAME and i >= 2:
             from_op = opargs[i-1]
             level_op = opargs[i-2]
-            if (from_op[0] in hasconst and
-                (level_op[0] in hasconst or level_op[0] == LOAD_SMALL_INT)):
+            if ((from_op[0] in hasconst or from_op[0] == LOAD_COMMON_CONSTANT) and
+                (level_op[0] in hasconst or level_op[0] == LOAD_SMALL_INT or
+                 level_op[0] == LOAD_COMMON_CONSTANT)):
                 level = _get_const_value(level_op[0], level_op[1], consts)
                 fromlist = _get_const_value(from_op[0], from_op[1], consts)
                 # IMPORT_NAME encodes lazy/eager flags in bits 0-1,

Lib/opcode.py

@@ -41,7 +41,10 @@
 _special_method_names = _opcode.get_special_method_names()
 _common_constants = [builtins.AssertionError, builtins.NotImplementedError,
                      builtins.tuple, builtins.all, builtins.any, builtins.list,
-                     builtins.set]
+                     builtins.set,
+                     # Append-only — must match CONSTANT_* in
+                     # Include/internal/pycore_opcode_utils.h.
+                     None, "", True, False, -1]
 _nb_ops = _opcode.get_nb_ops()
 
 hascompare = [opmap["COMPARE_OP"]]

Lib/test/test_annotationlib.py

@@ -1619,6 +1619,84 @@ def annotate(format, /):
             # Some non-Format value
             annotationlib.call_annotate_function(annotate, 7)
 
+    def test_basic_non_function_annotate(self):
+        class Annotate:
+            def __call__(self, format, /, __Format=Format,
+                         __NotImplementedError=NotImplementedError):
+                if format == __Format.VALUE:
+                    return {'x': str}
+                elif format == __Format.VALUE_WITH_FAKE_GLOBALS:
+                    return {'x': int}
+                elif format == __Format.STRING:
+                    return {'x': "float"}
+                else:
+                    raise __NotImplementedError(format)
+
+        annotations = annotationlib.call_annotate_function(Annotate(), Format.VALUE)
+        self.assertEqual(annotations, {"x": str})
+
+        annotations = annotationlib.call_annotate_function(Annotate(), Format.STRING)
+        self.assertEqual(annotations, {"x": "float"})
+
+        with self.assertRaises(AttributeError) as cm:
+            annotations = annotationlib.call_annotate_function(
+                Annotate(), Format.FORWARDREF
+            )
+
+        self.assertEqual(cm.exception.name, "__builtins__")
+        self.assertIsInstance(cm.exception.obj, Annotate)
+
+    def test_full_non_function_annotate(self):
+        def outer():
+            local = str
+
+            class Annotate:
+                called_formats = []
+
+                def __call__(self, format=None, *, _self=None):
+                    nonlocal local
+                    if _self is not None:
+                        self, format = _self, self
+
+                    self.called_formats.append(format)
+                    if format == 1:  # VALUE
+                        return {"x": MyClass, "y": int, "z": local}
+                    if format == 2:  # VALUE_WITH_FAKE_GLOBALS
+                        return {"w": unknown, "x": MyClass, "y": int, "z": local}
+                    raise NotImplementedError
+
+                __globals__ = {"MyClass": MyClass}
+                __builtins__ = {"int": int}
+                __closure__ = (types.CellType(str),)
+                __defaults__ = (None,)
+
+                __kwdefaults__ = property(lambda self: dict(_self=self))
+                __code__ = property(lambda self: self.__call__.__code__)
+
+            return Annotate()
+
+        annotate = outer()
+
+        self.assertEqual(
+            annotationlib.call_annotate_function(annotate, Format.VALUE),
+            {"x": MyClass, "y": int, "z": str}
+        )
+        self.assertEqual(annotate.called_formats[-1], Format.VALUE)
+
+        self.assertEqual(
+            annotationlib.call_annotate_function(annotate, Format.STRING),
+            {"w": "unknown", "x": "MyClass", "y": "int", "z": "local"}
+        )
+        self.assertIn(Format.STRING, annotate.called_formats)
+        self.assertEqual(annotate.called_formats[-1], Format.VALUE_WITH_FAKE_GLOBALS)
+
+        self.assertEqual(
+            annotationlib.call_annotate_function(annotate, Format.FORWARDREF),
+            {"w": support.EqualToForwardRef("unknown"), "x": MyClass, "y": int, "z": str}
+        )
+        self.assertIn(Format.FORWARDREF, annotate.called_formats)
+        self.assertEqual(annotate.called_formats[-1], Format.VALUE_WITH_FAKE_GLOBALS)
+
     def test_error_from_value_raised(self):
         # Test that the error from format.VALUE is raised
         # if all formats fail

Lib/test/test_ast/test_ast.py

@@ -2685,11 +2685,12 @@ def test_get_docstring(self):
 
     def get_load_const(self, tree):
         # Compile to bytecode, disassemble and get parameter of LOAD_CONST
-        # instructions
+        # and LOAD_COMMON_CONSTANT instructions
         co = compile(tree, '<string>', 'exec')
         consts = []
         for instr in dis.get_instructions(co):
-            if instr.opcode in dis.hasconst:
+            if instr.opcode in dis.hasconst or \
+                    instr.opname == 'LOAD_COMMON_CONSTANT':
                 consts.append(instr.argval)
         return consts
 

Lib/test/test_capi/test_opt.py

@@ -3446,6 +3446,27 @@ def testfunc(n):
         self.assertIn("_BUILD_LIST", uops)
         self.assertNotIn("_LOAD_COMMON_CONSTANT", uops)
 
+    def test_load_common_constant_new_literals(self):
+        def testfunc(n):
+            x = None
+            s = ""
+            t = True
+            f = False
+            m = -1
+            for _ in range(n):
+                x = None
+                s = ""
+                t = True
+                f = False
+                m = -1
+            return x, s, t, f, m
+        res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
+        self.assertEqual(res, (None, "", True, False, -1))
+        self.assertIsNotNone(ex)
+        uops = get_opnames(ex)
+        self.assertNotIn("_LOAD_COMMON_CONSTANT", uops)
+        self.assertIn("_LOAD_CONST_INLINE_BORROW", uops)
+
     def test_load_small_int(self):
         def testfunc(n):
             x = 0