gh-116021: Deprecate support for instantiating abstract AST nodes

Doc/deprecations/pending-removal-in-3.20.rst

@@ -30,3 +30,8 @@ Pending removal in Python 3.20
   - :mod:`zlib`
 
   (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.)
+
+* :mod:`ast`:
+
+  * Creating instances of abstract AST nodes (such as :class:`ast.AST`
+    or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20.

Doc/library/ast.rst

@@ -40,7 +40,7 @@ Node classes
 
 .. class:: AST
 
-   This is the base of all AST node classes.  The actual node classes are
+   This is the abstract base of all AST node classes.  The actual node classes are
    derived from the :file:`Parser/Python.asdl` file, which is reproduced
    :ref:`above <abstract-grammar>`.  They are defined in the :mod:`!_ast` C
    module and re-exported in :mod:`!ast`.
@@ -159,6 +159,15 @@ Node classes
    match any of the fields of the AST node. This behavior is deprecated and will
    be removed in Python 3.15.
 
+.. deprecated-removed:: next 3.20
+
+    In the :ref:`grammar above <abstract-grammar>`, the AST node classes that
+    correspond to production rules with variants (aka "sums") are abstract
+    classes. Previous versions of Python allowed for the creation of direct
+    instances of these abstract node classes. This behavior is deprecated and
+    will be removed in Python 3.20.
+
+
 .. note::
     The descriptions of the specific node classes displayed here
     were initially adapted from the fantastic `Green Tree

Doc/whatsnew/3.15.rst

@@ -1485,6 +1485,13 @@ Deprecated
 New deprecations
 ----------------
 
+* :mod:`ast`
+
+  * Creating instances of abstract AST nodes (such as :class:`ast.AST`
+    or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20.
+
+    (Contributed by Brian Schubert in :gh:`116021`.)
+
 * :mod:`base64`:
 
   * Accepting the ``+`` and ``/`` characters with an alternative alphabet in

Lib/test/test_ast/test_ast.py

@@ -1,4 +1,5 @@
 import _ast_unparse
+import _ast
 import ast
 import builtins
 import contextlib
@@ -85,7 +86,9 @@ def _assertTrueorder(self, ast_node, parent_pos):
         self.assertEqual(ast_node._fields, ast_node.__match_args__)
 
     def test_AST_objects(self):
-        x = ast.AST()
+        # Directly instantiating abstract node class AST is allowed (but deprecated)
+        with self.assertWarns(DeprecationWarning):
+            x = ast.AST()
         self.assertEqual(x._fields, ())
         x.foobar = 42
         self.assertEqual(x.foobar, 42)
@@ -94,7 +97,7 @@ def test_AST_objects(self):
         with self.assertRaises(AttributeError):
             x.vararg
 
-        with self.assertRaises(TypeError):
+        with self.assertRaises(TypeError), self.assertWarns(DeprecationWarning):
             # "ast.AST constructor takes 0 positional arguments"
             ast.AST(2)
 
@@ -110,15 +113,21 @@ def cleanup():
 
         msg = "type object 'ast.AST' has no attribute '_fields'"
         # Both examples used to crash:
-        with self.assertRaisesRegex(AttributeError, msg):
+        with (
+            self.assertRaisesRegex(AttributeError, msg),
+            self.assertWarns(DeprecationWarning),
+        ):
             ast.AST(arg1=123)
-        with self.assertRaisesRegex(AttributeError, msg):
+        with (
+            self.assertRaisesRegex(AttributeError, msg),
+            self.assertWarns(DeprecationWarning),
+        ):
             ast.AST()
 
-    def test_AST_garbage_collection(self):
+    def test_node_garbage_collection(self):
         class X:
             pass
-        a = ast.AST()
+        a = ast.Module()
         a.x = X()
         a.x.a = a
         ref = weakref.ref(a.x)
@@ -428,7 +437,15 @@ def _construct_ast_class(self, cls):
             elif typ is object:
                 kwargs[name] = b'capybara'
             elif isinstance(typ, type) and issubclass(typ, ast.AST):
-                kwargs[name] = self._construct_ast_class(typ)
+                if _ast._is_abstract(typ):
+                    # Use an arbitrary concrete subclass
+                    concrete = next((sub for sub in typ.__subclasses__()
+                                     if not _ast._is_abstract(sub)), None)
+                    msg = f"abstract node class {typ} has no concrete subclasses"
+                    self.assertIsNotNone(concrete, msg)
+                else:
+                    concrete = typ
+                kwargs[name] = self._construct_ast_class(concrete)
         return cls(**kwargs)
 
     def test_arguments(self):
@@ -574,14 +591,21 @@ def test_nodeclasses(self):
             x = ast.BinOp(1, 2, 3, foobarbaz=42)
         self.assertEqual(x.foobarbaz, 42)
 
+        # Directly instantiating abstract node types is allowed (but deprecated)
+        self.assertWarns(DeprecationWarning, ast.stmt)
+        self.assertWarns(DeprecationWarning, ast.expr_context)
+
     def test_no_fields(self):
         # this used to fail because Sub._fields was None
         x = ast.Sub()
         self.assertEqual(x._fields, ())
 
     def test_invalid_sum(self):
         pos = dict(lineno=2, col_offset=3)
-        m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], [])
+        with self.assertWarns(DeprecationWarning):
+            # Creating instances of ast.expr is deprecated
+            e = ast.expr(**pos)
+        m = ast.Module([ast.Expr(e, **pos)], [])
         with self.assertRaises(TypeError) as cm:
             compile(m, "<test>", "exec")
         self.assertIn("but got expr()", str(cm.exception))
@@ -1103,14 +1127,19 @@ class CopyTests(unittest.TestCase):
     def iter_ast_classes():
         """Iterate over the (native) subclasses of ast.AST recursively.
 
-        This excludes the special class ast.Index since its constructor
-        returns an integer.
+        This excludes:
+          * abstract AST nodes
+          * the special class ast.Index, since its constructor returns
+            an integer.
         """
         def do(cls):
             if cls.__module__ != 'ast':
                 return
             if cls is ast.Index:
                 return
+            # Don't attempt to create instances of abstract AST nodes
+            if _ast._is_abstract(cls):
+                return
 
             yield cls
             for sub in cls.__subclasses__():

Lib/test/test_sys.py

@@ -1897,7 +1897,7 @@ def test_pythontypes(self):
         check = self.check_sizeof
         # _ast.AST
         import _ast
-        check(_ast.AST(), size('P'))
+        check(_ast.Module(), size('3P'))
         try:
             raise TypeError
         except TypeError as e:

Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst

@@ -0,0 +1,2 @@
+Support for creating instances of abstract AST nodes from the :mod:`ast` module
+is deprecated and scheduled for removal in Python 3.20. Patch by Brian Schubert.

Parser/asdl_c.py

@@ -881,6 +881,19 @@ def visitModule(self, mod):
         return -1;
     }
 
+    int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self));
+    if (contains == -1) {
+        return -1;
+    }
+    else if (contains == 1) {
+        if (PyErr_WarnFormat(
+                PyExc_DeprecationWarning, 1,
+                "Instantiating abstract AST node class %T is deprecated. "
+                "This will become an error in Python 3.20", self) < 0) {
+            return -1;
+        }
+    }
+
     Py_ssize_t i, numfields = 0;
     int res = -1;
     PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL;
@@ -1887,6 +1900,13 @@ def visitModule(self, mod):
                 if (!state->AST_type) {
                     return -1;
                 }
+                state->abstract_types = PySet_New(NULL);
+                if (!state->abstract_types) {
+                    return -1;
+                }
+                if (PySet_Add(state->abstract_types, state->AST_type) < 0) {
+                    return -1;
+                }
                 if (add_ast_fields(state) < 0) {
                     return -1;
                 }
@@ -1928,6 +1948,7 @@ def visitSum(self, sum, name):
                             (name, name, len(sum.attributes)), 1)
         else:
             self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1)
+        self.emit("if (PySet_Add(state->abstract_types, state->%s_type) < 0) return -1;" % name, 1)
         self.emit_defaults(name, sum.attributes, 1)
         simple = is_simple(sum)
         for t in sum.types:
@@ -1960,6 +1981,30 @@ def emit_defaults(self, name, fields, depth):
 class ASTModuleVisitor(PickleVisitor):
 
     def visitModule(self, mod):
+        self.emit("""
+/* Helper for checking if a node class is abstract in the tests. */
+static PyObject *
+ast_is_abstract(PyObject *Py_UNUSED(module), PyObject *cls) {
+    struct ast_state *state = get_ast_state();
+    if (state == NULL) {
+        return NULL;
+    }
+    int contains = PySet_Contains(state->abstract_types, cls);
+    if (contains == -1) {
+        return NULL;
+    }
+    else if (contains == 1) {
+        Py_RETURN_TRUE;
+    }
+    Py_RETURN_FALSE;
+}
+
+static struct PyMethodDef astmodule_methods[] = {
+    {"_is_abstract", ast_is_abstract, METH_O, NULL},
+    {NULL}  /* Sentinel */
+};
+""".strip(), 0, reflow=False)
+        self.emit("", 0)
         self.emit("static int", 0)
         self.emit("astmodule_exec(PyObject *m)", 0)
         self.emit("{", 0)
@@ -2000,7 +2045,8 @@ def visitModule(self, mod):
     .m_name = "_ast",
     // The _ast module uses a per-interpreter state (PyInterpreterState.ast)
     .m_size = 0,
-    .m_slots = astmodule_slots,
+    .m_methods = astmodule_methods,
+    .m_slots = astmodule_slots
 };
 
 PyMODINIT_FUNC
@@ -2289,6 +2335,7 @@ def generate_module_def(mod, metadata, f, internal_h):
         "%s_type" % type
         for type in metadata.types
     )
+    module_state.add("abstract_types")
 
     state_strings = sorted(state_strings)
     module_state = sorted(module_state)