Checkpointing review changes

* flush_pth_start() -> process_startup_files()
* Move around some versionchanged and deprecated-removed sections
* Add: kw-only arg `defer_processing_start_files` in several places to ensure that the
  reverse-engineered semantics of the undocumented functions are preserved.

Author: warsaw

Doc/library/site.rst

@@ -91,13 +91,31 @@ and lines beginning with ``#`` are skipped.
 For backward compatibility, lines starting with ``import`` (followed by space
 or tab) are executed with :func:`exec`.
 
+.. versionchanged:: 3.13
+
+   The :file:`.pth` files are now decoded by UTF-8 at first and then by the
+   :term:`locale encoding` if it fails.
+
 .. versionchanged:: next
 
-   ``import`` lines in :file:`{name}.pth` are ignored when a :ref:`matching
-   <site-start-files>` :file:`{name}.start` file exists.
+   :file:`.pth` file lines starting with ``import`` are deprecated.  During
+   the deprecation period, such lines are still executed (except in the case
+   below), but a diagnostic message is emitted only when the :option:`-v` flag
+   is given.
+
+   ``import`` lines in :file:`{name}.pth` are silently ignored when a
+   :ref:`matching <site-start-files>` :file:`{name}.start` file exists.
+
+   Errors on individual lines no longer abort processing of the rest of the
+   file.  Each error is reported and the remaining lines continue to be
+   processed.
 
 .. deprecated-removed:: next 3.20
 
+   Decoding :file:`{name}.pth` files in any encoding other than ``utf-8-sig``
+   is deprecated in Python 3.15, and support for decoding from the locale
+   encoding will be removed in Python 3.20.
+
    ``import`` lines in :file:`{name}.pth` files are deprecated and will be
    silently ignored in Python 3.18 and 3.19.  In Python 3.20 a warning will be
    produced for ``import`` lines in :file:`{name}.pth` files.
@@ -155,30 +173,6 @@ the remaining entry points.
    :file:`{name}.start` files invoke :func:`pkgutil.resolve_name` with
    ``strict=True``, which requires the full ``pkg.mod:callable`` form.
 
-.. versionchanged:: 3.13
-
-   The :file:`.pth` files are now decoded by UTF-8 at first and then by the
-   :term:`locale encoding` if it fails.
-
-.. deprecated-removed:: next 3.20
-
-   Decoding :file:`{name}.pth` files in any encoding other than ``utf-8-sig``
-   is deprecated in Python 3.15, and support for decoding from the locale
-   encoding will be removed in Python 3.20.
-
-.. versionchanged:: next
-
-   :file:`.pth` file lines starting with ``import`` are deprecated.  During
-   the deprecation period, such lines are still executed, but a diagnostic
-   message is emitted when the :option:`-v` flag is given.  If a
-   :file:`{name}.start` file with the same base name exists, ``import`` lines
-   in :file:`{name}.pth` files are silently ignored.  See
-   :ref:`site-start-files` and :pep:`829`.
-
-   Errors on individual lines no longer abort processing of the rest of the
-   file.  Each error is reported and the remaining lines continue to be
-   processed.
-
 .. index::
    single: package
    triple: path; configuration; file
@@ -362,16 +356,21 @@ Module contents
       This function used to be called unconditionally.
 
 
-.. function:: addsitedir(sitedir, known_paths=None)
+.. function:: addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False)
 
-   Add a directory to sys.path and process its :file:`.pth` and
-   :file:`.start` files.  Typically used in :mod:`sitecustomize` or
+   Add a directory to sys.path and parse the :file:`.pth` and :file:`.start`
+   files found in that directory.  Typically used in :mod:`sitecustomize` or
    :mod:`usercustomize` (see above).
 
    The *known_paths* argument is an optional set of case-normalized paths
    used to prevent duplicate :data:`sys.path` entries.  When ``None`` (the
    default), the set is built from the current :data:`sys.path`.
 
+   While :file:`.pth` and :file:`.start` files are always parsed, set
+   *defer_processing_start_files* to ``True`` to prevent processing the
+   startup data found in those files, so that you can process them explicitly
+   (this is typically used by the :func:`main` function).
+
    .. versionchanged:: next
 
       Also processes :file:`.start` files.  See :ref:`site-start-files`.

Lib/site.py

@@ -32,8 +32,9 @@
   syntax.  These are resolved via pkgutil.resolve_name() and called with no
   arguments.
 
-All .pth path extensions are applied before any .start entry points are
-executed, ensuring that paths are available before startup code runs.
+When called from main(), all .pth path extensions are applied before any
+.start entry points are executed, ensuring that paths are available before
+startup code runs.
 
 See the documentation for the site module for full details:
 https://docs.python.org/3/library/site.html
@@ -352,7 +353,7 @@ def _execute_start_entrypoints():
                     exc)
 
 
-def flush_pth_start():
+def process_startup_files():
     """Flush all pending sys.path and entry points."""
     _extend_syspath()
     _exec_imports()
@@ -370,13 +371,13 @@ def addpackage(sitedir, name, known_paths):
     else:
         reset = False
     _read_pth_file(sitedir, name, known_paths)
-    flush_pth_start()
+    process_startup_files()
     if reset:
         known_paths = None
     return known_paths
 
 
-def addsitedir(sitedir, known_paths=None):
+def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False):
     """Add 'sitedir' argument to sys.path if missing and handle startup
     files."""
     _trace(f"Adding directory: {sitedir!r}")
@@ -415,8 +416,10 @@ def addsitedir(sitedir, known_paths=None):
 
     # If standalone call (not from main()), flush immediately
     # so the caller sees the effect.
+    if not defer_processing_start_files:
+        process_startup_files()
+
     if reset:
-        flush_pth_start()
         known_paths = None
 
     return known_paths
@@ -531,7 +534,7 @@ def getusersitepackages():
 
     return USER_SITE
 
-def addusersitepackages(known_paths):
+def addusersitepackages(known_paths, *, defer_processing_start_files=False):
     """Add a per user site-package to sys.path
 
     Each user has its own python directory with site-packages in the
@@ -543,7 +546,7 @@ def addusersitepackages(known_paths):
     user_site = getusersitepackages()
 
     if ENABLE_USER_SITE and os.path.isdir(user_site):
-        addsitedir(user_site, known_paths)
+        addsitedir(user_site, known_paths, defer_processing_start_files=defer_processing_start_files)
     return known_paths
 
 def getsitepackages(prefixes=None):
@@ -585,12 +588,12 @@ def getsitepackages(prefixes=None):
             sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
     return sitepackages
 
-def addsitepackages(known_paths, prefixes=None):
+def addsitepackages(known_paths, prefixes=None, *, defer_processing_start_files=False):
     """Add site-packages to sys.path"""
     _trace("Processing global site-packages")
     for sitedir in getsitepackages(prefixes):
         if os.path.isdir(sitedir):
-            addsitedir(sitedir, known_paths)
+            addsitedir(sitedir, known_paths, defer_processing_start_files=defer_processing_start_files)
 
     return known_paths
 
@@ -872,15 +875,15 @@ def main():
     known_paths = venv(known_paths)
     if ENABLE_USER_SITE is None:
         ENABLE_USER_SITE = check_enableusersite()
-    known_paths = addusersitepackages(known_paths)
-    known_paths = addsitepackages(known_paths)
+    known_paths = addusersitepackages(known_paths, defer_processing_start_files=True)
+    known_paths = addsitepackages(known_paths, defer_processing_start_files=True)
     # PEP 829: flush accumulated data from all .pth and .start files.
     # Paths are extended first, then deprecated import lines are exec'd,
     # and finally .start entry points are executed — ensuring sys.path is
-    # fully populated before any startup code runs.  flush_pth_start()
+    # fully populated before any startup code runs.  process_startup_files()
     # also clears the pending state so a later addsitedir() call does
     # not re-apply already-processed data.
-    flush_pth_start()
+    process_startup_files()
     setquit()
     setcopyright()
     sethelper()

Lib/test/test_site.py

@@ -191,17 +191,20 @@ def test_addsitedir(self):
             pth_file.cleanup()
 
     def test_addsitedir_explicit_flush(self):
-        # addsitedir() reads .pth files and, when called standalone
-        # (known_paths=None).  Flushes paths and import lines explicitly.
+        # addsitedir() reads .pth files and, with
+        # defer_processing_start_files=True, accumulates pending state
+        # without flushing.  A subsequent process_startup_files() call
+        # then applies the paths and runs the import lines.
         pth_file = PthFile()
-        pth_file.cleanup(prep=True) # Make sure that nothing is pre-existing
-                                    # that is tested for
+        # Ensure we have a clean slate.
+        pth_file.cleanup(prep=True)
         try:
             pth_file.create()
-            # Providing known_paths=set() prevents flushing.
-            site.addsitedir(pth_file.base_dir, set())
+            # Pass defer_processing_start_files=True to prevent flushing.
+            site.addsitedir(pth_file.base_dir, set(),
+                            defer_processing_start_files=True)
             self.assertNotIn(pth_file.imported, sys.modules)
-            site.flush_pth_start()
+            site.process_startup_files()
             self.pth_file_tests(pth_file)
         finally:
             pth_file.cleanup()