unix: use extended PGO training set

.github/workflows/linux.yml

@@ -329,12 +329,20 @@ jobs:
               unset LD_LIBRARY_PATH
             fi
 
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --junit-xml "${JUNIT_FILENAME}" --stdlib dist/*.tar.zst
           fi
         env:
+          JUNIT_FILENAME: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
           MATRIX_RUN: ${{ matrix.run }}
           MATRIX_LIBC: ${{ matrix.libc }}
 
+      - name: Upload Junit Test Results
+        if: ${{ matrix.run == 'true' }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
+
   build-1:
     needs:
       - generate-matrix

.github/workflows/macos.yml

@@ -181,7 +181,15 @@ jobs:
           build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks dist/*.tar.zst
 
           if [ "${MATRIX_RUN}" == "true" ]; then
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --junit-xml "${JUNIT_FILENAME}" --stdlib dist/*.tar.zst
           fi
         env:
+          JUNIT_FILENAME: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
           MATRIX_RUN: ${{ matrix.run }}
+
+      - name: Upload Junit Test Results
+        if: ${{ matrix.run == 'true' }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"

.github/workflows/windows.yml

@@ -186,4 +186,12 @@ jobs:
         run: |
           $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative
           .\pythonbuild.exe validate-distribution $Dists
-          uv run --no-dev test-distribution.py $Dists
+          uv run --no-dev test-distribution.py --junit-xml $env:JUNIT_FILENAME --stdlib $Dists
+        env:
+          JUNIT_FILENAME: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
+
+      - name: Upload Junit Test Results
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"

cpython-unix/build-cpython.sh

@@ -51,10 +51,14 @@ tar -xf "Python-${PYTHON_VERSION}.tar.xz"
 PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
 SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
 
-cat Setup.local
-mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
+# Put critical config files in logs to aid debugging.
+for f in Setup.local Makefile.extra stdlib-test-annotations.json profiling-training-ignores.txt; do
+  echo "BEGIN $f"
+  cat $f
+  echo "END $f"
+done
 
-cat Makefile.extra
+mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
 
 pushd "Python-${PYTHON_VERSION}"
 
@@ -595,7 +599,27 @@ if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}"  ]; then
 fi
 
 # Define the base PGO profiling task, which we'll extend below with ignores
-export PROFILE_TASK='-m test --pgo'
+#
+# --pgo-extended implies --pgo. And --pgo mode significantly changes the behavior
+# of the test harness. Notably, it disables richer reporting of test failures,
+# making it vastly more difficult to debug test failures. This behavior undermines
+# our ability to debug failures when running tests for PGO instrumentation.
+#
+# The only material downside to not using --pgo is some tests that self-skip
+# when run under PGO don't do so. When we audited for such tests in 2026-03,
+# all such tests had the reason "PGO isn't useful" or some such. There were
+# ~10 such tests. Since we run the entire test suite anyway, the inclusion of
+# such "worthless" tests isn't impactful. The test failure observability is
+# worth their loss.
+export PROFILE_TASK='-m test'
+
+# Display test output on failure. This helps immensely with debugging PGO
+# failures.
+PROFILE_TASK="${PROFILE_TASK} -W"
+
+# Force kill tests taking too long. This prevents deadlocks that can cause
+# CI jobs to run for potentially hours while doing nothing.
+PROFILE_TASK="${PROFILE_TASK} --timeout 300"
 
 # Run tests in parallel to reduce wall time.
 #
@@ -610,21 +634,13 @@ export PROFILE_TASK='-m test --pgo'
 # and there will be no loss in profile quality.
 PROFILE_TASK="${PROFILE_TASK} -j ${NUM_CPUS}"
 
-# On 3.14+ `test_strftime_y2k` fails when cross-compiling for `x86_64_v2` and `x86_64_v3` targets on
-# Linux, so we ignore it. See https://github.com/python/cpython/issues/128104
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" && -n "${CROSS_COMPILING}" && "${PYBUILD_PLATFORM}" != macos* ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test_strftime_y2k"
-fi
+PROFILE_TASK="${PROFILE_TASK} --ignorefile ${ROOT}/profiling-training-ignores.txt"
 
-# On 3.14+ `test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding` fails during
-# PGO due to RecursionError not being raised as expected. See https://github.com/python/cpython/issues/140125
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test_json"
-fi
-
-# PGO optimized / BOLT instrumented binaries segfault in a test_bytes test. Skip it.
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format"
+# Exclude whole modules from profiling based on stdlib test annotations.
+# Note: --exclude is a bool flag that changes behavior of positional arguments
+# to mean "exclude" instead of "include."
+if [ -n "${PROFILING_EXCLUDE_MODULES}" ]; then
+    PROFILE_TASK="${PROFILE_TASK} --exclude ${PROFILING_EXCLUDE_MODULES}"
 fi
 
 # ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL,
@@ -1392,14 +1408,18 @@ cp -av Modules/config.c.in "${ROOT}/out/python/build/Modules/"
 cp -av Python/frozen.c "${ROOT}/out/python/build/Python/"
 cp -av Modules/Setup* "${ROOT}/out/python/build/Modules/"
 
-# Copy the test hardness runner for convenience.
+# Copy the test harness runner for convenience.
 # As of Python 3.13, the test harness runner has been removed so we provide a compatibility script
 if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
     cp -av "${ROOT}/run_tests-13.py" "${ROOT}/out/python/build/run_tests.py"
 else
     cp -av Tools/scripts/run_tests.py "${ROOT}/out/python/build/"
 fi
 
+# Copy standard library test annotations so it is in the artifact and the
+# distribution self-describes expected test wonkiness.
+cp -av "${ROOT}/stdlib-test-annotations.json" "${ROOT}/out/python/build/stdlib-test-annotations.json"
+
 # Don't hard-code the build-time prefix into the pkg-config files. See
 # the description of `pcfiledir` in `man pkg-config`.
 find "${ROOT}/out/python/install/lib/pkgconfig" -name \*.pc -type f -exec \

cpython-unix/build.py

@@ -24,6 +24,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_setup_line,
+    stdlib_test_annotations,
 )
 from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
 from pythonbuild.downloads import DOWNLOADS
@@ -50,6 +51,7 @@
 SUPPORT = ROOT / "cpython-unix"
 EXTENSION_MODULES = SUPPORT / "extension-modules.yml"
 TARGETS_CONFIG = SUPPORT / "targets.yml"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LINUX_ALLOW_SYSTEM_LIBRARIES = {
     "c",
@@ -732,6 +734,13 @@ def build_cpython(
     setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
     pip_archive = download_entry("pip", DOWNLOADS_PATH)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     ems = extension_modules_config(EXTENSION_MODULES)
 
     setup = derive_setup_local(
@@ -804,12 +813,36 @@ def build_cpython(
 
             build_env.copy_file(fh.name, dest_name="Makefile.extra")
 
+        # Install the derived test annotations.
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+            test_annotations.json_dump(fh)
+            fh.flush()
+
+            build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json")
+
+        # Install a file with tests to skip during profile training.
+        # Also collect module excludes for the profiling run of the test harness.
+        profiling_exclude_modules = []
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+
+            for ann in test_annotations.annotations:
+                if ann.profile_training_skip():
+                    fh.write(f"{ann.name}\n")
+                if m := ann.profile_training_exclude_module():
+                    profiling_exclude_modules.append(m)
+
+            fh.flush()
+            build_env.copy_file(fh.name, dest_name="profiling-training-ignores.txt")
+
         env = {
             "PIP_VERSION": DOWNLOADS["pip"]["version"],
             "PYTHON_VERSION": python_version,
             "PYTHON_MAJMIN_VERSION": ".".join(python_version.split(".")[0:2]),
             "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
             "TOOLCHAIN": "clang-%s" % host_platform,
+            "PROFILING_EXCLUDE_MODULES": " ".join(profiling_exclude_modules),
         }
 
         # Set environment variables allowing convenient testing for Python
@@ -1070,7 +1103,7 @@ def main():
             write_dockerfiles(SUPPORT, BUILD)
         elif action == "makefiles":
             targets = get_targets(TARGETS_CONFIG)
-            write_triples_makefiles(targets, BUILD, SUPPORT)
+            write_triples_makefiles(targets, ROOT, BUILD, SUPPORT)
             write_target_settings(targets, BUILD / "targets")
             write_package_versions(BUILD / "versions")
 

cpython-unix/patch-pgo-file-pool-3.11.patch

@@ -25,7 +25,7 @@ index bd851bb626..b138499d1b 100644
      PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
      LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr"
 -    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
-+    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%128m.profclangr\""
++    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%8192m.profclangr\""
      LLVM_MERGED_DATA_FILE=code.profclangd
      if test $LLVM_PROF_FOUND = not-found
      then

cpython-unix/patch-pgo-file-pool.patch

@@ -26,7 +26,7 @@ index 82cf6f938a..13db5547e1 100644
      ")
 -    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\""
 -    LLVM_MERGED_DATA_FILE="\$(shell pwd)/code.profclangd"
-+     LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%128m.profclangr\""
++     LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%8192m.profclangr\""
 +     LLVM_MERGED_DATA_FILE="\$(shell pwd)/code.profclangd"
      if test $LLVM_PROF_FOUND = not-found
      then

cpython-windows/build.py

@@ -22,6 +22,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_config_c,
+    stdlib_test_annotations,
 )
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.utils import (
@@ -39,6 +40,7 @@
 BUILD = ROOT / "build"
 DIST = ROOT / "dist"
 SUPPORT = ROOT / "cpython-windows"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LOG_PREFIX = [None]
 LOG_FH = [None]
@@ -128,7 +130,6 @@
     "_zstd": ["zstd"],
 }
 
-
 # Tests to run during PGO profiling.
 #
 # This set was copied from test.libregrtest.pgo in the CPython source
@@ -1407,6 +1408,13 @@ def build_cpython(
     else:
         raise Exception("unhandled architecture: %s" % arch)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     tempdir_opts = (
         {"ignore_cleanup_errors": True} if sys.version_info >= (3, 12) else {}
     )
@@ -1763,6 +1771,12 @@ def build_cpython(
                 out_dir / "python" / "build" / "run_tests.py",
             )
 
+        # Install a JSON file annotating tests.
+        with (out_dir / "python" / "build" / "stdlib-test-annotations.json").open(
+            "w", encoding="utf-8"
+        ) as fh:
+            test_annotations.json_dump(fh)
+
         licenses_dir = out_dir / "python" / "licenses"
         licenses_dir.mkdir()
         for f in sorted(os.listdir(ROOT)):

pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
     "docker>=7.1.0",
     "jinja2>=3.1.5",
     "jsonschema>=4.23.0",
+    "junitparser>=4.0.2",
     "pyyaml>=6.0.2",
     "six>=1.17.0",
     "tomli>=2.2.1",