From c4f89fd01342c3728ff5032f921eae37929ef9f8 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 27 Apr 2026 21:10:04 +0100 Subject: [PATCH] Merge `cpython3` and `python3-libraries` projects --- projects/cpython3/Dockerfile | 26 -------- projects/cpython3/build.sh | 84 ------------------------- projects/cpython3/project.yaml | 18 ------ projects/cpython3/run_tests.sh | 22 ------- projects/python3-libraries/build.sh | 40 +++++++++++- projects/python3-libraries/project.yaml | 11 ++-- 6 files changed, 43 insertions(+), 158 deletions(-) delete mode 100644 projects/cpython3/Dockerfile delete mode 100644 projects/cpython3/build.sh delete mode 100644 projects/cpython3/project.yaml delete mode 100644 projects/cpython3/run_tests.sh diff --git a/projects/cpython3/Dockerfile b/projects/cpython3/Dockerfile deleted file mode 100644 index ab3038fa1e09..000000000000 --- a/projects/cpython3/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -FROM gcr.io/oss-fuzz-base/base-builder - -RUN apt-get update && \ - apt-get install -y build-essential libncursesw5-dev \ - libreadline-dev libssl-dev libgdbm-dev \ - libc6-dev libsqlite3-dev tk-dev libbz2-dev \ - zlib1g-dev libffi-dev - -RUN git clone https://github.com/python/cpython.git cpython3 -WORKDIR cpython3 -COPY run_tests.sh build.sh $SRC/ diff --git a/projects/cpython3/build.sh b/projects/cpython3/build.sh deleted file mode 100644 index 022c851feec2..000000000000 --- a/projects/cpython3/build.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -# Ignore memory leaks from python scripts invoked in the build -export ASAN_OPTIONS="detect_leaks=0" -export MSAN_OPTIONS="halt_on_error=0:exitcode=0:report_umrs=0" - -# Remove -pthread from CFLAGS, this trips up ./configure -# which thinks pthreads are available without any CLI flags -CFLAGS=${CFLAGS//"-pthread"/} - -# Ensure assert statements are enabled. It may help identify problems -# earlier if those fire. -CFLAGS="${CFLAGS} -UNDEBUG" - -# We use some internal CPython API. -CFLAGS="${CFLAGS} -IInclude/internal/" - -FLAGS=() -case $SANITIZER in - address) - FLAGS+=("--with-address-sanitizer") - ;; - memory) - FLAGS+=("--with-memory-sanitizer") - # installing ensurepip takes a while with MSAN instrumentation, so - # we disable it here - FLAGS+=("--without-ensurepip") - # -msan-keep-going is needed to allow MSAN's halt_on_error to function - FLAGS+=("CFLAGS=-mllvm -msan-keep-going=1") - ;; - undefined) - FLAGS+=("--with-undefined-behavior-sanitizer") - ;; -esac -./configure "${FLAGS[@]:-}" --prefix $OUT - -# We use altinstall to avoid having the Makefile create symlinks -make -j$(nproc) altinstall - -FUZZ_DIR=Modules/_xxtestfuzz -for fuzz_test in $(cat $FUZZ_DIR/fuzz_tests.txt) -do - # Build (but don't link) the fuzzing stub with a C compiler - $CC $CFLAGS $($OUT/bin/python*-config --cflags) $FUZZ_DIR/fuzzer.c \ - -D _Py_FUZZ_ONE -D _Py_FUZZ_$fuzz_test -c -Wno-unused-function \ - -o $WORK/$fuzz_test.o - # Link with C++ compiler to appease libfuzzer - $CXX $CXXFLAGS -rdynamic $WORK/$fuzz_test.o -o $OUT/$fuzz_test \ - $LIB_FUZZING_ENGINE $($OUT/bin/python*-config --ldflags --embed) - - # Zip up and copy any seed corpus - if [ -d "${FUZZ_DIR}/${fuzz_test}_corpus" ]; then - zip -j "${OUT}/${fuzz_test}_seed_corpus.zip" ${FUZZ_DIR}/${fuzz_test}_corpus/* - fi - # Copy over the dictionary for this test - if [ -e "${FUZZ_DIR}/dictionaries/${fuzz_test}.dict" ]; then - cp "${FUZZ_DIR}/dictionaries/${fuzz_test}.dict" "$OUT/${fuzz_test}.dict" - fi -done - -# A little bit hacky but we have to copy $OUT/include to -# $OUT/$OUT/include as the coverage build needs all source -# files used in execution and expects it to be there. -# See projects/tensorflow/build.sh for prior art -if [ "$SANITIZER" = "coverage" ] -then - mkdir -p $OUT/$OUT - cp -r $OUT/include $OUT/$OUT/ -fi diff --git a/projects/cpython3/project.yaml b/projects/cpython3/project.yaml deleted file mode 100644 index 5e59b1bdb3da..000000000000 --- a/projects/cpython3/project.yaml +++ /dev/null @@ -1,18 +0,0 @@ -homepage: "https://python.org/" -language: c++ -primary_contact: "seth@python.org" -main_repo: "https://github.com/python/cpython" -auto_ccs: - - "alex.gaynor@gmail.com" - - "ammar@ammaraskar.com" - - "greg@krypto.org" - - "pablogsal@python.org" - - "stanulbrych@gmail.com" -fuzzing_engines: - - afl - - honggfuzz - - libfuzzer -sanitizers: - - address - - memory - - undefined diff --git a/projects/cpython3/run_tests.sh b/projects/cpython3/run_tests.sh deleted file mode 100644 index 027dccbd58b9..000000000000 --- a/projects/cpython3/run_tests.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -# Ignore memory leaks from python scripts invoked in the build -export ASAN_OPTIONS="detect_leaks=0" - -# Skip network related unit test case -make test TESTOPTS="-u all,-network,-urlfetch" diff --git a/projects/python3-libraries/build.sh b/projects/python3-libraries/build.sh index 3fa2f9c49c6c..836decead0ce 100755 --- a/projects/python3-libraries/build.sh +++ b/projects/python3-libraries/build.sh @@ -23,6 +23,14 @@ export MSAN_OPTIONS="halt_on_error=0:exitcode=0:report_umrs=0" # which thinks pthreads are available without any CLI flags CFLAGS=${CFLAGS//"-pthread"/} +# Ensure assert statements are enabled. It may help identify problems +# earlier if those fire. +CFLAGS="${CFLAGS} -UNDEBUG" + +# Some fuzz tests in fuzzer.c use CPython internal API. Carried over from +# the former cpython3 project. +CFLAGS="${CFLAGS} -I${SRC}/cpython/Include/internal/" + FLAGS=() case $SANITIZER in address) @@ -30,6 +38,9 @@ case $SANITIZER in ;; memory) FLAGS+=("--with-memory-sanitizer") + # installing ensurepip takes a while with MSAN instrumentation, so + # we disable it here + FLAGS+=("--without-ensurepip") # -msan-keep-going is needed to allow MSAN's halt_on_error to function FLAGS+=("CFLAGS=-mllvm -msan-keep-going=1") ;; @@ -59,6 +70,7 @@ $OUT/cpython-install/bin/python3 -m pip install hypothesis cd $SRC/library-fuzzers make +# ------------------------- Python harness fuzzers ----------------------------# while read -r name fuzzer; do cp $SRC/library-fuzzers/fuzzer-$name $OUT/ cp $SRC/library-fuzzers/$fuzzer $OUT/ @@ -70,8 +82,32 @@ while read -r name fuzzer; do fi done < $SRC/library-fuzzers/fuzz_targets.txt -# Use CPython source code as seed corpus and use dict from cpython3 -cp $SRC/cpython/Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict $OUT/fuzzer-ast.dict +# Use CPython source code as seed corpus +cp $SRC/library-fuzzers/dictionaries/fuzz_pycompile.dict $OUT/fuzzer-ast.dict mkdir corp-ast/ find $SRC/cpython -type f -name '*.py' -size -4097c -exec cp {} corp-ast/ \; zip -j $OUT/fuzzer-ast_seed_corpus.zip corp-ast/* + +# ---------------------------- C harness fuzzers ------------------------------# +while read -r fuzz_test; do + cp $SRC/library-fuzzers/$fuzz_test $OUT/ + + # Zip up and copy any seed corpus + if [ -d "$SRC/library-fuzzers/${fuzz_test}_corpus" ]; then + zip -j "$OUT/${fuzz_test}_seed_corpus.zip" $SRC/library-fuzzers/${fuzz_test}_corpus/* + fi + # Copy over the dictionary for this test + if [ -e "$SRC/library-fuzzers/dictionaries/${fuzz_test}.dict" ]; then + cp "$SRC/library-fuzzers/dictionaries/${fuzz_test}.dict" "$OUT/${fuzz_test}.dict" + fi +done < $SRC/library-fuzzers/fuzz_tests.txt + +# A little bit hacky but we have to copy $OUT/include to +# $OUT/$OUT/include as the coverage build needs all source +# files used in execution and expects it to be there. +# See projects/tensorflow/build.sh for prior art +if [ "$SANITIZER" = "coverage" ] +then + mkdir -p $OUT/$OUT + cp -r $OUT/cpython-install/include $OUT/$OUT/ +fi diff --git a/projects/python3-libraries/project.yaml b/projects/python3-libraries/project.yaml index cbac71670d32..46d8c8db0283 100644 --- a/projects/python3-libraries/project.yaml +++ b/projects/python3-libraries/project.yaml @@ -1,6 +1,6 @@ homepage: "https://www.python.org/" main_repo: "https://github.com/python/cpython" -language: c +language: c++ primary_contact: "seth@python.org" auto_ccs: - "greg@krypto.org" @@ -9,11 +9,10 @@ auto_ccs: - "pablogsal@python.org" - "stanulbrych@gmail.com" fuzzing_engines: - - libfuzzer + - afl - honggfuzz + - libfuzzer sanitizers: - address - - memory: - experimental: True - - undefined: - experimental: True + - memory + - undefined