diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 00000000000000..6c92ad8d4c15a2 --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,8 @@ +FROM gcr.io/oss-fuzz-base/base-builder:v1 +RUN apt update && apt install -y build-essential gdb lcov pkg-config \ + libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ + libncurses5-dev libreadline6-dev libsqlite3-dev libssl-dev lzma \ + lzma-dev tk-dev uuid-dev zlib1g-dev +COPY . $SRC/cpython +WORKDIR cpython +COPY .clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100755 index 00000000000000..70e6d70f362f57 --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Script copied from oss-fuzz, please keep in sync with: +# https://github.com/google/oss-fuzz/blob/master/projects/cpython3/build.sh + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Ignore memory leaks from python scripts invoked in the build +export ASAN_OPTIONS="detect_leaks=0" +export MSAN_OPTIONS="halt_on_error=0:exitcode=0:report_umrs=0" + +# Remove -pthread from CFLAGS, this trips up ./configure +# which thinks pthreads are available without any CLI flags +CFLAGS=${CFLAGS//"-pthread"/} + +# Ensure assert statements are enabled. It may help identify problems +# earlier if those fire. +CFLAGS="${CFLAGS} -UNDEBUG" + +# We use some internal CPython API. +CFLAGS="${CFLAGS} -IInclude/internal/" + +FLAGS=() +case $SANITIZER in + address) + FLAGS+=("--with-address-sanitizer") + ;; + memory) + FLAGS+=("--with-memory-sanitizer") + # installing ensurepip takes a while with MSAN instrumentation, so + # we disable it here + FLAGS+=("--without-ensurepip") + # -msan-keep-going is needed to allow MSAN's halt_on_error to function + FLAGS+=("CFLAGS=-mllvm -msan-keep-going=1") + ;; + undefined) + FLAGS+=("--with-undefined-behavior-sanitizer") + ;; +esac +./configure "${FLAGS[@]:-}" --prefix $OUT + +# We use altinstall to avoid having the Makefile create symlinks +make -j$(nproc) altinstall + +FUZZ_DIR=Modules/_xxtestfuzz +for fuzz_test in $(cat $FUZZ_DIR/fuzz_tests.txt) +do + # Build (but don't link) the fuzzing stub with a C compiler + $CC $CFLAGS $($OUT/bin/python*-config --cflags) $FUZZ_DIR/fuzzer.c \ + -D _Py_FUZZ_ONE -D _Py_FUZZ_$fuzz_test -c -Wno-unused-function \ + -o $WORK/$fuzz_test.o + # Link with C++ compiler to appease libfuzzer + $CXX $CXXFLAGS -rdynamic $WORK/$fuzz_test.o -o $OUT/$fuzz_test \ + $LIB_FUZZING_ENGINE $($OUT/bin/python*-config --ldflags --embed) + + # Zip up and copy any seed corpus + if [ -d "${FUZZ_DIR}/${fuzz_test}_corpus" ]; then + zip -j "${OUT}/${fuzz_test}_seed_corpus.zip" ${FUZZ_DIR}/${fuzz_test}_corpus/* + fi + # Copy over the dictionary for this test + if [ -e "${FUZZ_DIR}/dictionaries/${fuzz_test}.dict" ]; then + cp "${FUZZ_DIR}/dictionaries/${fuzz_test}.dict" "$OUT/${fuzz_test}.dict" + fi +done + +# A little bit hacky but we have to copy $OUT/include to +# $OUT/$OUT/include as the coverage build needs all source +# files used in execution and expects it to be there. +# See projects/tensorflow/build.sh for prior art +if [ "$SANITIZER" = "coverage" ] +then + mkdir -p $OUT/$OUT + cp -r $OUT/include $OUT/$OUT/ +fi diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml new file mode 100644 index 00000000000000..b455aa397c7995 --- /dev/null +++ b/.clusterfuzzlite/project.yaml @@ -0,0 +1 @@ +language: c diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06551b13219c2a..bd8b1fee7410a5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,26 +3,7 @@ name: Tests # gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because # it prevents to mark a job as mandatory. A PR cannot be merged if a job is # mandatory but not scheduled because of "paths-ignore". -on: - workflow_dispatch: - push: - branches: - - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' - pull_request: - branches: - - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' +on: [push] permissions: contents: read @@ -520,6 +501,35 @@ jobs: - name: Tests run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + cluster_fuzz_line: + name: ClusterFuzzLite fuzzing + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + strategy: + fail-fast: false + matrix: + sanitizer: + - address + - undefined + - memory + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@v1 + with: + language: c + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@v1 + with: + fuzz-seconds: 300 + mode: 'code-change' + parallel-fuzzing: true + sanitizer: ${{ matrix.sanitizer }} + all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass if: always()