Fix & update benchmarks and fuzzers (#1541)

1. Fix and update (to the latest tag/release) the (nontrivial) benchmarks and fuzzers to adapt to `Ubuntu:20.04` and `Python3.10.8`. 2. Fix and update (to the latest tag/release) the (nontrivial) fuzzers to adapt to `Ubuntu:20.04` and `Python3.10.8`. 3. Delete the trivial/buggy benchmarks and fuzzers.
google · Nov 22, 2022 · 94cc1d3 · 94cc1d3
1 parent 4f2a965
commit 94cc1d3
Show file tree

Hide file tree

Showing 123 changed files with 1,292 additions and 1,226 deletions.
diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml
@@ -17,93 +17,98 @@ jobs:
       fail-fast: false
       matrix:
         fuzzer:
+          # Default fuzzers general comparison evaluation.
           - afl
           - aflfast
-          - afl_um_prioritize
-          - afl_um_random
-          - afl_um_parallel
           - aflplusplus
-          - aflplusplus_optimal
-          - aflplusplus_tracepc
-          - aflplusplus_um_prioritize
-          - aflplusplus_um_prioritize_75
-          - aflplusplus_um_random
-          - aflplusplus_um_random_75
-          - aflplusplus_um_random_3
-          - aflplusplus_um_random_6
-          - aflplusplus_um_parallel
           - aflsmart
           - centipede
-          - entropic
+          - eclipser
           - fairfuzz
           - honggfuzz
-          - honggfuzz_um_random
-          - honggfuzz_um_random_75
-          - honggfuzz_um_prioritize
-          - honggfuzz_um_prioritize_75
-          - honggfuzz_um_parallel
-          - lafintel
-          - klee
+          - libafl
           - libfuzzer
-          - libfuzzer_um_random
-          - libfuzzer_um_random_75
-          - libfuzzer_um_prioritize
-          - libfuzzer_um_prioritize_75
-          - libfuzzer_um_parallel
-          - libfuzzer_dataflow
-          - libfuzzer_dataflow_load
-          - libfuzzer_dataflow_store
-          - libfuzzer_dataflow_pre
           - mopt
-          - neuzz
-          - libafl
-          - libafl_text
-          - pythia_effect_bb
-          - pythia_bb
-          - fafuzz
-          - tortoisefuzz
           - wingfuzz
-          # Binary-only (greybox) fuzzers.
-          - eclipser
-          - eclipser_um_prioritize
-          - eclipser_um_prioritize_75
-          - eclipser_um_random
-          - eclipser_um_random_75
-          - eclipser_um_parallel
-          - afl_qemu
-          - honggfuzz_qemu
-          - weizz_qemu
-          - aflplusplus_qemu
-          - aflplusplus_frida
+          # - klee               # To Be Fixed.
+          # - tortoisefuzz       # To Be Fixed.
           # Concolic execution
-          - fuzzolic_aflplusplus_z3
-          - fuzzolic_aflplusplus_fuzzy
-          - eclipser_aflplusplus
-          - symqemu_aflplusplus
-          - symcc_aflplusplus
-          - symcc_aflplusplus_single
-          - symcc_afl
-          - symcc_afl_single
+          # - symcc_aflplusplus  # To Be Fixed.
           # Grammar fuzzers
-          - nautilus
-          - gramatron
-          - token_level
-          - grimoire
+          # - nautilus           # To Be Fixed.
+          # - gramatron          # To Be Fixed.
+          # - token_level        # To Be Fixed.
+          # - grimoire           # To Be Fixed.
           # Temporary variants.
-          - aflplusplus_dict2file
-          - afl_2_52_b
-          - aflplusplus_cmplog
-          - afl_random_favored
-          - entropic_execute_final
-          - libfuzzer_exeute_final
-          - introspector_driven_focus
-          - libfuzzer_fork_parallel
-          - centipede_function_filter
+          # - introspector_driven_focus
+          # - centipede_function_filter
+          # - aflplusplus_dict2file
+          # - afl_2_52_b
+          # - aflplusplus_cmplog
+          # - afl_random_favored
+          # - entropic_execute_final
+          # - libfuzzer_exeute_final
+          # - libfuzzer_fork_parallel
+          # - afl_um_prioritize
+          # - afl_um_random
+          # - afl_um_parallel
+          # - aflplusplus_optimal
+          # - aflplusplus_tracepc
+          # - aflplusplus_um_prioritize
+          # - aflplusplus_um_prioritize_75
+          # - aflplusplus_um_random
+          # - aflplusplus_um_random_75
+          # - aflplusplus_um_random_3
+          # - aflplusplus_um_random_6
+          # - aflplusplus_um_parallel
+          # - honggfuzz_um_random
+          # - honggfuzz_um_random_75
+          # - honggfuzz_um_prioritize
+          # - honggfuzz_um_prioritize_75
+          # - honggfuzz_um_parallel
+          # - libfuzzer_um_random
+          # - libfuzzer_um_random_75
+          # - libfuzzer_um_prioritize
+          # - libfuzzer_um_prioritize_75
+          # - libfuzzer_um_parallel
+          # - libfuzzer_dataflow
+          # - libfuzzer_dataflow_load
+          # - libfuzzer_dataflow_store
+          # - libfuzzer_dataflow_pre
+          # - libafl_text
+          # - pythia_effect_bb
+          ## Binary-only (greybox) fuzzers.
+          # - eclipser_um_prioritize
+          # - eclipser_um_prioritize_75
+          # - eclipser_um_random
+          # - eclipser_um_random_75
+          # - eclipser_um_parallel
+          ## Binary-only (greybox) fuzzers.
+          # - afl_qemu
+          # - honggfuzz_qemu
+          # - weizz_qemu
+          # - aflplusplus_qemu
+          # - aflplusplus_frida
+          ## Concolic fuzzers.
+          # - fuzzolic_aflplusplus_z3
+          # - fuzzolic_aflplusplus_fuzzy
+          # - eclipser_aflplusplus
+          # - symqemu_aflplusplus
+          # - symcc_aflplusplus_single
+          # - symcc_afl
+          ## Concolic execution
+          # - symcc_afl_single
+          ## Deprecated.
+          # - entropic
+          # - lafintel
+          # - neuzz
+          # - pythia_bb
+          # - fafuzz
 
         benchmark_type:
           - oss-fuzz
           - standard
-          - bug
+          # - bug
 
     steps:
     - uses: actions/checkout@v2

diff --git a/benchmarks/arrow_parquet-arrow-fuzz/Dockerfile b/benchmarks/arrow_parquet-arrow-fuzz/Dockerfile
@@ -17,8 +17,7 @@
 FROM gcr.io/oss-fuzz-base/base-builder@sha256:fb1a9a49752c9e504687448d1f1a048ec1e062e2e40f7e8a23e86b63ff3dad7c
 
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update -y -q && \
-    apt-get upgrade -y -q && \
+RUN apt-get update && \
     apt-get install -y -q --no-install-recommends \
         bison \
         build-essential \
@@ -28,6 +27,11 @@ RUN apt-get update -y -q && \
         ninja-build \
         python3
 
-RUN git clone --depth=1 https://github.com/apache/arrow.git $SRC/arrow
+RUN git clone \
+        --depth=1 \
+        --branch apache-arrow-10.0.0 \
+        --recurse-submodules \
+        https://github.com/apache/arrow.git \
+        $SRC/arrow
 
 COPY build.sh thrift.patch $SRC/
diff --git a/benchmarks/arrow_parquet-arrow-fuzz/benchmark.yaml b/benchmarks/arrow_parquet-arrow-fuzz/benchmark.yaml
@@ -1,9 +1,8 @@
-commit: fb8868d25570234f0f18e8bcdb6ccb0c3b63d0f0
-commit_date: 2020-02-25 02:36:00+00:00
 fuzz_target: parquet-arrow-fuzz
 project: arrow
-type: bug
 unsupported_fuzzers:
+  - honggfuzz  # To Be Fixed.
+  - libafl     # To Be Fixed.
   - aflcc
   - afl_qemu
   - aflplusplus_qemu

diff --git a/benchmarks/arrow_parquet-arrow-fuzz/build.sh b/benchmarks/arrow_parquet-arrow-fuzz/build.sh
@@ -53,7 +53,7 @@ cmake ${ARROW} -GNinja \
     -DPARQUET_BUILD_EXAMPLES=off \
     -DPARQUET_BUILD_EXECUTABLES=off \
     -DPARQUET_REQUIRE_ENCRYPTION=off \
-    -DARROW_WITH_BROTLI=off \
+    -DARROW_WITH_BROTLI=on \
     -DARROW_WITH_BZ2=off \
     -DARROW_WITH_LZ4=off \
     -DARROW_WITH_SNAPPY=off \

diff --git a/benchmarks/aspell_aspell_fuzzer/Dockerfile b/benchmarks/aspell_aspell_fuzzer/Dockerfile
@@ -18,8 +18,15 @@ FROM gcr.io/oss-fuzz-base/base-builder@sha256:fb1a9a49752c9e504687448d1f1a048ec1
 
 RUN apt-get update && apt-get upgrade -y && apt-get install -y pkg-config wget
 
-RUN git clone https://github.com/gnuaspell/aspell.git $SRC/aspell
-RUN git clone --depth 1 -b master https://github.com/gnuaspell/aspell-fuzz.git $SRC/aspell-fuzz
+RUN git clone \
+        --depth 1 \
+        --branch rel-0.60.8 \
+        https://github.com/gnuaspell/aspell.git \
+        $SRC/aspell
+
+RUN git clone \
+        https://github.com/gnuaspell/aspell-fuzz.git \
+        $SRC/aspell-fuzz
 
 # Suppress an immediate UBSan violation that prevents fuzzing
 RUN wget https://github.com/GNUAspell/aspell/commit/a2cd7ffd25e6213f36139cda4a911e2e03ed417c.patch -O $SRC/aspell/fix_aspell_ub.patch

diff --git a/benchmarks/aspell_aspell_fuzzer/benchmark.yaml b/benchmarks/aspell_aspell_fuzzer/benchmark.yaml
@@ -1,8 +1,5 @@
-commit: b503ec3e3e134dbc28bf129c012e5d6245a39472
-commit_date: 2019-08-05 23:54:02+00:00
 fuzz_target: aspell_fuzzer
 project: aspell
-type: bug
 unsupported_fuzzers:
   - aflcc
   - afl_qemu

diff --git a/benchmarks/bloaty_fuzz_target/Dockerfile b/benchmarks/bloaty_fuzz_target/Dockerfile
@@ -15,7 +15,18 @@
 ################################################################################
 
 FROM gcr.io/oss-fuzz-base/base-builder@sha256:fb1a9a49752c9e504687448d1f1a048ec1e062e2e40f7e8a23e86b63ff3dad7c
-RUN apt-get update && apt-get upgrade -y && apt-get install -y cmake ninja-build g++
-RUN git clone --depth 1 https://github.com/google/bloaty.git bloaty
+
+RUN apt-get update && \
+    apt-get install -y \
+        cmake \
+        ninja-build \
+        g++ \
+        libz-dev
+
+RUN git clone \
+    --depth 1 \
+    --branch v1.1\
+    https://github.com/google/bloaty.git
+
 WORKDIR bloaty
 COPY build.sh $SRC/
diff --git a/benchmarks/ffmpeg_ffmpeg_demuxer_fuzzer/Dockerfile b/benchmarks/ffmpeg_ffmpeg_demuxer_fuzzer/Dockerfile
@@ -15,36 +15,97 @@
 ################################################################################
 
 FROM gcr.io/oss-fuzz-base/base-builder@sha256:fb1a9a49752c9e504687448d1f1a048ec1e062e2e40f7e8a23e86b63ff3dad7c
-ADD bionic.list /etc/apt/sources.list.d/bionic.list
-ADD nasm_apt.pin /etc/apt/preferences
-RUN apt-get update && apt-get upgrade -y && apt-get install -y make autoconf automake libtool build-essential \
-    libass-dev libfreetype6-dev libsdl1.2-dev \
-    libvdpau-dev libxcb1-dev libxcb-shm0-dev \
-    pkg-config texinfo libbz2-dev zlib1g-dev yasm cmake mercurial wget \
-    xutils-dev libpciaccess-dev nasm
-
-RUN git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg
-
-RUN wget https://www.alsa-project.org/files/pub/lib/alsa-lib-1.1.0.tar.bz2
-RUN git clone -n https://gitlab.freedesktop.org/mesa/drm.git
-RUN cd drm; git checkout 5db0f7692d1fdf05f9f6c0c02ffa5a5f4379c1f3
-RUN git clone --depth 1 https://github.com/mstorsjo/fdk-aac.git
-ADD https://sourceforge.net/projects/lame/files/latest/download lame.tar.gz
-RUN git clone git://anongit.freedesktop.org/xorg/lib/libXext && \
-    cd /src/libXext && \
-    git checkout d965a1a8ce9331d2aaf1c697a29455ad55171b36
-
-RUN git clone -n git://anongit.freedesktop.org/git/xorg/lib/libXfixes
-RUN cd libXfixes; git checkout 174a94975af710247719310cfc53bd13e1f3b44d
-RUN git clone --depth 1 https://github.com/intel/libva
-RUN git clone --depth 1 -b libvdpau-1.2 git://people.freedesktop.org/~aplattner/libvdpau
-RUN git clone --depth 1 https://chromium.googlesource.com/webm/libvpx
-RUN git clone --depth 1 https://github.com/xiph/ogg
-RUN git clone --depth 1 https://github.com/xiph/opus
-RUN git clone --depth 1 https://github.com/xiph/theora
-RUN git clone --depth 1 https://github.com/xiph/vorbis
-RUN git clone --depth 1 https://code.videolan.org/videolan/x264.git
-RUN git clone --depth 1 https://bitbucket.org/multicoreware/x265_git.git
-RUN mv x265_git x265
+
+RUN apt-get update && \
+    apt-get install -y \
+        make \
+        autoconf \
+        libtool \
+        build-essential \
+        libass-dev \
+        libfreetype6-dev \
+        libsdl1.2-dev \
+        libvdpau-dev \
+        libxcb1-dev \
+        libxcb-shm0-dev \
+        libdrm-dev \
+        pkg-config \
+        texinfo \
+        libbz2-dev \
+        zlib1g-dev \
+        yasm \
+        cmake \
+        mercurial \
+        wget \
+        xutils-dev \
+        libpciaccess-dev \
+        nasm \
+        meson \
+        rsync && \
+    curl \
+        -LO \
+        http://mirrors.kernel.org/ubuntu/pool/main/a/automake-1.16/automake_1.16.5-1.3_all.deb && \
+    apt install ./automake_1.16.5-1.3_all.deb && \
+    rm automake_1.16.5-1.3_all.deb
+
+RUN git clone \
+        --branch v1.2.8 \
+        --depth 1 \
+        https://github.com/alsa-project/alsa-lib.git
+
+RUN git clone \
+        --branch v2.0.2 \
+        --depth 1 \
+        https://github.com/mstorsjo/fdk-aac.git
+
+RUN git clone \
+        --branch libXext-1.3.5 \
+        --depth 1 \
+        https://gitlab.freedesktop.org/xorg/lib/libxext.git
+
+RUN git clone \
+        --depth 1 \
+        --branch 2.16.0 \
+        https://github.com/intel/libva
+
+RUN git clone \
+        --depth 1 \
+        --branch libvdpau-1.2 \
+        https://gitlab.freedesktop.org/vdpau/libvdpau.git
+
+RUN git clone \
+        --depth 1 \
+        --branch v1.12.0 \
+        https://chromium.googlesource.com/webm/libvpx
+
+RUN git clone \
+    --depth 1 \
+    --branch v1.3.5 \
+    https://github.com/xiph/ogg
+
+RUN git clone \
+        --depth 1 \
+        --branch v1.3.1 \
+        https://github.com/xiph/opus
+
+RUN git clone \
+        --depth 1 \
+        --branch v1.1.1 \
+        https://github.com/xiph/theora
+
+RUN git clone \
+        --depth 1 \
+        --branch v1.3.7 \
+        https://github.com/xiph/vorbis
+
+RUN git clone \
+        --depth 1 \
+        --branch v2.10.3 \
+        https://gitlab.gnome.org/GNOME/libxml2.git
+
+RUN git clone \
+        --branch n5.1.2 \
+        --depth 1 \
+        https://git.ffmpeg.org/ffmpeg.git
 
 COPY build.sh group_seed_corpus.py $SRC/