afl++ version comparison (google#1798)

This run
https://www.fuzzbench.com/reports/experimental/2023-03-23-various/index.html
showed that v3.14c has a better performance than the current 4.06a which
is baffling. The difference is huge!

so this experiment tests all afl++ versions starting with 3.13 up to
current with the same llvm version (it needs version 13 because 3.14 is
obviously not updated to current llvm changes).

It might be the llvm version it might be something else ... no clue, I
need more data.
Curous that this did not come up earlier as I test this rigorously
before a release.

fuzzers/aflplusplus_313/builder.Dockerfile

@@ -0,0 +1,56 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+RUN apt-get update && \
+    apt-get install -y \
+        build-essential \
+        python3-dev \
+        python3-setuptools \
+        automake \
+        cmake \
+        git \
+        flex \
+        bison \
+        libglib2.0-dev \
+        libpixman-1-dev \
+        cargo \
+        libgtk-3-dev \
+        # for QEMU mode
+        ninja-build \
+        gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev \
+        libstdc++-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-dev
+
+# Download afl++.
+RUN git clone https://github.com/AFLplusplus/AFLplusplus /afl && \
+    cd /afl && \
+    git checkout cda62bab0837f1cbec2a1245de32b04a09e61af5 || \
+    true
+
+RUN apt install -y lsb-release wget software-properties-common gnupg
+
+RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 13
+
+ENV LLVM_CONFIG llvm-config-13
+
+# Build without Python support as we don't need it.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN cd /afl && \
+    unset CFLAGS CXXFLAGS && \
+    export CC=clang AFL_NO_X86=1 && \
+    PYTHON_INCLUDE=/ make && \
+    make -C utils/aflpp_driver && \
+    cp utils/aflpp_driver/libAFLDriver.a /

fuzzers/aflplusplus_313/description.md

@@ -0,0 +1,14 @@
+# aflplusplus
+
+AFL++ fuzzer instance that has the following config active for all benchmarks:
+  - PCGUARD instrumentation 
+  - cmplog feature
+  - dict2file feature
+  - "fast" power schedule
+  - persistent mode + shared memory test cases
+
+Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/)
+
+[builder.Dockerfile](builder.Dockerfile)
+[fuzzer.py](fuzzer.py)
+[runner.Dockerfile](runner.Dockerfile)

fuzzers/aflplusplus_313/fuzzer.py

@@ -0,0 +1,282 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Integration code for AFLplusplus fuzzer."""
+
+import os
+import shutil
+
+from fuzzers.afl import fuzzer as afl_fuzzer
+from fuzzers import utils
+
+
+def get_cmplog_build_directory(target_directory):
+    """Return path to CmpLog target directory."""
+    return os.path.join(target_directory, 'cmplog')
+
+
+def get_uninstrumented_build_directory(target_directory):
+    """Return path to CmpLog target directory."""
+    return os.path.join(target_directory, 'uninstrumented')
+
+
+def build(*args):  # pylint: disable=too-many-branches,too-many-statements
+    """Build benchmark."""
+    # BUILD_MODES is not already supported by fuzzbench, meanwhile we provide
+    # a default configuration.
+
+    build_modes = list(args)
+    if 'BUILD_MODES' in os.environ:
+        build_modes = os.environ['BUILD_MODES'].split(',')
+
+    # Placeholder comment.
+    build_directory = os.environ['OUT']
+
+    # If nothing was set this is the default:
+    if not build_modes:
+        build_modes = ['tracepc', 'cmplog', 'dict2file']
+
+    # For bug type benchmarks we have to instrument via native clang pcguard :(
+    build_flags = os.environ['CFLAGS']
+
+    if build_flags.find(
+            'array-bounds'
+    ) != -1 and 'qemu' not in build_modes and 'classic' not in build_modes:
+        if 'gcc' not in build_modes:
+            build_modes[0] = 'native'
+
+    # Instrumentation coverage modes:
+    if 'lto' in build_modes:
+        os.environ['CC'] = '/afl/afl-clang-lto'
+        os.environ['CXX'] = '/afl/afl-clang-lto++'
+        edge_file = build_directory + '/aflpp_edges.txt'
+        os.environ['AFL_LLVM_DOCUMENT_IDS'] = edge_file
+        if os.path.isfile('/usr/local/bin/llvm-ranlib-13'):
+            os.environ['RANLIB'] = 'llvm-ranlib-13'
+            os.environ['AR'] = 'llvm-ar-13'
+            os.environ['AS'] = 'llvm-as-13'
+        elif os.path.isfile('/usr/local/bin/llvm-ranlib-12'):
+            os.environ['RANLIB'] = 'llvm-ranlib-12'
+            os.environ['AR'] = 'llvm-ar-12'
+            os.environ['AS'] = 'llvm-as-12'
+        else:
+            os.environ['RANLIB'] = 'llvm-ranlib'
+            os.environ['AR'] = 'llvm-ar'
+            os.environ['AS'] = 'llvm-as'
+    elif 'qemu' in build_modes:
+        os.environ['CC'] = 'clang'
+        os.environ['CXX'] = 'clang++'
+    elif 'gcc' in build_modes:
+        os.environ['CC'] = 'afl-gcc-fast'
+        os.environ['CXX'] = 'afl-g++-fast'
+        if build_flags.find('array-bounds') != -1:
+            os.environ['CFLAGS'] = '-fsanitize=address -O1'
+            os.environ['CXXFLAGS'] = '-fsanitize=address -O1'
+        else:
+            os.environ['CFLAGS'] = ''
+            os.environ['CXXFLAGS'] = ''
+            os.environ['CPPFLAGS'] = ''
+    else:
+        os.environ['CC'] = '/afl/afl-clang-fast'
+        os.environ['CXX'] = '/afl/afl-clang-fast++'
+
+    print('AFL++ build: ')
+    print(build_modes)
+
+    if 'qemu' in build_modes or 'symcc' in build_modes:
+        os.environ['CFLAGS'] = ' '.join(utils.NO_SANITIZER_COMPAT_CFLAGS)
+        cxxflags = [utils.LIBCPLUSPLUS_FLAG] + utils.NO_SANITIZER_COMPAT_CFLAGS
+        os.environ['CXXFLAGS'] = ' '.join(cxxflags)
+
+    if 'tracepc' in build_modes or 'pcguard' in build_modes:
+        os.environ['AFL_LLVM_USE_TRACE_PC'] = '1'
+    elif 'classic' in build_modes:
+        os.environ['AFL_LLVM_INSTRUMENT'] = 'CLASSIC'
+    elif 'native' in build_modes:
+        os.environ['AFL_LLVM_INSTRUMENT'] = 'LLVMNATIVE'
+
+    # Instrumentation coverage options:
+    # Do not use a fixed map location (LTO only)
+    if 'dynamic' in build_modes:
+        os.environ['AFL_LLVM_MAP_DYNAMIC'] = '1'
+    # Use a fixed map location (LTO only)
+    if 'fixed' in build_modes:
+        os.environ['AFL_LLVM_MAP_ADDR'] = '0x10000'
+    # Generate an extra dictionary.
+    if 'dict2file' in build_modes or 'native' in build_modes:
+        os.environ['AFL_LLVM_DICT2FILE'] = build_directory + '/afl++.dict'
+        #os.environ['AFL_LLVM_DICT2FILE_NO_MAIN'] = '1'
+    # Enable context sentitivity for LLVM mode (non LTO only)
+    if 'ctx' in build_modes:
+        os.environ['AFL_LLVM_CTX'] = '1'
+    # Enable N-gram coverage for LLVM mode (non LTO only)
+    if 'ngram2' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '2'
+    elif 'ngram3' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '3'
+    elif 'ngram4' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '4'
+    elif 'ngram5' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '5'
+    elif 'ngram6' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '6'
+    elif 'ngram7' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '7'
+    elif 'ngram8' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '8'
+    elif 'ngram16' in build_modes:
+        os.environ['AFL_LLVM_NGRAM_SIZE'] = '16'
+    if 'ctx1' in build_modes:
+        os.environ['AFL_LLVM_CTX_K'] = '1'
+    elif 'ctx2' in build_modes:
+        os.environ['AFL_LLVM_CTX_K'] = '2'
+    elif 'ctx3' in build_modes:
+        os.environ['AFL_LLVM_CTX_K'] = '3'
+    elif 'ctx4' in build_modes:
+        os.environ['AFL_LLVM_CTX_K'] = '4'
+
+    # Only one of the following OR cmplog
+    # enable laf-intel compare splitting
+    if 'laf' in build_modes:
+        os.environ['AFL_LLVM_LAF_SPLIT_SWITCHES'] = '1'
+        os.environ['AFL_LLVM_LAF_SPLIT_COMPARES'] = '1'
+        os.environ['AFL_LLVM_LAF_SPLIT_FLOATS'] = '1'
+        if 'autodict' not in build_modes:
+            os.environ['AFL_LLVM_LAF_TRANSFORM_COMPARES'] = '1'
+
+    if 'eclipser' in build_modes:
+        os.environ['FUZZER_LIB'] = '/libStandaloneFuzzTarget.a'
+    else:
+        os.environ['FUZZER_LIB'] = '/libAFLDriver.a'
+
+    # Some benchmarks like lcms. (see:
+    # https://github.com/mm2/Little-CMS/commit/ab1093539b4287c233aca6a3cf53b234faceb792#diff-f0e6d05e72548974e852e8e55dffc4ccR212)
+    # fail to compile if the compiler outputs things to stderr in unexpected
+    # cases. Prevent these failures by using AFL_QUIET to stop afl-clang-fast
+    # from writing AFL specific messages to stderr.
+    os.environ['AFL_QUIET'] = '1'
+    os.environ['AFL_MAP_SIZE'] = '2621440'
+
+    src = os.getenv('SRC')
+    work = os.getenv('WORK')
+
+    with utils.restore_directory(src), utils.restore_directory(work):
+        # Restore SRC to its initial state so we can build again without any
+        # trouble. For some OSS-Fuzz projects, build_benchmark cannot be run
+        # twice in the same directory without this.
+        utils.build_benchmark()
+
+    if 'cmplog' in build_modes and 'qemu' not in build_modes:
+
+        # CmpLog requires an build with different instrumentation.
+        new_env = os.environ.copy()
+        new_env['AFL_LLVM_CMPLOG'] = '1'
+
+        # For CmpLog build, set the OUT and FUZZ_TARGET environment
+        # variable to point to the new CmpLog build directory.
+        cmplog_build_directory = get_cmplog_build_directory(build_directory)
+        os.mkdir(cmplog_build_directory)
+        new_env['OUT'] = cmplog_build_directory
+        fuzz_target = os.getenv('FUZZ_TARGET')
+        if fuzz_target:
+            new_env['FUZZ_TARGET'] = os.path.join(cmplog_build_directory,
+                                                  os.path.basename(fuzz_target))
+
+        print('Re-building benchmark for CmpLog fuzzing target')
+        utils.build_benchmark(env=new_env)
+
+    if 'symcc' in build_modes:
+
+        symcc_build_directory = get_uninstrumented_build_directory(
+            build_directory)
+        os.mkdir(symcc_build_directory)
+
+        # symcc requires an build with different instrumentation.
+        new_env = os.environ.copy()
+        new_env['CC'] = '/symcc/build/symcc'
+        new_env['CXX'] = '/symcc/build/sym++'
+        new_env['SYMCC_OUTPUT_DIR'] = '/tmp'
+        new_env['CXXFLAGS'] = new_env['CXXFLAGS'].replace('-stlib=libc++', '')
+        new_env['FUZZER_LIB'] = '/libfuzzer-harness.o'
+        new_env['OUT'] = symcc_build_directory
+        new_env['SYMCC_LIBCXX_PATH'] = '/libcxx_native_build'
+        new_env['SYMCC_NO_SYMBOLIC_INPUT'] = '1'
+        new_env['SYMCC_SILENT'] = '1'
+
+        # For symcc build, set the OUT and FUZZ_TARGET environment
+        # variable to point to the new symcc build directory.
+        new_env['OUT'] = symcc_build_directory
+        fuzz_target = os.getenv('FUZZ_TARGET')
+        if fuzz_target:
+            new_env['FUZZ_TARGET'] = os.path.join(symcc_build_directory,
+                                                  os.path.basename(fuzz_target))
+
+        print('Re-building benchmark for symcc fuzzing target')
+        utils.build_benchmark(env=new_env)
+
+    shutil.copy('/afl/afl-fuzz', build_directory)
+    if os.path.exists('/afl/afl-qemu-trace'):
+        shutil.copy('/afl/afl-qemu-trace', build_directory)
+    if os.path.exists('/aflpp_qemu_driver_hook.so'):
+        shutil.copy('/aflpp_qemu_driver_hook.so', build_directory)
+    if os.path.exists('/get_frida_entry.sh'):
+        shutil.copy('/afl/afl-frida-trace.so', build_directory)
+        shutil.copy('/get_frida_entry.sh', build_directory)
+
+
+# pylint: disable=too-many-arguments
+def fuzz(input_corpus,
+         output_corpus,
+         target_binary,
+         flags=tuple(),
+         skip=False,
+         no_cmplog=False):  # pylint: disable=too-many-arguments
+    """Run fuzzer."""
+    # Calculate CmpLog binary path from the instrumented target binary.
+    target_binary_directory = os.path.dirname(target_binary)
+    cmplog_target_binary_directory = (
+        get_cmplog_build_directory(target_binary_directory))
+    target_binary_name = os.path.basename(target_binary)
+    cmplog_target_binary = os.path.join(cmplog_target_binary_directory,
+                                        target_binary_name)
+
+    afl_fuzzer.prepare_fuzz_environment(input_corpus)
+    # decomment this to enable libdislocator.
+    # os.environ['AFL_ALIGNED_ALLOC'] = '1' # align malloc to max_align_t
+    # os.environ['AFL_PRELOAD'] = '/afl/libdislocator.so'
+
+    flags = list(flags)
+
+    if os.path.exists('./afl++.dict'):
+        flags += ['-x', './afl++.dict']
+
+    # Move the following to skip for upcoming _double tests:
+    if os.path.exists(cmplog_target_binary) and no_cmplog is False:
+        flags += ['-c', cmplog_target_binary]
+
+    #os.environ['AFL_IGNORE_TIMEOUTS'] = '1'
+    os.environ['AFL_IGNORE_UNKNOWN_ENVS'] = '1'
+    os.environ['AFL_FAST_CAL'] = '1'
+    os.environ['AFL_NO_WARN_INSTABILITY'] = '1'
+
+    if not skip:
+        os.environ['AFL_DISABLE_TRIM'] = '1'
+        os.environ['AFL_CMPLOG_ONLY_NEW'] = '1'
+        if 'ADDITIONAL_ARGS' in os.environ:
+            flags += os.environ['ADDITIONAL_ARGS'].split(' ')
+
+    afl_fuzzer.run_afl_fuzz(input_corpus,
+                            output_corpus,
+                            target_binary,
+                            additional_flags=flags)

fuzzers/aflplusplus_313/runner.Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/fuzzbench/base-image
+
+# This makes interactive docker runs painless:
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out"
+#ENV AFL_MAP_SIZE=2621440
+ENV PATH="$PATH:/out"
+ENV AFL_SKIP_CPUFREQ=1
+ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1
+ENV AFL_TESTCACHE_SIZE=2