Seed experiment

fuzzers/afl_empty_seeds/builder.Dockerfile

@@ -0,0 +1,33 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Download and compile AFL v2.57b.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN git clone \
+        --depth 1 \
+        --branch v2.57b \
+        https://github.com/google/AFL.git /afl && \
+    cd /afl && \
+    CFLAGS= CXXFLAGS= AFL_NO_X86=1 make
+
+# Use afl_driver.cpp from LLVM as our fuzzing library.
+RUN apt-get update && \
+    apt-get install wget -y && \
+    wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \
+    clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \
+    clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \
+    ar r /libAFL.a *.o

fuzzers/afl_empty_seeds/fuzzer.py

@@ -0,0 +1,152 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Integration code for AFL fuzzer."""
+
+import json
+import os
+import shutil
+import subprocess
+
+from fuzzers import utils
+
+
+def prepare_build_environment():
+    """Set environment variables used to build targets for AFL-based
+    fuzzers."""
+    cflags = ['-fsanitize-coverage=trace-pc-guard']
+    utils.append_flags('CFLAGS', cflags)
+    utils.append_flags('CXXFLAGS', cflags)
+
+    os.environ['CC'] = 'clang'
+    os.environ['CXX'] = 'clang++'
+    os.environ['FUZZER_LIB'] = '/libAFL.a'
+
+
+def build():
+    """Build benchmark."""
+    prepare_build_environment()
+
+    utils.build_benchmark()
+
+    print('[post_build] Copying afl-fuzz to $OUT directory')
+    # Copy out the afl-fuzz binary as a build artifact.
+    shutil.copy('/afl/afl-fuzz', os.environ['OUT'])
+
+
+def get_stats(output_corpus, fuzzer_log):  # pylint: disable=unused-argument
+    """Gets fuzzer stats for AFL."""
+    # Get a dictionary containing the stats AFL reports.
+    stats_file = os.path.join(output_corpus, 'fuzzer_stats')
+    if not os.path.exists(stats_file):
+        print('Can\'t find fuzzer_stats')
+        return '{}'
+    with open(stats_file, encoding='utf-8') as file_handle:
+        stats_file_lines = file_handle.read().splitlines()
+    stats_file_dict = {}
+    for stats_line in stats_file_lines:
+        key, value = stats_line.split(': ')
+        stats_file_dict[key.strip()] = value.strip()
+
+    # Report to FuzzBench the stats it accepts.
+    stats = {'execs_per_sec': float(stats_file_dict['execs_per_sec'])}
+    return json.dumps(stats)
+
+
+def prepare_fuzz_environment(input_corpus):
+    """Prepare to fuzz with AFL or another AFL-based fuzzer."""
+    # Tell AFL to not use its terminal UI so we get usable logs.
+    os.environ['AFL_NO_UI'] = '1'
+    # Skip AFL's CPU frequency check (fails on Docker).
+    os.environ['AFL_SKIP_CPUFREQ'] = '1'
+    # No need to bind affinity to one core, Docker enforces 1 core usage.
+    os.environ['AFL_NO_AFFINITY'] = '1'
+    # AFL will abort on startup if the core pattern sends notifications to
+    # external programs. We don't care about this.
+    os.environ['AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES'] = '1'
+    # Don't exit when crashes are found. This can happen when corpus from
+    # OSS-Fuzz is used.
+    os.environ['AFL_SKIP_CRASHES'] = '1'
+    # Shuffle the queue
+    os.environ['AFL_SHUFFLE_QUEUE'] = '1'
+
+    # AFL needs at least one non-empty seed to start.
+    utils.create_seed_file_for_empty_corpus(input_corpus)
+
+
+def check_skip_det_compatible(additional_flags):
+    """ Checks if additional flags are compatible with '-d' option"""
+    # AFL refuses to take in '-d' with '-M' or '-S' options for parallel mode.
+    # (cf. https://github.com/google/AFL/blob/8da80951/afl-fuzz.c#L7477)
+    if '-M' in additional_flags or '-S' in additional_flags:
+        return False
+    return True
+
+
+def prepare_empty_corpus(input_corpus):
+    if os.path.exists(input_corpus):
+        shutil.rmtree(input_corpus)
+    os.makedirs(input_corpus)
+    with open(os.path.join(input_corpus, 'a'), 'wb') as f:
+        f.write(b'a')
+
+
+def run_afl_fuzz(input_corpus,
+                 output_corpus,
+                 target_binary,
+                 additional_flags=None,
+                 hide_output=False):
+    """Run afl-fuzz."""
+
+    prepare_empty_corpus(input_corpus)
+    # Spawn the afl fuzzing process.
+    print('[run_afl_fuzz] Running target with afl-fuzz')
+    command = [
+        './afl-fuzz',
+        '-i',
+        input_corpus,
+        '-o',
+        output_corpus,
+        # Use no memory limit as ASAN doesn't play nicely with one.
+        '-m',
+        'none',
+        '-t',
+        '1000+',  # Use same default 1 sec timeout, but add '+' to skip hangs.
+    ]
+    # Use '-d' to skip deterministic mode, as long as it it compatible with
+    # additional flags.
+    if not additional_flags or check_skip_det_compatible(additional_flags):
+        command.append('-d')
+    if additional_flags:
+        command.extend(additional_flags)
+    dictionary_path = utils.get_dictionary_path(target_binary)
+
+    # if dictionary_path:
+    #     command.extend(['-x', dictionary_path])
+    command += [
+        '--',
+        target_binary,
+        # Pass INT_MAX to afl the maximize the number of persistent loops it
+        # performs.
+        '2147483647'
+    ]
+    print('[run_afl_fuzz] Running command: ' + ' '.join(command))
+    output_stream = subprocess.DEVNULL if hide_output else None
+    subprocess.check_call(command, stdout=output_stream, stderr=output_stream)
+
+
+def fuzz(input_corpus, output_corpus, target_binary):
+    """Run afl-fuzz on target."""
+    prepare_fuzz_environment(input_corpus)
+
+    run_afl_fuzz(input_corpus, output_corpus, target_binary)

fuzzers/afl_empty_seeds/runner.Dockerfile

@@ -0,0 +1,15 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/fuzzbench/base-image

fuzzers/aflplusplus_empty_seeds/builder.Dockerfile

@@ -0,0 +1,49 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+RUN apt-get update && \
+    apt-get install -y \
+        build-essential \
+        python3-dev \
+        python3-setuptools \
+        automake \
+        cmake \
+        git \
+        flex \
+        bison \
+        libglib2.0-dev \
+        libpixman-1-dev \
+        cargo \
+        libgtk-3-dev \
+        # for QEMU mode
+        ninja-build \
+        gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev \
+        libstdc++-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-dev
+
+# Download afl++.
+RUN git clone -b dev https://github.com/AFLplusplus/AFLplusplus /afl && \
+    cd /afl && \
+    git checkout 56d5aa3101945e81519a3fac8783d0d8fad82779 || \
+    true
+
+# Build without Python support as we don't need it.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN cd /afl && \
+    unset CFLAGS CXXFLAGS && \
+    export CC=clang AFL_NO_X86=1 && \
+    PYTHON_INCLUDE=/ make && \
+    cp utils/aflpp_driver/libAFLDriver.a /

fuzzers/aflplusplus_empty_seeds/description.md

@@ -0,0 +1,14 @@
+# aflplusplus
+
+AFL++ fuzzer instance that has the following config active for all benchmarks:
+  - PCGUARD instrumentation 
+  - cmplog feature
+  - dict2file feature
+  - "fast" power schedule
+  - persistent mode + shared memory test cases
+
+Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/)
+
+[builder.Dockerfile](builder.Dockerfile)
+[fuzzer.py](fuzzer.py)
+[runner.Dockerfile](runner.Dockerfile)