Update fuzzer Pastis (google#1843)

* Install PASTIS from PIP (remove packages).
* Update Honggfuzz to latest oss-fuzz version.
* Add support for dictionaries (AFL++ and Honggfuzz).
* Add support for cmplog (AFL++).

fuzzers/pastis/builder.Dockerfile

@@ -53,17 +53,17 @@ RUN apt-get update -y && \
 
 # Copy honggfuzz PASTIS patch.
 RUN mkdir /patches
-COPY patches/honggfuzz-b1e09be-pastis.patch /patches
+COPY patches/honggfuzz-3a8f2ae-pastis.patch /patches
 
-# Donwload honggfuzz oss-fuzz version (commit b1e09bec344728f52019ed78907236cabe1c6c8a)
+# Donwload honggfuzz oss-fuzz version (commit 3a8f2ae41604b6696e7bd5e5cdc0129ce49567c0)
 RUN git clone https://github.com/google/honggfuzz.git /honggfuzz && \
     cd /honggfuzz && \
-    git checkout b1e09bec344728f52019ed78907236cabe1c6c8a && \
+    git checkout 3a8f2ae41604b6696e7bd5e5cdc0129ce49567c0 && \
     cd ..
 
 # Apply PASTIS patch.
 RUN cd / && \
-    patch -s -p0 < /patches/honggfuzz-b1e09be-pastis.patch
+    patch -s -p0 < /patches/honggfuzz-3a8f2ae-pastis.patch
 
 # Set CFLAGS use honggfuzz's defaults except for -mnative which can build CPU
 # dependent code that may not work on the machines we actually fuzz on.

fuzzers/pastis/fuzzer.py

@@ -87,7 +87,7 @@ def build_aflpp():
         # Restore SRC to its initial state so we can build again without any
         # trouble. For some OSS-Fuzz projects, build_benchmark cannot be run
         # twice in the same directory without this.
-        aflplusplus_fuzzer.build('classic')
+        aflplusplus_fuzzer.build()
 
     os.environ['OUT'] = out_dir
 
@@ -211,6 +211,26 @@ def fuzz(input_corpus, output_corpus, target_binary):
     shutil.copy(os.path.join(os.environ['OUT'], target_binary_name),
                 os.path.join(targets_dir, target_binary_name + '_tt'))
 
+    # Copy and rename the dictionary file in case it exists (AFL++).
+    dictionary_path = os.path.join(aflpp_target_dir, 'afl++.dict')
+    if os.path.exists(dictionary_path):
+        shutil.copy(
+            dictionary_path,
+            os.path.join(targets_dir, target_binary_name + '_aflpp.dict'))
+
+    # Copy and rename the dictionary file in case it exists (Honggfuzz).
+    dictionary_path = utils.get_dictionary_path(target_binary)
+    if dictionary_path and os.path.exists(dictionary_path):
+        shutil.copy(dictionary_path,
+                    os.path.join(targets_dir, target_binary_name + '_hf.dict'))
+
+    # Copy cmplog directory if it exists.
+    cmplog_path = os.path.join(aflpp_target_dir, 'cmplog', target_binary_name)
+    if os.path.exists(cmplog_path):
+        shutil.copy(
+            cmplog_path,
+            os.path.join(targets_dir, target_binary_name + '_aflpp.cmplog'))
+
     # Prepare command-line string.
     command = [
         'pastis-benchmark',

fuzzers/pastis/patches/honggfuzz-b1e09be-pastis.patch → fuzzers/pastis/patches/honggfuzz-3a8f2ae-pastis.patch

@@ -1,6 +1,6 @@
-diff -ruN honggfuzz/cmdline.c honggfuzz-b1e09be-pastis/cmdline.c
+diff -ruN honggfuzz/cmdline.c honggfuzz-3a8f2ae-pastis/cmdline.c
 --- honggfuzz/cmdline.c	2022-06-23 17:27:05.000000000 -0300
-+++ honggfuzz-b1e09be-pastis/cmdline.c	2023-01-13 16:48:23.617159827 -0300
++++ honggfuzz-3a8f2ae-pastis/cmdline.c	2023-01-13 16:48:23.617159827 -0300
 @@ -525,6 +525,8 @@
          { { "export_feedback", no_argument, NULL, 0x10E }, "Export the coverage feedback structure as ./hfuzz-feedback" },
          { { "const_feedback", required_argument, NULL, 0x112 }, "Use constant integer/string values from fuzzed programs to mangle input files via a dynamic dictionary (default: true)" },
@@ -23,9 +23,9 @@ diff -ruN honggfuzz/cmdline.c honggfuzz-b1e09be-pastis/cmdline.c
              default:
                  cmdlineHelp(argv[0], custom_opts);
                  return false;
-diff -ruN honggfuzz/fuzz.c honggfuzz-b1e09be-pastis/fuzz.c
+diff -ruN honggfuzz/fuzz.c honggfuzz-3a8f2ae-pastis/fuzz.c
 --- honggfuzz/fuzz.c	2022-06-23 17:27:05.000000000 -0300
-+++ honggfuzz-b1e09be-pastis/fuzz.c	2023-01-13 16:48:50.349198188 -0300
++++ honggfuzz-3a8f2ae-pastis/fuzz.c	2023-01-13 16:48:50.349198188 -0300
 @@ -229,6 +229,39 @@
              softNewPC, softNewCmp, run->hwCnts.cpuInstrCnt, run->hwCnts.cpuBranchCnt,
              run->hwCnts.bbCnt, softCurEdge, softCurPC, softCurCmp);
@@ -66,9 +66,9 @@ diff -ruN honggfuzz/fuzz.c honggfuzz-b1e09be-pastis/fuzz.c
          /* Update per-input coverage metrics */
          run->dynfile->cov[0] = softCurEdge + softCurPC + run->hwCnts.bbCnt;
          run->dynfile->cov[1] = softCurCmp;
-diff -ruN honggfuzz/honggfuzz.c honggfuzz-b1e09be-pastis/honggfuzz.c
+diff -ruN honggfuzz/honggfuzz.c honggfuzz-3a8f2ae-pastis/honggfuzz.c
 --- honggfuzz/honggfuzz.c	2022-06-23 17:27:05.000000000 -0300
-+++ honggfuzz-b1e09be-pastis/honggfuzz.c	2023-01-13 16:49:10.965232496 -0300
++++ honggfuzz-3a8f2ae-pastis/honggfuzz.c	2023-01-13 16:49:10.965232496 -0300
 @@ -23,12 +23,14 @@
   */
 
@@ -125,9 +125,9 @@ diff -ruN honggfuzz/honggfuzz.c honggfuzz-b1e09be-pastis/honggfuzz.c
 
      printSummary(&hfuzz);
 
-diff -ruN honggfuzz/honggfuzz.h honggfuzz-b1e09be-pastis/honggfuzz.h
+diff -ruN honggfuzz/honggfuzz.h honggfuzz-3a8f2ae-pastis/honggfuzz.h
 --- honggfuzz/honggfuzz.h	2022-06-23 17:27:05.000000000 -0300
-+++ honggfuzz-b1e09be-pastis/honggfuzz.h	2023-01-13 16:49:18.817246608 -0300
++++ honggfuzz-3a8f2ae-pastis/honggfuzz.h	2023-01-13 16:49:18.817246608 -0300
 @@ -216,6 +216,9 @@
          dynfile_t*  dynfileq2Current;
          TAILQ_HEAD(dyns_t, _dynfile_t) dynfileq;
@@ -138,9 +138,9 @@ diff -ruN honggfuzz/honggfuzz.h honggfuzz-b1e09be-pastis/honggfuzz.h
      } io;
      struct {
          int                argc;
-diff -ruN honggfuzz/input.c honggfuzz-b1e09be-pastis/input.c
+diff -ruN honggfuzz/input.c honggfuzz-3a8f2ae-pastis/input.c
 --- honggfuzz/input.c	2022-06-23 17:27:05.000000000 -0300
-+++ honggfuzz-b1e09be-pastis/input.c	2023-01-13 16:49:38.961285357 -0300
++++ honggfuzz-3a8f2ae-pastis/input.c	2023-01-13 16:49:38.961285357 -0300
 @@ -31,6 +31,7 @@
  #include <stdio.h>
  #include <stdlib.h>
@@ -278,9 +278,9 @@ diff -ruN honggfuzz/input.c honggfuzz-b1e09be-pastis/input.c
  const uint8_t* input_getRandomInputAsBuf(run_t* run, size_t* len) {
      if (run->global->feedback.dynFileMethod == _HF_DYNFILE_NONE) {
          LOG_W(
-diff -ruN honggfuzz/input.h honggfuzz-b1e09be-pastis/input.h
+diff -ruN honggfuzz/input.h honggfuzz-3a8f2ae-pastis/input.h
 --- honggfuzz/input.h	2022-06-23 17:27:05.000000000 -0300
-+++ honggfuzz-b1e09be-pastis/input.h	2023-01-13 16:49:57.593324375 -0300
++++ honggfuzz-3a8f2ae-pastis/input.h	2023-01-13 16:49:57.593324375 -0300
 @@ -49,5 +49,7 @@
  extern bool           input_prepareExternalFile(run_t* run);
  extern bool           input_postProcessFile(run_t* run, const char* cmd);

fuzzers/pastis/runner.Dockerfile

@@ -26,14 +26,7 @@ RUN DEBIAN_FRONTEND="noninteractive" \
     apt-get install -y --no-install-suggests --no-install-recommends \
         libmagic-dev
 
-# Copy packages.
-COPY packages /packages
-COPY scripts/pastis-install-packages.sh /packages
-
-# Install PASTIS packages.
-RUN cd /packages && \
-    chmod +x /packages/pastis-install-packages.sh && \
-    /packages/pastis-install-packages.sh ./
+RUN pip install pastis-framework
 
 #
 # AFLplusplus